From 48f67a284e50da56f21dca5bf37951fb3122ad6a Mon Sep 17 00:00:00 2001 From: ITQ Date: Tue, 18 Aug 2026 23:11:31 +0300 Subject: [PATCH] feat(terraform): updated configs for yc lab --- .../main/configs/cloud_init/main_master.yaml | 20 +++++++++++++++ terraform/main/configs/cloud_init/master.yaml | 20 +++++++++++++++ terraform/main/configs/cloud_init/worker.yaml | 3 +++ terraform/main/{ => etc}/api_gateway.tf | 0 terraform/main/{ => etc}/certificate.tf | 0 terraform/main/{ => etc}/object_storage.tf | 0 terraform/main/instance.tf | 25 ++++++++++++------- terraform/main/instance_group.tf | 25 +++++++++++-------- terraform/main/loadbalancer.tf | 15 ++++++++--- terraform/main/network.tf | 10 ++++++++ terraform/main/outputs.tf | 2 +- terraform/main/variables.tf | 5 ++++ terraform/scripts/env | 1 + 13 files changed, 101 insertions(+), 25 deletions(-) rename terraform/main/{ => etc}/api_gateway.tf (100%) rename terraform/main/{ => etc}/certificate.tf (100%) rename terraform/main/{ => etc}/object_storage.tf (100%) diff --git a/terraform/main/configs/cloud_init/main_master.yaml b/terraform/main/configs/cloud_init/main_master.yaml index 9c2135c..0f61a73 100644 --- a/terraform/main/configs/cloud_init/main_master.yaml +++ b/terraform/main/configs/cloud_init/main_master.yaml @@ -44,10 +44,30 @@ runcmd: resize2fs $DISK - | curl -sfL https://get.k3s.io | INSTALL_K3S_CHANNEL=stable INSTALL_K3S_EXEC="server \ + -v 2 \ --cluster-init \ + --bind-address 0.0.0.0 \ + --https-listen-port 6443 \ --token=${k3s_token} \ --node-name=`hostname -f` \ --node-taint CriticalAddonsOnly=true:NoExecute \ + --cluster-cidr 10.96.0.0/16 \ + --service-cidr 10.97.0.0/16 \ + --cluster-dns 10.97.0.10 \ + --tls-san ${k3s_fqdn} \ + --tls-san-security \ + --cluster-domain cluster.local \ + --disable coredns,traefik,local-storage,metrics-server,runtimes \ + --disable-network-policy \ + --disable-kube-proxy \ + --flannel-backend none \ + --egress-selector-mode agent \ + --servicelb-namespace kube-system \ + --nonroot-devices \ + --secrets-encryption-provider secretbox \ + --lb-server-port 6444 \ + --write-kubeconfig-mode 0600 \ + --write-kubeconfig-group root \ --write-kubeconfig ${k3s_dir}/kubeconfig/config \ --image-credential-provider-bin-dir /var/lib/rancher/credentialprovider/bin \ --image-credential-provider-config /var/lib/rancher/credentialprovider/config.yaml \ diff --git a/terraform/main/configs/cloud_init/master.yaml b/terraform/main/configs/cloud_init/master.yaml index 1ca7d2b..1c5baca 100644 --- a/terraform/main/configs/cloud_init/master.yaml +++ b/terraform/main/configs/cloud_init/master.yaml @@ -44,9 +44,29 @@ runcmd: resize2fs $DISK - | curl -sfL https://get.k3s.io | INSTALL_K3S_CHANNEL=stable INSTALL_K3S_EXEC="server \ + -v 2 \ + --bind-address 0.0.0.0 \ + --https-listen-port 6443 \ --server=https://${k3s_master_ip}:6443 \ --token=${k3s_token} \ --node-name=`hostname -f` \ + --cluster-cidr 10.96.0.0/16 \ + --service-cidr 10.97.0.0/16 \ + --cluster-dns 10.97.0.10 \ + --tls-san ${k3s_fqdn} \ + --tls-san-security \ + --cluster-domain cluster.local \ + --disable coredns,traefik,local-storage,metrics-server,runtimes \ + --disable-network-policy \ + --disable-kube-proxy \ + --flannel-backend none \ + --egress-selector-mode agent \ + --servicelb-namespace kube-system \ + --nonroot-devices \ + --secrets-encryption-provider secretbox \ + --lb-server-port 6444 \ + --write-kubeconfig-mode 0600 \ + --write-kubeconfig-group root \ --write-kubeconfig ${k3s_dir}/kubeconfig/config \ --image-credential-provider-bin-dir /var/lib/rancher/credentialprovider/bin \ --image-credential-provider-config /var/lib/rancher/credentialprovider/config.yaml \ diff --git a/terraform/main/configs/cloud_init/worker.yaml b/terraform/main/configs/cloud_init/worker.yaml index 61fc6dc..baf847b 100644 --- a/terraform/main/configs/cloud_init/worker.yaml +++ b/terraform/main/configs/cloud_init/worker.yaml @@ -44,9 +44,12 @@ runcmd: resize2fs $DISK - | curl -sfL https://get.k3s.io | INSTALL_K3S_CHANNEL=stable INSTALL_K3S_EXEC="agent \ + -v 2 \ --server=https://${k3s_master_ip}:6443 \ --token=${k3s_token} \ --node-name=`hostname -f` \ + --nonroot-devices \ + --lb-server-port 6444 \ --image-credential-provider-bin-dir /var/lib/rancher/credentialprovider/bin \ --image-credential-provider-config /var/lib/rancher/credentialprovider/config.yaml \ --data-dir=${k3s_dir}" sh - diff --git a/terraform/main/api_gateway.tf b/terraform/main/etc/api_gateway.tf similarity index 100% rename from terraform/main/api_gateway.tf rename to terraform/main/etc/api_gateway.tf diff --git a/terraform/main/certificate.tf b/terraform/main/etc/certificate.tf similarity index 100% rename from terraform/main/certificate.tf rename to terraform/main/etc/certificate.tf diff --git a/terraform/main/object_storage.tf b/terraform/main/etc/object_storage.tf similarity index 100% rename from terraform/main/object_storage.tf rename to terraform/main/etc/object_storage.tf diff --git a/terraform/main/instance.tf b/terraform/main/instance.tf index ded7826..63dee15 100644 --- a/terraform/main/instance.tf +++ b/terraform/main/instance.tf @@ -1,3 +1,14 @@ +resource "terraform_data" "k8s_main_master_user_data" { + input = templatefile("${path.module}/configs/cloud_init/main_master.yaml", { + k3s_credential_provider_config = base64encode(file("${path.module}/configs/k3s/credentialprovider.yaml")) + k3s_credential_provider = base64encode(file("${path.module}/configs/k3s/yc-credential-provider")) + k3s_dir = local.k3s_data_dir + k3s_token = random_password.k3s_token.result + k3s_fqdn = var.cluster_domain + yc_cloud_id = var.cloud_id + }) +} + resource "yandex_compute_instance" "k8s_main_master" { name = "k8s-master" hostname = local.k8s_main_master_fqdn @@ -15,9 +26,11 @@ resource "yandex_compute_instance" "k8s_main_master" { } lifecycle { - prevent_destroy = true ignore_changes = [ - hostname, + hostname, boot_disk[0].initialize_params[0].image_id + ] + replace_triggered_by = [ + terraform_data.k8s_main_master_user_data ] } @@ -64,13 +77,7 @@ resource "yandex_compute_instance" "k8s_main_master" { enable-oslogin = true serial-port-enable = 1 install-unified-agent = 0 - user-data = templatefile("${path.module}/configs/cloud_init/main_master.yaml", { - k3s_credential_provider_config = base64encode(file("${path.module}/configs/k3s/credentialprovider.yaml")) - k3s_credential_provider = base64encode(file("${path.module}/configs/k3s/yc-credential-provider")) - k3s_dir = local.k3s_data_dir - k3s_token = random_password.k3s_token.result - yc_cloud_id = var.cloud_id - }) + user-data = terraform_data.k8s_main_master_user_data.output } metadata_options { diff --git a/terraform/main/instance_group.tf b/terraform/main/instance_group.tf index 16e8355..919ad4d 100644 --- a/terraform/main/instance_group.tf +++ b/terraform/main/instance_group.tf @@ -20,7 +20,7 @@ resource "yandex_compute_instance_group" "k8s_master" { platform_id = "standard-v3" resources { - core_fraction = 50 + core_fraction = 100 cores = 2 gpus = 0 memory = 4 @@ -39,8 +39,8 @@ resource "yandex_compute_instance_group" "k8s_master" { device_name = "data" initialize_params { description = "Data disk" - size = 20 - type = "network-hdd" + size = 10 + type = "network-ssd" } mode = "READ_WRITE" } @@ -51,6 +51,7 @@ resource "yandex_compute_instance_group" "k8s_master" { yandex_vpc_subnet.default_ru_central1_a.id, yandex_vpc_subnet.default_ru_central1_b.id, yandex_vpc_subnet.default_ru_central1_d.id, + yandex_vpc_subnet.default_ru_central1_e.id, ] nat = false } @@ -74,6 +75,7 @@ resource "yandex_compute_instance_group" "k8s_master" { k3s_dir = local.k3s_data_dir k3s_token = random_password.k3s_token.result k3s_master_ip = local.k8s_main_master_fqdn + k3s_fqdn = var.cluster_domain yc_cloud_id = var.cloud_id }) } @@ -81,11 +83,11 @@ resource "yandex_compute_instance_group" "k8s_master" { scale_policy { fixed_scale { - size = 3 + size = 2 } } allocation_policy { - zones = ["ru-central1-a", "ru-central1-b", "ru-central1-d"] + zones = ["ru-central1-a", "ru-central1-b", "ru-central1-d", "ru-central1-e"] } deploy_policy { max_creating = 3 @@ -144,10 +146,10 @@ resource "yandex_compute_instance_group" "k8s_worker" { platform_id = "standard-v3" resources { - core_fraction = 50 - cores = 2 + core_fraction = 100 + cores = 4 gpus = 0 - memory = 2 + memory = 8 } boot_disk { @@ -163,8 +165,8 @@ resource "yandex_compute_instance_group" "k8s_worker" { device_name = "data" initialize_params { description = "Data disk" - size = 20 - type = "network-hdd" + size = 10 + type = "network-ssd" } mode = "READ_WRITE" } @@ -175,6 +177,7 @@ resource "yandex_compute_instance_group" "k8s_worker" { yandex_vpc_subnet.default_ru_central1_a.id, yandex_vpc_subnet.default_ru_central1_b.id, yandex_vpc_subnet.default_ru_central1_d.id, + yandex_vpc_subnet.default_ru_central1_e.id, ] nat = false } @@ -209,7 +212,7 @@ resource "yandex_compute_instance_group" "k8s_worker" { } } allocation_policy { - zones = ["ru-central1-a", "ru-central1-b", "ru-central1-d"] + zones = ["ru-central1-a", "ru-central1-b", "ru-central1-d", "ru-central1-e"] } deploy_policy { max_creating = 3 diff --git a/terraform/main/loadbalancer.tf b/terraform/main/loadbalancer.tf index 23a700b..43dbc55 100644 --- a/terraform/main/loadbalancer.tf +++ b/terraform/main/loadbalancer.tf @@ -27,13 +27,21 @@ resource "yandex_lb_network_load_balancer" "k8s_lb" { ip_version = "ipv4" } } + listener { + name = "postgres-rw" + port = 5432 + external_address_spec { + address = yandex_vpc_address.nlb.external_ipv4_address[0].address + ip_version = "ipv4" + } + } attached_target_group { target_group_id = yandex_lb_target_group.k8s_main_master.id // Verifies that Traefik daeomnset has started on node healthcheck { - name = "main-master-traefik" + name = "main-master" healthy_threshold = 2 unhealthy_threshold = 2 interval = 5 @@ -47,9 +55,8 @@ resource "yandex_lb_network_load_balancer" "k8s_lb" { attached_target_group { target_group_id = yandex_compute_instance_group.k8s_master.load_balancer[0].target_group_id - // Verifies that Traefik daeomnset has started on node healthcheck { - name = "master-instance-group-traefik" + name = "master-instance-group" healthy_threshold = 2 unhealthy_threshold = 2 interval = 5 @@ -65,7 +72,7 @@ resource "yandex_lb_network_load_balancer" "k8s_lb" { // Verifies that Traefik daeomnset has started on node healthcheck { - name = "worker-instance-group-traefik" + name = "worker-instance-group" healthy_threshold = 2 unhealthy_threshold = 2 interval = 5 diff --git a/terraform/main/network.tf b/terraform/main/network.tf index c5a1849..4b52563 100644 --- a/terraform/main/network.tf +++ b/terraform/main/network.tf @@ -51,6 +51,16 @@ resource "yandex_vpc_subnet" "default_ru_central1_d" { zone = "ru-central1-d" } +resource "yandex_vpc_subnet" "default_ru_central1_e" { + name = "default-ru-central1-e" + description = "Default subnet for ru-central1-e" + + network_id = yandex_vpc_network.default.id + route_table_id = yandex_vpc_route_table.default.id + v4_cidr_blocks = ["10.5.0.0/16"] + zone = "ru-central1-e" +} + resource "yandex_vpc_address" "nlb" { name = "nlb" description = "External IP address for network lb" diff --git a/terraform/main/outputs.tf b/terraform/main/outputs.tf index 1eea07a..44fe2ef 100644 --- a/terraform/main/outputs.tf +++ b/terraform/main/outputs.tf @@ -45,7 +45,7 @@ output "registry_pull_sa_key" { # }) # sensitive = true # } -# + # output "lets_encrypt_challenges" { # value = yandex_cm_certificate.default.challenges # } diff --git a/terraform/main/variables.tf b/terraform/main/variables.tf index ad888f5..fd385de 100644 --- a/terraform/main/variables.tf +++ b/terraform/main/variables.tf @@ -7,6 +7,11 @@ variable "domain" { type = string } +variable "cluster_domain" { + type = string + default = "k8s.nlb.infra.itqdev.xyz" +} + variable "frontend_domain" { description = "Domain to serve frontend from (must equal domain or *.domain), default: domain" type = string diff --git a/terraform/scripts/env b/terraform/scripts/env index 56c3a30..be8f853 100644 --- a/terraform/scripts/env +++ b/terraform/scripts/env @@ -6,6 +6,7 @@ export YC_FOLDER_ID=$(yc config get folder-id) export YC_TOKEN=$(yc iam create-token) export TF_VAR_folder_id=$YC_FOLDER_ID +export TF_VAR_cloud_id=$YC_CLOUD_ID if [ -f ./.env ]; then export $(grep -v '^#' ./.env | xargs)