From 804444ab7492fb12708ad123a413f41e6dd5f6ad Mon Sep 17 00:00:00 2001 From: ITQ Date: Fri, 31 Oct 2025 15:12:35 +0300 Subject: [PATCH] chore(terraform): refactored configuration --- main/.env.template | 3 +- main/api_gateway.tf | 6 +- main/certificate.tf | 2 +- main/configs/api_gateway/spec.yaml | 2 +- main/configs/cloud_init/main_master.yaml | 2 + main/configs/cloud_init/master.yaml | 1 + main/configs/cloud_init/worker.yaml | 1 + main/disk.tf | 18 +---- main/iam.tf | 18 ++--- main/instance.tf | 88 +++--------------------- main/instance_group.tf | 33 ++++----- main/loadbalancer.tf | 88 +++++++++++++++++------- main/locals.tf | 7 +- main/logging.tf | 4 +- main/object_storage.tf | 4 +- main/outputs.tf | 2 +- main/templates.tf | 19 +++-- main/variables.tf | 16 +++-- 18 files changed, 143 insertions(+), 171 deletions(-) diff --git a/main/.env.template b/main/.env.template index 6853bb8..0654c60 100644 --- a/main/.env.template +++ b/main/.env.template @@ -9,7 +9,8 @@ YC_FOLDER_ID= TF_VAR_cloud_id= TF_VAR_folder_id= TF_VAR_domain= -TF_VAR_ssh_public_key_path= +TF_VAR_frontend_domain= +TF_VAR_frontend_develop_domain= AWS_BUCKET= AWS_BUCKET_KEY= diff --git a/main/api_gateway.tf b/main/api_gateway.tf index 6468df0..cfa765f 100644 --- a/main/api_gateway.tf +++ b/main/api_gateway.tf @@ -8,14 +8,14 @@ resource "yandex_api_gateway" "frontend" { } custom_domains { - fqdn = var.domain + fqdn = local.final_frontend_domain certificate_id = yandex_cm_certificate.default.id } log_options { - log_group_id = yandex_logging_group.default.id + log_group_id = yandex_logging_group.api_gateway.id min_level = "INFO" } - spec = data.template_file.api_gateway_spec.rendered + spec = data.template_file.frontend_api_gateway_spec.rendered } diff --git a/main/certificate.tf b/main/certificate.tf index 13b06fe..d7f8874 100644 --- a/main/certificate.tf +++ b/main/certificate.tf @@ -1,5 +1,5 @@ resource "yandex_cm_certificate" "default" { - name = "default" + name = replace(lower(var.domain), ".", "-") description = "Default certificate for all resources" deletion_protection = true diff --git a/main/configs/api_gateway/spec.yaml b/main/configs/api_gateway/spec.yaml index 0e670e7..21d0f8f 100644 --- a/main/configs/api_gateway/spec.yaml +++ b/main/configs/api_gateway/spec.yaml @@ -31,7 +31,7 @@ paths: http_headers: "Content-Type": "application/json" content: - "*": '{"backend_url": "https://backend.interview.yandex.itqdev.xyz"}' + "*": '{"backend_url": "https://some.backend.example.com"}' responses: "200": description: Config returned. diff --git a/main/configs/cloud_init/main_master.yaml b/main/configs/cloud_init/main_master.yaml index 4e0be65..9c2135c 100644 --- a/main/configs/cloud_init/main_master.yaml +++ b/main/configs/cloud_init/main_master.yaml @@ -41,10 +41,12 @@ runcmd: fi echo "$DISK $K3S_DIR ext4 defaults 0 0" >> /etc/fstab mount -a + resize2fs $DISK - | curl -sfL https://get.k3s.io | INSTALL_K3S_CHANNEL=stable INSTALL_K3S_EXEC="server \ --cluster-init \ --token=${k3s_token} \ + --node-name=`hostname -f` \ --node-taint CriticalAddonsOnly=true:NoExecute \ --write-kubeconfig ${k3s_dir}/kubeconfig/config \ --image-credential-provider-bin-dir /var/lib/rancher/credentialprovider/bin \ diff --git a/main/configs/cloud_init/master.yaml b/main/configs/cloud_init/master.yaml index 5459497..1ca7d2b 100644 --- a/main/configs/cloud_init/master.yaml +++ b/main/configs/cloud_init/master.yaml @@ -41,6 +41,7 @@ runcmd: fi echo "$DISK $K3S_DIR ext4 defaults 0 0" >> /etc/fstab mount -a + resize2fs $DISK - | curl -sfL https://get.k3s.io | INSTALL_K3S_CHANNEL=stable INSTALL_K3S_EXEC="server \ --server=https://${k3s_master_ip}:6443 \ diff --git a/main/configs/cloud_init/worker.yaml b/main/configs/cloud_init/worker.yaml index 60ba79b..61fc6dc 100644 --- a/main/configs/cloud_init/worker.yaml +++ b/main/configs/cloud_init/worker.yaml @@ -41,6 +41,7 @@ runcmd: fi echo "$DISK $K3S_DIR ext4 defaults 0 0" >> /etc/fstab mount -a + resize2fs $DISK - | curl -sfL https://get.k3s.io | INSTALL_K3S_CHANNEL=stable INSTALL_K3S_EXEC="agent \ --server=https://${k3s_master_ip}:6443 \ diff --git a/main/disk.tf b/main/disk.tf index 664a204..0812052 100644 --- a/main/disk.tf +++ b/main/disk.tf @@ -1,6 +1,6 @@ -resource "yandex_compute_disk" "master_disk" { - name = "master_disk" - description = "Disk for master persistent storage" +resource "yandex_compute_disk" "k8s_main_master_disk" { + name = "k8s_main_master_disk" + description = "Disk for K8S main master persistent storage" zone = "ru-central1-d" size = 10 @@ -9,15 +9,3 @@ resource "yandex_compute_disk" "master_disk" { allow_recreate = false } - -resource "yandex_compute_disk" "worker_minecraft_disk" { - name = "worker_minecraft_disk" - description = "Disk for worker persistent storage" - - zone = "ru-central1-d" - size = 20 - type = "network-ssd" - block_size = 4096 - - allow_recreate = false -} diff --git a/main/iam.tf b/main/iam.tf index f8373cf..39bd6cd 100644 --- a/main/iam.tf +++ b/main/iam.tf @@ -29,29 +29,29 @@ resource "yandex_iam_service_account_iam_binding" "instance_group_vm_sa_admin" { ] } -resource "yandex_iam_service_account" "master_vm_sa" { - name = "master-vm-sa" +resource "yandex_iam_service_account" "k8s_main_master_vm_sa" { + name = "k8s-master-vm-sa" description = "Service account for master vm" } -resource "yandex_compute_instance_iam_binding" "master_vm_sa_editor" { - instance_id = yandex_compute_instance.master.id +resource "yandex_compute_instance_iam_binding" "k8s_main_master_vm_sa_editor" { + instance_id = yandex_compute_instance.k8s_main_master.id role = "editor" members = [ - "serviceAccount:${yandex_iam_service_account.master_vm_sa.id}" + "serviceAccount:${yandex_iam_service_account.k8s_main_master_vm_sa.id}" ] } -resource "yandex_resourcemanager_folder_iam_member" "master_vm_sa_viewer" { +resource "yandex_resourcemanager_folder_iam_member" "k8s_main_master_vm_sa_viewer" { folder_id = var.folder_id role = "compute.viewer" - member = "serviceAccount:${yandex_iam_service_account.master_vm_sa.id}" + member = "serviceAccount:${yandex_iam_service_account.k8s_main_master_vm_sa.id}" } -resource "yandex_resourcemanager_folder_iam_member" "master_vm_sa_oslogin" { +resource "yandex_resourcemanager_folder_iam_member" "k8s_main_master_vm_sa_oslogin" { folder_id = var.folder_id role = "compute.osAdminLogin" - member = "serviceAccount:${yandex_iam_service_account.master_vm_sa.id}" + member = "serviceAccount:${yandex_iam_service_account.k8s_main_master_vm_sa.id}" } resource "yandex_iam_service_account" "registry_push_sa" { diff --git a/main/instance.tf b/main/instance.tf index 2d21266..edbceba 100644 --- a/main/instance.tf +++ b/main/instance.tf @@ -1,10 +1,10 @@ -resource "yandex_compute_instance" "master" { - name = "master" - hostname = "master" +resource "yandex_compute_instance" "k8s_main_master" { + name = "k8s-master" + hostname = local.k8s_main_master_fqdn description = "Salt, ansible and kubernetes master, provisions other nodes" platform_id = "standard-v3" zone = "ru-central1-d" - service_account_id = yandex_iam_service_account.master_vm_sa.id + service_account_id = yandex_iam_service_account.k8s_main_master_vm_sa.id allow_recreate = true allow_stopping_for_update = true @@ -24,7 +24,7 @@ resource "yandex_compute_instance" "master" { boot_disk { initialize_params { image_id = data.yandex_compute_image.ubuntu_2404.id - description = "Boot disk for master" + description = "Boot disk for K8S main master" block_size = 4096 size = 10 type = "network-hdd" @@ -34,7 +34,7 @@ resource "yandex_compute_instance" "master" { } secondary_disk { - disk_id = yandex_compute_disk.master_disk.id + disk_id = yandex_compute_disk.k8s_main_master_disk.id device_name = "data" mode = "READ_WRITE" auto_delete = false @@ -42,11 +42,11 @@ resource "yandex_compute_instance" "master" { network_interface { subnet_id = yandex_vpc_subnet.default_ru_central1_d.id - ip_address = local.master_ip + ip_address = local.k8s_main_master_ip ipv4 = true nat = true dns_record { - fqdn = local.master_fqdn + fqdn = "${local.k8s_main_master_fqdn}." ptr = true ttl = 10 } @@ -57,7 +57,7 @@ resource "yandex_compute_instance" "master" { enable-oslogin = true serial-port-enable = 1 install-unified-agent = 0 - user-data = data.template_file.main_master_cloud_init.rendered + user-data = data.template_file.k8s_main_master_cloud_init.rendered } metadata_options { @@ -71,73 +71,3 @@ resource "yandex_compute_instance" "master" { placement_group_partition = 0 } } - -resource "yandex_compute_instance" "worker-minecraft" { - name = "worker-minecraft" - hostname = "worker-minecraft.internal" - description = "Worker node for Kubernetes cluster" - labels = { - "kubernetes" = "worker" - "role" = "worker" - "salt" = "minion" - } - - service_account_id = yandex_iam_service_account.instance_group_vm_sa.id - platform_id = "standard-v3" - zone = "ru-central1-d" - - resources { - core_fraction = 100 - cores = 2 - gpus = 0 - memory = 8 - } - - boot_disk { - initialize_params { - image_id = data.yandex_compute_image.ubuntu_2404.id - description = "Boot disk" - size = 10 - type = "network-hdd" - } - mode = "READ_WRITE" - } - - secondary_disk { - disk_id = yandex_compute_disk.worker_minecraft_disk.id - device_name = "data" - mode = "READ_WRITE" - auto_delete = false - } - - network_interface { - subnet_id = yandex_vpc_subnet.default_ru_central1_d.id - ipv4 = true - nat = false - } - network_acceleration_type = "standard" - - - metadata = { - enable-oslogin = true - serial-port-enable = 1 - install-unified-agent = 0 - user-data = data.template_file.worker_cloud_init.rendered - } - - metadata_options { - aws_v1_http_endpoint = 1 - aws_v1_http_token = 2 - gce_http_endpoint = 1 - gce_http_token = 1 - } - placement_policy { - host_affinity_rules = [] - placement_group_partition = 0 - } - - depends_on = [ - yandex_compute_instance.master, - yandex_iam_service_account.instance_group_vm_sa - ] -} diff --git a/main/instance_group.tf b/main/instance_group.tf index 8c18938..6a6be44 100644 --- a/main/instance_group.tf +++ b/main/instance_group.tf @@ -1,5 +1,5 @@ -resource "yandex_compute_instance_group" "master" { - name = "master" +resource "yandex_compute_instance_group" "k8s_master" { + name = "k8s-master" description = "Masters instance group for Kubernetes cluster" service_account_id = yandex_iam_service_account.instance_group_sa.id deletion_protection = true @@ -9,8 +9,8 @@ resource "yandex_compute_instance_group" "master" { } instance_template { - name = "master-{instance.index_in_zone}-{instance.zone_id}" - hostname = "master-{instance.index_in_zone}.{instance.zone_id}" + name = "k8s-master-{instance.index_in_zone}-{instance.zone_id}" + hostname = "master-{instance.index_in_zone}.{instance.zone_id}.k8s.internal" labels = { "instance-group" = "master" "kubernetes" = "control-plane" @@ -28,7 +28,8 @@ resource "yandex_compute_instance_group" "master" { boot_disk { initialize_params { - image_id = data.yandex_compute_image.ubuntu_2404.id + # image_id = data.yandex_compute_image.ubuntu_2404.id + image_id = "f2enm07snmcg7jhhcgst" description = "Boot disk" size = 10 type = "network-hdd" @@ -68,7 +69,7 @@ resource "yandex_compute_instance_group" "master" { enable-oslogin = true serial-port-enable = 1 install-unified-agent = 0 - user-data = data.template_file.master_cloud_init.rendered + user-data = data.template_file.k8s_master_cloud_init.rendered } } @@ -90,7 +91,7 @@ resource "yandex_compute_instance_group" "master" { } load_balancer { - target_group_name = "master-instance-group" + target_group_name = "k8s-master-instance-group" target_group_description = "Load balancer target group for k8s masters" max_opening_traffic_duration = 600 ignore_health_checks = true @@ -108,15 +109,15 @@ resource "yandex_compute_instance_group" "master" { max_checking_health_duration = 600 depends_on = [ - yandex_compute_instance.master, + yandex_compute_instance.k8s_main_master, yandex_iam_service_account.instance_group_sa, yandex_resourcemanager_folder_iam_member.instance_group_sa_compute_editor, yandex_resourcemanager_folder_iam_member.instance_group_sa_lb_editor ] } -resource "yandex_compute_instance_group" "worker" { - name = "worker" +resource "yandex_compute_instance_group" "k8s_worker" { + name = "k8s-worker" description = "Workers instance group for Kubernetes cluster" service_account_id = yandex_iam_service_account.instance_group_sa.id deletion_protection = true @@ -126,8 +127,8 @@ resource "yandex_compute_instance_group" "worker" { } instance_template { - name = "worker-{instance.index_in_zone}-{instance.zone_id}" - hostname = "worker-{instance.index_in_zone}.{instance.zone_id}.internal" + name = "k8s-worker-{instance.index_in_zone}-{instance.zone_id}" + hostname = "worker-{instance.index_in_zone}.{instance.zone_id}.k8s.internal" labels = { "instance-group" = "worker" "kubernetes" = "worker" @@ -185,13 +186,13 @@ resource "yandex_compute_instance_group" "worker" { enable-oslogin = true serial-port-enable = 1 install-unified-agent = 0 - user-data = data.template_file.worker_cloud_init.rendered + user-data = data.template_file.k8s_worker_cloud_init.rendered } } scale_policy { fixed_scale { - size = 0 + size = 1 } } allocation_policy { @@ -207,7 +208,7 @@ resource "yandex_compute_instance_group" "worker" { } load_balancer { - target_group_name = "worker-instance-group" + target_group_name = "k8s-worker-instance-group" target_group_description = "Load balancer target group for k8s workers" max_opening_traffic_duration = 600 ignore_health_checks = true @@ -225,7 +226,7 @@ resource "yandex_compute_instance_group" "worker" { max_checking_health_duration = 600 depends_on = [ - yandex_compute_instance.master, + yandex_compute_instance.k8s_main_master, yandex_iam_service_account.instance_group_sa, yandex_resourcemanager_folder_iam_member.instance_group_sa_compute_editor, yandex_resourcemanager_folder_iam_member.instance_group_sa_lb_editor diff --git a/main/loadbalancer.tf b/main/loadbalancer.tf index 00be048..659bdd9 100644 --- a/main/loadbalancer.tf +++ b/main/loadbalancer.tf @@ -1,36 +1,72 @@ -# resource "yandex_lb_network_load_balancer" "instance_group_lb" { -# name = "instance-group-lb" -# allow_zonal_shift = true -# deletion_protection = true +resource "yandex_lb_network_load_balancer" "k8s_lb" { + name = "k8s-lb" + allow_zonal_shift = true + deletion_protection = true -# listener { -# name = "test" -# port = 8080 -# external_address_spec { -# # address = yandex_vpc_address.default.external_ipv4_address[0].address -# ip_version = "ipv4" -# } -# } + listener { + name = "kubeapi" + port = 6443 + external_address_spec { + address = yandex_vpc_address.nlb.external_ipv4_address[0].address + ip_version = "ipv4" + } + } -# attached_target_group { -# target_group_id = yandex_compute_instance_group.default.load_balancer[0].target_group_id + attached_target_group { + target_group_id = yandex_lb_target_group.k8s_main_master.id -# healthcheck { -# name = "http" -# http_options { -# port = 8080 -# path = "/ping" -# } -# } -# } -# } + // Verifies that Traefik daeomnset has started on node + healthcheck { + name = "main-master-traefik" + healthy_threshold = 2 + unhealthy_threshold = 2 + interval = 5 + timeout = 4 + tcp_options { + port = 80 + } + } + } -resource "yandex_lb_target_group" "main_master" { - name = "main-master" + attached_target_group { + target_group_id = yandex_compute_instance_group.k8s_master.load_balancer[0].target_group_id + + // Verifies that Traefik daeomnset has started on node + healthcheck { + name = "master-instance-group-traefik" + healthy_threshold = 2 + unhealthy_threshold = 2 + interval = 5 + timeout = 4 + tcp_options { + port = 80 + } + } + } + + attached_target_group { + target_group_id = yandex_compute_instance_group.k8s_worker.load_balancer[0].target_group_id + + // Verifies that Traefik daeomnset has started on node + healthcheck { + name = "worker-instance-group-traefik" + healthy_threshold = 2 + unhealthy_threshold = 2 + interval = 5 + timeout = 4 + tcp_options { + port = 80 + } + } + } +} + +resource "yandex_lb_target_group" "k8s_main_master" { + name = "k8s-main-master" region_id = "ru-central1" target { subnet_id = yandex_vpc_subnet.default_ru_central1_d.id - address = yandex_compute_instance.master.network_interface.0.ip_address + address = yandex_compute_instance.k8s_main_master.network_interface.0.ip_address } } diff --git a/main/locals.tf b/main/locals.tf index b1ad54c..038fabe 100644 --- a/main/locals.tf +++ b/main/locals.tf @@ -1,7 +1,10 @@ locals { - master_ip = "10.4.0.3" - master_fqdn = "master." + k8s_main_master_ip = "10.4.0.3" + k8s_main_master_fqdn = "master.k8s.internal" k3s_data_dir = "/mnt/k3s" k3s_channel = "stable" + + final_frontend_domain = var.frontend_domain != null ? var.frontend_domain : var.domain + final_frontend_develop_domain = var.frontend_develop_domain != null ? var.frontend_develop_domain : "dev.${var.domain}" } diff --git a/main/logging.tf b/main/logging.tf index ce2acc2..759975f 100644 --- a/main/logging.tf +++ b/main/logging.tf @@ -1,5 +1,5 @@ -resource "yandex_logging_group" "default" { - name = "default" +resource "yandex_logging_group" "api_gateway" { + name = "api-gateway" retention_period = "336h0m0s" } diff --git a/main/object_storage.tf b/main/object_storage.tf index d6408e7..63525c9 100644 --- a/main/object_storage.tf +++ b/main/object_storage.tf @@ -1,5 +1,5 @@ resource "yandex_storage_bucket" "frontend" { - bucket = var.domain + bucket = local.final_frontend_domain default_storage_class = "STANDARD" force_destroy = false max_size = 104857600 @@ -12,7 +12,7 @@ resource "yandex_storage_bucket" "frontend" { } resource "yandex_storage_bucket" "frontend_develop" { - bucket = "dev.${var.domain}" + bucket = local.final_frontend_develop_domain default_storage_class = "STANDARD" force_destroy = false max_size = 104857600 diff --git a/main/outputs.tf b/main/outputs.tf index 28e9cf1..4436ab1 100644 --- a/main/outputs.tf +++ b/main/outputs.tf @@ -4,7 +4,7 @@ output "registry_data" { } } -output "api_gateway_data" { +output "frontend_api_gateway_data" { value = { domain = yandex_api_gateway.frontend.domain } diff --git a/main/templates.tf b/main/templates.tf index 3c30af0..f21b11f 100644 --- a/main/templates.tf +++ b/main/templates.tf @@ -1,4 +1,4 @@ -data "template_file" "main_master_cloud_init" { +data "template_file" "k8s_main_master_cloud_init" { template = file("${path.module}/configs/cloud_init/main_master.yaml") vars = { k3s_credential_provider_config = base64encode(file("${path.module}/configs/k3s/credentialprovider.yaml")) @@ -9,33 +9,40 @@ data "template_file" "main_master_cloud_init" { } } -data "template_file" "master_cloud_init" { +data "template_file" "k8s_master_cloud_init" { template = file("${path.module}/configs/cloud_init/master.yaml") vars = { k3s_credential_provider_config = base64encode(file("${path.module}/configs/k3s/credentialprovider.yaml")) k3s_credential_provider = base64encode(file("${path.module}/configs/k3s/yc-credential-provider")) k3s_dir = local.k3s_data_dir k3s_token = random_password.k3s_token.result - k3s_master_ip = local.master_ip + k3s_master_ip = local.k8s_main_master_fqdn yc_cloud_id = var.cloud_id } } -data "template_file" "worker_cloud_init" { +data "template_file" "k8s_worker_cloud_init" { template = file("${path.module}/configs/cloud_init/worker.yaml") vars = { k3s_credential_provider_config = base64encode(file("${path.module}/configs/k3s/credentialprovider.yaml")) k3s_credential_provider = base64encode(file("${path.module}/configs/k3s/yc-credential-provider")) k3s_dir = local.k3s_data_dir k3s_token = random_password.k3s_token.result - k3s_master_ip = local.master_ip + k3s_master_ip = local.k8s_main_master_fqdn yc_cloud_id = var.cloud_id } } -data "template_file" "api_gateway_spec" { +data "template_file" "frontend_api_gateway_spec" { template = file("${path.module}/configs/api_gateway/spec.yaml") vars = { bucket_name = yandex_storage_bucket.frontend.bucket } } + +data "template_file" "frontend_develop_api_gateway_spec" { + template = file("${path.module}/configs/api_gateway/spec.yaml") + vars = { + bucket_name = yandex_storage_bucket.frontend_develop.bucket + } +} diff --git a/main/variables.tf b/main/variables.tf index 3301ddd..ad888f5 100644 --- a/main/variables.tf +++ b/main/variables.tf @@ -2,17 +2,19 @@ variable "folder_id" {} variable "cloud_id" {} -variable "resources_prefix" { - type = string - default = "" -} - variable "domain" { description = "Domain to issue TLS certificates for with Let's Encrypt" type = string } -variable "ssh_public_key_path" { - description = "This public key will be placed on all nodes" +variable "frontend_domain" { + description = "Domain to serve frontend from (must equal domain or *.domain), default: domain" type = string + default = null +} + +variable "frontend_develop_domain" { + description = "Domain to serve develop version of frontend from (must equal domain or *.domain), default: dev.domain" + type = string + default = null }