fix(): fixed healthcheck intervals

terraform/main/configs/cloud_init/main_master.yaml

@@ -44,15 +44,38 @@ runcmd:
     resize2fs $DISK
   - |
     curl -sfL https://get.k3s.io | INSTALL_K3S_CHANNEL=stable INSTALL_K3S_EXEC="server \
+        -v 2 \
         --cluster-init \
+        --bind-address 0.0.0.0 \
+        --https-listen-port 6443 \
         --token=${k3s_token} \
         --node-name=`hostname -f` \
         --node-taint CriticalAddonsOnly=true:NoExecute \
+        --cluster-cidr 10.96.0.0/16 \
+        --service-cidr 10.97.0.0/16 \
+        --cluster-dns 10.97.0.10 \
+        --tls-san ${k3s_fqdn} \
+        --tls-san-security \
+        --cluster-domain cluster.local \
+        --disable coredns,traefik,local-storage,metrics-server,runtimes \
+        --disable-network-policy \
+        --disable-kube-proxy \
+        --disable-cloud-controller \
+        --disable-helm-controller \
+        --flannel-backend none \
+        --egress-selector-mode agent \
+        --servicelb-namespace kube-system \
+        --nonroot-devices \
+        --secrets-encryption \
+        --secrets-encryption-provider secretbox \
+        --lb-server-port 6444 \
+        --etcd-expose-metrics \
+        --write-kubeconfig-mode 0600 \
+        --write-kubeconfig-group root \
         --write-kubeconfig ${k3s_dir}/kubeconfig/config \
         --image-credential-provider-bin-dir /var/lib/rancher/credentialprovider/bin \
         --image-credential-provider-config /var/lib/rancher/credentialprovider/config.yaml \
-        --data-dir=${k3s_dir} \
-        --default-local-storage-path=${k3s_dir}/storage" sh -
+        --data-dir=${k3s_dir}" sh -
   - |
     curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
   - |

terraform/main/configs/cloud_init/master.yaml

@@ -44,14 +44,37 @@ runcmd:
     resize2fs $DISK
   - |
     curl -sfL https://get.k3s.io | INSTALL_K3S_CHANNEL=stable INSTALL_K3S_EXEC="server \
+        -v 2 \
+        --bind-address 0.0.0.0 \
+        --https-listen-port 6443 \
         --server=https://${k3s_master_ip}:6443 \
         --token=${k3s_token} \
         --node-name=`hostname -f` \
+        --cluster-cidr 10.96.0.0/16 \
+        --service-cidr 10.97.0.0/16 \
+        --cluster-dns 10.97.0.10 \
+        --tls-san ${k3s_fqdn} \
+        --tls-san-security \
+        --cluster-domain cluster.local \
+        --disable coredns,traefik,local-storage,metrics-server,runtimes \
+        --disable-network-policy \
+        --disable-kube-proxy \
+        --disable-cloud-controller \
+        --disable-helm-controller \
+        --flannel-backend none \
+        --egress-selector-mode agent \
+        --servicelb-namespace kube-system \
+        --nonroot-devices \
+        --secrets-encryption \
+        --secrets-encryption-provider secretbox \
+        --lb-server-port 6444 \
+        --etcd-expose-metrics \
+        --write-kubeconfig-mode 0600 \
+        --write-kubeconfig-group root \
         --write-kubeconfig ${k3s_dir}/kubeconfig/config \
         --image-credential-provider-bin-dir /var/lib/rancher/credentialprovider/bin \
         --image-credential-provider-config /var/lib/rancher/credentialprovider/config.yaml \
-        --data-dir=${k3s_dir} \
-        --default-local-storage-path=${k3s_dir}/storage" sh -
+        --data-dir=${k3s_dir}" sh -
   - |
     mkdir -p /home/ubuntu/.kube
     cp ${k3s_dir}/kubeconfig/config /home/ubuntu/.kube/config

terraform/main/configs/cloud_init/worker.yaml

@@ -44,9 +44,12 @@ runcmd:
     resize2fs $DISK
   - |
     curl -sfL https://get.k3s.io | INSTALL_K3S_CHANNEL=stable INSTALL_K3S_EXEC="agent \
+        -v 2 \
         --server=https://${k3s_master_ip}:6443 \
         --token=${k3s_token} \
         --node-name=`hostname -f` \
+        --nonroot-devices \
+        --lb-server-port 6444 \
         --image-credential-provider-bin-dir /var/lib/rancher/credentialprovider/bin \
         --image-credential-provider-config /var/lib/rancher/credentialprovider/config.yaml \
         --data-dir=${k3s_dir}" sh -

terraform/main/instance.tf

@@ -1,5 +1,16 @@
+resource "terraform_data" "k8s_main_master_user_data" {
+  input = templatefile("${path.module}/configs/cloud_init/main_master.yaml", {
+    k3s_credential_provider_config = base64encode(file("${path.module}/configs/k3s/credentialprovider.yaml"))
+    k3s_credential_provider        = base64encode(file("${path.module}/configs/k3s/yc-credential-provider"))
+    k3s_dir                        = local.k3s_data_dir
+    k3s_token                      = random_password.k3s_token.result
+    k3s_fqdn                       = var.cluster_domain
+    yc_cloud_id                    = var.cloud_id
+  })
+}
+
 resource "yandex_compute_instance" "k8s_main_master" {
-  name               = "k8s-master"
+  name               = "master-0-ru-central1-d-k8s"
   hostname           = local.k8s_main_master_fqdn
   description        = "Salt, ansible and kubernetes master, provisions other nodes"
   platform_id        = "standard-v3"
@@ -15,9 +26,11 @@ resource "yandex_compute_instance" "k8s_main_master" {
   }
 
   lifecycle {
-    prevent_destroy       = true
     ignore_changes = [
-      hostname,
+      hostname, boot_disk[0].initialize_params[0].image_id
+    ]
+    replace_triggered_by = [
+      terraform_data.k8s_main_master_user_data
     ]
   }
 
@@ -64,13 +77,7 @@ resource "yandex_compute_instance" "k8s_main_master" {
     enable-oslogin        = true
     serial-port-enable    = 1
     install-unified-agent = 0
-    user-data = templatefile("${path.module}/configs/cloud_init/main_master.yaml", {
-      k3s_credential_provider_config = base64encode(file("${path.module}/configs/k3s/credentialprovider.yaml"))
-      k3s_credential_provider        = base64encode(file("${path.module}/configs/k3s/yc-credential-provider"))
-      k3s_dir                        = local.k3s_data_dir
-      k3s_token                      = random_password.k3s_token.result
-      yc_cloud_id                    = var.cloud_id
-    })
+    user-data             = terraform_data.k8s_main_master_user_data.output
   }
 
   metadata_options {

terraform/main/instance_group.tf

@@ -9,7 +9,7 @@ resource "yandex_compute_instance_group" "k8s_master" {
   }
 
   instance_template {
-    name     = "k8s-master-{instance.index_in_zone}-{instance.zone_id}"
+    name     = "master-{instance.index_in_zone}-{instance.zone_id}-k8s"
     hostname = "master-{instance.index_in_zone}.{instance.zone_id}.k8s.internal"
     labels = {
       "instance-group" = "master"
@@ -20,7 +20,7 @@ resource "yandex_compute_instance_group" "k8s_master" {
 
     platform_id = "standard-v3"
     resources {
-      core_fraction = 50
+      core_fraction = 100
       cores         = 2
       gpus          = 0
       memory        = 4
@@ -39,8 +39,8 @@ resource "yandex_compute_instance_group" "k8s_master" {
       device_name = "data"
       initialize_params {
         description = "Data disk"
-        size        = 20
-        type        = "network-hdd"
+        size        = 10
+        type        = "network-ssd"
       }
       mode = "READ_WRITE"
     }
@@ -51,6 +51,7 @@ resource "yandex_compute_instance_group" "k8s_master" {
         yandex_vpc_subnet.default_ru_central1_a.id,
         yandex_vpc_subnet.default_ru_central1_b.id,
         yandex_vpc_subnet.default_ru_central1_d.id,
+        yandex_vpc_subnet.default_ru_central1_e.id,
       ]
       nat = false
     }
@@ -74,6 +75,7 @@ resource "yandex_compute_instance_group" "k8s_master" {
         k3s_dir                        = local.k3s_data_dir
         k3s_token                      = random_password.k3s_token.result
         k3s_master_ip                  = local.k8s_main_master_fqdn
+        k3s_fqdn                       = var.cluster_domain
         yc_cloud_id                    = var.cloud_id
       })
     }
@@ -81,11 +83,11 @@ resource "yandex_compute_instance_group" "k8s_master" {
 
   scale_policy {
     fixed_scale {
-      size = 3
+      size = 2
     }
   }
   allocation_policy {
-    zones = ["ru-central1-a", "ru-central1-b", "ru-central1-d"]
+    zones = ["ru-central1-a", "ru-central1-b", "ru-central1-d", "ru-central1-e"]
   }
   deploy_policy {
     max_creating     = 3
@@ -103,10 +105,10 @@ resource "yandex_compute_instance_group" "k8s_master" {
     ignore_health_checks         = true
   }
   health_check {
-    healthy_threshold   = 2
-    unhealthy_threshold = 2
-    interval            = 2
-    timeout             = 1
+    healthy_threshold   = 3
+    unhealthy_threshold = 3
+    interval            = 10
+    timeout             = 9
 
     tcp_options {
       port = 6443
@@ -133,7 +135,7 @@ resource "yandex_compute_instance_group" "k8s_worker" {
   }
 
   instance_template {
-    name     = "k8s-worker-{instance.index_in_zone}-{instance.zone_id}"
+    name     = "worker-{instance.index_in_zone}-{instance.zone_id}-k8s"
     hostname = "worker-{instance.index_in_zone}.{instance.zone_id}.k8s.internal"
     labels = {
       "instance-group" = "worker"
@@ -144,10 +146,10 @@ resource "yandex_compute_instance_group" "k8s_worker" {
 
     platform_id = "standard-v3"
     resources {
-      core_fraction = 50
-      cores         = 2
+      core_fraction = 100
+      cores         = 4
       gpus          = 0
-      memory        = 2
+      memory        = 8
     }
 
     boot_disk {
@@ -163,8 +165,8 @@ resource "yandex_compute_instance_group" "k8s_worker" {
       device_name = "data"
       initialize_params {
         description = "Data disk"
-        size        = 20
-        type        = "network-hdd"
+        size        = 10
+        type        = "network-ssd"
       }
       mode = "READ_WRITE"
     }
@@ -175,6 +177,7 @@ resource "yandex_compute_instance_group" "k8s_worker" {
         yandex_vpc_subnet.default_ru_central1_a.id,
         yandex_vpc_subnet.default_ru_central1_b.id,
         yandex_vpc_subnet.default_ru_central1_d.id,
+        yandex_vpc_subnet.default_ru_central1_e.id,
       ]
       nat = false
     }
@@ -209,7 +212,7 @@ resource "yandex_compute_instance_group" "k8s_worker" {
     }
   }
   allocation_policy {
-    zones = ["ru-central1-a", "ru-central1-b", "ru-central1-d"]
+    zones = ["ru-central1-a", "ru-central1-b", "ru-central1-d", "ru-central1-e"]
   }
   deploy_policy {
     max_creating     = 3
@@ -227,10 +230,10 @@ resource "yandex_compute_instance_group" "k8s_worker" {
     ignore_health_checks         = true
   }
   health_check {
-    healthy_threshold   = 2
-    unhealthy_threshold = 2
-    interval            = 2
-    timeout             = 1
+    healthy_threshold   = 3
+    unhealthy_threshold = 3
+    interval            = 10
+    timeout             = 9
 
     tcp_options {
       port = 80

terraform/main/loadbalancer.tf

@@ -27,13 +27,21 @@ resource "yandex_lb_network_load_balancer" "k8s_lb" {
       ip_version = "ipv4"
     }
   }
+  listener {
+    name = "postgres-rw"
+    port = 5432
+    external_address_spec {
+      address    = yandex_vpc_address.nlb.external_ipv4_address[0].address
+      ip_version = "ipv4"
+    }
+  }
 
   attached_target_group {
     target_group_id = yandex_lb_target_group.k8s_main_master.id
 
     // Verifies that Traefik daeomnset has started on node
     healthcheck {
-      name                = "main-master-traefik"
+      name                = "main-master"
       healthy_threshold   = 2
       unhealthy_threshold = 2
       interval            = 5
@@ -47,9 +55,8 @@ resource "yandex_lb_network_load_balancer" "k8s_lb" {
   attached_target_group {
     target_group_id = yandex_compute_instance_group.k8s_master.load_balancer[0].target_group_id
 
-    // Verifies that Traefik daeomnset has started on node
     healthcheck {
-      name                = "master-instance-group-traefik"
+      name                = "master-instance-group"
       healthy_threshold   = 2
       unhealthy_threshold = 2
       interval            = 5
@@ -65,7 +72,7 @@ resource "yandex_lb_network_load_balancer" "k8s_lb" {
 
     // Verifies that Traefik daeomnset has started on node
     healthcheck {
-      name                = "worker-instance-group-traefik"
+      name                = "worker-instance-group"
       healthy_threshold   = 2
       unhealthy_threshold = 2
       interval            = 5

terraform/main/network.tf

@@ -51,6 +51,16 @@ resource "yandex_vpc_subnet" "default_ru_central1_d" {
   zone           = "ru-central1-d"
 }
 
+resource "yandex_vpc_subnet" "default_ru_central1_e" {
+  name        = "default-ru-central1-e"
+  description = "Default subnet for ru-central1-e"
+
+  network_id     = yandex_vpc_network.default.id
+  route_table_id = yandex_vpc_route_table.default.id
+  v4_cidr_blocks = ["10.5.0.0/16"]
+  zone           = "ru-central1-e"
+}
+
 resource "yandex_vpc_address" "nlb" {
   name                = "nlb"
   description         = "External IP address for network lb"

terraform/main/outputs.tf

@@ -45,7 +45,7 @@ output "registry_pull_sa_key" {
 #   })
 #   sensitive = true
 # }
-# 
+
 # output "lets_encrypt_challenges" {
 #   value = yandex_cm_certificate.default.challenges
 # }

terraform/main/variables.tf

@@ -7,6 +7,11 @@ variable "domain" {
   type        = string
 }
 
+variable "cluster_domain" {
+  type = string
+  default = "k8s.nlb.infra.itqdev.xyz"
+}
+
 variable "frontend_domain" {
   description = "Domain to serve frontend from (must equal domain or *.domain), default: domain"
   type        = string

terraform/scripts/env

@@ -6,6 +6,7 @@ export YC_FOLDER_ID=$(yc config get folder-id)
 export YC_TOKEN=$(yc iam create-token)
 
 export TF_VAR_folder_id=$YC_FOLDER_ID
+export TF_VAR_cloud_id=$YC_CLOUD_ID
 
 if [ -f ./.env ]; then
   export $(grep -v '^#' ./.env | xargs)