diff --git a/kubespray/group_vars/all/net.yaml b/kubespray/group_vars/all/net.yaml new file mode 100644 index 0000000..cd2fb49 --- /dev/null +++ b/kubespray/group_vars/all/net.yaml @@ -0,0 +1,13 @@ +--- +ipv6_stack: true + +kube_pods_subnet_ipv6: fd85:ee78:d8a6:8607::1:0000/112 +kube_service_addresses_ipv6: fd85:ee78:d8a6:8607::1000/116 +calico_ipv6pool_cidr: "{{ kube_pods_subnet_ipv6 }}" +calico_pool_blocksize_ipv6: 116 + +enable_dns_autoscaler: false +enable_nodelocaldns: false + +helm_enabled: true +local_path_provisioner_enabled: true diff --git a/kubespray/inventory.ini b/kubespray/inventory.ini new file mode 100644 index 0000000..cad858f --- /dev/null +++ b/kubespray/inventory.ini @@ -0,0 +1,13 @@ +control-plane ansible_host=10.6.6.10 ip=10.6.6.10 ipv6=2001:db8:ca2:2::10 +worker-1 ansible_host=10.6.6.20 ip=10.6.6.20 ipv6=2001:db8:ca2:2::20 + +[kube_control_plane] +control-plane + +[etcd] +control-plane + +[kube_node] +worker-1 + +[calico_rr] diff --git a/kubespray/tests/check-network-adv.yaml b/kubespray/tests/check-network-adv.yaml new file mode 100644 index 0000000..3d56dd1 --- /dev/null +++ b/kubespray/tests/check-network-adv.yaml @@ -0,0 +1,149 @@ +--- +- name: Test tunl0 routes + command: "/sbin/ip route" + register: routes + failed_when: routes.stdout_lines + | select('contains', '/' ~ calico_pool_blocksize|d(26)) + | select('contains', 'tunl0') | length == 0 + when: + - ('kube_node' in group_names) + - (calico_ipip_mode is defined and calico_ipip_mode != 'Never') + - kube_network_plugin | default('calico') == 'calico' + +- import_role: # noqa name[missing] + name: cluster-dump + +- name: Wait for netchecker server + command: "{{ bin_dir }}/kubectl get pods --field-selector=status.phase==Running -o jsonpath-as-json={.items[*].metadata.name} --namespace {{ netcheck_namespace }}" + register: pods_json + until: + - pods_json.stdout | from_json | select('match', 'netchecker-server.*') | length == 1 + - (pods_json.stdout | from_json | select('match', 'netchecker-agent.*') | length) + >= (groups['k8s_cluster'] | intersect(ansible_play_hosts) | length * 2) + retries: 3 + delay: 10 + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: Get netchecker pods + command: "{{ bin_dir }}/kubectl -n {{ netcheck_namespace }} describe pod -l app={{ item }}" + run_once: true + delegate_to: "{{ groups['kube_control_plane'][0] }}" + with_items: + - netchecker-agent + - netchecker-agent-hostnet + when: not pods_json is success + +- name: Perform netchecker tests + run_once: true + delegate_to: "{{ groups['kube_control_plane'][0] }}" + block: + - name: Get netchecker agents + uri: + url: "http://{{ (ansible_default_ipv6.address if not (ipv4_stack | default(true)) else ansible_default_ipv4.address) | ansible.utils.ipwrap }}:{{ netchecker_port }}/api/v1/agents/" + return_content: true + headers: + Accept: application/json + register: agents + retries: 18 + delay: "{{ agent_report_interval }}" + until: + - agents is success + - (agents.content | from_json | length) == (groups['k8s_cluster'] | length * 2) + + - name: Check netchecker status + uri: + url: "http://{{ (ansible_default_ipv6.address if not (ipv4_stack | default(true)) else ansible_default_ipv4.address) | ansible.utils.ipwrap }}:{{ netchecker_port }}/api/v1/connectivity_check" + return_content: true + headers: + Accept: application/json + register: connectivity_check + retries: 3 + delay: "{{ agent_report_interval }}" + until: + - connectivity_check is success + - connectivity_check.content | from_json + + rescue: + - name: Get kube-proxy logs + command: "{{ bin_dir }}/kubectl -n kube-system logs -l k8s-app=kube-proxy" + + - name: Get logs from other apps + command: "{{ bin_dir }}/kubectl -n kube-system logs -l k8s-app={{ item }} --all-containers" + with_items: + - kube-router + - flannel + - canal-node + - calico-node + - cilium + + - name: Netchecker tests failed + fail: + msg: "netchecker tests failed" + +- name: Check connectivity with all netchecker agents + vars: + connectivity_check_result: "{{ connectivity_check.content | from_json }}" + agents_check_result: "{{ agents.content | from_json }}" + assert: + that: + - agents_check_result is defined + - connectivity_check_result is defined + - agents_check_result.keys() | length > 0 + - not connectivity_check_result.Absent + - not connectivity_check_result.Outdated + msg: "Connectivity check to netchecker agents failed" + delegate_to: "{{ groups['kube_control_plane'][0] }}" + run_once: true + +- name: Create macvlan network conf + command: + cmd: "{{ bin_dir }}/kubectl create -f -" + stdin: | + apiVersion: "k8s.cni.cncf.io/v1" + kind: NetworkAttachmentDefinition + metadata: + name: macvlan-conf + spec: + config: '{ + "cniVersion": "0.4.0", + "type": "macvlan", + "master": "eth0", + "mode": "bridge", + "ipam": { + "type": "host-local", + "subnet": "192.168.1.0/24", + "rangeStart": "192.168.1.200", + "rangeEnd": "192.168.1.216", + "routes": [ + { "dst": "0.0.0.0/0" } + ], + "gateway": "192.168.1.1" + } + }' + --- + apiVersion: v1 + kind: Pod + metadata: + name: samplepod + annotations: + k8s.v1.cni.cncf.io/networks: macvlan-conf + spec: + containers: + - name: samplepod + command: ["/bin/bash", "-c", "sleep 2000000000000"] + image: dougbtv/centos-network + delegate_to: groups['kube_control_plane'][0] + run_once: true + when: + - kube_network_plugin_multus | default(false) | bool + +- name: Check secondary macvlan interface + command: "{{ bin_dir }}/kubectl exec samplepod -- ip addr show dev net1" + register: output + until: output.rc == 0 + retries: 90 + changed_when: false + delegate_to: groups['kube_control_plane'][0] + run_once: true + when: + - kube_network_plugin_multus | default(false) | bool