From 83c6ce35137d4cf5da998d2ab7fadc956ef98a1e Mon Sep 17 00:00:00 2001 From: NaeiKinDus Date: Sun, 4 Feb 2024 14:48:13 +0000 Subject: [PATCH] feat: k3s role --- .../roles/common/tasks/main.yml | 1 + .../roles/k3s/defaults/main.yml | 8 ++ .../roles/k3s/handlers/main.yml | 14 +++ .../infrastructure/roles/k3s/meta/main.yml | 20 ++++ .../infrastructure/roles/k3s/tasks/agent.yml | 18 ++++ .../infrastructure/roles/k3s/tasks/main.yml | 92 +++++++++++++++++++ .../infrastructure/roles/k3s/tasks/server.yml | 44 +++++++++ .../templates/nftables.d/k3s_agents.nft.j2 | 22 +++++ .../templates/nftables.d/k3s_servers.nft.j2 | 44 +++++++++ .../infrastructure/roles/k3s/tests/inventory | 2 + .../infrastructure/roles/k3s/vars/main.yml | 1 + inventory/group_vars/all/vars.yml | 1 + inventory/group_vars/internal/vars.yml | 4 + inventory/host_vars/actinium/vars.yml | 4 + inventory/host_vars/lithium/vars.yml | 3 + playbooks/internal.yml | 7 ++ 16 files changed, 285 insertions(+) create mode 100644 collections/ansible_collections/nullified/infrastructure/roles/k3s/defaults/main.yml create mode 100644 collections/ansible_collections/nullified/infrastructure/roles/k3s/handlers/main.yml create mode 100644 collections/ansible_collections/nullified/infrastructure/roles/k3s/meta/main.yml create mode 100644 collections/ansible_collections/nullified/infrastructure/roles/k3s/tasks/agent.yml create mode 100644 collections/ansible_collections/nullified/infrastructure/roles/k3s/tasks/main.yml create mode 100644 collections/ansible_collections/nullified/infrastructure/roles/k3s/tasks/server.yml create mode 100644 collections/ansible_collections/nullified/infrastructure/roles/k3s/templates/nftables.d/k3s_agents.nft.j2 create mode 100644 collections/ansible_collections/nullified/infrastructure/roles/k3s/templates/nftables.d/k3s_servers.nft.j2 create mode 100644 collections/ansible_collections/nullified/infrastructure/roles/k3s/tests/inventory create mode 100644 collections/ansible_collections/nullified/infrastructure/roles/k3s/vars/main.yml diff --git a/collections/ansible_collections/nullified/infrastructure/roles/common/tasks/main.yml b/collections/ansible_collections/nullified/infrastructure/roles/common/tasks/main.yml index 1f1ed53..48bec6e 100644 --- a/collections/ansible_collections/nullified/infrastructure/roles/common/tasks/main.yml +++ b/collections/ansible_collections/nullified/infrastructure/roles/common/tasks/main.yml @@ -60,6 +60,7 @@ - bzip2 - catimg - cron + - curl - dateutils - emacs-nox - firmware-misc-nonfree diff --git a/collections/ansible_collections/nullified/infrastructure/roles/k3s/defaults/main.yml b/collections/ansible_collections/nullified/infrastructure/roles/k3s/defaults/main.yml new file mode 100644 index 0000000..b70e4fe --- /dev/null +++ b/collections/ansible_collections/nullified/infrastructure/roles/k3s/defaults/main.yml @@ -0,0 +1,8 @@ +--- +k3s_cluster_name: default +k3s_cluster_role: server +k3s_kube_context: default +k3s_extra_args: '' +k3s_operator_ips: [] +k3s_cluster_cidr: '10.42.0.0/16' +k3s_service_cidr: '10.43.0.0/16' diff --git a/collections/ansible_collections/nullified/infrastructure/roles/k3s/handlers/main.yml b/collections/ansible_collections/nullified/infrastructure/roles/k3s/handlers/main.yml new file mode 100644 index 0000000..446755e --- /dev/null +++ b/collections/ansible_collections/nullified/infrastructure/roles/k3s/handlers/main.yml @@ -0,0 +1,14 @@ +--- +- name: restart firewall service + become: true + ansible.builtin.systemd_service: + name: nftables.service + enabled: true + state: restarted + +- name: restart k3s service + become: true + ansible.builtin.systemd_service: + name: k3s.service + enabled: true + state: restarted diff --git a/collections/ansible_collections/nullified/infrastructure/roles/k3s/meta/main.yml b/collections/ansible_collections/nullified/infrastructure/roles/k3s/meta/main.yml new file mode 100644 index 0000000..6cc7854 --- /dev/null +++ b/collections/ansible_collections/nullified/infrastructure/roles/k3s/meta/main.yml @@ -0,0 +1,20 @@ +--- +galaxy_info: + author: Florian L. + namespace: nullified + description: Install and configure K3S and related tools + # issue_tracker_url: http://example.com/issue/tracker + license: MIT + min_ansible_version: 2.15 + + # https://galaxy.ansible.com/api/v1/platforms/ + platforms: + - name: Debian + versions: + - bookworm + + galaxy_tags: + - kubernetes + - k3s + +dependencies: [] diff --git a/collections/ansible_collections/nullified/infrastructure/roles/k3s/tasks/agent.yml b/collections/ansible_collections/nullified/infrastructure/roles/k3s/tasks/agent.yml new file mode 100644 index 0000000..630c9d1 --- /dev/null +++ b/collections/ansible_collections/nullified/infrastructure/roles/k3s/tasks/agent.yml @@ -0,0 +1,18 @@ +--- +# TODO: implement +# TODO: disable swap + +- name: operation not supported + ansible.builtin.debug: + msg: Operation currently not supported + failed_when: true + +- name: setup firewall rules + become: true + ansible.builtin.template: + src: ../templates/nftables.d/k3s_agents.nft.j2 + dest: /etc/nftables.d/k3s_agents.nft + mode: '0600' + notify: + - 'k3s : restart firewall service' + - 'k3s : restart k3s service' diff --git a/collections/ansible_collections/nullified/infrastructure/roles/k3s/tasks/main.yml b/collections/ansible_collections/nullified/infrastructure/roles/k3s/tasks/main.yml new file mode 100644 index 0000000..26df51e --- /dev/null +++ b/collections/ansible_collections/nullified/infrastructure/roles/k3s/tasks/main.yml @@ -0,0 +1,92 @@ +--- +- name: gather facts if not already done + ansible.builtin.setup: + gather_subset: + - user_id + +- name: group by cluster name + ansible.builtin.group_by: + key: "k3s_clusters_{{ k3s_cluster_name }}_{{ k3s_cluster_role }}" + changed_when: false + +- name: determine cluster type and members + ansible.builtin.set_fact: + k3s_cluster_type: "{{ 'ha' if groups['k3s_clusters_' ~ k3s_cluster_name ~ '_' ~ k3s_cluster_role] | length > 1 else 'single' }}" + k3s_cluster_servers: "{{ groups['k3s_clusters_' ~ k3s_cluster_name ~ '_server'] }}" + k3s_cluster_agents: "{{ groups['k3s_clusters_' ~ k3s_cluster_name ~ '_agent'] | default([]) }}" + k3s_nft_servers4: "{{ groups['k3s_clusters_' ~ k3s_cluster_name ~ '_server'] | default([]) | map('extract', hostvars, ['k3s_cluster_ip']) | ansible.utils.ipv4 }}" + k3s_nft_agents4: "{{ groups['k3s_clusters_' ~ k3s_cluster_name ~ '_agent'] | default([]) | map('extract', hostvars, ['k3s_cluster_ip']) | ansible.utils.ipv4 }}" + k3s_nft_servers6: "{{ groups['k3s_clusters_' ~ k3s_cluster_name ~ '_server'] | default([]) | map('extract', hostvars, ['k3s_cluster_ip']) | ansible.utils.ipv6 }}" + k3s_nft_agents6: "{{ groups['k3s_clusters_' ~ k3s_cluster_name ~ '_agent'] | default([]) | map('extract', hostvars, ['k3s_cluster_ip']) | ansible.utils.ipv6 }}" + k3s_nft_operators4: "{{ k3s_operator_ips | ansible.utils.ipv4 }}" + k3s_nft_operators6: "{{ k3s_operator_ips | ansible.utils.ipv6 }}" + changed_when: false + +- name: get local controller account information + connection: local + ansible.builtin.getent: + database: passwd + key: "{{ ansible_facts.user_id }}" + split: ":" + changed_when: false + when: ansible_facts['getent_passwd'] is undefined or ansible_facts['user_id'] not in ansible_facts['getent_passwd'] + +- name: set controller environment variables + ansible.builtin.set_fact: + controller_user_home: "{{ ansible_facts['getent_passwd'][ansible_facts['user_id']][4] }}" + k3sup_binary: "{{ ansible_facts['getent_passwd'][ansible_facts['user_id']][4] }}/.local/bin/k3sup" + kubeconfig_repository: "{{ ansible_facts['getent_passwd'][ansible_facts['user_id']][4] }}/.kubeconfig_repository" + changed_when: false + +- name: retrieve k3sup on Ansible controller + connection: local + nullified.infrastructure.github_artifact: + asset_name: k3sup + asset_type: release + repository: alexellis/k3sup + creates: '{{ k3sup_binary }}' + cmds: + - mkdir -p $HOME/.local/bin + - "install --mode=750 {asset_dirname}/{asset_filename} {{ k3sup_binary }}" + +- name: setup kubeconfig repository + connection: local + ansible.builtin.file: + path: "{{ kubeconfig_repository }}" + state: directory + mode: '0700' + +- name: setup permissions + become: true + block: + - name: install sudo + ansible.builtin.apt: + update_cache: true + force_apt_get: true + cache_valid_time: 3600 + pkg: [ sudo ] + state: present + - name: add operator to sudoers + ansible.builtin.lineinfile: + backup: true + path: /etc/sudoers + regexp: "^{{ k3s_operator_username }}\b.+$" + line: "{{ k3s_operator_username }} ALL=(ALL) NOPASSWD: ALL" + state: present + register: backup_sudoers + changed_when: false + +- name: setup server role + include_tasks: server.yml + when: k3s_cluster_role is match("server") +- name: setup agent role + include_tasks: agent.yml + when: k3s_cluster_role is match("agent") + +- name: reset permissions + become: true + command: + cmd: "mv {{ backup_sudoers.backup }} /etc/sudoers" + removes: "{{ backup_sudoers.backup }}" + when: backup_sudoers.backup + changed_when: false diff --git a/collections/ansible_collections/nullified/infrastructure/roles/k3s/tasks/server.yml b/collections/ansible_collections/nullified/infrastructure/roles/k3s/tasks/server.yml new file mode 100644 index 0000000..e6221b1 --- /dev/null +++ b/collections/ansible_collections/nullified/infrastructure/roles/k3s/tasks/server.yml @@ -0,0 +1,44 @@ +--- +# TODO: disable swap + +- name: setup firewall rules + become: true + ansible.builtin.template: + src: ../templates/nftables.d/k3s_servers.nft.j2 + dest: /etc/nftables.d/k3s_servers.nft + mode: '0600' + notify: + - 'k3s : restart firewall service' + - 'k3s : restart k3s service' + +- name: flush handlers + ansible.builtin.meta: flush_handlers + +- name: install K3S cluster, single server + connection: local + ansible.builtin.command: + argv: + - "{{ k3sup_binary }}" + - install + - "--merge" + - "--local-path" + - "{{ kubeconfig_repository }}/{{ k3s_cluster_name }}.kubeconfig" + - "--context" + - "{{ k3s_kube_context }}" + - "--k3s-extra-args" + - "{{ k3s_extra_args }}" + - "--user" + - "{{ k3s_operator_username }}" + - "--ssh-key" + - "{{ k3s_operator_ssh_key_path }}" + - "--host" + - "{{ inventory_hostname }}.{{ global_dns_domainname }}" + register: k3s_init + when: k3s_cluster_type is match("single") + changed_when: not "No change detected so skipping service start" in k3s_init.stdout + +- name: install K3S cluster, HA + connection: local + debug: msg="Not supported yet" + when: k3s_cluster_type is match("ha") + failed_when: true diff --git a/collections/ansible_collections/nullified/infrastructure/roles/k3s/templates/nftables.d/k3s_agents.nft.j2 b/collections/ansible_collections/nullified/infrastructure/roles/k3s/templates/nftables.d/k3s_agents.nft.j2 new file mode 100644 index 0000000..e336a68 --- /dev/null +++ b/collections/ansible_collections/nullified/infrastructure/roles/k3s/templates/nftables.d/k3s_agents.nft.j2 @@ -0,0 +1,22 @@ +# K3S source: agents +table inet filter { + chain input { + # inter-node communication + ## UDP + {%+ if k3s_nft_servers4 or k3s_nft_agents4 %}ip saddr { {{ (k3s_nft_servers4 + k3s_nft_agents4) | join(', ') }} } udp dport { 8472, 51820 } accept{%- endif +%} + {%+ if k3s_nft_servers6 or k3s_nft_agents6 %}ip6 saddr { {{ (k3s_nft_servers6 + k3s_nft_agents6) | join(', ') }} } udp dport { 8472, 51821 } accept{%- endif +%} + ## TCP + {%+ if k3s_nft_servers4 or k3s_nft_agents4 %}ip saddr { {{ (k3s_nft_servers4 + k3s_nft_agents4) | join(', ') }} } tcp dport { 5001, 6443, 10250 } accept{%- endif +%} + {%+ if k3s_nft_servers6 or k3s_nft_agents6 %}ip6 saddr { {{ (k3s_nft_servers6 + k3s_nft_agents6) | join(', ') }} } tcp dport { 5001, 6443, 10250 } accept{%- endif +%} + } + + chain output { + # inter-node communication + ## UDP + {%+ if k3s_nft_servers4 or k3s_nft_agents4 %}ip daddr { {{ (k3s_nft_servers4 + k3s_nft_agents4) | join(', ') }} } udp dport { 8472, 51820 } accept{%- endif +%} + {%+ if k3s_nft_servers6 or k3s_nft_agents6 %}ip6 daddr { {{ (k3s_nft_servers6 + k3s_nft_agents6) | join(', ') }} } udp dport { 8472, 51821 } accept{%- endif +%} + ## TCP + {%+ if k3s_nft_servers4 or k3s_nft_agents4 %}ip daddr { {{ (k3s_nft_servers4 + k3s_nft_agents4) | join(', ') }} } tcp dport { 5001, 6443, 10250 } accept{%- endif +%} + {%+ if k3s_nft_servers6 or k3s_nft_agents6 %}ip6 daddr { {{ (k3s_nft_servers6 + k3s_nft_agents6) | join(', ') }} } tcp dport { 5001, 6443, 10250 } accept{%- endif +%} + } +} diff --git a/collections/ansible_collections/nullified/infrastructure/roles/k3s/templates/nftables.d/k3s_servers.nft.j2 b/collections/ansible_collections/nullified/infrastructure/roles/k3s/templates/nftables.d/k3s_servers.nft.j2 new file mode 100644 index 0000000..cf04da6 --- /dev/null +++ b/collections/ansible_collections/nullified/infrastructure/roles/k3s/templates/nftables.d/k3s_servers.nft.j2 @@ -0,0 +1,44 @@ +# K3S source: servers +table inet filter { + chain input { + # operators access + {%+ if k3s_nft_operators4 %}ip saddr { {{ k3s_nft_operators4 | join(', ') }} } tcp dport { 6443 } accept{%- endif +%} + {%+ if k3s_nft_operators6 %}ip saddr { {{ k3s_nft_operators6 | join(', ') }} } tcp dport { 6443 } accept{%- endif +%} + + # required only for HA with embedded etcd + {%+ if k3s_nft_servers4 %}ip saddr { {{ k3s_nft_servers4 | join(',') }} } tcp dport { 2379, 2380 } accept{%- endif +%} + {%+ if k3s_nft_servers6 %}ip6 saddr { {{ k3s_nft_servers6 | join(',') }} } tcp dport { 2379, 2380 } accept{%- endif +%} + + # inter-node communication + ## UDP + {%+ if k3s_nft_servers4 or k3s_nft_agents4 %}ip saddr { {{ (k3s_nft_servers4 + k3s_nft_agents4) | join(', ') }} } udp dport { 8472, 51820 } accept{%- endif +%} + {%+ if k3s_nft_servers6 or k3s_nft_agents6 %}ip6 saddr { {{ (k3s_nft_servers6 + k3s_nft_agents6) | join(', ') }} } udp dport { 8472, 51821 } accept{%- endif +%} + ## TCP + {%+ if k3s_nft_servers4 or k3s_nft_agents4 %}ip saddr { {{ (k3s_nft_servers4 + k3s_nft_agents4) | join(', ') }} } tcp dport { 5001, 6443, 10250 } accept{%- endif +%} + {%+ if k3s_nft_servers6 or k3s_nft_agents6 %}ip6 saddr { {{ (k3s_nft_servers6 + k3s_nft_agents6) | join(', ') }} } tcp dport { 5001, 6443, 10250 } accept{%- endif +%} + + {%+ if k3s_cluster_cidr | ansible.utils.ipv4 %}ip saddr {{ k3s_cluster_cidr }} meta l4proto { tcp, udp } accept{%- endif +%} + {%+ if k3s_cluster_cidr | ansible.utils.ipv6 %}ip6 saddr {{ k3s_cluster_cidr }} meta l4proto { tcp, udp } accept{%- endif +%} + {%+ if k3s_service_cidr | ansible.utils.ipv4 %}ip saddr {{ k3s_service_cidr }} meta l4proto { tcp, udp } accept{%- endif +%} + {%+ if k3s_service_cidr | ansible.utils.ipv6 %}ip6 saddr {{ k3s_service_cidr }} meta l4proto { tcp, udp } accept{%- endif +%} + } + + chain output { + # required only for HA with embedded etcd + {%+ if k3s_nft_servers4 %}ip daddr { {{ k3s_nft_servers4 | join(',') }} } tcp dport { 2379, 2380 } accept{%- endif +%} + {%+ if k3s_nft_servers6 %}ip6 daddr { {{ k3s_nft_servers6 | join(',') }} } tcp dport { 2379, 2380 } accept{%- endif +%} + + # inter-node communication + ## UDP + {%+ if k3s_nft_servers4 or k3s_nft_agents4 %}ip daddr { {{ (k3s_nft_servers4 + k3s_nft_agents4) | join(', ') }} } udp dport { 8472, 51820 } accept{%- endif +%} + {%+ if k3s_nft_servers6 or k3s_nft_agents6 %}ip6 daddr { {{ (k3s_nft_servers6 + k3s_nft_agents6) | join(', ') }} } udp dport { 8472, 51821 } accept{%- endif +%} + ## TCP + {%+ if k3s_nft_servers4 or k3s_nft_agents4 %}ip daddr { {{ (k3s_nft_servers4 + k3s_nft_agents4) | join(', ') }} } tcp dport { 5001, 6443, 10250 } accept{%- endif +%} + {%+ if k3s_nft_servers6 or k3s_nft_agents6 %}ip6 daddr { {{ (k3s_nft_servers6 + k3s_nft_agents6) | join(', ') }} } tcp dport { 5001, 6443, 10250 } accept{%- endif +%} + + {%+ if k3s_cluster_cidr | ansible.utils.ipv4 %}ip daddr {{ k3s_cluster_cidr }} meta l4proto { tcp, udp } accept{%- endif +%} + {%+ if k3s_cluster_cidr | ansible.utils.ipv6 %}ip6 daddr {{ k3s_cluster_cidr }} meta l4proto { tcp, udp } accept{%- endif +%} + {%+ if k3s_service_cidr | ansible.utils.ipv4 %}ip daddr {{ k3s_service_cidr }} meta l4proto { tcp, udp } accept{%- endif +%} + {%+ if k3s_service_cidr | ansible.utils.ipv6 %}ip6 daddr {{ k3s_service_cidr }} meta l4proto { tcp, udp } accept{%- endif +%} + } +} diff --git a/collections/ansible_collections/nullified/infrastructure/roles/k3s/tests/inventory b/collections/ansible_collections/nullified/infrastructure/roles/k3s/tests/inventory new file mode 100644 index 0000000..878877b --- /dev/null +++ b/collections/ansible_collections/nullified/infrastructure/roles/k3s/tests/inventory @@ -0,0 +1,2 @@ +localhost + diff --git a/collections/ansible_collections/nullified/infrastructure/roles/k3s/vars/main.yml b/collections/ansible_collections/nullified/infrastructure/roles/k3s/vars/main.yml new file mode 100644 index 0000000..ed97d53 --- /dev/null +++ b/collections/ansible_collections/nullified/infrastructure/roles/k3s/vars/main.yml @@ -0,0 +1 @@ +--- diff --git a/inventory/group_vars/all/vars.yml b/inventory/group_vars/all/vars.yml index 7c96535..93de873 100644 --- a/inventory/group_vars/all/vars.yml +++ b/inventory/group_vars/all/vars.yml @@ -6,6 +6,7 @@ custom_github_token: "{{ vault_custom_github_token | default('') }}" # global (hosts' system parameters) ## TODO: move to a CMDB global_dns_type: "dot" +global_dns_domainname: "{{ vault_global_dns_domainname }}" # empty values for dns{4,6} servers mean that servers will be retrieved dynamically from /etc/resolv.conf global_dns_udp_dns4: "{{ vault_global_dns_udp_dns4 }}" global_dns_udp_dns6: "{{ vault_global_dns_udp_dns6 }}" diff --git a/inventory/group_vars/internal/vars.yml b/inventory/group_vars/internal/vars.yml index d8e1701..1e5be3d 100644 --- a/inventory/group_vars/internal/vars.yml +++ b/inventory/group_vars/internal/vars.yml @@ -6,3 +6,7 @@ global_ip_dualstack: false # security role security_firewall_mangle_drop_privatenets: false +global_dns_domainname: "{{ vault_global_dns_domainname }}" +k3s_operator_ssh_key_path: "{{ vault_k3s_operator_ssh_key_path }}" +k3s_operator_ips: "{{ vault_k3s_operator_ips }}" +k3s_operator_username: "{{ vault_k3s_operator_username }}" diff --git a/inventory/host_vars/actinium/vars.yml b/inventory/host_vars/actinium/vars.yml index f6f5a58..57a5f10 100644 --- a/inventory/host_vars/actinium/vars.yml +++ b/inventory/host_vars/actinium/vars.yml @@ -4,3 +4,7 @@ ansible_user: "{{ vault_ssh_user }}" security_firewall_mangle_drop_privatenets: false security_firewall_mangle_policy_forward: accept + +k3s_cluster_name: internal +k3s_cluster_role: server +k3s_cluster_ip: "{{ vault_cluster_ip }}" diff --git a/inventory/host_vars/lithium/vars.yml b/inventory/host_vars/lithium/vars.yml index e7c8a02..15f65fc 100644 --- a/inventory/host_vars/lithium/vars.yml +++ b/inventory/host_vars/lithium/vars.yml @@ -1,3 +1,6 @@ ansible_become_password: "{{ vault_root_pass }}" ansible_host: "{{ vault_ansible_host }}" ansible_user: "{{ vault_ssh_user }}" + +k3s_cluster_name: pounce +k3s_cluster_role: server diff --git a/playbooks/internal.yml b/playbooks/internal.yml index 23315f4..96394ff 100644 --- a/playbooks/internal.yml +++ b/playbooks/internal.yml @@ -33,3 +33,10 @@ - name: include gaming role ansible.builtin.include_role: name: nullified.infrastructure.gaming + +- name: install K3S + hosts: internal:&k3s + tasks: + - name: include k3s role + ansible.builtin.include_role: + name: nullified.infrastructure.k3s