feat!(tasks): move tasks to include; move dependencies installation to a task;...

.gitignore

@@ -24,7 +24,7 @@ vault.yml
 *.pyo
 *.pyc
 __pycache__/
-venv
+.venv
 # various
 TODO.md
 !.gitkeep

README.md

@@ -19,57 +19,41 @@ thoroughly reviewed it, especially the system hardening and security tweaks that
 
 # Usage
 ## Prerequisites
-***Required dependencies***
+- curl or similar,
-- Python3.9+,
+- python3.9+,
-- PIP,
+- [go-task](https://github.com/go-task/task) (or follow the instructions below to install it),
-- Python3 venv
 
-***Dependencies installed using the `Installation` instructions***
-- [Task](https://taskfile.dev/),
-- Debian packages:
-  - curl
-  - libcurl4-openssl-dev,
-  - libssl-dev,
-  - libcairo2,
-  - libcairo2-dev,
-  - libffi-dev,
-  - python3-dev,
-  - python3-venv
-
-***Optional, dev-related dependencies***
-- Docker
-
-## Installation
 ```shell
-# Debian amd64
+# To install Task for a Debian (amd64) based system you can execute the following
-sudo apt install -y \
+TEMP_WORKDIR=$(mktemp -d)
-  curl \
+cd "${TEMP_WORKDIR}"
-  libcurl4-openssl-dev \
+# retrieve the latest available version's tag
-  libssl-dev \
-  libcairo2 \
-  libcairo2-dev \
-  libffi-dev \
-  python3-venv \
-  python3-dev;
 TASK_VERSION=$(curl -fsSL -XGET https://api.github.com/repos/go-task/task/releases/latest | grep tag_name | tr -d ' ",' | cut -d ':' -f 2)
-curl -fsSLO https://github.com/go-task/task/releases/download/"${TASK_VERSION}"/task_linux_amd64.deb;
+# retrieve the Debian archive
-sudo dpkg -i task_linux_amd64.deb;
+curl -fsSLO https://github.com/go-task/task/releases/download/"${TASK_VERSION}"/task_linux_amd64.deb
-rm task_linux_amd64.deb;
+# retrieve the checksums file
+curl -fsSLO https://github.com/go-task/task/releases/download/"${TASK_VERSION}/"task_checksums.txt
+# ensure the downloaded archive is OK
+sha256sum -c task_checksums.txt --ignore-missing
+# install the tool
+sudo dpkg -i ./task_linux_amd64.deb
+cd -
+rm -r "${TEMP_WORKDIR}"
+unset TEMP_WORKDIR
 ```
 
 ## Setup
 ```shell
 # Generate default ansible configuration
-./scripts/generate_ansible_config.sh > "${HOME}"/.ansible.cfg
+./scripts/generate_ansible_config.sh ./ansible.cfg
 
-# Setup Python virtual env
+# Setup the project
-task venv:setup
+task setup
+# additionally install the dev dependencies if you need them
+task setup:dev
 
 # Prepare and edit your inventory as needed
 cp inventory/inventory.yml.dist inventory/inventory.yml
-
-# Prepare and edit the global vault as needed
-cp inventory/vault.yml.dist inventory/vault.yml
 ```
 
 ## Using Tasks
@@ -145,3 +129,6 @@ Global variables are defined in `inventory/group_vars/all/vars.yml` and are used
 - inventory: all inventory related files are stored here;
 - playbooks: top level playbooks, describe the way the infrastructure is laid out;
 - scripts: various scripts and helpers;
+
+# TODO / Issues
+- gpg setup for HC Vault should be done with current user or root only, not mixed

Taskfile.yml

@@ -1,5 +1,9 @@
 version: '3'
 
+includes:
+  setup: ./tasks/setup_{{OS}}.yml
+  test: ./tasks/tests.yml
+
 env:
   DOCKER_REPOSITORY: pouncetech/molecule
 
@@ -9,71 +13,13 @@ vars:
   COLLECTIONS_DIR: '{{.ROOT_DIR}}/collections/ansible_collections'
 
 tasks:
-  setup:venv:
-    desc: install a Python3 virtualenv and all the required ansible / molecule dependencies.
-    cmds:
-      - |
-        set -e
-        rm -rf ./venv || true
-        python3 -m virtualenv --download venv
-        ./venv/bin/python3 -m pip install --upgrade -r requirements.txt
-    status:
-      - test -d venv
-
-  setup:ansible:
-    desc: install ansible galaxy collections
-    cmds:
-      - '{{.PYTHON_WRAPPER}} ansible-galaxy collection install -r ansible_galaxy-requirements.yml'
-
   setup:
-    desc: set up environment for Python virtualenv and ansible dependencies
+    desc: install requirements, set up a Python virtualenv and install ansible dependencies
     cmds:
+      - task: 'setup:requirements'
       - task: 'setup:venv'
       - task: 'setup:ansible'
-
+      - task: 'setup:galaxy'
-  test:modules:
-    desc: run `ansible-test sanity` on collections to find common issues for modules and collections
-    dir: collections/ansible_collections
-    vars:
-      ANSIBLE_COLLECTIONS:
-        sh: find -mindepth 2 -maxdepth 2 -type d
-    cmds:
-      - for: { var: ANSIBLE_COLLECTIONS }
-        task: 'test:module:sanity'
-        vars:
-          COLLECTION_PATH: 'collections/ansible_collections/{{.ITEM}}'
-
-  test:module:sanity:
-    internal: true
-    dir: '{{.COLLECTION_PATH}}'
-    cmds:
-      - 'test -d tests && {{.PYTHON_WRAPPER}} ansible-test sanity --venv || echo - ignored $(pwd)'
-
-  test:collections:
-    desc: run molecule tests for all roles and collections.
-    dir: collections/ansible_collections
-    vars:
-      ANSIBLE_COLLECTIONS:
-        sh: find -mindepth 2 -maxdepth 2 -type d
-    cmds:
-      - for: { var: ANSIBLE_COLLECTIONS }
-        task: 'test:collection:molecule'
-        vars:
-          COLLECTION_PATH: 'collections/ansible_collections/{{.ITEM}}'
-
-  test:collection:molecule:
-    internal: true
-    dir: '{{.COLLECTION_PATH}}/extensions'
-    cmds:
-      - 'test -d molecule && {{.PYTHON_WRAPPER}} molecule test || echo - ignored $(pwd)'
-
-  module:github_artifact:
-    desc: run a module from the collection for testing purposes
-    vars:
-      PLUGINS_DIR: '{{.ROOT_DIR}}/collections/ansible_collections/nullified/infrastructure/plugins'
-    cmd: |
-      {{.PYTHON_WRAPPER}} python3 {{.PLUGINS_DIR}}/modules/github_artifact.py {{.PLUGINS_DIR}}/tests/github_artifact.json |
-        {{.PYTHON_WRAPPER}} python3 -m json.tool | {{.PYTHON_WRAPPER}} pygmentize -l json
 
   docker:build:
     desc: build docker images locally.

scripts/python_wrapper.sh

@@ -3,7 +3,7 @@
 set -e
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
-test -d "$SCRIPT_DIR"/../venv || (printf "Python virtualenv not found (%s)\n" "$SCRIPT_DIR"/../venv && exit 1)
+test -d "$SCRIPT_DIR"/../.venv || (printf "Python virtualenv not found (%s)\n" "$SCRIPT_DIR"/../.venv && exit 1)
-source "$SCRIPT_DIR"/../venv/bin/activate
+source "$SCRIPT_DIR"/../.venv/bin/activate
 
 exec "$@"

scripts/setup_dev_env.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+set -e -o noclobber -o pipefail
+
+sudo apt install --no-install-recommends --assume-yes \
+  libvirt0 \
+  libvirt-clients \
+  libvirt-clients-qemu \
+  libvirt-daemon \
+  libvirt-daemon-config-network \
+  libvirt-daemon-driver-qemu \
+  libvirt-daemon-system \
+  libvirt-daemon-system-systemd \
+  libvirt-dev \
+  vagrant \
+  vagrant-libvirt
+
+./.venv/bin/python3 -m pip install --upgrade -r requirements-dev.txt
+[ -d /etc/nftables.d ] || sudo mkdir /etc/nftables.d
+
+sudo tee /etc/nftables.d/vagrant.nft <<'EOF' > /dev/null
+define virt_if = { "virbr0", "virbr1", "virbr2" }
+
+table inet filter {
+    chain input {
+        iifname $virt_if accept
+    }
+    chain output {
+        oifname $virt_if accept
+    }
+}
+EOF
+
+sudo /usr/sbin/nft -f /etc/nftables.d/vagrant.nft

tasks/setup_linux.yml

@@ -0,0 +1,45 @@
+version: '3'
+
+tasks:
+  requirements:
+    desc: install required packages
+    cmds:
+      - |
+        sudo apt install --no-install-recommends --assume-yes \
+          coreutils \
+          curl \
+          libcurl4-openssl-dev \
+          python3-virtualenv \
+          python3-dev
+
+  venv:
+    desc: install a Python3 virtualenv and all the required ansible / molecule dependencies.
+    cmds:
+      - |
+        rm -rf ./.venv || true
+        python3 -m virtualenv --download ./.venv
+        ./.venv/bin/python3 -m pip install --upgrade -r requirements.txt
+    status:
+      - test -d ./.venv
+
+  ansible:
+    desc: create or update an ansible.cfg file
+    cmds:
+      - ./scripts/generate_ansible_config.sh -o ansible.cfg
+    status:
+      - test -f ansible.cfg
+
+  galaxy:
+    desc: install ansible galaxy collections
+    cmds:
+      - "{{.PYTHON_WRAPPER}} ansible-galaxy collection install -r ansible_galaxy-requirements.yml"
+
+  dev:
+    desc: install required packages to run molecule, vagrant and other dev tools
+    cmds:
+      - ./scripts/setup_dev_env.sh
+    preconditions:
+      - sh: test -f ./.venv/bin/python3
+        msg: virtualenv must be set up first
+      - sh: test -x /usr/sbin/nft
+        msg: "nftables is required (tested bin path: /usr/sbin/nft)"

tasks/tests.yml

@@ -0,0 +1,46 @@
+version: '3'
+
+tasks:
+  modules:
+    desc: run `ansible-test sanity` on collections to find common issues for modules and collections
+    dir: collections/ansible_collections
+    vars:
+      ANSIBLE_COLLECTIONS:
+        sh: find -mindepth 2 -maxdepth 2 -type d
+    cmds:
+      - for: { var: ANSIBLE_COLLECTIONS }
+        task: 'test:module:sanity'
+        vars:
+          COLLECTION_PATH: 'collections/ansible_collections/{{.ITEM}}'
+
+  module:sanity:
+    internal: true
+    dir: '{{.COLLECTION_PATH}}'
+    cmds:
+      - 'test -d tests && {{.PYTHON_WRAPPER}} ansible-test sanity --venv || echo - ignored $(pwd)'
+
+  collections:
+    desc: run molecule tests for all roles and collections.
+    dir: collections/ansible_collections
+    vars:
+      ANSIBLE_COLLECTIONS:
+        sh: find -mindepth 2 -maxdepth 2 -type d
+    cmds:
+      - for: { var: ANSIBLE_COLLECTIONS }
+        task: 'test:collection:molecule'
+        vars:
+          COLLECTION_PATH: 'collections/ansible_collections/{{.ITEM}}'
+
+  collection:molecule:
+    internal: true
+    dir: '{{.COLLECTION_PATH}}/extensions'
+    cmds:
+      - 'test -d molecule && {{.PYTHON_WRAPPER}} molecule test || echo - ignored $(pwd)'
+
+  module:github_artifact:
+    desc: run a module from the collection for testing purposes
+    vars:
+      PLUGINS_DIR: '{{.ROOT_DIR}}/collections/ansible_collections/nullified/infrastructure/plugins'
+    cmd: |
+      {{.PYTHON_WRAPPER}} python3 {{.PLUGINS_DIR}}/modules/github_artifact.py {{.PLUGINS_DIR}}/tests/github_artifact.json |
+        {{.PYTHON_WRAPPER}} python3 -m json.tool | {{.PYTHON_WRAPPER}} pygmentize -l json