diff --git a/.gitignore b/.gitignore index d178462..c19b82b 100644 --- a/.gitignore +++ b/.gitignore @@ -24,7 +24,7 @@ vault.yml *.pyo *.pyc __pycache__/ -venv +.venv # various TODO.md !.gitkeep diff --git a/README.md b/README.md index ff2ea77..c2f4f73 100644 --- a/README.md +++ b/README.md @@ -19,57 +19,41 @@ thoroughly reviewed it, especially the system hardening and security tweaks that # Usage ## Prerequisites -***Required dependencies*** -- Python3.9+, -- PIP, -- Python3 venv +- curl or similar, +- python3.9+, +- [go-task](https://github.com/go-task/task) (or follow the instructions below to install it), -***Dependencies installed using the `Installation` instructions*** -- [Task](https://taskfile.dev/), -- Debian packages: - - curl - - libcurl4-openssl-dev, - - libssl-dev, - - libcairo2, - - libcairo2-dev, - - libffi-dev, - - python3-dev, - - python3-venv - -***Optional, dev-related dependencies*** -- Docker - -## Installation ```shell -# Debian amd64 -sudo apt install -y \ - curl \ - libcurl4-openssl-dev \ - libssl-dev \ - libcairo2 \ - libcairo2-dev \ - libffi-dev \ - python3-venv \ - python3-dev; +# To install Task for a Debian (amd64) based system you can execute the following +TEMP_WORKDIR=$(mktemp -d) +cd "${TEMP_WORKDIR}" +# retrieve the latest available version's tag TASK_VERSION=$(curl -fsSL -XGET https://api.github.com/repos/go-task/task/releases/latest | grep tag_name | tr -d ' ",' | cut -d ':' -f 2) -curl -fsSLO https://github.com/go-task/task/releases/download/"${TASK_VERSION}"/task_linux_amd64.deb; -sudo dpkg -i task_linux_amd64.deb; -rm task_linux_amd64.deb; +# retrieve the Debian archive +curl -fsSLO https://github.com/go-task/task/releases/download/"${TASK_VERSION}"/task_linux_amd64.deb +# retrieve the checksums file +curl -fsSLO https://github.com/go-task/task/releases/download/"${TASK_VERSION}/"task_checksums.txt +# ensure the downloaded archive is OK +sha256sum -c task_checksums.txt --ignore-missing +# install the tool +sudo dpkg -i ./task_linux_amd64.deb +cd - +rm -r "${TEMP_WORKDIR}" +unset TEMP_WORKDIR ``` ## Setup ```shell # Generate default ansible configuration -./scripts/generate_ansible_config.sh > "${HOME}"/.ansible.cfg +./scripts/generate_ansible_config.sh ./ansible.cfg -# Setup Python virtual env -task venv:setup +# Setup the project +task setup +# additionally install the dev dependencies if you need them +task setup:dev # Prepare and edit your inventory as needed cp inventory/inventory.yml.dist inventory/inventory.yml - -# Prepare and edit the global vault as needed -cp inventory/vault.yml.dist inventory/vault.yml ``` ## Using Tasks @@ -145,3 +129,6 @@ Global variables are defined in `inventory/group_vars/all/vars.yml` and are used - inventory: all inventory related files are stored here; - playbooks: top level playbooks, describe the way the infrastructure is laid out; - scripts: various scripts and helpers; + +# TODO / Issues +- gpg setup for HC Vault should be done with current user or root only, not mixed diff --git a/Taskfile.yml b/Taskfile.yml index 12420bf..2905361 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -1,5 +1,9 @@ version: '3' +includes: + setup: ./tasks/setup_{{OS}}.yml + test: ./tasks/tests.yml + env: DOCKER_REPOSITORY: pouncetech/molecule @@ -9,71 +13,13 @@ vars: COLLECTIONS_DIR: '{{.ROOT_DIR}}/collections/ansible_collections' tasks: - setup:venv: - desc: install a Python3 virtualenv and all the required ansible / molecule dependencies. - cmds: - - | - set -e - rm -rf ./venv || true - python3 -m virtualenv --download venv - ./venv/bin/python3 -m pip install --upgrade -r requirements.txt - status: - - test -d venv - - setup:ansible: - desc: install ansible galaxy collections - cmds: - - '{{.PYTHON_WRAPPER}} ansible-galaxy collection install -r ansible_galaxy-requirements.yml' - setup: - desc: set up environment for Python virtualenv and ansible dependencies + desc: install requirements, set up a Python virtualenv and install ansible dependencies cmds: + - task: 'setup:requirements' - task: 'setup:venv' - task: 'setup:ansible' - - test:modules: - desc: run `ansible-test sanity` on collections to find common issues for modules and collections - dir: collections/ansible_collections - vars: - ANSIBLE_COLLECTIONS: - sh: find -mindepth 2 -maxdepth 2 -type d - cmds: - - for: { var: ANSIBLE_COLLECTIONS } - task: 'test:module:sanity' - vars: - COLLECTION_PATH: 'collections/ansible_collections/{{.ITEM}}' - - test:module:sanity: - internal: true - dir: '{{.COLLECTION_PATH}}' - cmds: - - 'test -d tests && {{.PYTHON_WRAPPER}} ansible-test sanity --venv || echo - ignored $(pwd)' - - test:collections: - desc: run molecule tests for all roles and collections. - dir: collections/ansible_collections - vars: - ANSIBLE_COLLECTIONS: - sh: find -mindepth 2 -maxdepth 2 -type d - cmds: - - for: { var: ANSIBLE_COLLECTIONS } - task: 'test:collection:molecule' - vars: - COLLECTION_PATH: 'collections/ansible_collections/{{.ITEM}}' - - test:collection:molecule: - internal: true - dir: '{{.COLLECTION_PATH}}/extensions' - cmds: - - 'test -d molecule && {{.PYTHON_WRAPPER}} molecule test || echo - ignored $(pwd)' - - module:github_artifact: - desc: run a module from the collection for testing purposes - vars: - PLUGINS_DIR: '{{.ROOT_DIR}}/collections/ansible_collections/nullified/infrastructure/plugins' - cmd: | - {{.PYTHON_WRAPPER}} python3 {{.PLUGINS_DIR}}/modules/github_artifact.py {{.PLUGINS_DIR}}/tests/github_artifact.json | - {{.PYTHON_WRAPPER}} python3 -m json.tool | {{.PYTHON_WRAPPER}} pygmentize -l json + - task: 'setup:galaxy' docker:build: desc: build docker images locally. diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..e69de29 diff --git a/scripts/python_wrapper.sh b/scripts/python_wrapper.sh index 0b9f254..b10a534 100755 --- a/scripts/python_wrapper.sh +++ b/scripts/python_wrapper.sh @@ -3,7 +3,7 @@ set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -test -d "$SCRIPT_DIR"/../venv || (printf "Python virtualenv not found (%s)\n" "$SCRIPT_DIR"/../venv && exit 1) -source "$SCRIPT_DIR"/../venv/bin/activate +test -d "$SCRIPT_DIR"/../.venv || (printf "Python virtualenv not found (%s)\n" "$SCRIPT_DIR"/../.venv && exit 1) +source "$SCRIPT_DIR"/../.venv/bin/activate exec "$@" diff --git a/scripts/setup_dev_env.sh b/scripts/setup_dev_env.sh new file mode 100755 index 0000000..718865c --- /dev/null +++ b/scripts/setup_dev_env.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +set -e -o noclobber -o pipefail + +sudo apt install --no-install-recommends --assume-yes \ + libvirt0 \ + libvirt-clients \ + libvirt-clients-qemu \ + libvirt-daemon \ + libvirt-daemon-config-network \ + libvirt-daemon-driver-qemu \ + libvirt-daemon-system \ + libvirt-daemon-system-systemd \ + libvirt-dev \ + vagrant \ + vagrant-libvirt + +./.venv/bin/python3 -m pip install --upgrade -r requirements-dev.txt +[ -d /etc/nftables.d ] || sudo mkdir /etc/nftables.d + +sudo tee /etc/nftables.d/vagrant.nft <<'EOF' > /dev/null +define virt_if = { "virbr0", "virbr1", "virbr2" } + +table inet filter { + chain input { + iifname $virt_if accept + } + chain output { + oifname $virt_if accept + } +} +EOF + +sudo /usr/sbin/nft -f /etc/nftables.d/vagrant.nft diff --git a/tasks/setup_linux.yml b/tasks/setup_linux.yml new file mode 100644 index 0000000..addbfa2 --- /dev/null +++ b/tasks/setup_linux.yml @@ -0,0 +1,45 @@ +version: '3' + +tasks: + requirements: + desc: install required packages + cmds: + - | + sudo apt install --no-install-recommends --assume-yes \ + coreutils \ + curl \ + libcurl4-openssl-dev \ + python3-virtualenv \ + python3-dev + + venv: + desc: install a Python3 virtualenv and all the required ansible / molecule dependencies. + cmds: + - | + rm -rf ./.venv || true + python3 -m virtualenv --download ./.venv + ./.venv/bin/python3 -m pip install --upgrade -r requirements.txt + status: + - test -d ./.venv + + ansible: + desc: create or update an ansible.cfg file + cmds: + - ./scripts/generate_ansible_config.sh -o ansible.cfg + status: + - test -f ansible.cfg + + galaxy: + desc: install ansible galaxy collections + cmds: + - "{{.PYTHON_WRAPPER}} ansible-galaxy collection install -r ansible_galaxy-requirements.yml" + + dev: + desc: install required packages to run molecule, vagrant and other dev tools + cmds: + - ./scripts/setup_dev_env.sh + preconditions: + - sh: test -f ./.venv/bin/python3 + msg: virtualenv must be set up first + - sh: test -x /usr/sbin/nft + msg: "nftables is required (tested bin path: /usr/sbin/nft)" diff --git a/tasks/tests.yml b/tasks/tests.yml new file mode 100644 index 0000000..4a291c2 --- /dev/null +++ b/tasks/tests.yml @@ -0,0 +1,46 @@ +version: '3' + +tasks: + modules: + desc: run `ansible-test sanity` on collections to find common issues for modules and collections + dir: collections/ansible_collections + vars: + ANSIBLE_COLLECTIONS: + sh: find -mindepth 2 -maxdepth 2 -type d + cmds: + - for: { var: ANSIBLE_COLLECTIONS } + task: 'test:module:sanity' + vars: + COLLECTION_PATH: 'collections/ansible_collections/{{.ITEM}}' + + module:sanity: + internal: true + dir: '{{.COLLECTION_PATH}}' + cmds: + - 'test -d tests && {{.PYTHON_WRAPPER}} ansible-test sanity --venv || echo - ignored $(pwd)' + + collections: + desc: run molecule tests for all roles and collections. + dir: collections/ansible_collections + vars: + ANSIBLE_COLLECTIONS: + sh: find -mindepth 2 -maxdepth 2 -type d + cmds: + - for: { var: ANSIBLE_COLLECTIONS } + task: 'test:collection:molecule' + vars: + COLLECTION_PATH: 'collections/ansible_collections/{{.ITEM}}' + + collection:molecule: + internal: true + dir: '{{.COLLECTION_PATH}}/extensions' + cmds: + - 'test -d molecule && {{.PYTHON_WRAPPER}} molecule test || echo - ignored $(pwd)' + + module:github_artifact: + desc: run a module from the collection for testing purposes + vars: + PLUGINS_DIR: '{{.ROOT_DIR}}/collections/ansible_collections/nullified/infrastructure/plugins' + cmd: | + {{.PYTHON_WRAPPER}} python3 {{.PLUGINS_DIR}}/modules/github_artifact.py {{.PLUGINS_DIR}}/tests/github_artifact.json | + {{.PYTHON_WRAPPER}} python3 -m json.tool | {{.PYTHON_WRAPPER}} pygmentize -l json