From 1d2983c36980204d08f013d9bc8103e63840d398 Mon Sep 17 00:00:00 2001 From: NaeiKinDus Date: Fri, 23 May 2025 00:00:00 +0000 Subject: [PATCH 1/4] fix(valkey): install systemd unit in proper dir; check local and git versions to avoid unnecessary recompiles --- .../infrastructure/roles/valkey/defaults/main.yml | 4 ++-- .../infrastructure/roles/valkey/tasks/gather_facts.yml | 9 ++++++++- .../nullified/infrastructure/roles/valkey/tasks/main.yml | 9 +++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/ansible_collections/nullified/infrastructure/roles/valkey/defaults/main.yml b/ansible_collections/nullified/infrastructure/roles/valkey/defaults/main.yml index ade04eb..96fe037 100644 --- a/ansible_collections/nullified/infrastructure/roles/valkey/defaults/main.yml +++ b/ansible_collections/nullified/infrastructure/roles/valkey/defaults/main.yml @@ -11,8 +11,8 @@ valkey_generate_cert: true # if set to false you must override valkey_tls_{certf valkey_enable_unixsocket: false # TLS configuration -valkey_tls_certfile: ~ -valkey_tls_keyfile: ~ +valkey_tls_certfile: /etc/valkey/valkey.crt +valkey_tls_keyfile: /etc/valkey/valkey.key valkey_tls_keyfile_pass: ~ # Connection configuration diff --git a/ansible_collections/nullified/infrastructure/roles/valkey/tasks/gather_facts.yml b/ansible_collections/nullified/infrastructure/roles/valkey/tasks/gather_facts.yml index 265d0b3..a133a40 100644 --- a/ansible_collections/nullified/infrastructure/roles/valkey/tasks/gather_facts.yml +++ b/ansible_collections/nullified/infrastructure/roles/valkey/tasks/gather_facts.yml @@ -1,7 +1,7 @@ --- - name: find systemd unit directory become: true - ansible.builtin.command: pkg-config systemd --variable=systemdsystemunitdir + ansible.builtin.command: pkg-config systemd --variable=systemd_system_conf_dir changed_when: false register: systemd_unit_directory_cmd - name: find systemd version @@ -10,7 +10,14 @@ systemctl --version | awk '{if($1=="systemd" && $2~"^[0-9]+$"){print $2}}' changed_when: false register: systemd_version_cmd +- name: find local valkey version + ansible.builtin.shell: > + valkey-cli --version | cut -d ' ' -f 2 + failed_when: false + changed_when: false + register: valkey_local_version_cmd - name: set facts ansible.builtin.set_fact: systemd_unit_directory: "{{ systemd_unit_directory_cmd.stdout }}" systemd_version: "{{ systemd_version_cmd.stdout | int }}" + valkey_installed_version: "{{ (valkey_local_version_cmd.stdout_lines[0] | default('0')) if valkey_local_version_cmd.rc == 0 else '0' }}" diff --git a/ansible_collections/nullified/infrastructure/roles/valkey/tasks/main.yml b/ansible_collections/nullified/infrastructure/roles/valkey/tasks/main.yml index 68f66bd..58a9335 100644 --- a/ansible_collections/nullified/infrastructure/roles/valkey/tasks/main.yml +++ b/ansible_collections/nullified/infrastructure/roles/valkey/tasks/main.yml @@ -17,6 +17,7 @@ - libssl-dev - libsystemd-dev - make + - pkgconf - python3-cryptography - ssl-cert - name: gather facts @@ -48,8 +49,16 @@ cmds: - tar -zxf {asset_dirname}/{asset_filename} - 'mv $(find . -maxdepth 1 -name valkey-io-valkey\* -type d) {{ valkey_install_dir }}/src' +- name: fetch source version + become: true + ansible.builtin.command: + cmd: sed -En 's/#define VALKEY_VERSION "([0-9._-]+)"/\1/p' ./src/version.h + chdir: '{{ valkey_install_dir }}/src' + register: source_version_cmd + changed_when: false - name: compile Valkey (some tweaks taken from Valkey's Dockerfile become: true + when: source_version_cmd.stdout_lines[0] != valkey_installed_version args: chdir: '{{ valkey_install_dir }}/src' ansible.builtin.shell: '{{ item }}' From ade0cea349a95ca86550007c030d811168dfc0bc Mon Sep 17 00:00:00 2001 From: NaeiKinDus Date: Fri, 23 May 2025 00:00:00 +0000 Subject: [PATCH 2/4] feat(molecule): added switch to install or ignore faeture-only roles (e.g. workstation, deluge, ...) when running converge --- .../extensions/molecule/default/converge.yml | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/ansible_collections/nullified/infrastructure/extensions/molecule/default/converge.yml b/ansible_collections/nullified/infrastructure/extensions/molecule/default/converge.yml index 76998a3..6f34456 100644 --- a/ansible_collections/nullified/infrastructure/extensions/molecule/default/converge.yml +++ b/ansible_collections/nullified/infrastructure/extensions/molecule/default/converge.yml @@ -1,5 +1,4 @@ --- -# playbook file that contains the call for your role - name: Fail if molecule group is missing hosts: localhost tasks: @@ -8,10 +7,12 @@ that: "'molecule' in groups" fail_msg: | molecule group was not found inside inventory groups: {{ groups }} - - name: Converge hosts: molecule gather_facts: true + vars: + bypass_reqs: false + bypass_misc: true vars_files: - ../../../../../../inventory/group_vars/all/vars.yml - ../../../../../../inventory/group_vars/all/vault.yml @@ -43,6 +44,7 @@ autoremove: true install_recommends: false upgrade: full + changed_when: false - meta: flush_handlers - name: Enable guest console access become: true @@ -50,39 +52,54 @@ name: serial-getty@ttyS0.service enabled: true state: restarted + changed_when: false - name: Testing provisioner variables loading + when: bypass_reqs is falsy ansible.builtin.include_role: name: nullified.infrastructure.provisioner tasks_from: load_facts.yml - name: Testing security role + when: bypass_reqs is falsy ansible.builtin.include_role: name: nullified.infrastructure.security tasks_from: main.yml - name: Testing common role + when: bypass_reqs is falsy ansible.builtin.include_role: name: nullified.infrastructure.common tasks_from: main.yml - name: Testing server role + when: bypass_reqs is falsy ansible.builtin.include_role: name: nullified.infrastructure.server tasks_from: main.yml - name: Testing development role + when: bypass_misc is falsy ansible.builtin.include_role: name: nullified.infrastructure.development tasks_from: main.yml - name: Testing workstation role + when: bypass_misc is falsy ansible.builtin.include_role: name: nullified.infrastructure.workstation tasks_from: main.yml - name: Testing gaming role + when: bypass_misc is falsy ansible.builtin.include_role: name: nullified.infrastructure.gaming tasks_from: main.yml - name: Testing deluge role + when: bypass_misc is falsy ansible.builtin.include_role: name: nullified.infrastructure.deluge tasks_from: main.yml + - name: Testing nginx role + when: bypass_reqs is falsy + ansible.builtin.include_role: + name: nullified.infrastructure.nginx + tasks_from: main.yml - name: Testing valkey role + when: bypass_reqs is falsy ansible.builtin.include_role: name: nullified.infrastructure.valkey tasks_from: main.yml From 768a8133b5f96d65520da327593241f28ad20168 Mon Sep 17 00:00:00 2001 From: NaeiKinDus Date: Fri, 23 May 2025 00:00:00 +0000 Subject: [PATCH 3/4] fix(deluge): install systemd unit file in proper directory --- .../infrastructure/roles/deluge/tasks/gather_facts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible_collections/nullified/infrastructure/roles/deluge/tasks/gather_facts.yml b/ansible_collections/nullified/infrastructure/roles/deluge/tasks/gather_facts.yml index d724c51..b9deda9 100644 --- a/ansible_collections/nullified/infrastructure/roles/deluge/tasks/gather_facts.yml +++ b/ansible_collections/nullified/infrastructure/roles/deluge/tasks/gather_facts.yml @@ -1,7 +1,7 @@ --- - name: find systemd unit directory become: true - ansible.builtin.command: pkg-config systemd --variable=systemdsystemunitdir + ansible.builtin.command: pkg-config systemd --variable=systemd_system_conf_dir changed_when: false register: systemd_unit_directory_cmd - name: find systemd version From 63eab11b8553b6008764da940051ee325102f205 Mon Sep 17 00:00:00 2001 From: NaeiKinDus Date: Fri, 23 May 2025 00:00:00 +0000 Subject: [PATCH 4/4] feat(searxng): added new role to install and configure searxng --- .../extensions/molecule/default/converge.yml | 4 + .../molecule/default/platform_vars.yml | 23 + .../infrastructure/roles/searxng/README.md | 31 ++ .../roles/searxng/defaults/main.yml | 34 ++ .../roles/searxng/handlers/main.yml | 11 + .../roles/searxng/meta/main.yml | 13 + .../searxng/tasks/backend_uwsgi_setup.yml | 48 ++ .../roles/searxng/tasks/gather_facts.yml | 17 + .../roles/searxng/tasks/main.yml | 106 ++++ .../roles/searxng/templates/.gitkeep | 0 .../roles/searxng/templates/limiter.toml.j2 | 23 + .../roles/searxng/templates/settings.yml.j2 | 482 ++++++++++++++++++ .../templates/systemd/searxng.service.j2 | 60 +++ .../templates/systemd/searxng.socket.j2 | 15 + .../searxng/templates/uwsgi/uwsgi.ini.j2 | 35 ++ .../roles/searxng/tests/inventory | 2 + .../roles/searxng/tests/test.yml | 5 + .../roles/searxng/vars/main.yml | 4 + playbooks/external.yml | 10 + 19 files changed, 923 insertions(+) create mode 100644 ansible_collections/nullified/infrastructure/roles/searxng/README.md create mode 100644 ansible_collections/nullified/infrastructure/roles/searxng/defaults/main.yml create mode 100644 ansible_collections/nullified/infrastructure/roles/searxng/handlers/main.yml create mode 100644 ansible_collections/nullified/infrastructure/roles/searxng/meta/main.yml create mode 100644 ansible_collections/nullified/infrastructure/roles/searxng/tasks/backend_uwsgi_setup.yml create mode 100644 ansible_collections/nullified/infrastructure/roles/searxng/tasks/gather_facts.yml create mode 100644 ansible_collections/nullified/infrastructure/roles/searxng/tasks/main.yml create mode 100644 ansible_collections/nullified/infrastructure/roles/searxng/templates/.gitkeep create mode 100644 ansible_collections/nullified/infrastructure/roles/searxng/templates/limiter.toml.j2 create mode 100644 ansible_collections/nullified/infrastructure/roles/searxng/templates/settings.yml.j2 create mode 100644 ansible_collections/nullified/infrastructure/roles/searxng/templates/systemd/searxng.service.j2 create mode 100644 ansible_collections/nullified/infrastructure/roles/searxng/templates/systemd/searxng.socket.j2 create mode 100644 ansible_collections/nullified/infrastructure/roles/searxng/templates/uwsgi/uwsgi.ini.j2 create mode 100644 ansible_collections/nullified/infrastructure/roles/searxng/tests/inventory create mode 100644 ansible_collections/nullified/infrastructure/roles/searxng/tests/test.yml create mode 100644 ansible_collections/nullified/infrastructure/roles/searxng/vars/main.yml diff --git a/ansible_collections/nullified/infrastructure/extensions/molecule/default/converge.yml b/ansible_collections/nullified/infrastructure/extensions/molecule/default/converge.yml index 6f34456..66d9dd7 100644 --- a/ansible_collections/nullified/infrastructure/extensions/molecule/default/converge.yml +++ b/ansible_collections/nullified/infrastructure/extensions/molecule/default/converge.yml @@ -103,5 +103,9 @@ ansible.builtin.include_role: name: nullified.infrastructure.valkey tasks_from: main.yml + - name: Testing SearxNG role + ansible.builtin.include_role: + name: nullified.infrastructure.searxng + tasks_from: main.yml - meta: flush_handlers diff --git a/ansible_collections/nullified/infrastructure/extensions/molecule/default/platform_vars.yml b/ansible_collections/nullified/infrastructure/extensions/molecule/default/platform_vars.yml index a07a3df..939dc4d 100644 --- a/ansible_collections/nullified/infrastructure/extensions/molecule/default/platform_vars.yml +++ b/ansible_collections/nullified/infrastructure/extensions/molecule/default/platform_vars.yml @@ -12,3 +12,26 @@ external_provisioner_source_ips: - '198.18.0.0/15' security_sysctl_configuration: 'vm.overcommit_memory': 1 + +searxng_conf_redis_url: "rediss://127.0.0.1:6379?ssl_cert_reqs=none" +searxng_limiter_botdetection_pass_ip: + - '172.16.0.0/12' + - '192.168.0.0/16' +nginx_sites: + - name: searxng + content: |- + server { + listen 80; + + location / { + uwsgi_pass unix:///run/uwsgi/searxng.socket; + + include uwsgi_params; + + uwsgi_param HTTP_HOST $host; + uwsgi_param HTTP_CONNECTION $http_connection; + uwsgi_param HTTP_X_SCHEME $scheme; + uwsgi_param HTTP_X_REAL_IP $remote_addr; + uwsgi_param HTTP_X_FORWARDED_FOR $proxy_add_x_forwarded_for; + } + } diff --git a/ansible_collections/nullified/infrastructure/roles/searxng/README.md b/ansible_collections/nullified/infrastructure/roles/searxng/README.md new file mode 100644 index 0000000..80014a2 --- /dev/null +++ b/ansible_collections/nullified/infrastructure/roles/searxng/README.md @@ -0,0 +1,31 @@ +Role Name +========= + +@todo + +Requirements +------------ + +- Redis +- uWSGI +@todo + +Role Variables +-------------- + +@todo + +Dependencies +------------ + +@todo + +Example Playbook +---------------- + +@todo + +License +------- + +[MIT](https://opensource.org/license/mit) diff --git a/ansible_collections/nullified/infrastructure/roles/searxng/defaults/main.yml b/ansible_collections/nullified/infrastructure/roles/searxng/defaults/main.yml new file mode 100644 index 0000000..f688438 --- /dev/null +++ b/ansible_collections/nullified/infrastructure/roles/searxng/defaults/main.yml @@ -0,0 +1,34 @@ +--- +searxng_conf_redis_enabled: true +searxng_conf_redis_url: +searxng_install_uwsgi: true +searxng_uwsgi_socket_dir: "/run/uwsgi" +searxng_uwsgi_config_filepath: "/etc/uwgsi/apps/searxng.ini" +searxng_git_dir: "/srv/git/searxng.git" +searxng_git_version: "HEAD" +searxng_user: "searxng" +searxng_group: "searxng" +searxng_socket_group: "nginx" +searxng_install_dir: "/etc/searxng" +# settings.yml values +searxng_conf_enable_open_metrics: 'true' +searxng_conf_open_metrics_password: '' +searxng_conf_search_languages: ['all', 'en', 'fr', 'de', 'es', 'it'] +searxng_conf_search_ban_time_on_fail: 10 +searxng_conf_search_favicon_resolver: '' +searxng_conf_search_max_ban_time_on_fail: 120 +searxng_conf_server_port: 8888 +searxng_conf_server_bind_address: "127.0.0.1" +searxng_conf_server_limiter: true +searxng_conf_server_public_instance: true +searxng_conf_server_image_proxy: true +searxng_conf_ui_default_locale: +searxng_conf_engines_invidious_base_url: + - https://invidious.nerdvpn.de + - https://inv.nadeko.net + - https://yewtu.be +# limiter.toml +searxng_limiter_realip_ipv4_prefix: 32 +searxng_limiter_realip_ipv6_prefix: 64 +searxng_limiter_botdetection_block_ip: [] +searxng_limiter_botdetection_pass_ip: [] diff --git a/ansible_collections/nullified/infrastructure/roles/searxng/handlers/main.yml b/ansible_collections/nullified/infrastructure/roles/searxng/handlers/main.yml new file mode 100644 index 0000000..3ed716d --- /dev/null +++ b/ansible_collections/nullified/infrastructure/roles/searxng/handlers/main.yml @@ -0,0 +1,11 @@ +--- +- name: restart service + become: true + ansible.builtin.systemd: + name: '{{ item }}' + enabled: true + state: restarted + daemon_reload: true + loop: + - searxng.socket + - searxng.service diff --git a/ansible_collections/nullified/infrastructure/roles/searxng/meta/main.yml b/ansible_collections/nullified/infrastructure/roles/searxng/meta/main.yml new file mode 100644 index 0000000..b87b2a1 --- /dev/null +++ b/ansible_collections/nullified/infrastructure/roles/searxng/meta/main.yml @@ -0,0 +1,13 @@ +--- +galaxy_info: + author: Florian L. + namespace: nullified + description: + license: MIT + min_ansible_version: 2.15 + platforms: + - name: Debian + versions: + - bookworm + galaxy_tags: [] +dependencies: [] diff --git a/ansible_collections/nullified/infrastructure/roles/searxng/tasks/backend_uwsgi_setup.yml b/ansible_collections/nullified/infrastructure/roles/searxng/tasks/backend_uwsgi_setup.yml new file mode 100644 index 0000000..57dd90b --- /dev/null +++ b/ansible_collections/nullified/infrastructure/roles/searxng/tasks/backend_uwsgi_setup.yml @@ -0,0 +1,48 @@ +--- +- name: setup uWSGI + become: true + when: searxng_install_uwsgi is truthy + block: + - name: setup directories + ansible.builtin.file: + path: '{{ item }}' + owner: '{{ searxng_user }}' + group: '{{ searxng_group }}' + mode: '0700' + state: directory + loop: + - '{{ searxng_uwsgi_config_filepath | dirname }}' + - name: install dependencies + ansible.builtin.apt: + update_cache: true + force_apt_get: true + state: present + cache_valid_time: 3600 + pkg: + - libpcre3 + - libpcre3-dev + - name: install uWSGI + ansible.builtin.pip: + name: + - 'uwsgi{{ searxng_uwsgi_version_constraint if searxng_uwsgi_version_constraint is truthy }}' + virtualenv: '{{ searxng_install_dir }}/.venv' + - name: install uWSGI configuration + ansible.builtin.template: + src: ../templates/uwsgi/uwsgi.ini.j2 + dest: '{{ searxng_uwsgi_config_filepath }}' + owner: '{{ searxng_user }}' + group: '{{ searxng_group }}' + mode: 'u=rw,g=r,o=' +- name: install systemd unit files + become: true + ansible.builtin.template: + src: '../templates/systemd/searxng.{{ item }}.j2' + dest: '{{ systemd_unit_directory }}/searxng.{{ item }}' + owner: root + group: root + mode: 'u=rwX,g=rX,o=' + notify: + - 'searxng : restart service' + loop: + - service + - socket diff --git a/ansible_collections/nullified/infrastructure/roles/searxng/tasks/gather_facts.yml b/ansible_collections/nullified/infrastructure/roles/searxng/tasks/gather_facts.yml new file mode 100644 index 0000000..688c75f --- /dev/null +++ b/ansible_collections/nullified/infrastructure/roles/searxng/tasks/gather_facts.yml @@ -0,0 +1,17 @@ +--- +- name: find systemd unit directory + become: true + ansible.builtin.command: pkg-config systemd --variable=systemd_system_conf_dir + changed_when: false + register: systemd_unit_directory_cmd +- name: find systemd version + become: true + ansible.builtin.shell: > + systemctl --version | awk '{if($1=="systemd" && $2~"^[0-9]+$"){print $2}}' + changed_when: false + register: systemd_version_cmd +- name: set facts + ansible.builtin.set_fact: + systemd_unit_directory: "{{ systemd_unit_directory_cmd.stdout }}" + systemd_version: "{{ systemd_version_cmd.stdout | int }}" + searxng_conf_server_secret_key: "{{ searxng_conf_server_secret_key | default(lookup('ansible.builtin.password', '/dev/null', length=64), true) }}" diff --git a/ansible_collections/nullified/infrastructure/roles/searxng/tasks/main.yml b/ansible_collections/nullified/infrastructure/roles/searxng/tasks/main.yml new file mode 100644 index 0000000..3ef909b --- /dev/null +++ b/ansible_collections/nullified/infrastructure/roles/searxng/tasks/main.yml @@ -0,0 +1,106 @@ +--- +- name: install dependencies + become: true + ansible.builtin.apt: + update_cache: true + force_apt_get: true + state: present + cache_valid_time: 3600 + pkg: + - build-essential + - git + - libffi-dev + - libssl-dev + - libxslt-dev + - pkgconf + - python3-babel + - python3-dev + - python3-virtualenv + - python3-yaml + - zlib1g-dev +- name: gather facts + ansible.builtin.include_tasks: gather_facts.yml +- name: create service group + become: true + ansible.builtin.group: + name: '{{ searxng_group }}' + system: true + state: present +- name: create service user + become: true + ansible.builtin.user: + name: '{{ searxng_user }}' + group: '{{ searxng_group }}' + shell: '/usr/bin/bash' + home: '{{ searxng_install_dir }}' + create_home: true + system: true + state: present +- name: mark git repository as safe + become: true + ansible.builtin.shell: > + git config --global --get safe.directory {{ searxng_git_dir }} || + git config --global --add safe.directory {{ searxng_git_dir }} +- name: clone repository + become: true + ansible.builtin.git: + repo: '{{ searxng_git_repository }}' + dest: '{{ searxng_git_dir }}' + single_branch: true + version: '{{ searxng_git_version }}' +- name: setup virtualenv + become: true + ansible.builtin.pip: + requirements: '{{ searxng_git_dir }}/requirements.txt' + virtualenv: '{{ searxng_install_dir }}/.venv' + virtualenv_site_packages: true + extra_args: '--use-pep517 --no-build-isolation -e {{ searxng_git_dir }}' +- name: set ownership + become: true + ansible.builtin.file: + path: '{{ searxng_install_dir }}' + owner: '{{ searxng_user }}' + group: '{{ searxng_group }}' + mode: 'u=rwX,g=rX,o=' + recurse: yes + follow: false +- name: setup shell environment + become: true + ansible.builtin.lineinfile: + path: '{{ searxng_install_dir }}/.profile' + line: 'source {{ searxng_install_dir }}/.venv/bin/activate' + search_string: 'source {{ searxng_install_dir }}/.venv/bin/activate' + create: true + mode: '0640' + owner: '{{ searxng_user }}' + group: '{{ searxng_group }}' + state: present +- name: setup configuration file + become: true + ansible.builtin.template: + src: '../templates/settings.yml.j2' + dest: '{{ searxng_install_dir }}/settings.yml' + owner: '{{ searxng_user }}' + group: '{{ searxng_group }}' + mode: 'u=rw,g=r,o=' +- name: setup limiter configuration + become: true + when: (searxng_conf_server_limiter | default(True, True)) is truthy + ansible.builtin.template: + src: '../templates/limiter.toml.j2' + dest: '{{ searxng_install_dir }}/limiter.toml' + owner: '{{ searxng_user }}' + group: '{{ searxng_group }}' + mode: 'u=rw,g=r,o=' +- name: setup backend + ansible.builtin.include_tasks: 'backend_{{ searxng_backend }}_setup.yml' +- name: fix permissions + become: true + ansible.builtin.file: + path: '{{ searxng_git_dir }}' + owner: '{{ searxng_user }}' + group: '{{ searxng_group }}' + recurse: true + state: directory +- name: flush handlers + ansible.builtin.meta: flush_handlers diff --git a/ansible_collections/nullified/infrastructure/roles/searxng/templates/.gitkeep b/ansible_collections/nullified/infrastructure/roles/searxng/templates/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/ansible_collections/nullified/infrastructure/roles/searxng/templates/limiter.toml.j2 b/ansible_collections/nullified/infrastructure/roles/searxng/templates/limiter.toml.j2 new file mode 100644 index 0000000..e1ea334 --- /dev/null +++ b/ansible_collections/nullified/infrastructure/roles/searxng/templates/limiter.toml.j2 @@ -0,0 +1,23 @@ +[real_ip] +ipv4_prefix = {{ searxng_limiter_realip_ipv4_prefix | default(32, True) }} +ipv6_prefix = {{ searxng_limiter_realip_ipv6_prefix | default(64, True) }} + +[botdetection.ip_limit] +link_token = true + +[botdetection.ip_lists] +{% set datalist = [] -%} +{%- for item in (searxng_limiter_botdetection_block_ip | default([])) -%} + {{- datalist.append("'{}'".format(item)) if item -}} +{%- endfor %} +block_ip = [ + {{ datalist | join(', ') | wordwrap(60, wrapstring="\n\t", break_long_words=False) }} +] + +{% set datalist = [] -%} +{%- for item in (searxng_limiter_botdetection_pass_ip | default([])) -%} + {{- datalist.append("'{}'".format(item)) if item -}} +{%- endfor %} +pass_ip = [ + {{ datalist | join(', ') | wordwrap(60, wrapstring="\n\t", break_long_words=False) }} +] diff --git a/ansible_collections/nullified/infrastructure/roles/searxng/templates/settings.yml.j2 b/ansible_collections/nullified/infrastructure/roles/searxng/templates/settings.yml.j2 new file mode 100644 index 0000000..9a2a698 --- /dev/null +++ b/ansible_collections/nullified/infrastructure/roles/searxng/templates/settings.yml.j2 @@ -0,0 +1,482 @@ +use_default_settings: true + +general: + instance_name: "SearXNG" + enable_metrics: {{ searxng_conf_enable_open_metrics | default(false, True) | bool }} + open_metrics: {{ searxng_conf_open_metrics_password | default('', True) | quote }} + +server: + base_url: {{ searxng_conf_server_base_url | default(False, True) | bool }} + limiter: {{ searxng_conf_server_limiter | default(true, True) | bool }} + public_instance: {{ searxng_conf_server_public_instance | default(true, True) | bool }} + secret_key: {{ searxng_conf_server_secret_key | to_yaml }} + image_proxy: {{ searxng_conf_server_image_proxy | default(true, True) | bool }} + method: "POST" + default_http_headers: + X-Content-Type-Options: nosniff + X-Download-Options: noopen + X-Robots-Tag: noindex, nofollow + Referrer-Policy: no-referrer + +{% if searxng_conf_redis_enabled | default(True, True) is truthy %} +redis: + url: {{ searxng_conf_redis_url }} +{% endif %} + +search: + safe_search: 0 + autocomplete: {{ searxng_conf_search_autocomplete | default('duckduckgo', True) | quote }} + autocomplete_min: 4 + favicon_resolver: {{ searxng_conf_search_favicon_resolver | default('', True) | quote }} + default_lang: {{ searxng_conf_search_default_lang | default('auto', True) | quote }} + languages: {{ searxng_conf_search_languages | default(['all', 'en', 'fr', 'de'], True) | to_yaml }} + ban_time_on_fail: {{ searxng_conf_search_ban_time_on_fail | default(10, True) }} + max_ban_time_on_fail: {{ searxng_conf_search_max_ban_time_on_fail | default(120, True) }} + suspended_times: + SearxEngineAccessDenied: 86400 + SearxEngineCaptcha: 86400 + SearxEngineTooManyRequests: 3600 + cf_SearxEngineCaptcha: 1296000 + cf_SearxEngineAccessDenied: 86400 + recaptcha_SearxEngineCaptcha: 604800 + formats: + - html + +ui: + query_in_title: false + infinite_scroll: true + default_theme: simple + center_alignment: false + default_locale: {{ searxng_conf_ui_default_locale | default("", True) | to_yaml }} + results_on_new_tab: true + theme_args: + simple_style: auto + hotkeys: default + url_formatting: pretty + +# preferences: +# lock: +# - categories +# - language +# - autocomplete +# - favicon +# - safesearch +# - method +# - doi_resolver +# - locale +# - theme +# - results_on_new_tab +# - infinite_scroll +# - search_on_category_select +# - method +# - image_proxy +# - query_in_title + +# searx supports result proxification using an external service: +# https://github.com/asciimoo/morty uncomment below section if you have running +# morty proxy the key is base64 encoded (keep the !!binary notation) +# Note: since commit af77ec3, morty accepts a base64 encoded key. +# +# result_proxy: +# url: http://127.0.0.1:3000/ +# # the key is a base64 encoded string, the YAML !!binary prefix is optional +# key: !!binary "your_morty_proxy_key" +# # [true|false] enable the "proxy" button next to each result +# proxify_results: true + +outgoing: + request_timeout: 3.0 + max_request_timeout: 10.0 + useragent_suffix: "" + pool_connections: 100 + pool_maxsize: 20 + enable_http2: true + # uncomment below section if you want to use a custom server certificate + # see https://www.python-httpx.org/advanced/#changing-the-verification-defaults + # and https://www.python-httpx.org/compatibility/#ssl-configuration + # verify: ~/.mitmproxy/mitmproxy-ca-cert.cer + # + # uncomment below section if you want to use a proxyq see: SOCKS proxies + # https://2.python-requests.org/en/latest/user/advanced/#proxies + # are also supported: see + # https://2.python-requests.org/en/latest/user/advanced/#socks + # + # proxies: + # all://: + # - http://proxy1:8080 + # - http://proxy2:8080 + # + # using_tor_proxy: true + # + # Extra seconds to add in order to account for the time taken by the proxy + # + # extra_proxy_timeout: 10 + # + # uncomment below section only if you have more than one network interface + # which can be the source of outgoing search requests + # + # source_ips: + # - 1.1.1.1 + # - 1.1.1.2 + # - fe80::/126 + +# Plugin configuration, for more details see +# https://docs.searxng.org/admin/settings/settings_plugins.html +plugins: + searx.plugins.calculator.SXNGPlugin: + active: true + searx.plugins.hash_plugin.SXNGPlugin: + active: true + searx.plugins.self_info.SXNGPlugin: + active: true + searx.plugins.unit_converter.SXNGPlugin: + active: true + searx.plugins.ahmia_filter.SXNGPlugin: + active: true + searx.plugins.hostnames.SXNGPlugin: + active: true + searx.plugins.oa_doi_rewrite.SXNGPlugin: + active: false + searx.plugins.tor_check.SXNGPlugin: + active: false + searx.plugins.tracker_url_remover.SXNGPlugin: + active: true + +# Configuration of the "Hostnames plugin": +# hostnames: +# replace: +# '(.*\.)?youtube\.com$': 'invidious.example.com' +# '(.*\.)?youtu\.be$': 'invidious.example.com' +# '(.*\.)?reddit\.com$': 'teddit.example.com' +# '(.*\.)?redd\.it$': 'teddit.example.com' +# '(www\.)?twitter\.com$': 'nitter.example.com' +# remove: +# - '(.*\.)?facebook.com$' +# low_priority: +# - '(.*\.)?google(\..*)?$' +# high_priority: +# - '(.*\.)?wikipedia.org$' +# +# Alternatively you can use external files for configuring the "Hostnames plugin": +# +# hostnames: +# replace: 'rewrite-hosts.yml' +# +# Content of 'rewrite-hosts.yml' (place the file in the same directory as 'settings.yml'): +# '(.*\.)?youtube\.com$': 'invidious.example.com' +# '(.*\.)?youtu\.be$': 'invidious.example.com' +# + +checker: + # disable checker when in debug mode + off_when_debug: true + additional_tests: + rosebud: &test_rosebud + matrix: + query: rosebud + lang: en + result_container: + - not_empty + - ['one_title_contains', 'citizen kane'] + test: + - unique_results + android: &test_android + matrix: + query: ['android'] + lang: ['en', 'de', 'fr', 'zh-CN'] + result_container: + - not_empty + - ['one_title_contains', 'google'] + test: + - unique_results + tests: + infobox: &tests_infobox + infobox: + matrix: + query: ["linux", "new york", "bbc"] + result_container: + - has_infobox + +categories_as_tabs: + general: + images: + videos: + news: + map: + music: + it: + science: + files: + +engines: + - name: annas archive + engine: annas_archive + disabled: true + shortcut: aa + - name: arxiv + engine: arxiv + shortcut: arx + timeout: 4.0 + - name: wikipedia + engine: wikipedia + shortcut: wp + # add "list" to the array to get results in the results list + display_type: ["infobox"] + categories: [general] + - name: bing + engine: bing + shortcut: bi + disabled: true + - name: bing images + engine: bing_images + shortcut: bii + - name: bitbucket + engine: xpath + paging: true + search_url: https://bitbucket.org/repo/all/{pageno}?name={query} + url_xpath: //article[@class="repo-summary"]//a[@class="repo-link"]/@href + title_xpath: //article[@class="repo-summary"]//a[@class="repo-link"] + content_xpath: //article[@class="repo-summary"]/p + categories: [it, repos] + timeout: 4.0 + disabled: true + shortcut: bb + about: + website: https://bitbucket.org/ + wikidata_id: Q2493781 + official_api_documentation: https://developer.atlassian.com/bitbucket + use_official_api: false + require_api_key: false + results: HTML + - name: media.ccc.de + engine: ccc_media + shortcut: c3tv + # We don't set language: de here because media.ccc.de is not just + # for a German audience. It contains many English videos and many + # German videos have English subtitles. + disabled: true + - name: ddg definitions + engine: duckduckgo_definitions + shortcut: ddd + weight: 2 + disabled: true + tests: *tests_infobox + - name: docker hub + engine: docker_hub + shortcut: dh + categories: [it, packages] + - name: wikidata + engine: wikidata + shortcut: wd + timeout: 3.0 + weight: 2 + # add "list" to the array to get results in the results list + display_type: ["infobox"] + tests: *tests_infobox + categories: [general] + - name: duckduckgo + engine: duckduckgo + shortcut: ddg + - name: duckduckgo images + engine: duckduckgo_extra + categories: [images, web] + ddg_category: images + shortcut: ddi + disabled: true + - name: duckduckgo videos + engine: duckduckgo_extra + categories: [videos, web] + ddg_category: videos + shortcut: ddv + disabled: true + - name: duckduckgo news + engine: duckduckgo_extra + categories: [news, web] + ddg_category: news + shortcut: ddn + disabled: true + - name: tineye + engine: tineye + shortcut: tin + timeout: 9.0 + disabled: true + - name: 1x + engine: www1x + shortcut: 1x + timeout: 3.0 + disabled: true + - name: github + engine: github + shortcut: gh + - name: goodreads + engine: goodreads + shortcut: good + timeout: 4.0 + disabled: true + - name: google + engine: google + shortcut: go + # additional_tests: + # android: *test_android + - name: google images + engine: google_images + shortcut: goi + # additional_tests: + # android: *test_android + # dali: + # matrix: + # query: ['Dali Christ'] + # lang: ['en', 'de', 'fr', 'zh-CN'] + # result_container: + # - ['one_title_contains', 'Salvador'] + - name: google scholar + engine: google_scholar + shortcut: gos + - name: material icons + engine: material_icons + categories: images + shortcut: mi + disabled: true + - name: hackernews + engine: hackernews + shortcut: hn + disabled: false + - name: invidious + engine: invidious + # Instances will be selected randomly, see https://api.invidious.io/ for + # instances that are stable (good uptime) and close to you. + base_url: {{ searxng_conf_engines_invidious_base_url | default([], True) | to_yaml }} + shortcut: iv + timeout: 3.0 + disabled: true + - name: library genesis + engine: xpath + # search_url: https://libgen.is/search.php?req={query} + search_url: https://libgen.rs/search.php?req={query} + url_xpath: //a[contains(@href,"book/index.php?md5")]/@href + title_xpath: //a[contains(@href,"book/")]/text()[1] + content_xpath: //td/a[1][contains(@href,"=author")]/text() + categories: files + timeout: 7.0 + disabled: true + shortcut: lg + about: + website: https://libgen.fun/ + wikidata_id: Q22017206 + official_api_documentation: + use_official_api: false + require_api_key: false + results: HTML + - name: z-library + engine: zlibrary + shortcut: zlib + categories: files + timeout: 7.0 + - name: openstreetmap + engine: openstreetmap + shortcut: osm + - name: qwant + qwant_categ: web + engine: qwant + shortcut: qw + categories: [general, web] + additional_tests: + rosebud: *test_rosebud + - name: qwant news + qwant_categ: news + engine: qwant + shortcut: qwn + categories: news + network: qwant + - name: qwant images + qwant_categ: images + engine: qwant + shortcut: qwi + categories: [images, web] + network: qwant + - name: qwant videos + qwant_categ: videos + engine: qwant + shortcut: qwv + categories: [videos, web] + network: qwant + - name: startpage + engine: startpage + shortcut: sp + startpage_categ: web + categories: [general, web] + additional_tests: + rosebud: *test_rosebud + - name: startpage news + engine: startpage + startpage_categ: news + categories: [news, web] + shortcut: spn + - name: startpage images + engine: startpage + startpage_categ: images + categories: [images, web] + shortcut: spi + - name: wikibooks + engine: mediawiki + weight: 0.5 + shortcut: wb + categories: [general, wikimedia] + base_url: "https://{language}.wikibooks.org/" + search_type: text + disabled: true + about: + website: https://www.wikibooks.org/ + wikidata_id: Q367 + - name: wikiquote + engine: mediawiki + weight: 0.5 + shortcut: wq + categories: [general, wikimedia] + base_url: "https://{language}.wikiquote.org/" + search_type: text + disabled: true + additional_tests: + rosebud: *test_rosebud + about: + website: https://www.wikiquote.org/ + wikidata_id: Q369 + - name: wikisource + engine: mediawiki + weight: 0.5 + shortcut: ws + categories: [general, wikimedia] + base_url: "https://{language}.wikisource.org/" + search_type: text + disabled: true + about: + website: https://www.wikisource.org/ + wikidata_id: Q263 + - name: wiktionary + engine: mediawiki + shortcut: wt + categories: [dictionaries, wikimedia] + base_url: "https://{language}.wiktionary.org/" + search_type: text + about: + website: https://www.wiktionary.org/ + wikidata_id: Q151 + - name: peertube + engine: peertube + shortcut: ptb + paging: true + # alternatives see: https://instances.joinpeertube.org/instances + # base_url: https://tube.4aem.com + categories: videos + disabled: true + timeout: 6.0 + +doi_resolvers: + oadoi.org: 'https://oadoi.org/' + doi.org: 'https://doi.org/' + doai.io: 'https://dissem.in/' + sci-hub.se: 'https://sci-hub.se/' + sci-hub.st: 'https://sci-hub.st/' + sci-hub.ru: 'https://sci-hub.ru/' + +default_doi_resolver: 'oadoi.org' diff --git a/ansible_collections/nullified/infrastructure/roles/searxng/templates/systemd/searxng.service.j2 b/ansible_collections/nullified/infrastructure/roles/searxng/templates/systemd/searxng.service.j2 new file mode 100644 index 0000000..f252195 --- /dev/null +++ b/ansible_collections/nullified/infrastructure/roles/searxng/templates/systemd/searxng.service.j2 @@ -0,0 +1,60 @@ +[Unit] +Description=SearxNG service +#Documentation= +After=network-online.target +Wants=searxng.socket + +[Service] +User={{ searxng_user }} +Group={{ searxng_group }} +Type=notify +NotifyAccess=all +UMask=007 +ExecStart={{ searxng_install_dir }}/.venv/bin/uwsgi \ + --ini "{{ searxng_uwsgi_config_filepath }}" \ + --socket "{{ searxng_uwsgi_socket_dir }}/searxng.socket" +Restart=on-failure +RestartSec=3 +TimeoutStopSec=300 +WorkingDirectory=~ +KillSignal=SIGQUIT +StandardError=journal + +# Security Hardening +PrivateTmp=true +CapabilityBoundingSet=CAP_SYS_RESOURCE +{% if systemd_version | int >= 187 %} +NoNewPrivileges=true +SystemCallFilter=@system-service +{% endif %} +{%+ if systemd_version | int >= 209 %}SystemCallArchitectures=native{%- endif +%} +{% if systemd_version | int >= 214 %} +ProtectHome=true +ProtectSystem=true +{% endif %} +{% if systemd_version | int >= 231 %} +ReadOnlyPaths=/ +ReadWritePaths={{ searxng_install_dir }} {{ searxng_uwsgi_socket_dir }} {{ searxng_uwsgi_config_filepath }} {{ searxng_git_dir }} +RestrictRealtime=true +{% endif %} +{% if systemd_version | int >= 232 %} +ProtectControlGroups=true +ProtectKernelModules=true +ProtectKernelTunables=true +RemoveIPC=true +{% endif %} +{% if systemd_version | int >= 233 %} +MountAPIVFS=true +RestrictNamespaces=ipc net mnt pid +{% endif %} +{%+ if systemd_version | int >= 235 %}LockPersonality=true{%- endif +%} +{% if systemd_version | int >= 242 %} +ProtectHostname=true +RestrictSUIDSGID=true +{% endif %} +{%+ if systemd_version | int >= 244 %}ProtectKernelLogs=true{%- endif +%} +{%+ if systemd_version | int >= 245 %}ProtectClock=true{%- endif +%} +{%+ if systemd_version | int >= 247 %}ProtectProc=invisible{%- endif +%} + +[Install] +WantedBy=multi-user.target diff --git a/ansible_collections/nullified/infrastructure/roles/searxng/templates/systemd/searxng.socket.j2 b/ansible_collections/nullified/infrastructure/roles/searxng/templates/systemd/searxng.socket.j2 new file mode 100644 index 0000000..2e25710 --- /dev/null +++ b/ansible_collections/nullified/infrastructure/roles/searxng/templates/systemd/searxng.socket.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=SearxNG uWSGI socket +#Documentation= +After=network-online.target + +[Socket] +ListenStream={{ searxng_uwsgi_socket_dir }}/searxng.socket +SocketMode=0660 +{% if systemd_version | int >= 214 %} +SocketUser={{ searxng_user }} +SocketGroup={{ searxng_socket_group | default(searxng_group, True) }} +{% endif %} + +[Install] +WantedBy=sockets.target diff --git a/ansible_collections/nullified/infrastructure/roles/searxng/templates/uwsgi/uwsgi.ini.j2 b/ansible_collections/nullified/infrastructure/roles/searxng/templates/uwsgi/uwsgi.ini.j2 new file mode 100644 index 0000000..7ceb6d2 --- /dev/null +++ b/ansible_collections/nullified/infrastructure/roles/searxng/templates/uwsgi/uwsgi.ini.j2 @@ -0,0 +1,35 @@ +# -*- mode: conf; coding: utf-8 -*- +[uwsgi] +uid = {{ searxng_user }} +gid = {{ searxng_group }} +master = True +lazy-apps = true +cheap = True +idle = 600 +die-on-idle = False +manage-script-name = True + +env = LANG=C.UTF-8 +env = LANGUAGE=C.UTF-8 +env = LC_ALL=C.UTF-8 +env = SEARXNG_SETTINGS_PATH={{ searxng_install_dir }}/settings.yml + +virtualenv = {{ searxng_install_dir }}/.venv +pythonpath = {{ searxng_git_dir }} + +chdir = {{ searxng_git_dir }}/searx +disable-logging = true +chmod-socket = 660 +single-interpreter = true +enable-threads = true +workers = %k +threads = 4 + +static-map = /static={{ searxng_git_dir }}/searx/static +static-gzip-all = True +offload-threads = %k + +socket = {{ searxng_uwsgi_socket_dir }}/searxng.socket +buffer-size = 8192 + +module = searx.webapp diff --git a/ansible_collections/nullified/infrastructure/roles/searxng/tests/inventory b/ansible_collections/nullified/infrastructure/roles/searxng/tests/inventory new file mode 100644 index 0000000..878877b --- /dev/null +++ b/ansible_collections/nullified/infrastructure/roles/searxng/tests/inventory @@ -0,0 +1,2 @@ +localhost + diff --git a/ansible_collections/nullified/infrastructure/roles/searxng/tests/test.yml b/ansible_collections/nullified/infrastructure/roles/searxng/tests/test.yml new file mode 100644 index 0000000..ec10bb2 --- /dev/null +++ b/ansible_collections/nullified/infrastructure/roles/searxng/tests/test.yml @@ -0,0 +1,5 @@ +--- +- hosts: localhost + remote_user: root + roles: + - test diff --git a/ansible_collections/nullified/infrastructure/roles/searxng/vars/main.yml b/ansible_collections/nullified/infrastructure/roles/searxng/vars/main.yml new file mode 100644 index 0000000..7b4ab92 --- /dev/null +++ b/ansible_collections/nullified/infrastructure/roles/searxng/vars/main.yml @@ -0,0 +1,4 @@ +--- +searxng_git_repository: "https://github.com/searxng/searxng.git" +searxng_uwsgi_version_constraint: +searxng_backend: uwsgi diff --git a/playbooks/external.yml b/playbooks/external.yml index 6a685ac..b893de6 100644 --- a/playbooks/external.yml +++ b/playbooks/external.yml @@ -67,3 +67,13 @@ apply: tags: [valkey] tags: [valkey] + +- name: setup searxng instances + hosts: external:&searxng + tasks: + - name: include searxng role + ansible.builtin.include_role: + name: nullified.infrastructure.searxng + apply: + tags: [searxng] + tags: [searxng]