Run basic lint CI against Ubuntu noble / Python 3.12

.github/workflows/ci.yml

@@ -8,7 +8,13 @@ defaults:
 jobs:
   lint:
     runs-on: ubuntu-latest
-    container: ubuntu:focal
+    strategy:
+      fail-fast: false
+      matrix:
+        ubuntu_version:
+          - focal
+          - noble
+    container: ubuntu:${{ matrix.ubuntu_version }}
     steps:
       - name: Install Git
         run: |
@@ -17,7 +23,7 @@ jobs:
       - name: Install dependencies
         run: |
           DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install --yes --no-install-recommends \
-            build-essential python3-virtualenv python3-dev enchant file apache2-dev jq libarchive-tools
+            build-essential python3-virtualenv python3-dev enchant-2 file apache2-dev jq libarchive-tools
           virtualenv .venv
           # TODO: this should be one step, but there are too many conflicting dependencies
           ./.venv/bin/pip install -r securedrop/requirements/python3/test-requirements.txt

install_files/ansible-base/roles/ossec/tasks/configure_server.yml

@@ -23,11 +23,9 @@
 - name: Add the OSSEC GPG public key to the OSSEC manager keyring.
   # multiline format for command module, since this is a long command
   command: >
-    gpg
+    sudo -u {{ ossec_group }} gpg
     --homedir /var/ossec/.gnupg
     --import /var/ossec/{{ item }}
-  become: yes
-  become_user: "{{ ossec_group }}"
   register: add_ossec_gpg_key_result
   changed_when: "'imported: 1' in add_ossec_gpg_key_result.stderr"
   when: item != ''

molecule/libvirt-staging-focal/create.yml

@@ -5,7 +5,7 @@
   vars:
     molecule_file: "{{ lookup('env', 'MOLECULE_FILE') }}"
     molecule_instance_config: "{{ lookup('env', 'MOLECULE_INSTANCE_CONFIG') }}"
-    molecule_yml: "{{ lookup('file', molecule_file) | molecule_from_yaml }}"
+    molecule_yml: "{{ lookup('file', molecule_file) | from_yaml }}"
   tasks:
 
     - name: Create molecule instance(s)
@@ -26,7 +26,7 @@
 
         state: up
       register: server
-      with_items: "{{ molecule_yml.platforms }}"
+      loop: "{{ molecule_yml.platforms | flatten(levels=1) }}"
 
     # Mandatory configuration for Molecule to function.
 
@@ -38,7 +38,7 @@
           'user': "{{ item.User }}",
           'port': "{{ item.Port }}",
           'identity_file': "{{ item.IdentityFile }}", }
-      with_items: "{{ server.results }}"
+      loop: "{{ server.results | flatten(levels=1) }}"
       register: instance_config_dict
       when: server.changed | bool
 
@@ -49,8 +49,8 @@
 
     - name: Dump instance config
       copy:
-        # NOTE(retr0h): Workaround for Ansible 2.2.
-        #               https://github.com/ansible/ansible/issues/20885
-        content: "{{ instance_conf | to_json | from_json | molecule_to_yaml | molecule_header }}"
+        content: |
+          # Molecule managed
+          {{ instance_conf | to_json | from_json }}
         dest: "{{ molecule_instance_config }}"
       when: server.changed | bool

molecule/libvirt-staging-focal/destroy.yml

@@ -6,7 +6,7 @@
   vars:
     molecule_file: "{{ lookup('env', 'MOLECULE_FILE') }}"
     molecule_instance_config: "{{ lookup('env',' MOLECULE_INSTANCE_CONFIG') }}"
-    molecule_yml: "{{ lookup('file', molecule_file) | molecule_from_yaml }}"
+    molecule_yml: "{{ lookup('file', molecule_file) | from_yaml }}"
     molecule_ephemeral_directory: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}"
   tasks:
     - name: Destroy molecule instance(s)
@@ -18,7 +18,7 @@
 
         state: destroy
       register: server
-      with_items: "{{ molecule_yml.platforms }}"
+      loop: "{{ molecule_yml.platforms | flatten(levels=1) }}"
 
     # Mandatory configuration for Molecule to function.
 
@@ -28,8 +28,8 @@
 
     - name: Dump instance config
       copy:
-        # NOTE(retr0h): Workaround for Ansible 2.2.
-        #               https://github.com/ansible/ansible/issues/20885
-        content: "{{ instance_conf | to_json | from_json | molecule_to_yaml | molecule_header }}"
+        content: |
+          # Molecule managed
+          {{ instance_conf | to_json | from_json | to_yaml }}
         dest: "{{ molecule_instance_config }}"
       when: server.changed | bool

molecule/libvirt-staging-focal/molecule.yml

@@ -47,7 +47,7 @@ provisioner:
     defaults:
       interpreter_python: auto
   options:
-    e: "@ansible-override-vars.yml"
+    e: "@molecule/libvirt-staging-focal/ansible-override-vars.yml"
   playbooks:
     converge: ../../install_files/ansible-base/securedrop-staging.yml
     create: create.yml
@@ -71,6 +71,6 @@ verifier:
   options:
     n: auto
     v: 2
-    junit-xml: ../../junit/testinfra-results.xml
+    junit-xml: junit/testinfra-results.xml
   env:
     SECUREDROP_TARGET_DISTRIBUTION: focal

molecule/qubes-staging-focal/create.yml

@@ -5,7 +5,7 @@
   vars:
     molecule_file: "{{ lookup('env', 'MOLECULE_FILE') }}"
     molecule_instance_config: "{{ lookup('env', 'MOLECULE_INSTANCE_CONFIG') }}"
-    molecule_yml: "{{ lookup('file', molecule_file) | molecule_from_yaml }}"
+    molecule_yml: "{{ lookup('file', molecule_file) | from_yaml }}"
   tasks:
     - name: Check that Qubes admin tools are installed
       shell: >
@@ -23,7 +23,7 @@
         clone_result.rc != 0 and "qvm-clone: error: VM "+item.vm_name+" already exists" not in clone_result.stderr_lines
       changed_when: >-
         clone_result.rc == 0 and clone_result.stdout == ""
-      with_items: "{{ molecule_yml.platforms }}"
+      loop: "{{ molecule_yml.platforms | flatten(levels=1) }}"
 
     - name: Start Qubes VMs
       command: qvm-start {{ item.vm_name }}
@@ -32,7 +32,7 @@
         start_result.rc != 0 and "domain "+item.vm_name+" is already running" not in start_result.stderr_lines
       changed_when: >-
         start_result.rc == 0 and start_result.stdout == ""
-      with_items: "{{ molecule_yml.platforms }}"
+      loop: "{{ molecule_yml.platforms | flatten(levels=1) }}"
 
     - name: Wait for VMs to boot
       pause:
@@ -45,7 +45,7 @@
       changed_when: false
       # Not necessary, using pipe lookup to avoid convoluted Jinja logic.
       when: false
-      with_items: "{{ molecule_yml.platforms }}"
+      loop: "{{ molecule_yml.platforms | flatten(levels=1) }}"
 
     # Mandatory configuration for Molecule to function.
 
@@ -59,7 +59,7 @@
           # Hardcoded username, must match the username manually configured during
           # base VM creation (see developer documentation).
           user: "sdadmin"
-      with_items: "{{ molecule_yml.platforms }}"
+      loop: "{{ molecule_yml.platforms | flatten(levels=1) }}"
       register: instance_config_dict
       when: start_result.changed | bool
 
@@ -78,8 +78,8 @@
 
     - name: Dump instance config
       copy:
-        # NOTE(retr0h): Workaround for Ansible 2.2.
-        #               https://github.com/ansible/ansible/issues/20885
-        content: "{{ instance_conf | to_json | from_json | molecule_to_yaml | molecule_header }}"
+        content: |
+          # Molecule managed
+          {{ instance_conf | to_json | from_json | to_yaml }}
         dest: "{{ molecule_instance_config }}"
       when: start_result.changed | bool

molecule/qubes-staging-focal/destroy.yml

@@ -6,7 +6,7 @@
   vars:
     molecule_file: "{{ lookup('env', 'MOLECULE_FILE') }}"
     molecule_instance_config: "{{ lookup('env',' MOLECULE_INSTANCE_CONFIG') }}"
-    molecule_yml: "{{ lookup('file', molecule_file) | molecule_from_yaml }}"
+    molecule_yml: "{{ lookup('file', molecule_file) | from_yaml }}"
     molecule_ephemeral_directory: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}"
   tasks:
     - name: Check that Qubes admin tools are installed
@@ -21,14 +21,14 @@
       register: server
       failed_when: >-
          server.rc != 0 and "qvm-shutdown: error: no such domain: '"+item.vm_name+"'" not in server.stderr_lines
-      with_items: "{{ molecule_yml.platforms }}"
+      loop: "{{ molecule_yml.platforms | flatten(levels=1) }}"
 
     - name: Destroy molecule instance(s)
       command: qvm-remove --force "{{ item.vm_name }}"
       register: server
       failed_when: >-
          server.rc != 0 and "qvm-remove: error: no such domain: '"+item.vm_name+"'" not in server.stderr_lines
-      with_items: "{{ molecule_yml.platforms }}"
+      loop: "{{ molecule_yml.platforms | flatten(levels=1) }}"
 
     # Mandatory configuration for Molecule to function.
 
@@ -38,8 +38,8 @@
 
     - name: Dump instance config
       copy:
-        # NOTE(retr0h): Workaround for Ansible 2.2.
-        #               https://github.com/ansible/ansible/issues/20885
-        content: "{{ instance_conf | to_json | from_json | molecule_to_yaml | molecule_header }}"
+        content: |
+          # Molecule managed
+          {{ instance_conf | to_json | from_json | to_yaml }}
         dest: "{{ molecule_instance_config }}"
       when: server.changed | bool

molecule/qubes-staging-focal/molecule.yml

@@ -33,7 +33,7 @@ provisioner:
       callback_whitelist: "profile_tasks, timer"
       interpreter_python: auto
   options:
-    e: "@qubes-vars.yml"
+    e: "@molecule/qubes-staging-focal/qubes-vars.yml"
   playbooks:
     converge: ../../install_files/ansible-base/securedrop-staging.yml
   env:

securedrop/pretty_bad_protocol/_parsers.py

@@ -393,7 +393,7 @@ def _check_option(arg, value):  # type: ignore[no-untyped-def]
     def is_flag(x):  # type: ignore[no-untyped-def]
         return x.startswith("--")
 
-    def _make_filo(args_string):  # type: ignore[no-untyped-def]
+    def _make_filo(arg):  # type: ignore[no-untyped-def]
         filo = arg.split(" ")
         filo.reverse()
         log.debug("_make_filo(): Converted to reverse list: %s" % filo)

securedrop/pylintrc

@@ -54,79 +54,13 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=print-statement,
-        parameter-unpacking,
-        unpacking-in-except,
-        old-raise-syntax,
-        backtick,
-        long-suffix,
-        old-ne-operator,
-        old-octal-literal,
-        import-star-module-level,
-        non-ascii-bytes-literal,
-        raw-checker-failed,
+disable=raw-checker-failed,
         bad-inline-option,
         locally-disabled,
-        locally-enabled,
         file-ignored,
         suppressed-message,
         useless-suppression,
-        deprecated-pragma,
-        apply-builtin,
-        basestring-builtin,
-        buffer-builtin,
-        cmp-builtin,
-        coerce-builtin,
-        execfile-builtin,
-        file-builtin,
-        long-builtin,
-        raw_input-builtin,
-        reduce-builtin,
-        standarderror-builtin,
-        unicode-builtin,
-        xrange-builtin,
-        coerce-method,
-        delslice-method,
-        getslice-method,
-        setslice-method,
-        no-absolute-import,
-        old-division,
-        dict-iter-method,
-        dict-view-method,
-        next-method-called,
-        metaclass-assignment,
-        indexing-exception,
-        raising-string,
-        reload-builtin,
-        oct-method,
-        hex-method,
-        nonzero-method,
-        cmp-method,
-        input-builtin,
-        round-builtin,
-        intern-builtin,
-        unichr-builtin,
-        map-builtin-not-iterating,
-        zip-builtin-not-iterating,
-        range-builtin-not-iterating,
-        filter-builtin-not-iterating,
-        using-cmp-argument,
-        eq-without-hash,
-        div-method,
-        idiv-method,
-        rdiv-method,
-        exception-message-attribute,
-        invalid-str-codec,
-        sys-max-int,
-        bad-python3-import,
-        deprecated-string-function,
-        deprecated-str-translate-call,
-        deprecated-itertools-function,
-        deprecated-types-field,
-        next-method-defined,
-        dict-items-not-iterating,
-        dict-keys-not-iterating,
-        dict-values-not-iterating
+        deprecated-pragma
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
@@ -204,13 +138,6 @@ max-line-length=100
 # Maximum number of lines in a module
 max-module-lines=1000
 
-# List of optional constructs for which whitespace checking is disabled. `dict-
-# separator` is used to allow tabulation in dicts, etc.: {1  : 1,\n222: 2}.
-# `trailing-comma` allows a space between comma and closing bracket: (a, ).
-# `empty-line` allows space-only lines.
-no-space-check=trailing-comma,
-               dict-separator
-
 # Allow the body of a class to be on the same line as the declaration if body
 # contains single statement.
 single-line-class-stmt=no
@@ -535,4 +462,4 @@ min-public-methods=2
 
 # Exceptions that will emit a warning when being caught. Defaults to
 # "Exception"
-overgeneral-exceptions=Exception
+overgeneral-exceptions=builtins.Exception

securedrop/requirements/python3/bootstrap-requirements.in

@@ -1,4 +1,4 @@
-pip>=21.3
+pip>=24.2
 setuptools>=70.0.0
 setuptools-scm>=8.0.0
 wheel>=0.38.1

securedrop/requirements/python3/bootstrap-requirements.txt

@@ -4,9 +4,9 @@ packaging==24.1 \
     --hash=sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002 \
     --hash=sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124
     # via setuptools-scm
-pip==21.3 \
-    --hash=sha256:4a1de8f97884ecfc10b48fe61c234f7e7dcf4490a37217011ad9369d899ad5a6 \
-    --hash=sha256:741a61baab1dbce2d8ca415effa48a2b6a964564f81a9f4f1fce4c433346c034
+pip==24.2 \
+    --hash=sha256:2cd581cf58ab7fcfca4ce8efa6dcacd0de5bf8d0a3eb9ec927e07405f4d9e2a2 \
+    --hash=sha256:5b5e490b5e9cb275c879595064adce9ebd31b854e3e803740b72f9ccf34a45b8
     # via -r requirements/python3/bootstrap-requirements.in
 setuptools==70.3.0 \
     --hash=sha256:f171bab1dfbc86b132997f26a119f6056a57950d058587841a0082e8830f9dc5 \

securedrop/requirements/python3/develop-requirements.in

@@ -13,23 +13,23 @@ html-linter
 importlib-resources
 jinja2>=3.1.3
 markupsafe>=2.1.2
-molecule>=3.0.1,<3.0.3
-molecule-vagrant==0.3
+molecule>=3.0.1,<4
+molecule-vagrant>=1,<2
 # Needed for ansible network filter
 # http://docs.ansible.com/ansible/latest/playbooks_filters_ipaddr.html
 netaddr
-pip
+pip>=24.2
 polib
 prompt_toolkit==2.0.9
 psutil>=5.6.6
-pylint>=2.7.0
+pylint>=3
 # pyenchant is via pylint[spelling]
 pyenchant>=3.2.1
 pynacl>=1.4.0
 pytest>=7.2.0
 pytest-xdist>=3.0.2
 python-vagrant
-pyyaml>=5.4.1
+pyyaml>=6.0.1
 requests>=2.31.0
 ruamel.yaml>=0.16.10
 ruff