Merge pull request #95 from DrDaveD/add-ci

Add ci checks, fix el7 epel, update to 4.41

.github/workflows/build.yml

@@ -0,0 +1,43 @@
+name: Make distributions and test
+
+on:
+  # run workflows on main master and release/** branches
+  push:
+    branches:
+      - main
+      - master
+      - release/**
+  # run workflows on pull requests against the same branches
+  pull_request:
+    branches:
+      - main
+      - master
+      - release/**
+
+# automatically cancel redundant builds
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  makeandtest:
+    name: ${{ matrix.distro }}:${{ matrix.version }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - distro: eurolinux/centos-7
+            version: latest
+          - distro: rockylinux
+            version: 8
+          - distro: rockylinux
+            version: 9
+          - distro: opensuse/leap
+            version: 15
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Get source code
+        uses: actions/checkout@v3
+
+      - name: Run make-and-test under docker
+        run: ./ci/docker-run ${{ matrix.distro }}:${{ matrix.version }}

ChangeLog

@@ -1,3 +1,8 @@
+cvmfsexec-4.41 - 9 September 2024
+- Switch to the archive for el7 epel packages.
+- Add CI checks on every pull request for a fairly through test of
+  all the combinations of modes, distributions, and machine types.
+
 cvmfsexec-4.40 - 5 July 2024
 - Switch to vault.centos.org for el7 fuse-libs package.
 

ci/docker-run

@@ -0,0 +1,24 @@
+#!/bin/bash -e
+# Run docker as shown at
+#  https://djw8605.github.io/2016/05/03/building-centos-packages-on-travisci/
+#
+# This more complicated setup is needed for github actions too because 
+#  they do not provide a mechanism for reliably enabling user namespaces.
+# Github actions does at least start a VM with docker already running.
+
+# Assumes running on Ubuntu 24+
+
+DOCKER_HUB_URI="$1"
+docker pull "$DOCKER_HUB_URI"
+
+DOCKER_CONTAINER_NAME="test_${OS_TYPE##*/}_${OS_VERSION//./_}"
+
+set -x
+docker run --privileged --network=host -v "$(pwd):/source:rw" \
+  -e DOCKER_HUB_URI="$DOCKER_HUB_URI" \
+  --name "$DOCKER_CONTAINER_NAME" "$DOCKER_HUB_URI" /bin/bash -exc \
+	"cd /source && ./ci/privileged-run"
+
+docker ps -a
+docker stop "$DOCKER_CONTAINER_NAME"
+docker rm -v "$DOCKER_CONTAINER_NAME"

ci/make-and-test

@@ -0,0 +1,61 @@
+#!/bin/bash
+# Now running as an unprivileged user with user namespaces enabled in a
+# container and required packages installed. Make distributions and test.
+
+. /etc/os-release
+
+set -ex
+for DIST in default osg egi; do
+    if [ "$DIST" = egi ] && [[ "$VERSION_ID" != 7* ]]; then
+        # egi not yet supported for el8 or el9
+        continue
+    fi
+    rm -rf dist /tmp/cvmfsexec
+    : test makedist
+    ./makedist $DIST
+    : test mode 3
+    ./cvmfsexec atlas.cern.ch -- ls /cvmfs/atlas.cern.ch/repo
+    : test self-extracting distribution
+    ./makedist -o /tmp/cvmfsexec
+    /tmp/cvmfsexec atlas.cern.ch -- ls /cvmfs/atlas.cern.ch/repo
+    ./makedist -o /tmp/cvmfsexec
+    : test mode 1
+    rm -rf dist/var/lib/cvmfs/shared
+    ./mountrepo `cd dist/cvmfs; echo *config*`
+    ./mountrepo atlas.cern.ch
+    ls dist/cvmfs/atlas.cern.ch/repo
+    if [[ "$ID" == *suse* ]]; then
+        # the sle15 cvmfs suse build as of 2.11.5 does not support fuse3
+        ./umountrepo -a
+        continue
+    fi
+    : test mode 4
+    if [ ! -d apptainer ]; then
+        # get singularity or apptainer from oasis cvmfs
+        ./mountrepo oasis.opensciencegrid.org
+        if [[ "$VERSION_ID" == 7* ]]; then
+            APPT=dist/cvmfs/oasis.opensciencegrid.org/mis/singularity/current
+        else
+            APPT=dist/cvmfs/oasis.opensciencegrid.org/mis/apptainer/current
+        fi
+        mkdir apptainer
+        cp -r $APPT/`arch` $APPT/bin apptainer
+    fi
+    ./umountrepo -a
+    rm -rf dist /tmp/cvmfsexec
+    ./makedist -s $DIST
+    ./makedist -s -o /tmp/cvmfsexec
+    SINGCVMFS_REPOSITORIES=atlas.cern.ch PATH=$PATH:$PWD/apptainer/bin \
+        ./singcvmfs exec -cip docker://$DOCKER_HUB_URI ls /cvmfs/atlas.cern.ch/repo
+    if [[ "$VERSION_ID" == 8* ]]; then
+        : try to at least make other supported architecture distributions
+        for arch in aarch64 ppc64le; do
+            rm -rf dist /tmp/cvmfsexec
+            ./makedist -m rhel8-$arch $DIST
+            ./makedist -o /tmp/cvmfsexec
+            rm -rf dist /tmp/cvmfsexec
+            ./makedist -s -m rhel8-$arch $DIST
+            ./makedist -s -o /tmp/cvmfsexec
+        done
+    fi
+done

ci/privileged-run

@@ -0,0 +1,25 @@
+#!/bin/bash
+# Running in a privileged container.  Install required packages and
+# switch to an unprivileged user to run the tests.
+
+set -ex
+if [ -f /usr/bin/zypper ]; then
+    # suse
+    zypper install -y tar gzip openssl-1_1 fuse fuse3
+else
+    # rhel
+    yum install -y procps-ng cpio findutils fuse fuse3
+    if [[ $DOCKER_HUB_URI == *:8 ]]; then
+        yum install -y yum-utils
+    fi
+fi
+
+# because host kernel is Ubuntu 24+, this enables user namespaces
+sysctl kernel.apparmor_restrict_unprivileged_userns=0
+
+# switch to an unprivileged user
+useradd -u 1000 --create-home -s /bin/bash testuser
+# leave .git as original owner for post job cleanup
+chown testuser .
+chown -R testuser *
+su testuser -c ci/make-and-test

cvmfsexec

@@ -9,7 +9,7 @@
 #set -x
 #PS4='c$$+ '
 
-VERSION=4.40
+VERSION=4.41
 
 usage()
 {

makedist

@@ -185,6 +185,10 @@ fi
 echo "Making $SINGMSG$DISTTYPE distribution for $MACHTYPE"
 
 getcoprurl() {
+    if [ ! -f /usr/bin/yumdownloader ]; then
+        echo "yumdownloader not found, skipping trying to get $1 from copr" >&2
+        return
+    fi
     typeset TMPF=$(mktemp)
     typeset REPONAME=makedist-$1
     cat >$TMPF <<!EOF!
@@ -285,7 +289,7 @@ if $INCLUDEHELPER; then
     fi
     URLS="$URLS $BASEURL/$HELPER"
     if [ "$EL" -lt 8 ]; then
-        EPELURL="https://download.fedoraproject.org/pub/epel/$EL/$ARCH/Packages"
+        EPELURL="https://archives.fedoraproject.org/pub/archive/epel/$EL/$ARCH/Packages"
     else
         EPELURL="https://download.fedoraproject.org/pub/epel/$EL/Everything/$ARCH/Packages"
     fi
@@ -302,7 +306,7 @@ CVMFSFUSE3URL="$CVMFSURL/`echo $CVMFSRPMNAME|sed 's/cvmfs-/cvmfs-fuse3-/'`"
 if $SING; then
     URLS="$URLS $CVMFSFUSE3URL"
     if [ "$EL" -lt 8 ]; then
-        FUSEURL="https://download.fedoraproject.org/pub/epel/$EL/$ARCH/Packages"
+        FUSEURL="https://archives.fedoraproject.org/pub/archive/epel/$EL/$ARCH/Packages"
         FUSESUBDIR=true
     elif [ "$EL" -eq 8 ]; then
         FUSEURL="https://repo.almalinux.org/almalinux/$EL/BaseOS/$ARCH/os/Packages"

singcvmfs

@@ -3,7 +3,7 @@
 #   with the singularity --fusemount option.
 # Written by Dave Dykstra March 2020
 
-VERSION=4.40
+VERSION=4.41
 
 ME="`basename $0`"