confd: add support for fetching container images over ftp/http/https

 - Anonymous FTP, or URL encoded ftp://user:hostname@addr/oci.tar.gz
 - HTTP/HTTPS fetched with curl, optional credentials support
 - Verify download against an optional sha256 checksum

Ensure the unpacked directory name does not contain a ':', it is a
restricted character and cannot be part of the file name.  If this
syntax is used we retain it as the name and retag it after load.

Fix #801

Signed-off-by: Joachim Wiberg <troglobit@gmail.com>

board/common/rootfs/usr/sbin/container

@@ -1,6 +1,8 @@
 #!/bin/sh
-
+DOWNLOADS=/var/lib/containers/oci
+BUILTIN=/lib/oci
 all=""
+sha=""
 env=""
 port=""
 force=
@@ -10,6 +12,72 @@ log()
     logger -I $PPID -t container -p local1.notice -- "$*"
 }
 
+check()
+{
+    file=$1
+
+    if [ -z "$sha" ]; then
+	log "no checksum to verify $file against, continuing."
+	return 0
+    fi
+
+    if echo "${sha}  ${file}" | sha256sum -c -s; then
+        log "$file checksum verified OK."
+	return 0
+    fi
+
+    got=$(sha256sum "${file}" | awk '{print $1}')
+    log "$file checksum mismatch, got $got, expected $sha, removing file."
+    rm -f "$file"
+
+    return 1
+}
+
+# Fetch an OCI image over ftp/http/https.  Use wget for FTP, which curl
+# empirically does not work well with.  Log progress+ & error to syslog.
+fetch()
+{
+    url=$1
+    file=$(basename "$url")
+    dst="$DOWNLOADS/$file"
+
+    cd "$DOWNLOADS" || return
+    if [ -e "$file" ]; then
+	log "$file already available."
+	if check "$file"; then
+	    echo "$dst"
+	    return 0
+	fi
+    fi
+
+    log "Fetching $url"
+
+    if echo "$url" | grep -qE "^ftp://"; then
+        cmd="wget -q $url"
+    elif echo "$url" | grep -qE "^https?://"; then
+        cmd="curl $creds -sSL --fail -o \"$file\" $url"
+    else
+        log "Unsupported URL scheme: $url"
+        return 1
+    fi
+
+    if out=$(eval "$cmd" 2>&1); then
+	log "$file downloaded successfully."
+	if check "$file"; then
+	    echo "$dst"
+	    return 0
+	fi
+    fi
+
+    # log error message from backend
+    while IFS= read -r line; do
+	log "$line"
+    done <<EOF
+$out
+EOF
+    return 1
+}
+
 # Unpacks a given oci-archive.tar[.gz] in the current directory.  Sanity
 # checks, at least one index.json in the top-level dir of the archive.
 # If there are more index files, this function does not handle them.
@@ -26,17 +94,22 @@ unpack_archive()
 	oci-archive:*)		# Packed OCI image, .tar or .tar.gz format
 	    file=${image#oci-archive:}
 	    ;;
+	ftp://* | http://* | https://*)
+            if ! file=$(fetch "$image"); then
+		return 1
+	    fi
+            ;;
 	*)			# docker://*, docker-archive:*, or URL
 	    echo "$image"
 	    return 0
 	    ;;
     esac
 
     if [ ! -e "$file" ]; then
-	if [ -e "/var/lib/containers/oci/$file" ]; then
-	    file="/var/lib/containers/oci/$file"
-	elif [ -e "/lib/oci/$file" ]; then
-	    file="/lib/oci/$file"
+	if [ -e "$DOWNLOADS/$file" ]; then
+	    file="$DOWNLOADS/$file"
+	elif [ -e "$BUILTIN/$file" ]; then
+	    file="$BUILTIN/$file"
 	else
 	    log "Error: cannot find OCI archive $file in search path."
 	    exit 1
@@ -58,10 +131,18 @@ unpack_archive()
 
 	[ -n "$quiet" ] || log "Extracting OCI archive $file ..."
         tar xf "$file" || (log "Error: failed unpacking $file in $(pwd)"; exit 1)
-	remove=true
     fi
 
     dir=$(dirname "$index")
+    if echo "$dir" | grep -q ":"; then
+        if [ -z "$name" ]; then
+            name="$dir"
+        fi
+        sanitized_dir=$(echo "$dir" | cut -d':' -f1)
+        mv "$dir" "$sanitized_dir" || (log "Error: failed renaming $dir to $sanitized_dir"; exit 1)
+        dir="$sanitized_dir"
+    fi
+
     [ -n "$quiet" ] || log "Loading OCI image $dir ..."
     podman load -qi "$dir"        >/dev/null
 
@@ -73,10 +154,6 @@ unpack_archive()
 	name=$dir
     fi
 
-    if [ "$remove" = "true" ]; then
-	rm -rf "$file"
-    fi
-
     echo "$name"
 }
 
@@ -102,7 +179,9 @@ create()
 
     # Unpack and load docker-archive/oci/oci-archive, returning image
     # name, or return docker:// URL for download.
-    image=$(unpack_archive "$image")
+    if ! image=$(unpack_archive "$image"); then
+	exit 1
+    fi
 
     if [ -z "$logging" ]; then
 	logging="--log-driver k8s-file --log-opt path=/run/containers/$name.fifo"
@@ -247,6 +326,7 @@ options:
       --dns-search LIST    Set host lookup search list when creating container
       --cap-add CAP        Add capability to unprivileged container
       --cap-drop CAP       Drop capability, for privileged containter
+      --checksum SHA       Use sha256sym to verify images from ftp/http/https
   -c, --creds USR[:PWD]    Credentials to pass to curl -u for remote ops
   -d, --detach             Detach a container started with 'run IMG [CMD]'
   -e, --env FILE           Environment variables when creating container
@@ -306,6 +386,10 @@ while [ "$1" != "" ]; do
 	    shift
 	    caps="$caps --cap-drop=$1"
 	    ;;
+	--checksum)
+	    shift
+	    sha="$1"
+	    ;;
 	-c | --creds)
 	    shift
 	    creds="-u $1"
@@ -350,7 +434,6 @@ while [ "$1" != "" ]; do
 	--log-path)
 	    shift
 	    logging="$logging --log-opt path=$1"
-	    log_path="$1"
 	    ;;
 	-m | --mount)
 	    shift
@@ -457,10 +540,10 @@ case $cmd in
     load)
 	url=$1
 	name=$2
+
 	# shellcheck disable=SC2086
-	if echo "$url" | grep -q "://"; then
-	    file=$(basename "$url")
-            curl -k $creds -Lo "$file" "$url"
+	if echo "$url" | grep -qE "^(ftp|http|https)://"; then
+            file=$(fetch "$url")
 	else
             file="$url"
 	fi
@@ -500,7 +583,7 @@ case $cmd in
 		podman images $all --format "{{.Repository}}:{{.Tag}}"
 		;;
 	    oci)
-		find /lib/oci /var/lib/containers/oci -type f 2>/dev/null
+		find $BUILTIN $DOWNLOADS -type f 2>/dev/null
 		;;
 	    *)		
 		podman ps $all --format "{{.Names}}"

src/confd/src/infix-containers.c

@@ -175,6 +175,9 @@ static int add(const char *name, struct lyd_node *cif)
 	if (lydx_is_enabled(cif, "manual"))
 		fprintf(fp, " --manual");
 
+ 	if ((string = lydx_get_cattr(cif, "checksum")))
+		fprintf(fp, " --checksum %s", string);
+
 	fprintf(fp, " create %s %s", name, image);
 
  	if ((string = lydx_get_cattr(cif, "command")))

src/confd/yang/containers.inc

@@ -2,5 +2,5 @@
 # REMEMBER TO UPDATE infix-interfaces ALSO IN confd.inc
 MODULES=(
     "infix-interfaces@2024-11-12.yang -e vlan-filtering -e containers"
-    "infix-containers@2024-10-14.yang"
+    "infix-containers@2024-11-15.yang"
 )

src/confd/yang/infix-containers.yang

@@ -26,24 +26,26 @@ module infix-containers {
     prefix infix-sys;
   }
 
-  revision 2024-11-12  {
-    description "Deprecate read-only, it is now always true.";
+  revision 2024-11-15 {
+    description "Two major changes:
+                  - Add support for ftp/http/https images with checksum
+                  - Deprecate read-only, it is now always true";
     reference "internal";
   }
 
-  revision 2024-10-14  {
+  revision 2024-10-14 {
     description "Two major changes:
                   - Allow changing name of host interfaces inside container
                   - Support hostname format specifiers, like ietf-system";
     reference "internal";
   }
 
-  revision 2024-03-27  {
+  revision 2024-03-27 {
     description "Add support for capabilities.";
     reference "internal";
   }
 
-  revision 2024-02-01  {
+  revision 2024-02-01 {
     description "Initial revision";
     reference "internal";
   }
@@ -138,13 +140,25 @@ module infix-containers {
                      oci-archive:/lib/oci/archive -- Use archive:latest from OCI archive
                                                      May be in .tar or .tar.gz format
 
+                     Additionally, the following URIs are also supported for setups
+                     that do not use a HUB or similar.  Recommend using 'checksum'!
+
+                     ftp://addr/path/to/archive   -- Downloaded using wget
+                     http://addr/path/to/archive  -- Downloaded using curl
+                     https://addr/path/to/archive -- Downloaded using curl
+
                      Note: if a remote repository cannot be reached, the creation of the
                            container will be put on a queue that retries pull every time
                            there is a route change in the host's system.";
 	mandatory true;
 	type string;
       }
 
+      leaf checksum {
+	description "Checksum for ftp/http/https OCI archives (sha256sum)";
+	type string;
+      }
+
       leaf image-id {
         description "Docker image ID, exact hash used.";
         config false;

src/klish-plugin-infix/xml/containers.xml

@@ -75,11 +75,15 @@
       <COMMAND name="creds" help="Credentials for remote access">
         <PARAM name="creds" ptype="/STRING" help="username[:password]"/>
       </COMMAND>
+      <COMMAND name="checksum" help="Checksum to verify against (SHA56)">
+        <PARAM name="sha" ptype="/STRING" help="SHA256"/>
+      </COMMAND>
     </SWITCH>
     <ACTION sym="script" in="tty" out="tty" interrupt="true">
       creds=${KLISH_PARAM_creds:+-c $KLISH_PARAM_creds}
+      sha=${KLISH_PARAM_sha:+--checksum $KLISH_PARAM_sha}
       cd /var/lib/containers/oci
-      doas container $creds load $KLISH_PARAM_url $KLISH_PARAM_name
+      doas container $creds $sha load $KLISH_PARAM_url $KLISH_PARAM_name
     </ACTION>
   </COMMAND>