Merge branch 'master' into 4256-deduplicate-docs

Signed-off-by: Ben Sherman <bentshermann@gmail.com>

.github/workflows/build.yml

@@ -92,7 +92,7 @@ jobs:
       any_changed: ${{ steps.changed-files.outputs.any_changed }}
 
   test:
-    if: ${{ !contains(github.event.head_commit.message, '[ci fast]') }}
+    if: ${{ !contains(github.event.head_commit.message, '[ci fast]') && needs.build.outputs.any_changed == 'true' }}
     needs: build
     runs-on: ubuntu-latest
     strategy:

VERSION

@@ -1 +1 @@
-23.08.1-edge
+23.09.1-edge

changelog.txt

@@ -1,5 +1,56 @@
 NEXTFLOW CHANGE-LOG
 ===================
+23.09.1-edge - 11 Sep 2023
+- Revert "Allow setting shell directive when using the trace file (#4210)" [9f9edcdc]
+
+23.09.0-edge - 10 Sep 2023
+- Add check for latest version (#4194) [3e8cd488]
+- Add inspect command (#4069) [090c31ce]
+- Add maxSubmitAwait (#3736) [5686bf1d]
+- Add scripts logging for troubleshooting [c056a74e]
+- Add support for Spack to Singularity builds [23c4ec1d]
+- Add support for Wave native build for Singularity [8a434893]
+- Add support for inputs and outputs arity [42504d3c]
+- Add support for remote debug (#4266) [87e0648a]
+- Add warning about using clusterOptions with process directives (#4248) [a68c1a3e]
+- Add which to dockerfile build [817aa05b]
+- Align Singularity experience with Docker (#4230) [a1e1d3ca] [c1cc60dd]
+- Allow setting shell directive when using the trace file (#4210) [7d6ad624]
+- Always emit publish event for cached task outputs (#4227) [62686ce8]
+- Deprecated Wave report feature [80c5cb27]
+- Disable staging script for remote work dir (#4282) [80f7cd46]
+- Disable version check on CI tests [db79e13f]
+- Docs improvement to clarify the usage of the bin dir (#4121) [2daa4172]
+- Document API differences of process path inputs (#4189) [c37e00bc]
+- Document use of local variables in closures (#4251) [05ff784a]
+- Fix IOException should be thrown when failing to creare Azure directory [b0bdfd79]
+- Fix Parallel execution of Conda corrupts data and packages (#4253) [976c282c]
+- Fix Parse negative CLI params as numbers (#4238) [1ae70d5d]
+- Fix Too long Http connection pool timeout [fa961e7f]
+- Fix Wave build for Singularity files [a60ef72b]
+- Fix Wave build when Conda package name is quoted [d19cb0b7]
+- Fix failing test [2785ffe9]
+- Fix fs cp command with remote file [366eedec]
+- Fix printf command with negative exit code (#4213) [465468b0]
+- Fix security deps in nf-azure plugin [c30d5211]
+- Fix setting `executor.queueSize = 0` is ignored (#4228) [6664b578]
+- Improve Wave handing of Conda envs [736ab9bb]
+- Improve pod options documentation (#4274) [c3aa26e1]
+- Make TraceRecord@store field public accessible [07582f0b]
+- Remove -dsl1 and -dsl1 rub cli options [b6721b71]
+- Remove experimental -dockerize option /2 [7def5542]
+- Remove experimental -dockerize option [937c8fb7]
+- Report an error on duplicate workflow name definitions (#4088) [fce9702e]
+- Undocument internal NXF_DEBUG variable [7955db8d]
+- Update AZ File share doc (#4235) [69d317b6]
+- Update docs about splitCsv() operator (#4163) [1dfb621c]
+- Update documentation of NXF_DEBUG (#4187) [a88a4245]
+- Use sipHash24 in place of deprecated murmur32 for script aliasing [bb96763f]
+- Bump groovy 3.0.19 [cb411208]
+- Bump nf-amazon@2.1.2 [7e5d414e]
+- Bump nf-azure@1.3.1 [83410f39]
+- Bump nf-wave@0.12.0 [a9f6dd65]
+
 23.08.1-edge - 17 Aug 2023
 - Add 429 http status code to Wave retriable errors [8eb5f305]
 - Add resource labels support for Azure Batch (#4178) [7b5e50a1]

docker/Dockerfile

@@ -1,5 +1,5 @@
 FROM amazoncorretto:17.0.8
-RUN yum install -y procps-ng shadow-utils
+RUN yum install -y procps-ng shadow-utils which
 
 ENV NXF_HOME=/.nextflow
 ARG TARGETPLATFORM=linux/amd64
@@ -11,7 +11,6 @@ COPY nextflow /usr/local/bin/nextflow
 
 # download runtime
 RUN mkdir /.nextflow \
- && touch /.nextflow/dockerized \
  && chmod 755 /usr/local/bin/nextflow \
  && chmod 755 /usr/local/bin/entry.sh \
  && nextflow info

docs/.dockerignore

@@ -1 +1,2 @@
+**
 !requirements.txt

docs/_static/theme.css

@@ -1,6 +1,18 @@
 
 /* navbar -- Seqera logo */
 
+.wy-side-nav-search {
+    background-color: #27ae60;
+}
+
+.wy-menu-vertical p.caption {
+    color: #27ae60;
+}
+
+.wy-menu-vertical a:active {
+    background-color: #27ae60;
+}
+
 .nav-footer-logo {
     width: 300px;
     margin-top: 2rem;

docs/azure.md

@@ -63,7 +63,7 @@ azure {
 }
 ```
 
-The files in the File share are available to the task in the directory: `<YOUR MOUNT DESTINATION>/<YOUR SOURCE FILE SHARE NAME>`.
+The files in the File share are available to the task in the directory: `<YOUR MOUNT DESTINATION>`.
 
 For instance, given the following configuration:
 
@@ -73,15 +73,20 @@ azure {
         // ...
 
         fileShares {
-            dir1 {
-                mountPath = "/mnt/mydata/"
+          rnaseqResources {
+                mountPath = "/mnt/mydata/myresources"
             }
         }
     }
 }
 ```
 
-The task can access the File share in `/mnt/mydata/dir1`.
+The task can access the File share in `/mnt/mydata/myresources`. Note: The string `rnaseqResources` in the above config can be any name of your choice, and it does not affect the underlying mount. 
+
+:::{warning}
+Azure File shares do not support authentication and management with Active Directory. The storage account key must be 
+set in the configuration if a share is mounted.
+:::
 
 (azure-batch)=
 

docs/cli.md

@@ -42,6 +42,9 @@ Available options:
 `-q, -quiet`
 : Do not print information messages.
 
+`-remote-debug`
+: Enable JVM interactive remote debugging (experimental).
+
 `-syslog`
 : Send logs to syslog server (e.g. localhost:514).
 

docs/conf.py

@@ -29,6 +29,7 @@
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = [
   'sphinx.ext.mathjax',
+  'sphinxcontrib.mermaid',
   'sphinx_rtd_theme',
   'myst_parser'
 ]
@@ -58,9 +59,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '23.08'
+version = '23.09'
 # The full version, including alpha/beta/rc tags.
-release = '23.08.1-edge'
+release = '23.09.1-edge'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/config.md

@@ -503,18 +503,28 @@ Read the {ref}`conda-page` page to learn more about how to use Conda environment
 
 ### Scope `dag`
 
-The `dag` scope controls the layout of the execution graph diagram generated by Nextflow.
+The `dag` scope controls the workflow diagram generated by Nextflow.
 
 The following settings are available:
 
 `dag.enabled`
 : When `true` turns on the generation of the DAG file (default: `false`).
 
+`dag.depth`
+: :::{versionadded} 23.10.0
+  :::
+: Controls the maximum depth at which to render sub-workflows (default: no limit; only supported by the Mermaid render).
+
 `dag.file`
 : Graph file name (default: `dag-<timestamp>.dot`).
 
 `dag.overwrite`
-: When `true` overwrites any existing DAG file with the same name.
+: When `true` overwrites any existing DAG file with the same name (default: `false`).
+
+`dag.verbose`
+: :::{versionadded} 23.10.0
+  :::
+: When `false`, channel names are omitted, operators are collapsed, and empty workflow inputs are removed (default: `false`; only supported by the Mermaid render).
 
 Read the {ref}`dag-visualisation` page to learn more about the execution graph that can be generated by Nextflow.
 
@@ -640,7 +650,7 @@ The following settings are available:
 : Determines how job status is retrieved. When `false` only the queue associated with the job execution is queried. When `true` the job status is queried globally i.e. irrespective of the submission queue (default: `false`).
 
 `executor.queueSize`
-: The number of tasks the executor will handle in a parallel manner. Default varies for each executor (see below).
+: The number of tasks the executor will handle in a parallel manner. A queue size of zero corresponds to no limit. Default varies for each executor (see below).
 
 `executor.queueStatInterval`
 : Determines how often to fetch the queue status from the scheduler (default: `1min`). Used only by grid executors.
@@ -1542,9 +1552,6 @@ The following environment variables control the configuration of the Nextflow ru
   :::
 : Enable the use of Conda recipes defined by using the {ref}`process-conda` directive. (default: `false`).
 
-`NXF_DEBUG`
-: Defines scripts debugging level: `1` dump task environment variables in the task log file; `2` enables command script execution tracing; `3` enables command wrapper execution tracing.
-
 `NXF_DEFAULT_DSL`
 : :::{versionadded} 22.03.0-edge
   :::

docs/developer/diagrams/.gitignore

@@ -0,0 +1 @@
+nextflow-merged.mmd

docs/developer/diagrams/README.md

@@ -0,0 +1,17 @@
+# Class Diagrams
+
+This directory contains class diagrams of the Nextflow source code, abridged and annotated for relevance and ease of use.
+
+Each node is a class. Fields are selectively documented in order to show only core data structures and the classes that "own" them. Methods are not explicitly documented, but they are mentioned in certain links where appropriate.
+
+Links between classes denote one of the following relationships:
+
+- Inheritance (`A <|-- B`): `B` is a subclass of `A`
+- Composition (`A --* B`): `A` contains `B`
+- Instantiation (`A --> B : f`): `A` creates instance(s) of `B` at runtime via `A::f()`
+
+Some links are commented out or not included at all, in order to focus on the most important classes and relationships. You can view these "hidden" links by simply uncommenting them, but I have found that their significance is sufficiently clear from the description files.
+
+A separate diagram description is provided for each package. These files are interoperable, which means that you can combine any subset of files into a larger diagram description. The `merge-diagrams.sh` can create a merged file for you automatically, and it includes a sensible default set of packages.
+
+You can use the [Mermaid Live Editor](https://mermaid-js.github.io/mermaid-live-editor/edit) or the [Mermaid CLI](https://github.com/mermaid-js/mermaid-cli) to render the diagram in a variety of image formats.

docs/developer/diagrams/merge-diagrams.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+packages=()
+packages+=("nextflow")
+# packages+=("nextflow.ast")
+packages+=("nextflow.cache")
+packages+=("nextflow.cli")
+# packages+=("nextflow.cloud.aws")
+# packages+=("nextflow.cloud.aws.nio")
+# packages+=("nextflow.cloud.azure")
+# packages+=("nextflow.cloud.google")
+# packages+=("nextflow.config")
+# packages+=("nextflow.container")
+packages+=("nextflow.dag")
+# packages+=("nextflow.executor")
+# packages+=("nextflow.extension")
+# packages+=("nextflow.ga4gh")
+# packages+=("nextflow.k8s")
+# packages+=("nextflow.plugin")
+packages+=("nextflow.processor")
+# packages+=("nextflow.scm")
+packages+=("nextflow.script")
+# packages+=("nextflow.secret")
+# packages+=("nextflow.trace")
+
+outfile="nextflow-merged.mmd"
+
+echo "classDiagram" > ${outfile}
+
+for package in "${packages[@]}"; do
+    echo "${package}"
+
+    tail -n +2 "${package}.mmd" >> ${outfile}
+    echo >> ${outfile}
+done

docs/developer/diagrams/nextflow.ast.mmd

@@ -0,0 +1,7 @@
+classDiagram
+    %%
+    %% nextflow.ast
+    %%
+    ScriptParser --> NextflowDSLImpl : parse
+    ScriptParser --> NextflowXformImpl : parse
+    ScriptParser --> OpXformImpl : parse

docs/developer/diagrams/nextflow.cache.mmd

@@ -0,0 +1,23 @@
+classDiagram
+    %%
+    %% nextflow.cache
+    %%
+    Session --* CacheDB
+    %% CacheFactory --> CacheDB : createInstance
+
+    CacheDB --* CacheStore
+
+    CacheStore <|-- DefaultCacheStore
+    CacheStore <|-- CloudCacheStore
+
+    class DefaultCacheStore {
+        uniqueId : UUID
+        runName : String
+        baseDir : Path
+    }
+
+    class CloudCacheStore {
+        uniqueId : UUID
+        runName : String
+        basePath : Path
+    }

docs/developer/diagrams/nextflow.cli.mmd

@@ -0,0 +1,29 @@
+classDiagram
+    %%
+    %% nextflow.cli
+    %%
+    class Launcher {
+        cliOptions : CliOptions
+        command : CmdBase
+    } 
+    Launcher --* CliOptions
+    Launcher --* CmdBase
+
+    %% CmdBase <|-- CmdClean
+    %% CmdBase <|-- CmdClone
+    %% CmdBase <|-- CmdConfig
+    CmdBase <|-- CmdConsole
+    %% CmdBase <|-- CmdDrop
+    %% CmdBase <|-- CmdFs
+    CmdBase <|-- CmdHelp
+    CmdBase <|-- CmdInfo
+    %% CmdBase <|-- CmdKubeRun
+    %% CmdBase <|-- CmdList
+    %% CmdBase <|-- CmdLog
+    %% CmdBase <|-- CmdNode
+    %% CmdBase <|-- CmdPlugin
+    %% CmdBase <|-- CmdPull
+    CmdBase <|-- CmdRun
+    %% CmdBase <|-- CmdSecret
+    %% CmdBase <|-- CmdSelfUpdate
+    %% CmdBase <|-- CmdView

docs/developer/diagrams/nextflow.cloud.aws.mmd

@@ -0,0 +1,21 @@
+classDiagram
+    %%
+    %% nextflow.cloud.aws
+    %%
+    Executor <|-- AwsBatchExecutor
+    TaskHandler <|-- AwsBatchTaskHandler
+    BashWrapperBuilder <|-- AwsBatchScriptLauncher
+
+    AwsBatchExecutor --* AwsOptions
+    AwsOptions --* AwsConfig
+    AwsConfig --* AwsBatchConfig
+    AwsConfig --* AwsS3Config
+
+    AwsBatchExecutor --> ParallelPollingMonitor : init
+    AwsBatchExecutor --> AwsBatchTaskHandler : submit
+    AwsBatchTaskHandler --> AwsBatchScriptLauncher : submit
+
+    %% TaskPollingMonitor <|-- ParallelPollingMonitor
+
+    SimpleFileCopyStrategy <|-- AwsBatchFileCopyStrategy
+    AwsBatchScriptLauncher --* AwsBatchFileCopyStrategy

docs/developer/diagrams/nextflow.cloud.aws.nio.mmd

@@ -0,0 +1,31 @@
+classDiagram
+    %%
+    %% nextflow.cloud.aws.nio
+    %%
+    S3FileSystemProvider --> S3FileSystem : newFileSystem
+
+    class S3FileSystem {
+        client : S3Client
+        endpoint : String
+    }
+    S3FileSystem --* S3Client
+
+    class S3Client {
+        client : AmazonS3
+        cannedAcl : CannedAccessControlList
+        kmsKeyId : String
+        storageEncryption : SSEAlgorithm
+        transferManager : TransferManager
+        transferPool : ExecutorService
+        uploadChunkSize : Long
+        uploadMaxThreads : Integer
+    }
+
+    Path <|-- S3Path
+
+    class S3Path {
+        bucket : String
+        parts : List~String~
+        fileSystem : S3FileSystem
+    }
+    S3Path --* S3FileSystem