Improve samplesheet generation

.github/workflows/ci.yml

@@ -33,7 +33,7 @@ jobs:
       matrix:
         NXF_VER:
           - "24.04.2"
-          - "latest-everything"
+          - "latest-stable"
         profile:
           - "docker"
           - "singularity"
@@ -102,8 +102,6 @@ jobs:
       - name: Run Tests (Shard ${{ matrix.shard }}/6)
         run: |
           nf-test test \
-              --debug \
-              --verbose \
               --ci \
               --shard ${{ matrix.shard }}/6 \
               --changed-since HEAD^ \

CHANGELOG.md

@@ -3,7 +3,17 @@
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## dev
+## 1.5.2dev - (UNRELEASED)
+
+### `Added`
+
+### `Changed`
+
+- [#276](https://github.com/nf-core/demultiplex/pull/276) Template update for nf-core/tools v3.0.2
+
+### `Fixed`
+
+- [#277](https://github.com/nf-core/demultiplex/pull/277) Improved samplesheet generation to always produce all types of samplesheets, added the ability to explicitly set strandedness, and fixed output paths to correctly reflect the `publishDir` subdirectory structure.
 
 ## 1.5.2 - 2024-10-07
 

modules/local/fastq_to_samplesheet/main.nf

@@ -14,15 +14,18 @@ process FASTQ_TO_SAMPLESHEET {
 
     exec:
 
+    // Calculate the dynamic output directory based on meta.lane
+    def outputDir = meta.lane ? "${params.outdir}/${meta.id}/L00${meta.lane}" : "${params.outdir}/${meta.id}"
+
     // Add relevant fields to the map
     def pipeline_map = [
         sample  : meta.samplename,
-        fastq_1 : meta.fastq_1
+        fastq_1 : outputDir + '/' + file(meta.fastq_1).fileName
     ]
 
     // Add fastq_2 if it's a paired-end sample
     if (!meta.single_end) {
-        pipeline_map.fastq_2 = meta.fastq_2
+        pipeline_map.fastq_2 = outputDir + '/' + file(meta.fastq_2).fileName
     }
 
     // Add pipeline-specific entries

nextflow.config

@@ -27,7 +27,7 @@ params {
     kraken_db                   = null         // file .tar.gz
 
     // Downstream Nextflow pipeline
-    downstream_pipeline         = "default"     // enum string [rnaseq, atacseq, taxprofiler, default]
+    strandedness                = "auto"        // enum string [auto, reverse, forward, unstranded]
 
     // Options: CheckQC
     checkqc_config              = []            // file .yaml

nextflow_schema.json

@@ -35,12 +35,6 @@
                     "type": "string",
                     "format": "file-path",
                     "description": "Path to Illumina v2 samplesheet validator .json file"
-                },
-                "downstream_pipeline": {
-                    "type": "string",
-                    "description": "Name of downstream nf-core pipeline (one of: rnaseq, atacseq, taxprofiler or default). Used to produce the input samplesheet for that pipeline.",
-                    "default": "default",
-                    "enum": ["rnaseq", "atacseq", "taxprofiler", "default"]
                 }
             }
         },
@@ -276,5 +270,11 @@
         {
             "$ref": "#/$defs/generic_options"
         }
-    ]
+    ],
+    "properties": {
+        "strandedness": {
+            "type": "string",
+            "default": "auto"
+        }
+    }
 }

tests/.nftignore

@@ -22,7 +22,7 @@ multiqc/multiqc_data/multiqc_sources.txt
 multiqc/multiqc_plots/{svg,pdf,png}/*.{svg,pdf,png}
 multiqc/multiqc_report.html
 pipeline_info/*.{html,json,txt,yml}
-samplesheet/samplesheet.csv
+samplesheet/*.csv
 sim-data/*-Bases2Fastq-Sim_QC.html
 sim-data/*.fastp.fastq.gz
 sim-data/*.fastp.fastq.gz.md5

tests/bases2fastq.nf.test.snap

@@ -1,7 +1,7 @@
 {
     "Bases2Fastq": {
         "content": [
-            8,
+            10,
             {
                 "BASES2FASTQ": {
                     "bases2fastq": "1.8.0.1260801529, use subject to license available at elementbiosciences.com"
@@ -23,7 +23,7 @@
                 }
             },
             [
-        "fastq",
+                "fastq",
                 "fastq/DefaultSample.samplesheet.csv",
                 "multiqc",
                 "multiqc/multiqc_data",
@@ -148,7 +148,9 @@
                 "pipeline_info",
                 "pipeline_info/nf_core_pipeline_software_mqc_versions.yml",
                 "samplesheet",
-                "samplesheet/samplesheet.csv",
+                "samplesheet/atacseq_samplesheet.csv",
+                "samplesheet/rnaseq_samplesheet.csv",
+                "samplesheet/taxprofiler_samplesheet.csv",
                 "sim-data",
                 "sim-data.csv",
                 "sim-data/20230404-Bases2Fastq-Sim_QC.html",
@@ -210,8 +212,8 @@
         ],
         "meta": {
             "nf-test": "0.9.0",
-            "nextflow": "24.09.2"
+            "nextflow": "24.04.4"
         },
-        "timestamp": "2024-10-15T16:37:55.855747"
+        "timestamp": "2024-10-17T23:29:10.785062467"
     }
-}
+}

tests/bcl2fastq.nf.test.snap

@@ -1,7 +1,7 @@
 {
     "Bcl2Fastq": {
         "content": [
-            6,
+            8,
             {
                 "BCL2FASTQ": {
                     "bcl2fastq": "2.20.0.422"
@@ -191,7 +191,9 @@
                 "pipeline_info",
                 "pipeline_info/nf_core_pipeline_software_mqc_versions.yml",
                 "samplesheet",
-                "samplesheet/samplesheet.csv"
+                "samplesheet/atacseq_samplesheet.csv",
+                "samplesheet/rnaseq_samplesheet.csv",
+                "samplesheet/taxprofiler_samplesheet.csv"
             ],
             [
                 "220422_M11111_0222_000000000-K9H97.lane1_no_adapters.csv:md5,ee5db2e12754e069998b0a96e535238c",
@@ -237,8 +239,8 @@
         ],
         "meta": {
             "nf-test": "0.9.0",
-            "nextflow": "24.09.2"
+            "nextflow": "24.04.4"
         },
-        "timestamp": "2024-10-15T16:39:25.304794"
+        "timestamp": "2024-10-17T23:30:33.296858681"
     }
-}
+}

tests/bclconvert.nf.test.snap

@@ -1,7 +1,7 @@
 {
     "BCL-CONVERT": {
         "content": [
-            6,
+            8,
             {
                 "BCLCONVERT": {
                     "bclconvert": "4.3.6"
@@ -177,7 +177,9 @@
                 "pipeline_info",
                 "pipeline_info/nf_core_pipeline_software_mqc_versions.yml",
                 "samplesheet",
-                "samplesheet/samplesheet.csv"
+                "samplesheet/atacseq_samplesheet.csv",
+                "samplesheet/rnaseq_samplesheet.csv",
+                "samplesheet/taxprofiler_samplesheet.csv"
             ],
             [
                 "220422_M11111_0222_000000000-K9H97.lane1_no_adapters.csv:md5,ee5db2e12754e069998b0a96e535238c",
@@ -233,8 +235,8 @@
         ],
         "meta": {
             "nf-test": "0.9.0",
-            "nextflow": "24.09.2"
+            "nextflow": "24.04.4"
         },
-        "timestamp": "2024-10-15T16:41:01.01928"
+        "timestamp": "2024-10-17T23:31:58.719385372"
     }
-}
+}

tests/bclconvert_mini.nf.test.snap

@@ -1,7 +1,7 @@
 {
     "BCL-CONVERT-mini": {
         "content": [
-            26,
+            38,
             {
                 "BCLCONVERT": {
                     "bclconvert": "4.3.6"
@@ -248,7 +248,9 @@
                 "pipeline_info",
                 "pipeline_info/nf_core_pipeline_software_mqc_versions.yml",
                 "samplesheet",
-                "samplesheet/samplesheet.csv"
+                "samplesheet/atacseq_samplesheet.csv",
+                "samplesheet/rnaseq_samplesheet.csv",
+                "samplesheet/taxprofiler_samplesheet.csv"
             ],
             [
                 "HBRR1_S1_L001.fastp.fastq.gz:md5,451657490d4816dd5ff904e34ecc7fdf",
@@ -346,8 +348,8 @@
         ],
         "meta": {
             "nf-test": "0.9.0",
-            "nextflow": "24.09.2"
+            "nextflow": "24.04.4"
         },
-        "timestamp": "2024-10-16T10:59:08.175426"
+        "timestamp": "2024-10-17T23:34:50.670465424"
     }
-}
+}

tests/fqtk.nf.test

@@ -4,6 +4,7 @@ nextflow_pipeline {
     script "../main.nf"
     profile "test_fqtk"
     tag "fqtk"
+    tag "pipeline"
 
     test("FQTK") {
 

tests/fqtk.nf.test.snap

@@ -1,7 +1,7 @@
 {
     "FQTK": {
         "content": [
-            129,
+            179,
             {
                 "CSV2TSV": {
                     "sed": 4.8
@@ -169,7 +169,9 @@
                 "pipeline_info",
                 "pipeline_info/nf_core_pipeline_software_mqc_versions.yml",
                 "samplesheet",
-                "samplesheet/samplesheet.csv",
+                "samplesheet/atacseq_samplesheet.csv",
+                "samplesheet/rnaseq_samplesheet.csv",
+                "samplesheet/taxprofiler_samplesheet.csv",
                 "test",
                 "test.csv",
                 "test/demux-metrics.txt",
@@ -728,8 +730,8 @@
         ],
         "meta": {
             "nf-test": "0.9.0",
-            "nextflow": "24.09.2"
+            "nextflow": "24.04.4"
         },
-        "timestamp": "2024-10-15T16:56:01.494868"
+        "timestamp": "2024-10-17T23:37:31.252912431"
     }
-}
+}

tests/kraken.nf.test.snap

@@ -1,7 +1,7 @@
 {
     "kraken2": {
         "content": [
-            9,
+            11,
             {
                 "BCL2FASTQ": {
                     "bcl2fastq": "2.20.0.422"
@@ -205,7 +205,9 @@
                 "pipeline_info",
                 "pipeline_info/nf_core_pipeline_software_mqc_versions.yml",
                 "samplesheet",
-                "samplesheet/samplesheet.csv"
+                "samplesheet/atacseq_samplesheet.csv",
+                "samplesheet/rnaseq_samplesheet.csv",
+                "samplesheet/taxprofiler_samplesheet.csv"
             ],
             [
                 "220422_M11111_0222_000000000-K9H97.lane1_no_adapters.csv:md5,ee5db2e12754e069998b0a96e535238c",
@@ -255,8 +257,8 @@
         ],
         "meta": {
             "nf-test": "0.9.0",
-            "nextflow": "24.09.2"
+            "nextflow": "24.04.4"
         },
-        "timestamp": "2024-10-15T16:57:22.70902"
+        "timestamp": "2024-10-17T23:39:05.363547947"
     }
-}
+}

tests/mkfastq.nf.test.snap

@@ -1,7 +1,7 @@
 {
     "MKFASTQ": {
         "content": [
-            10,
+            14,
             {
                 "CELLRANGER_MKFASTQ": {
                     "cellranger": "8.0.0"
@@ -161,7 +161,9 @@
                 "pipeline_info",
                 "pipeline_info/nf_core_pipeline_software_mqc_versions.yml",
                 "samplesheet",
-                "samplesheet/samplesheet.csv"
+                "samplesheet/atacseq_samplesheet.csv",
+                "samplesheet/rnaseq_samplesheet.csv",
+                "samplesheet/taxprofiler_samplesheet.csv"
             ],
             [
                 "cellranger-tiny-bcl-simple.lane1_no_adapters.csv:md5,86a945ab9d579c7bc98d43f76a07118b",
@@ -200,8 +202,8 @@
         ],
         "meta": {
             "nf-test": "0.9.0",
-            "nextflow": "24.09.2"
+            "nextflow": "24.04.4"
         },
-        "timestamp": "2024-10-15T14:34:23.682439"
+        "timestamp": "2024-10-17T23:40:20.21203835"
     }
-}
+}

tests/sgdemux.nf.test.snap

@@ -1,7 +1,7 @@
 {
     "SGDEMUX": {
         "content": [
-            128,
+            178,
             {
                 "FALCO": {
                     "falco": "1.2.1"
@@ -204,7 +204,9 @@
                 "pipeline_info",
                 "pipeline_info/nf_core_pipeline_software_mqc_versions.yml",
                 "samplesheet",
-                "samplesheet/samplesheet.csv",
+                "samplesheet/atacseq_samplesheet.csv",
+                "samplesheet/rnaseq_samplesheet.csv",
+                "samplesheet/taxprofiler_samplesheet.csv",
                 "sim-data",
                 "sim-data.csv",
                 "sim-data/Undetermined_S25_L001.fastp.html",
@@ -580,6 +582,6 @@
             "nf-test": "0.9.0",
             "nextflow": "24.04.4"
         },
-        "timestamp": "2024-10-15T16:07:45.042862"
+        "timestamp": "2024-10-18T00:54:45.989490742"
     }
-}
+}