From 11a5b8a379212e6bb2a0bd5f2b9e303604a9b217 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Mon, 1 Sep 2025 13:27:28 +0200 Subject: [PATCH 1/4] first pass at POC for topics update for bwamem2 and dependencies --- modules/nf-core/bwamem2/index/main.nf | 24 +-- modules/nf-core/bwamem2/index/meta.yml | 7 - .../nf-core/bwamem2/index/tests/main.nf.test | 33 +++- .../bwamem2/index/tests/main.nf.test.snap | 89 ++++++---- modules/nf-core/bwamem2/mem/main.nf | 61 +++---- modules/nf-core/bwamem2/mem/meta.yml | 7 - .../nf-core/bwamem2/mem/tests/main.nf.test | 30 ++-- .../bwamem2/mem/tests/main.nf.test.snap | 153 ++++++++-------- subworkflows/nf-core/fasta_index_dna/main.nf | 106 ++++++----- subworkflows/nf-core/fastq_align_dna/main.nf | 108 +++++------ .../fastq_align_dna/tests/main.nf.test | 24 ++- .../fastq_align_dna/tests/main.nf.test.snap | 168 ++++++++++++++---- .../fastq_create_umi_consensus_fgbio/main.nf | 123 +++++++------ .../tests/main.nf.test.snap | 24 ++- 14 files changed, 558 insertions(+), 399 deletions(-) diff --git a/modules/nf-core/bwamem2/index/main.nf b/modules/nf-core/bwamem2/index/main.nf index 240343b55697..1c03cc6f1b5f 100644 --- a/modules/nf-core/bwamem2/index/main.nf +++ b/modules/nf-core/bwamem2/index/main.nf @@ -1,20 +1,20 @@ process BWAMEM2_INDEX { - tag "$fasta" + tag "${fasta}" // NOTE Requires 28N GB memory where N is the size of the reference sequence // source: https://github.com/bwa-mem2/bwa-mem2/issues/9 memory { 28.B * fasta.size() } conda "${moduleDir}/environment.yml" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/9a/9ac054213e67b3c9308e409b459080bbe438f8fd6c646c351bc42887f35a42e7/data' : - 'community.wave.seqera.io/library/bwa-mem2_htslib_samtools:e1f420694f8e42bd' }" + container "${workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container + ? 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/9a/9ac054213e67b3c9308e409b459080bbe438f8fd6c646c351bc42887f35a42e7/data' + : 'community.wave.seqera.io/library/bwa-mem2_htslib_samtools:e1f420694f8e42bd'}" input: tuple val(meta), path(fasta) output: tuple val(meta), path("bwamem2"), emit: index - path "versions.yml" , emit: versions + tuple val("${task.process}"), val('bwa-mem2'), eval("bwa-mem2 version 2>&1 | tail -1"), topic: versions when: task.ext.when == null || task.ext.when @@ -26,14 +26,9 @@ process BWAMEM2_INDEX { mkdir bwamem2 bwa-mem2 \\ index \\ - $args \\ + ${args} \\ -p bwamem2/${prefix} \\ - $fasta - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - bwamem2: \$(echo \$(bwa-mem2 version 2>&1) | sed 's/.* //') - END_VERSIONS + ${fasta} """ stub: @@ -46,10 +41,5 @@ process BWAMEM2_INDEX { touch bwamem2/${prefix}.pac touch bwamem2/${prefix}.amb touch bwamem2/${prefix}.bwt.2bit.64 - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - bwamem2: \$(echo \$(bwa-mem2 version 2>&1) | sed 's/.* //') - END_VERSIONS """ } diff --git a/modules/nf-core/bwamem2/index/meta.yml b/modules/nf-core/bwamem2/index/meta.yml index b2aa45fb6583..b58e3be8c55f 100644 --- a/modules/nf-core/bwamem2/index/meta.yml +++ b/modules/nf-core/bwamem2/index/meta.yml @@ -39,13 +39,6 @@ output: pattern: "*.{0123,amb,ann,bwt.2bit.64,pac}" ontologies: - edam: "http://edamontology.org/data_3210" # Genome index - versions: - - versions.yml: - type: file - description: File containing software versions - pattern: "versions.yml" - ontologies: - - edam: http://edamontology.org/format_3750 # YAML authors: - "@maxulysse" maintainers: diff --git a/modules/nf-core/bwamem2/index/tests/main.nf.test b/modules/nf-core/bwamem2/index/tests/main.nf.test index dbf11132c78d..273f77e89a1a 100644 --- a/modules/nf-core/bwamem2/index/tests/main.nf.test +++ b/modules/nf-core/bwamem2/index/tests/main.nf.test @@ -22,9 +22,38 @@ nextflow_process { } then { + assert process.success assertAll( - { assert process.success }, - { assert snapshot(process.out).match() } + { assert snapshot( + process.out.index, + ['versions', process.out[1]] + ).match() } + ) + } + } + + test("BWAMEM2 index - stub") { + + options "-stub" + + when { + process { + """ + input[0] = [ + [id: 'test'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ] + """ + } + } + + then { + assert process.success + assertAll( + { assert snapshot( + process.out.index, + ['versions', process.out[1]] + ).match() } ) } } diff --git a/modules/nf-core/bwamem2/index/tests/main.nf.test.snap b/modules/nf-core/bwamem2/index/tests/main.nf.test.snap index 69b268ee4638..36bcf5fe3b11 100644 --- a/modules/nf-core/bwamem2/index/tests/main.nf.test.snap +++ b/modules/nf-core/bwamem2/index/tests/main.nf.test.snap @@ -1,47 +1,68 @@ { "BWAMEM2 index": { "content": [ - { - "0": [ + [ + [ + { + "id": "test" + }, [ - { - "id": "test" - }, - [ - "genome.fasta.0123:md5,b02870de80106104abcb03cd9463e7d8", - "genome.fasta.amb:md5,3a68b8b2287e07dd3f5f95f4344ba76e", - "genome.fasta.ann:md5,c32e11f6c859f166c7525a9c1d583567", - "genome.fasta.bwt.2bit.64:md5,d097a1b82dee375d41a1ea69895a9216", - "genome.fasta.pac:md5,983e3d2cd6f36e2546e6d25a0da78d66" - ] + "genome.fasta.0123:md5,b02870de80106104abcb03cd9463e7d8", + "genome.fasta.amb:md5,3a68b8b2287e07dd3f5f95f4344ba76e", + "genome.fasta.ann:md5,c32e11f6c859f166c7525a9c1d583567", + "genome.fasta.bwt.2bit.64:md5,d097a1b82dee375d41a1ea69895a9216", + "genome.fasta.pac:md5,983e3d2cd6f36e2546e6d25a0da78d66" ] - ], - "1": [ - "versions.yml:md5,9ffd13d12e7108ed15c58566bc4717d6" - ], - "index": [ + ] + ], + [ + "versions", + [ + [ + "BWAMEM2_INDEX", + "bwa-mem2", + "2.2.1" + ] + ] + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "25.04.6" + }, + "timestamp": "2025-09-01T13:01:42.615223977" + }, + "BWAMEM2 index - stub": { + "content": [ + [ + [ + { + "id": "test" + }, + [ + "genome.fasta.0123:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.amb:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.ann:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.bwt.2bit.64:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.pac:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + [ + "versions", + [ [ - { - "id": "test" - }, - [ - "genome.fasta.0123:md5,b02870de80106104abcb03cd9463e7d8", - "genome.fasta.amb:md5,3a68b8b2287e07dd3f5f95f4344ba76e", - "genome.fasta.ann:md5,c32e11f6c859f166c7525a9c1d583567", - "genome.fasta.bwt.2bit.64:md5,d097a1b82dee375d41a1ea69895a9216", - "genome.fasta.pac:md5,983e3d2cd6f36e2546e6d25a0da78d66" - ] + "BWAMEM2_INDEX", + "bwa-mem2", + "2.2.1" ] - ], - "versions": [ - "versions.yml:md5,9ffd13d12e7108ed15c58566bc4717d6" ] - } + ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.02.0" + "nf-test": "0.9.2", + "nextflow": "25.04.6" }, - "timestamp": "2024-03-18T12:59:39.132616" + "timestamp": "2025-09-01T13:01:52.2043579" } } \ No newline at end of file diff --git a/modules/nf-core/bwamem2/mem/main.nf b/modules/nf-core/bwamem2/mem/main.nf index eab662a87ad9..62f758ecdf11 100644 --- a/modules/nf-core/bwamem2/mem/main.nf +++ b/modules/nf-core/bwamem2/mem/main.nf @@ -1,25 +1,26 @@ process BWAMEM2_MEM { - tag "$meta.id" + tag "${meta.id}" label 'process_high' conda "${moduleDir}/environment.yml" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/9a/9ac054213e67b3c9308e409b459080bbe438f8fd6c646c351bc42887f35a42e7/data' : - 'community.wave.seqera.io/library/bwa-mem2_htslib_samtools:e1f420694f8e42bd' }" + container "${workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container + ? 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/9a/9ac054213e67b3c9308e409b459080bbe438f8fd6c646c351bc42887f35a42e7/data' + : 'community.wave.seqera.io/library/bwa-mem2_htslib_samtools:e1f420694f8e42bd'}" input: tuple val(meta), path(reads) tuple val(meta2), path(index) tuple val(meta3), path(fasta) - val sort_bam + val sort_bam output: - tuple val(meta), path("*.sam") , emit: sam , optional:true - tuple val(meta), path("*.bam") , emit: bam , optional:true - tuple val(meta), path("*.cram") , emit: cram, optional:true - tuple val(meta), path("*.crai") , emit: crai, optional:true - tuple val(meta), path("*.csi") , emit: csi , optional:true - path "versions.yml" , emit: versions + tuple val(meta), path("*.sam"), emit: sam, optional: true + tuple val(meta), path("*.bam"), emit: bam, optional: true + tuple val(meta), path("*.cram"), emit: cram, optional: true + tuple val(meta), path("*.crai"), emit: crai, optional: true + tuple val(meta), path("*.csi"), emit: csi, optional: true + tuple val("${task.process}"), val('bwa-mem2'), eval("bwa-mem2 version 2>&1 | tail -1"), topic: versions + tuple val("${task.process}"), val('samtools'), eval("samtools --version 2>&1 | head -1 | sed 's/^.*samtools //'"), topic: versions when: task.ext.when == null || task.ext.when @@ -31,53 +32,45 @@ process BWAMEM2_MEM { def samtools_command = sort_bam ? 'sort' : 'view' def extension_pattern = /(--output-fmt|-O)+\s+(\S+)/ - def extension_matcher = (args2 =~ extension_pattern) + def extension_matcher = (args2 =~ extension_pattern) def extension = extension_matcher.getCount() > 0 ? extension_matcher[0][2].toLowerCase() : "bam" - def reference = fasta && extension=="cram" ? "--reference ${fasta}" : "" - if (!fasta && extension=="cram") error "Fasta reference is required for CRAM output" + def reference = fasta && extension == "cram" ? "--reference ${fasta}" : "" + if (!fasta && extension == "cram") { + error("Fasta reference is required for CRAM output") + } """ INDEX=`find -L ./ -name "*.amb" | sed 's/\\.amb\$//'` bwa-mem2 \\ mem \\ - $args \\ - -t $task.cpus \\ + ${args} \\ + -t ${task.cpus} \\ \$INDEX \\ - $reads \\ - | samtools $samtools_command $args2 -@ $task.cpus ${reference} -o ${prefix}.${extension} - - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - bwamem2: \$(echo \$(bwa-mem2 version 2>&1) | sed 's/.* //') - samtools: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//') - END_VERSIONS + ${reads} \\ + | samtools ${samtools_command} ${args2} -@ ${task.cpus} ${reference} -o ${prefix}.${extension} - """ stub: - def args2 = task.ext.args2 ?: '' def prefix = task.ext.prefix ?: "${meta.id}" def extension_pattern = /(--output-fmt|-O)+\s+(\S+)/ - def extension_matcher = (args2 =~ extension_pattern) + def extension_matcher = (args2 =~ extension_pattern) def extension = extension_matcher.getCount() > 0 ? extension_matcher[0][2].toLowerCase() : "bam" - if (!fasta && extension=="cram") error "Fasta reference is required for CRAM output" + if (!fasta && extension == "cram") { + error("Fasta reference is required for CRAM output") + } def create_index = "" if (extension == "cram") { create_index = "touch ${prefix}.crai" - } else if (extension == "bam") { + } + else if (extension == "bam") { create_index = "touch ${prefix}.csi" } """ touch ${prefix}.${extension} ${create_index} - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - bwamem2: \$(echo \$(bwa-mem2 version 2>&1) | sed 's/.* //') - samtools: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//') - END_VERSIONS """ } diff --git a/modules/nf-core/bwamem2/mem/meta.yml b/modules/nf-core/bwamem2/mem/meta.yml index 6c7d17284617..b57fe10fd1b6 100644 --- a/modules/nf-core/bwamem2/mem/meta.yml +++ b/modules/nf-core/bwamem2/mem/meta.yml @@ -118,13 +118,6 @@ output: description: Index file for BAM file pattern: "*.{csi}" ontologies: [] - versions: - - versions.yml: - type: file - description: File containing software versions - pattern: "versions.yml" - ontologies: - - edam: http://edamontology.org/format_3750 # YAML authors: - "@maxulysse" - "@matthdsm" diff --git a/modules/nf-core/bwamem2/mem/tests/main.nf.test b/modules/nf-core/bwamem2/mem/tests/main.nf.test index 9e0ab14aec59..8568d3b62ebf 100644 --- a/modules/nf-core/bwamem2/mem/tests/main.nf.test +++ b/modules/nf-core/bwamem2/mem/tests/main.nf.test @@ -41,16 +41,15 @@ nextflow_process { } then { + assert process.success assertAll( - { assert process.success }, { assert snapshot( bam(process.out.bam[0][1]).getHeaderMD5(), bam(process.out.bam[0][1]).getReadsMD5(), - process.out.versions + ['versions', process.out[5], process.out[6]] ).match() } ) } - } test("sarscov2 - fastq, index, fasta, true") { @@ -70,16 +69,15 @@ nextflow_process { } then { + assert process.success assertAll( - { assert process.success }, { assert snapshot( bam(process.out.bam[0][1]).getHeaderMD5(), bam(process.out.bam[0][1]).getReadsMD5(), - process.out.versions + ['versions', process.out[5], process.out[6]] ).match() } ) } - } test("sarscov2 - [fastq1, fastq2], index, fasta, false") { @@ -102,16 +100,15 @@ nextflow_process { } then { + assert process.success assertAll( - { assert process.success }, { assert snapshot( bam(process.out.bam[0][1]).getHeaderMD5(), bam(process.out.bam[0][1]).getReadsMD5(), - process.out.versions + ['versions', process.out[5], process.out[6]] ).match() } ) } - } test("sarscov2 - [fastq1, fastq2], index, fasta, true") { @@ -134,16 +131,15 @@ nextflow_process { } then { + assert process.success assertAll( - { assert process.success }, { assert snapshot( bam(process.out.bam[0][1]).getHeaderMD5(), bam(process.out.bam[0][1]).getReadsMD5(), - process.out.versions + ['versions', process.out[5], process.out[6]] ).match() } ) } - } test("sarscov2 - [fastq1, fastq2], index, fasta, true - stub") { @@ -168,12 +164,14 @@ nextflow_process { } then { + assert process.success assertAll( - { assert process.success }, - { assert snapshot(process.out).match() } + { assert snapshot( + bam(process.out.bam[0][1]).getHeaderMD5(), + bam(process.out.bam[0][1]).getReadsMD5(), + ['versions', process.out[5], process.out[6]] + ).match() } ) } - } - } diff --git a/modules/nf-core/bwamem2/mem/tests/main.nf.test.snap b/modules/nf-core/bwamem2/mem/tests/main.nf.test.snap index ec90701f33e5..45cbe4124692 100644 --- a/modules/nf-core/bwamem2/mem/tests/main.nf.test.snap +++ b/modules/nf-core/bwamem2/mem/tests/main.nf.test.snap @@ -4,126 +4,139 @@ "9505760d66e1d5a5d34ab79a98228c6", "57aeef88ed701a8ebc8e2f0a381b2a6", [ - "versions.yml:md5,ca1b1dcf82b92fb0751816fca16a477a" + "versions", + [ + [ + "BWAMEM2_MEM", + "bwa-mem2", + "2.2.1" + ] + ], + [ + [ + "BWAMEM2_MEM", + "samtools", + "1.21" + ] + ] ] ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.5" + "nextflow": "25.04.6" }, - "timestamp": "2025-03-27T10:57:52.782442426" + "timestamp": "2025-09-01T13:00:06.640548717" }, "sarscov2 - [fastq1, fastq2], index, fasta, true - stub": { "content": [ - { - "0": [ - - ], - "1": [ - [ - { - "id": "test", - "single_end": false - }, - "test.bam:md5,d41d8cd98f00b204e9800998ecf8427e" - ] - ], - "2": [ - - ], - "3": [ - - ], - "4": [ - [ - { - "id": "test", - "single_end": false - }, - "test.csi:md5,d41d8cd98f00b204e9800998ecf8427e" - ] - ], - "5": [ - "versions.yml:md5,ca1b1dcf82b92fb0751816fca16a477a" - ], - "bam": [ + "895266757df4950e5f81c3bc8fca5b47", + "d41d8cd98f00b204e9800998ecf8427e", + [ + "versions", + [ [ - { - "id": "test", - "single_end": false - }, - "test.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + "BWAMEM2_MEM", + "bwa-mem2", + "2.2.1" ] ], - "crai": [ - - ], - "cram": [ - - ], - "csi": [ + [ [ - { - "id": "test", - "single_end": false - }, - "test.csi:md5,d41d8cd98f00b204e9800998ecf8427e" + "BWAMEM2_MEM", + "samtools", + "1.21" ] - ], - "sam": [ - - ], - "versions": [ - "versions.yml:md5,ca1b1dcf82b92fb0751816fca16a477a" ] - } + ] ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.5" + "nextflow": "25.04.6" }, - "timestamp": "2025-03-27T10:58:37.140104324" + "timestamp": "2025-09-01T13:00:35.963137028" }, "sarscov2 - [fastq1, fastq2], index, fasta, true": { "content": [ "e0c38d5772ca5f4d5d9999f4477e933c", "af8628d9df18b2d3d4f6fd47ef2bb872", [ - "versions.yml:md5,ca1b1dcf82b92fb0751816fca16a477a" + "versions", + [ + [ + "BWAMEM2_MEM", + "bwa-mem2", + "2.2.1" + ] + ], + [ + [ + "BWAMEM2_MEM", + "samtools", + "1.21" + ] + ] ] ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.5" + "nextflow": "25.04.6" }, - "timestamp": "2025-03-27T10:58:19.047052261" + "timestamp": "2025-09-01T13:00:22.040417418" }, "sarscov2 - fastq, index, fasta, false": { "content": [ "58d05395bbb819e929885bde415947ae", "798439cbd7fd81cbcc5078022dc5479d", [ - "versions.yml:md5,ca1b1dcf82b92fb0751816fca16a477a" + "versions", + [ + [ + "BWAMEM2_MEM", + "bwa-mem2", + "2.2.1" + ] + ], + [ + [ + "BWAMEM2_MEM", + "samtools", + "1.21" + ] + ] ] ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.5" + "nextflow": "25.04.6" }, - "timestamp": "2025-03-27T10:56:53.456559296" + "timestamp": "2025-09-01T12:59:36.649629007" }, "sarscov2 - fastq, index, fasta, true": { "content": [ "276189f6f003f99a87664e74fad2893d", "94fcf617f5b994584c4e8d4044e16b4f", [ - "versions.yml:md5,ca1b1dcf82b92fb0751816fca16a477a" + "versions", + [ + [ + "BWAMEM2_MEM", + "bwa-mem2", + "2.2.1" + ] + ], + [ + [ + "BWAMEM2_MEM", + "samtools", + "1.21" + ] + ] ] ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.5" + "nextflow": "25.04.6" }, - "timestamp": "2025-03-27T10:57:21.949711746" + "timestamp": "2025-09-01T12:59:51.468221516" } } \ No newline at end of file diff --git a/subworkflows/nf-core/fasta_index_dna/main.nf b/subworkflows/nf-core/fasta_index_dna/main.nf index 3274f8aa7a85..f77cd9ff0e20 100644 --- a/subworkflows/nf-core/fasta_index_dna/main.nf +++ b/subworkflows/nf-core/fasta_index_dna/main.nf @@ -4,71 +4,67 @@ // FASTA_INDEX_DNA: Build aligner specific index for fasta files // -include { BOWTIE2_BUILD } from '../../../modules/nf-core/bowtie2/build/main' -include { BWA_INDEX as BWAMEM1_INDEX } from '../../../modules/nf-core/bwa/index/main' -include { BWAMEM2_INDEX } from '../../../modules/nf-core/bwamem2/index/main' -include { DRAGMAP_HASHTABLE } from '../../../modules/nf-core/dragmap/hashtable/main' -include { SNAPALIGNER_INDEX as SNAP_INDEX } from '../../../modules/nf-core/snapaligner/index/main' +include { BOWTIE2_BUILD } from '../../../modules/nf-core/bowtie2/build/main' +include { BWA_INDEX as BWAMEM1_INDEX } from '../../../modules/nf-core/bwa/index/main' +include { BWAMEM2_INDEX } from '../../../modules/nf-core/bwamem2/index/main' +include { DRAGMAP_HASHTABLE } from '../../../modules/nf-core/dragmap/hashtable/main' +include { SNAPALIGNER_INDEX as SNAP_INDEX } from '../../../modules/nf-core/snapaligner/index/main' workflow FASTA_INDEX_DNA { - take: - ch_fasta // channel: [mandatory] [ val(meta), path(fasta) ] - ch_altliftover // channel: [mandatory, if aligner is bwamem or bwamem2 or snap] [ val(meta), path(altliftover) ] - val_aligner // string: [mandatory] aligner [bowtie2, bwamem, bwamem2, dragmap, snap] + ch_fasta // channel: [mandatory] [ val(meta), path(fasta) ] + ch_altliftover // channel: [mandatory, if aligner is bwamem or bwamem2 or snap] [ val(meta), path(altliftover) ] + val_aligner // string: [mandatory] aligner [bowtie2, bwamem, bwamem2, dragmap, snap] main: - ch_aligner_index = Channel.empty() - ch_versions = Channel.empty() - - switch (val_aligner) { - case 'bowtie2': - BOWTIE2_BUILD(ch_fasta ) // if aligner is bowtie2 - ch_aligner_index = ch_aligner_index.mix(BOWTIE2_BUILD.out.index) - ch_versions = ch_versions.mix(BOWTIE2_BUILD.out.versions) - break - case 'bwamem': - BWAMEM1_INDEX(ch_fasta) // If aligner is bwa-mem - ch_aligner_index = ch_aligner_index - .mix( - BWAMEM1_INDEX.out.index - .join(ch_altliftover) - .map{meta, index, alt -> [meta, index + alt]} - ) + ch_aligner_index = Channel.empty() + ch_versions = Channel.empty() - ch_versions = ch_versions.mix(BWAMEM1_INDEX.out.versions) - break - case 'bwamem2': - BWAMEM2_INDEX(ch_fasta) // If aligner is bwa-mem2 - ch_aligner_index = ch_aligner_index - .mix( - BWAMEM2_INDEX.out.index - .join(ch_altliftover) - .map{meta, index, alt -> [meta, index + alt]} - ) + if (val_aligner == 'bowtie2') { + BOWTIE2_BUILD(ch_fasta) + // if aligner is bowtie2 + ch_aligner_index = ch_aligner_index.mix(BOWTIE2_BUILD.out.index) + ch_versions = ch_versions.mix(BOWTIE2_BUILD.out.versions) + } + else if (val_aligner == 'bwamem') { + BWAMEM1_INDEX(ch_fasta) + // If aligner is bwa-mem + ch_aligner_index = ch_aligner_index.mix( + BWAMEM1_INDEX.out.index.join(ch_altliftover).map { meta, index, alt -> [meta, index + alt] } + ) - ch_versions = ch_versions.mix(BWAMEM2_INDEX.out.versions) - break - case 'dragmap': - DRAGMAP_HASHTABLE(ch_fasta) // If aligner is dragmap - ch_aligner_index = ch_aligner_index.mix(DRAGMAP_HASHTABLE.out.hashmap) - ch_versions = ch_versions.mix(DRAGMAP_HASHTABLE.out.versions) - break - case 'snap': - ch_snap_reference = ch_fasta - .join(ch_altliftover) - .map {meta, fasta, alt -> [meta, fasta, [], [], alt]} + ch_versions = ch_versions.mix(BWAMEM1_INDEX.out.versions) + } + else if (val_aligner == 'bwamem2') { + BWAMEM2_INDEX(ch_fasta) + // If aligner is bwa-mem2 + ch_aligner_index = ch_aligner_index.mix( + BWAMEM2_INDEX.out.index.join(ch_altliftover).map { meta, index, alt -> [meta, index + alt] } + ) + } + else if (val_aligner == 'dragmap') { + DRAGMAP_HASHTABLE(ch_fasta) + // If aligner is dragmap + ch_aligner_index = ch_aligner_index.mix(DRAGMAP_HASHTABLE.out.hashmap) + ch_versions = ch_versions.mix(DRAGMAP_HASHTABLE.out.versions) + } + else if (val_aligner == 'snap') { + ch_snap_reference = ch_fasta + .join(ch_altliftover) + .map { meta, fasta, alt -> [meta, fasta, [], [], alt] } - SNAP_INDEX(ch_snap_reference) // If aligner is snap - ch_aligner_index = ch_aligner_index.mix(SNAP_INDEX.out.index) - ch_versions = ch_versions.mix(SNAP_INDEX.out.versions) - break - default: - error "Unknown aligner: ${val_aligner}" + SNAP_INDEX(ch_snap_reference) + // If aligner is snap + ch_aligner_index = ch_aligner_index.mix(SNAP_INDEX.out.index) + ch_versions = ch_versions.mix(SNAP_INDEX.out.versions) + } + else { + error("Unknown aligner: ${val_aligner}") } emit: - index = ch_aligner_index // channel: [ val(meta), path(index) ] - versions = ch_versions // channel: [ path(versions.yml) ] + index = ch_aligner_index // channel: [ val(meta), path(index) ] + topic_versions = channel.topic('versions') + versions = ch_versions // channel: [ path(versions.yml) ] } diff --git a/subworkflows/nf-core/fastq_align_dna/main.nf b/subworkflows/nf-core/fastq_align_dna/main.nf index f3ee73d442a4..8cd76a53ea38 100644 --- a/subworkflows/nf-core/fastq_align_dna/main.nf +++ b/subworkflows/nf-core/fastq_align_dna/main.nf @@ -5,65 +5,67 @@ // -include { BOWTIE2_ALIGN } from "../../../modules/nf-core/bowtie2/align/main" -include { BWA_MEM as BWAMEM1_MEM } from '../../../modules/nf-core/bwa/mem/main' -include { BWAMEM2_MEM as BWAMEM2_MEM } from '../../../modules/nf-core/bwamem2/mem/main' -include { DRAGMAP_ALIGN } from "../../../modules/nf-core/dragmap/align/main" -include { SNAPALIGNER_ALIGN as SNAP_ALIGN } from '../../../modules/nf-core/snapaligner/align/main' - - - +include { BOWTIE2_ALIGN } from "../../../modules/nf-core/bowtie2/align/main" +include { BWA_MEM as BWAMEM1_MEM } from '../../../modules/nf-core/bwa/mem/main' +include { BWAMEM2_MEM as BWAMEM2_MEM } from '../../../modules/nf-core/bwamem2/mem/main' +include { DRAGMAP_ALIGN } from "../../../modules/nf-core/dragmap/align/main" +include { SNAPALIGNER_ALIGN as SNAP_ALIGN } from '../../../modules/nf-core/snapaligner/align/main' workflow FASTQ_ALIGN_DNA { take: - ch_reads // channel: [mandatory] meta, reads - ch_aligner_index // channel: [mandatory] aligner index - ch_fasta // channel: [mandatory] fasta file - aligner // string: [mandatory] aligner [bowtie2, bwamem, bwamem2, dragmap, snap] - sort // boolean: [mandatory] true -> sort, false -> don't sort + ch_reads // channel: [mandatory] meta, reads + ch_aligner_index // channel: [mandatory] aligner index + ch_fasta // channel: [mandatory] fasta file + aligner // string: [mandatory] aligner [bowtie2, bwamem, bwamem2, dragmap, snap] + sort // boolean: [mandatory] true -> sort, false -> don't sort main: - ch_bam_index = Channel.empty() - ch_bam = Channel.empty() - ch_reports = Channel.empty() - ch_versions = Channel.empty() + ch_bam_index = Channel.empty() + ch_bam = Channel.empty() + ch_reports = Channel.empty() + ch_versions = Channel.empty() - // Align fastq files to reference genome and (optionally) sort - if (aligner == 'bowtie2') { - BOWTIE2_ALIGN(ch_reads, ch_aligner_index, ch_fasta, false, sort) // if aligner is bowtie2 - ch_bam = ch_bam.mix(BOWTIE2_ALIGN.out.bam) - ch_versions = ch_versions.mix(BOWTIE2_ALIGN.out.versions) - } - else if (aligner == 'bwamem'){ - BWAMEM1_MEM (ch_reads, ch_aligner_index, ch_fasta, sort) // If aligner is bwa-mem - ch_bam = ch_bam.mix(BWAMEM1_MEM.out.bam) - ch_bam_index = ch_bam_index.mix(BWAMEM1_MEM.out.csi) - ch_versions = ch_versions.mix(BWAMEM1_MEM.out.versions) - } - else if (aligner == 'bwamem2'){ - BWAMEM2_MEM (ch_reads, ch_aligner_index, ch_fasta, sort) // If aligner is bwa-mem2 - ch_bam = ch_bam.mix(BWAMEM2_MEM.out.bam) - ch_versions = ch_versions.mix(BWAMEM2_MEM.out.versions) - } - else if (aligner == 'dragmap'){ - DRAGMAP_ALIGN(ch_reads, ch_aligner_index, ch_fasta, sort) // If aligner is dragmap - ch_bam = ch_bam.mix(DRAGMAP_ALIGN.out.bam) - ch_reports = ch_reports.mix(DRAGMAP_ALIGN.out.log) - ch_versions = ch_versions.mix(DRAGMAP_ALIGN.out.versions) - } - else if (aligner == 'snap'){ - SNAP_ALIGN (ch_reads, ch_aligner_index) // If aligner is snap - ch_bam = ch_bam.mix(SNAP_ALIGN.out.bam) - ch_bam_index.mix(SNAP_ALIGN.out.bai) - ch_versions = ch_versions.mix(SNAP_ALIGN.out.versions) - } - else { - error "Unknown aligner: ${aligner}" - } + // Align fastq files to reference genome and (optionally) sort + if (aligner == 'bowtie2') { + BOWTIE2_ALIGN(ch_reads, ch_aligner_index, ch_fasta, false, sort) + // if aligner is bowtie2 + ch_bam = ch_bam.mix(BOWTIE2_ALIGN.out.bam) + ch_versions = ch_versions.mix(BOWTIE2_ALIGN.out.versions) + } + else if (aligner == 'bwamem') { + BWAMEM1_MEM(ch_reads, ch_aligner_index, ch_fasta, sort) + // If aligner is bwa-mem + ch_bam = ch_bam.mix(BWAMEM1_MEM.out.bam) + ch_bam_index = ch_bam_index.mix(BWAMEM1_MEM.out.csi) + ch_versions = ch_versions.mix(BWAMEM1_MEM.out.versions) + } + else if (aligner == 'bwamem2') { + BWAMEM2_MEM(ch_reads, ch_aligner_index, ch_fasta, sort) + // If aligner is bwa-mem2 + ch_bam = ch_bam.mix(BWAMEM2_MEM.out.bam) + } + else if (aligner == 'dragmap') { + DRAGMAP_ALIGN(ch_reads, ch_aligner_index, ch_fasta, sort) + // If aligner is dragmap + ch_bam = ch_bam.mix(DRAGMAP_ALIGN.out.bam) + ch_reports = ch_reports.mix(DRAGMAP_ALIGN.out.log) + ch_versions = ch_versions.mix(DRAGMAP_ALIGN.out.versions) + } + else if (aligner == 'snap') { + SNAP_ALIGN(ch_reads, ch_aligner_index) + // If aligner is snap + ch_bam = ch_bam.mix(SNAP_ALIGN.out.bam) + ch_bam_index.mix(SNAP_ALIGN.out.bai) + ch_versions = ch_versions.mix(SNAP_ALIGN.out.versions) + } + else { + error("Unknown aligner: ${aligner}") + } emit: - bam = ch_bam // channel: [ [meta], bam ] - bam_index = ch_bam_index // channel: [ [meta], csi/bai ] - reports = ch_reports // channel: [ [meta], log ] - versions = ch_versions // channel: [ versions.yml ] + bam = ch_bam // channel: [ [meta], bam ] + bam_index = ch_bam_index // channel: [ [meta], csi/bai ] + reports = ch_reports // channel: [ [meta], log ] + versions = ch_versions // channel: [ versions.yml ] + topic_versions = channel.topic('versions') } diff --git a/subworkflows/nf-core/fastq_align_dna/tests/main.nf.test b/subworkflows/nf-core/fastq_align_dna/tests/main.nf.test index 48988102919a..e640206ce191 100644 --- a/subworkflows/nf-core/fastq_align_dna/tests/main.nf.test +++ b/subworkflows/nf-core/fastq_align_dna/tests/main.nf.test @@ -51,7 +51,8 @@ nextflow_workflow { { assert workflow.success}, { assert snapshot( file(workflow.out.bam[0][1]).name, - workflow.out.versions[0] + workflow.out.versions.collect{ path(it).yaml }, + workflow.out.topic_versions ).match() } ) } @@ -85,7 +86,8 @@ nextflow_workflow { { assert workflow.success}, { assert snapshot( file(workflow.out.bam[0][1]).name, - workflow.out.versions[0] + workflow.out.versions.collect{ path(it).yaml }, + workflow.out.topic_versions ).match() } ) } @@ -120,7 +122,8 @@ nextflow_workflow { file(workflow.out.bam[0][1]).name, workflow.out.bam_index, workflow.out.reports, - workflow.out.versions, + workflow.out.versions.collect{ path(it).yaml }, + workflow.out.topic_versions ).match() } ) @@ -156,7 +159,8 @@ nextflow_workflow { file(workflow.out.bam[0][1]).name, workflow.out.bam_index, workflow.out.reports, - workflow.out.versions, + workflow.out.versions.collect{ path(it).yaml }, + workflow.out.topic_versions ).match() } ) @@ -192,7 +196,8 @@ nextflow_workflow { file(workflow.out.bam[0][1]).name, workflow.out.bam_index, workflow.out.reports, - workflow.out.versions, + workflow.out.versions.collect{ path(it).yaml }, + workflow.out.topic_versions ).match() } ) @@ -228,7 +233,8 @@ nextflow_workflow { file(workflow.out.bam[0][1]).name, workflow.out.bam_index, workflow.out.reports, - workflow.out.versions, + workflow.out.versions.collect{ path(it).yaml }, + workflow.out.topic_versions ).match() } ) @@ -263,7 +269,8 @@ nextflow_workflow { { assert snapshot( file(workflow.out.bam[0][1]).name, file(workflow.out.reports[0][1]).readLines().findAll { it.startsWith("decompHash") }, - file(workflow.out.versions[0]).name + workflow.out.versions.collect{ path(it).yaml }, + workflow.out.topic_versions ).match() } ) } @@ -297,7 +304,8 @@ nextflow_workflow { { assert snapshot( file(workflow.out.bam[0][1]).name, file(workflow.out.reports[0][1]).readLines().findAll { it.startsWith("decompHash") }, - file(workflow.out.versions[0]).name + workflow.out.versions.collect{ path(it).yaml }, + workflow.out.topic_versions ).match() } ) } diff --git a/subworkflows/nf-core/fastq_align_dna/tests/main.nf.test.snap b/subworkflows/nf-core/fastq_align_dna/tests/main.nf.test.snap index a4fd38bc23b5..0f83d7b3371c 100644 --- a/subworkflows/nf-core/fastq_align_dna/tests/main.nf.test.snap +++ b/subworkflows/nf-core/fastq_align_dna/tests/main.nf.test.snap @@ -9,14 +9,31 @@ ], [ - "versions.yml:md5,0e1a9cd2ce7baf650bec3dd365fbced7" + + ], + [ + [ + "BWAMEM2_INDEX", + "bwa-mem2", + "2.2.1" + ], + [ + "FASTQ_ALIGN_DNA:BWAMEM2_MEM", + "bwa-mem2", + "2.2.1" + ], + [ + "FASTQ_ALIGN_DNA:BWAMEM2_MEM", + "samtools", + "1.21" + ] ] ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.5" + "nextflow": "25.04.6" }, - "timestamp": "2025-03-27T08:57:56.74202649" + "timestamp": "2025-09-01T13:16:56.604786099" }, "test_fastq_align_dragmap_PE": { "content": [ @@ -29,13 +46,24 @@ "decompHashTableAutoHits...", "decompHashTableSetFlags..." ], - "versions.yml" + [ + { + "FASTQ_ALIGN_DNA:DRAGMAP_ALIGN": { + "dragmap": "1.2.1", + "samtools": "1.19.2", + "pigz": "2.3.4" + } + } + ], + [ + + ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.2", + "nextflow": "25.04.6" }, - "timestamp": "2024-03-14T08:28:25.283436546" + "timestamp": "2025-09-01T13:17:57.365007941" }, "test_fastq_align_bwa_mem_SE": { "content": [ @@ -47,14 +75,22 @@ ], [ - "versions.yml:md5,83f7314fe48e4c905a39e47723c78039" + { + "FASTQ_ALIGN_DNA:BWAMEM1_MEM": { + "bwa": "0.7.18-r1243-dirty", + "samtools": 1.21 + } + } + ], + [ + ] ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.5" + "nextflow": "25.04.6" }, - "timestamp": "2025-03-27T08:56:27.900928171" + "timestamp": "2025-09-01T13:16:17.903317919" }, "test_fastq_align_bwamem2_SE": { "content": [ @@ -66,14 +102,31 @@ ], [ - "versions.yml:md5,0e1a9cd2ce7baf650bec3dd365fbced7" + + ], + [ + [ + "BWAMEM2_INDEX", + "bwa-mem2", + "2.2.1" + ], + [ + "FASTQ_ALIGN_DNA:BWAMEM2_MEM", + "bwa-mem2", + "2.2.1" + ], + [ + "FASTQ_ALIGN_DNA:BWAMEM2_MEM", + "samtools", + "1.21" + ] ] ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.5" + "nextflow": "25.04.6" }, - "timestamp": "2025-03-27T08:57:28.080810832" + "timestamp": "2025-09-01T13:16:42.777381479" }, "test_fastq_align_bwa_mem_PE": { "content": [ @@ -85,25 +138,44 @@ ], [ - "versions.yml:md5,83f7314fe48e4c905a39e47723c78039" + { + "FASTQ_ALIGN_DNA:BWAMEM1_MEM": { + "bwa": "0.7.18-r1243-dirty", + "samtools": 1.21 + } + } + ], + [ + ] ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.5" + "nextflow": "25.04.6" }, - "timestamp": "2025-03-27T08:56:55.897418514" + "timestamp": "2025-09-01T13:16:27.96904962" }, "test_fastq_align_bowtie2_SE": { "content": [ "test.bam", - "versions.yml:md5,ef191e9624d747934f15d368715d87db" + [ + { + "FASTQ_ALIGN_DNA:BOWTIE2_ALIGN": { + "bowtie2": "2.5.4", + "samtools": 1.21, + "pigz": 2.8 + } + } + ], + [ + + ] ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.4" + "nextflow": "25.04.6" }, - "timestamp": "2025-03-24T14:20:30.468391" + "timestamp": "2025-09-01T13:14:42.1079798" }, "test_fastq_align_dragmap_SE": { "content": [ @@ -116,24 +188,46 @@ "decompHashTableAutoHits...", "decompHashTableSetFlags..." ], - "versions.yml" + [ + { + "FASTQ_ALIGN_DNA:DRAGMAP_ALIGN": { + "dragmap": "1.2.1", + "samtools": "1.19.2", + "pigz": "2.3.4" + } + } + ], + [ + + ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.2", + "nextflow": "25.04.6" }, - "timestamp": "2024-03-14T08:28:12.991034375" + "timestamp": "2025-09-01T13:17:48.771253082" }, "test_fastq_align_bowtie2_PE": { "content": [ "test.bam", - "versions.yml:md5,ef191e9624d747934f15d368715d87db" + [ + { + "FASTQ_ALIGN_DNA:BOWTIE2_ALIGN": { + "bowtie2": "2.5.4", + "samtools": 1.21, + "pigz": 2.8 + } + } + ], + [ + + ] ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.4" + "nextflow": "25.04.6" }, - "timestamp": "2025-03-24T14:25:41.372083" + "timestamp": "2025-09-01T13:16:06.75021421" }, "test_fastq_align_snapaligner_PE": { "content": [ @@ -155,6 +249,9 @@ ], "3": [ "versions.yml:md5,ef9883fd373e293fbf6a98985d3c0308" + ], + "4": [ + ], "bam": [ [ @@ -170,6 +267,9 @@ ], "reports": [ + ], + "topic_versions": [ + ], "versions": [ "versions.yml:md5,ef9883fd373e293fbf6a98985d3c0308" @@ -177,10 +277,10 @@ } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.2", + "nextflow": "25.04.6" }, - "timestamp": "2024-03-14T08:19:36.298315938" + "timestamp": "2025-09-01T13:18:23.086332913" }, "test_fastq_align_snapaligner_SE": { "content": [ @@ -202,6 +302,9 @@ ], "3": [ "versions.yml:md5,ef9883fd373e293fbf6a98985d3c0308" + ], + "4": [ + ], "bam": [ [ @@ -217,6 +320,9 @@ ], "reports": [ + ], + "topic_versions": [ + ], "versions": [ "versions.yml:md5,ef9883fd373e293fbf6a98985d3c0308" @@ -225,8 +331,8 @@ ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.5" + "nextflow": "25.04.6" }, - "timestamp": "2025-03-25T08:46:43.073714962" + "timestamp": "2025-09-01T13:18:12.649489873" } } \ No newline at end of file diff --git a/subworkflows/nf-core/fastq_create_umi_consensus_fgbio/main.nf b/subworkflows/nf-core/fastq_create_umi_consensus_fgbio/main.nf index 4dac538a5f82..9058d7de4a9a 100644 --- a/subworkflows/nf-core/fastq_create_umi_consensus_fgbio/main.nf +++ b/subworkflows/nf-core/fastq_create_umi_consensus_fgbio/main.nf @@ -4,35 +4,34 @@ // use the mapped information to group UMIs and generate consensus reads // -include { BWA_INDEX as BWAMEM1_INDEX } from '../../../modules/nf-core/bwa/index/main.nf' -include { BWA_MEM as BWAMEM1_MEM_PRE } from '../../../modules/nf-core/bwa/mem/main.nf' -include { BWA_MEM as BWAMEM1_MEM_POST } from '../../../modules/nf-core/bwa/mem/main.nf' -include { BWAMEM2_INDEX } from '../../../modules/nf-core/bwamem2/index/main.nf' -include { BWAMEM2_MEM as BWAMEM2_MEM_PRE } from '../../../modules/nf-core/bwamem2/mem/main.nf' -include { BWAMEM2_MEM as BWAMEM2_MEM_POST } from '../../../modules/nf-core/bwamem2/mem/main.nf' -include { FGBIO_CALLMOLECULARCONSENSUSREADS as CALLUMICONSENSUS } from '../../../modules/nf-core/fgbio/callmolecularconsensusreads/main.nf' -include { FGBIO_CALLDUPLEXCONSENSUSREADS as CALLDUPLEXCONSENSUS } from '../../../modules/nf-core/fgbio/callduplexconsensusreads/main.nf' -include { FGBIO_FASTQTOBAM as FASTQTOBAM } from '../../../modules/nf-core/fgbio/fastqtobam/main.nf' -include { FGBIO_FILTERCONSENSUSREADS as FILTERCONSENSUS } from '../../../modules/nf-core/fgbio/filterconsensusreads/main.nf' -include { FGBIO_GROUPREADSBYUMI as GROUPREADSBYUMI } from '../../../modules/nf-core/fgbio/groupreadsbyumi/main.nf' -include { FGBIO_ZIPPERBAMS as ZIPPERBAMS_PRE } from '../../../modules/nf-core/fgbio/zipperbams/main.nf' -include { FGBIO_ZIPPERBAMS as ZIPPERBAMS_POST } from '../../../modules/nf-core/fgbio/zipperbams/main.nf' -include { SAMTOOLS_FASTQ as BAM2FASTQ_PRE } from '../../../modules/nf-core/samtools/fastq/main.nf' -include { SAMTOOLS_FASTQ as BAM2FASTQ_POST } from '../../../modules/nf-core/samtools/fastq/main.nf' -include { SAMTOOLS_SORT as SORTBAM } from '../../../modules/nf-core/samtools/sort/main.nf' +include { BWA_INDEX as BWAMEM1_INDEX } from '../../../modules/nf-core/bwa/index/main.nf' +include { BWA_MEM as BWAMEM1_MEM_PRE } from '../../../modules/nf-core/bwa/mem/main.nf' +include { BWA_MEM as BWAMEM1_MEM_POST } from '../../../modules/nf-core/bwa/mem/main.nf' +include { BWAMEM2_INDEX } from '../../../modules/nf-core/bwamem2/index/main.nf' +include { BWAMEM2_MEM as BWAMEM2_MEM_PRE } from '../../../modules/nf-core/bwamem2/mem/main.nf' +include { BWAMEM2_MEM as BWAMEM2_MEM_POST } from '../../../modules/nf-core/bwamem2/mem/main.nf' +include { FGBIO_CALLMOLECULARCONSENSUSREADS as CALLUMICONSENSUS } from '../../../modules/nf-core/fgbio/callmolecularconsensusreads/main.nf' +include { FGBIO_CALLDUPLEXCONSENSUSREADS as CALLDUPLEXCONSENSUS } from '../../../modules/nf-core/fgbio/callduplexconsensusreads/main.nf' +include { FGBIO_FASTQTOBAM as FASTQTOBAM } from '../../../modules/nf-core/fgbio/fastqtobam/main.nf' +include { FGBIO_FILTERCONSENSUSREADS as FILTERCONSENSUS } from '../../../modules/nf-core/fgbio/filterconsensusreads/main.nf' +include { FGBIO_GROUPREADSBYUMI as GROUPREADSBYUMI } from '../../../modules/nf-core/fgbio/groupreadsbyumi/main.nf' +include { FGBIO_ZIPPERBAMS as ZIPPERBAMS_PRE } from '../../../modules/nf-core/fgbio/zipperbams/main.nf' +include { FGBIO_ZIPPERBAMS as ZIPPERBAMS_POST } from '../../../modules/nf-core/fgbio/zipperbams/main.nf' +include { SAMTOOLS_FASTQ as BAM2FASTQ_PRE } from '../../../modules/nf-core/samtools/fastq/main.nf' +include { SAMTOOLS_FASTQ as BAM2FASTQ_POST } from '../../../modules/nf-core/samtools/fastq/main.nf' +include { SAMTOOLS_SORT as SORTBAM } from '../../../modules/nf-core/samtools/sort/main.nf' workflow FASTQ_CREATE_UMI_CONSENSUS_FGBIO { - take: - reads // channel: [mandatory] [ val(meta), [ reads ] ] - fasta // channel: [mandatory] [ val(meta), /path/to/reference/fasta ] - bwa_index // channel: [optional] [ val(meta), /path/to/reference/bwaindex ] - dict // channel: [mandatory] [ val(meta), /path/to/reference/dictionary ] - groupreadsbyumi_strategy // string: [mandatory] grouping strategy - default: "Adjacency" - aligner // string: [mandatory] "bwa-mem" or "bwa-mem2" - duplex // bool: [mandatory] true or false depending on UMI structure - min_reads // [mandatory] One integer (for non-duplex) or a string of up-to three space-separated numbers for duplex sequencing - min_baseq // integer: [mandatory] - max_base_error_rate // integer: [mandatory] Maximum base error rate for consensus building + reads // channel: [mandatory] [ val(meta), [ reads ] ] + fasta // channel: [mandatory] [ val(meta), /path/to/reference/fasta ] + bwa_index // channel: [optional] [ val(meta), /path/to/reference/bwaindex ] + dict // channel: [mandatory] [ val(meta), /path/to/reference/dictionary ] + groupreadsbyumi_strategy // string: [mandatory] grouping strategy - default: "Adjacency" + aligner // string: [mandatory] "bwa-mem" or "bwa-mem2" + duplex // bool: [mandatory] true or false depending on UMI structure + min_reads // [mandatory] One integer (for non-duplex) or a string of up-to three space-separated numbers for duplex sequencing + min_baseq // integer: [mandatory] + max_base_error_rate // integer: [mandatory] Maximum base error rate for consensus building main: @@ -47,11 +46,11 @@ workflow FASTQ_CREATE_UMI_CONSENSUS_FGBIO { // if the UMIs are present in read names instead of inline sequences // please make sure you adjust your config to include --extract-umis-from-read-names with ext.args // of the following step - FASTQTOBAM ( reads ) + FASTQTOBAM(reads) ch_versions = ch_versions.mix(FASTQTOBAM.out.versions) // in order to map uBAM using BWA MEM, we need to convert uBAM to FASTQ - BAM2FASTQ_PRE ( FASTQTOBAM.out.bam, false ) + BAM2FASTQ_PRE(FASTQTOBAM.out.bam, false) ch_versions = ch_versions.mix(BAM2FASTQ_PRE.out.versions) // the user can choose here to use either bwa-mem (default) or bwa-mem2 @@ -59,39 +58,38 @@ workflow FASTQ_CREATE_UMI_CONSENSUS_FGBIO { if (aligner == "bwa-mem") { - if(!bwa_index){ - BWAMEM1_INDEX ( fasta ) + if (!bwa_index) { + BWAMEM1_INDEX(fasta) ch_versions = ch_versions.mix(BWAMEM1_INDEX.out.versions) } // sets bwaindex to correct input - bwaindex = bwa_index ?: BWAMEM1_INDEX.out.index + bwaindex = bwa_index ?: BWAMEM1_INDEX.out.index // appropriately tagged interleaved FASTQ reads are mapped to the reference // the aligner should be set with the following parameters "-p -K 150000000 -Y" // to be configured in ext.args of your config - BWAMEM1_MEM_PRE ( BAM2FASTQ_PRE.out.fastq, bwaindex, fasta, false ) + BWAMEM1_MEM_PRE(BAM2FASTQ_PRE.out.fastq, bwaindex, fasta, false) ch_versions = ch_versions.mix(BWAMEM1_MEM_PRE.out.versions) aligned_bam = aligned_bam.mix(BWAMEM1_MEM_PRE.out.bam) - } else { + } + else { - if(!bwa_index){ - BWAMEM2_INDEX ( fasta ) - ch_versions = ch_versions.mix(BWAMEM2_INDEX.out.versions) + if (!bwa_index) { + BWAMEM2_INDEX(fasta) } // sets bwaindex to correct input - bwaindex = bwa_index ?: BWAMEM2_INDEX.out.index + bwaindex = bwa_index ?: BWAMEM2_INDEX.out.index // appropriately tagged interleaved FASTQ reads are mapped to the reference // the aligner should be set with the following parameters "-p -K 150000000 -Y" // to be configured in ext.args of your config - BWAMEM2_MEM_PRE ( BAM2FASTQ_PRE.out.fastq, bwaindex, fasta, false ) - ch_versions = ch_versions.mix(BWAMEM2_MEM_PRE.out.versions) + BWAMEM2_MEM_PRE(BAM2FASTQ_PRE.out.fastq, bwaindex, fasta, false) aligned_bam = BWAMEM2_MEM_PRE.out.bam } // in order to tag mates information in the BAM file // FGBIO tool ZipperBams is used to merge info from mapped and unmapped BAM files - ZIPPERBAMS_PRE ( FASTQTOBAM.out.bam, aligned_bam, fasta, dict ) + ZIPPERBAMS_PRE(FASTQTOBAM.out.bam, aligned_bam, fasta, dict) ch_versions = ch_versions.mix(ZIPPERBAMS_PRE.out.versions) // appropriately tagged reads are now grouped by UMI information @@ -101,60 +99,61 @@ workflow FASTQ_CREATE_UMI_CONSENSUS_FGBIO { // For multiplex PCR and similar data where reads' genomic positions are fixed by the primers // it is recommended to use --strategy Identity to reduce runtime at the expense of lower accuracy // For duplex UMIs reads MUST be grouped using --strategy paired - GROUPREADSBYUMI ( ZIPPERBAMS_PRE.out.bam, groupreadsbyumi_strategy ) + GROUPREADSBYUMI(ZIPPERBAMS_PRE.out.bam, groupreadsbyumi_strategy) ch_versions = ch_versions.mix(GROUPREADSBYUMI.out.versions) // prepare output channel independently on UMI structure consensus_bam = Channel.empty() - if (duplex){ + if (duplex) { // this is executed if the library contains duplex UMIs - CALLDUPLEXCONSENSUS ( GROUPREADSBYUMI.out.bam, min_reads, min_baseq ) + CALLDUPLEXCONSENSUS(GROUPREADSBYUMI.out.bam, min_reads, min_baseq) ch_versions = ch_versions.mix(CALLDUPLEXCONSENSUS.out.versions) - consensus_bam = CALLDUPLEXCONSENSUS.out.bam - - } else { + consensus_bam = CALLDUPLEXCONSENSUS.out.bam + } + else { // using the above created groups, a consensus across reads in the same group // can be called // this will emit a consensus BAM file - CALLUMICONSENSUS ( GROUPREADSBYUMI.out.bam, min_reads, min_baseq ) + CALLUMICONSENSUS(GROUPREADSBYUMI.out.bam, min_reads, min_baseq) ch_versions = ch_versions.mix(CALLUMICONSENSUS.out.versions) - consensus_bam = CALLUMICONSENSUS.out.bam + consensus_bam = CALLUMICONSENSUS.out.bam } - FILTERCONSENSUS ( consensus_bam, fasta, min_reads, min_baseq, max_base_error_rate ) + FILTERCONSENSUS(consensus_bam, fasta, min_reads, min_baseq, max_base_error_rate) ch_versions = ch_versions.mix(FILTERCONSENSUS.out.versions) // now the consensus uBAM needs to be converted into FASTQ again // to be aligned - BAM2FASTQ_POST ( FILTERCONSENSUS.out.bam, false ) + BAM2FASTQ_POST(FILTERCONSENSUS.out.bam, false) ch_versions = ch_versions.mix(BAM2FASTQ_POST.out.versions) if (aligner == "bwa-mem") { // index made available through previous steps - BWAMEM1_MEM_POST ( BAM2FASTQ_POST.out.fastq, bwaindex, fasta, false ) + BWAMEM1_MEM_POST(BAM2FASTQ_POST.out.fastq, bwaindex, fasta, false) ch_versions = ch_versions.mix(BWAMEM1_MEM_POST.out.versions) aligned_bam_post = BWAMEM1_MEM_POST.out.bam - } else { + } + else { // index made available through previous steps - BWAMEM2_MEM_POST ( BAM2FASTQ_POST.out.fastq, bwaindex, fasta, false ) - ch_versions = ch_versions.mix(BWAMEM2_MEM_POST.out.versions) + BWAMEM2_MEM_POST(BAM2FASTQ_POST.out.fastq, bwaindex, fasta, false) aligned_bam_post = BWAMEM2_MEM_POST.out.bam } // in order to tag mates information in the BAM file // FGBIO tool ZipperBams is used to merge info from mapped and unmapped BAM files - ZIPPERBAMS_POST ( consensus_bam, aligned_bam_post, fasta, dict ) + ZIPPERBAMS_POST(consensus_bam, aligned_bam_post, fasta, dict) ch_versions = ch_versions.mix(ZIPPERBAMS_POST.out.versions) // finally sort bam file - SORTBAM ( ZIPPERBAMS_POST.out.bam, fasta, '' ) + SORTBAM(ZIPPERBAMS_POST.out.bam, fasta, '') ch_versions = ch_versions.mix(SORTBAM.out.versions) emit: - ubam = FASTQTOBAM.out.bam // channel: [ val(meta), [ bam ] ] - groupbam = GROUPREADSBYUMI.out.bam // channel: [ val(meta), [ bam ] ] - consensusbam = consensus_bam // channel: [ val(meta), [ bam ] ] - mappedconsensusbam = SORTBAM.out.bam // channel: [ val(meta), [ bam ] ] - versions = ch_versions // channel: [ versions.yml ] + ubam = FASTQTOBAM.out.bam // channel: [ val(meta), [ bam ] ] + groupbam = GROUPREADSBYUMI.out.bam // channel: [ val(meta), [ bam ] ] + consensusbam = consensus_bam // channel: [ val(meta), [ bam ] ] + mappedconsensusbam = SORTBAM.out.bam // channel: [ val(meta), [ bam ] ] + versions = ch_versions // channel: [ versions.yml ] + topic_versions = channel.topic('versions') } diff --git a/subworkflows/nf-core/fastq_create_umi_consensus_fgbio/tests/main.nf.test.snap b/subworkflows/nf-core/fastq_create_umi_consensus_fgbio/tests/main.nf.test.snap index 61bb2beee239..6a7f7975ffec 100644 --- a/subworkflows/nf-core/fastq_create_umi_consensus_fgbio/tests/main.nf.test.snap +++ b/subworkflows/nf-core/fastq_create_umi_consensus_fgbio/tests/main.nf.test.snap @@ -51,6 +51,9 @@ "versions.yml:md5,b9f0e09d5635dd473bfa884ceae60585", "versions.yml:md5,c384e160ec22d3eb16113fdd66189a48", "versions.yml:md5,f3af9af3b919b4673fe433d256273c51" + ], + "5": [ + ], "consensusbam": [ [ @@ -78,6 +81,9 @@ }, "test_single.bam:md5,1742f2216a9367de155bed596081f1d1" ] + ], + "topic_versions": [ + ], "ubam": [ [ @@ -108,7 +114,7 @@ "nf-test": "0.9.2", "nextflow": "25.04.6" }, - "timestamp": "2025-08-13T14:19:31.778999414" + "timestamp": "2025-09-01T13:21:21.198100221" }, "single_umi - stub": { "content": [ @@ -162,6 +168,9 @@ "versions.yml:md5,b9f0e09d5635dd473bfa884ceae60585", "versions.yml:md5,c384e160ec22d3eb16113fdd66189a48", "versions.yml:md5,f3af9af3b919b4673fe433d256273c51" + ], + "5": [ + ], "consensusbam": [ [ @@ -189,6 +198,9 @@ }, "test_single.bam:md5,d41d8cd98f00b204e9800998ecf8427e" ] + ], + "topic_versions": [ + ], "ubam": [ [ @@ -219,7 +231,7 @@ "nf-test": "0.9.2", "nextflow": "25.04.6" }, - "timestamp": "2025-08-13T16:53:08.915617697" + "timestamp": "2025-09-01T13:24:40.400566492" }, "duplex_umi": { "content": [ @@ -273,6 +285,9 @@ "versions.yml:md5,b9f0e09d5635dd473bfa884ceae60585", "versions.yml:md5,c384e160ec22d3eb16113fdd66189a48", "versions.yml:md5,f3af9af3b919b4673fe433d256273c51" + ], + "5": [ + ], "consensusbam": [ [ @@ -300,6 +315,9 @@ }, "test_single.bam:md5,096694a04711c7ade82787b8b61807ed" ] + ], + "topic_versions": [ + ], "ubam": [ [ @@ -330,6 +348,6 @@ "nf-test": "0.9.2", "nextflow": "25.04.6" }, - "timestamp": "2025-08-13T14:22:11.283480391" + "timestamp": "2025-09-01T13:24:01.864213352" } } \ No newline at end of file From fc1de4ee69306afb1c14fcd223b0113ef5eeeaf3 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Mon, 1 Sep 2025 13:45:04 +0200 Subject: [PATCH 2/4] update minimal nextflow version --- .github/workflows/lint.yml | 2 +- .github/workflows/nf-test-gpu.yml | 2 +- .github/workflows/nf-test.yml | 2 +- .github/workflows/pytest-workflow.yml | 2 +- .github/workflows/update-gpu-snapshot.yml | 2 +- .github/workflows/update-sentieon-snapshot.yml | 2 +- .github/workflows/wave.yml | 2 +- tests/config/nextflow.config | 2 +- tests/config/nf-test.config | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d080486e4e17..becccfaedda6 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -29,7 +29,7 @@ env: NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # renovate: datasource=github-releases depName=nextflow/nextflow versioning=semver - NXF_VER: "24.10.1" + NXF_VER: "25.04.6" jobs: pre-commit: diff --git a/.github/workflows/nf-test-gpu.yml b/.github/workflows/nf-test-gpu.yml index 1c9f7de543ea..06878bcb351a 100644 --- a/.github/workflows/nf-test-gpu.yml +++ b/.github/workflows/nf-test-gpu.yml @@ -30,7 +30,7 @@ env: NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity # renovate: datasource=github-releases depName=nextflow-io/nextflow versioning=semver - NXF_VER: "24.10.2" + NXF_VER: "25.04.6" jobs: nf-test-changes: diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index e24bb9e293b7..653631ef184e 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -28,7 +28,7 @@ env: NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity # renovate: datasource=github-releases depName=nextflow/nextflow versioning=semver - NXF_VER: "24.10.2" + NXF_VER: "25.04.6" jobs: nf-test-changes: diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index 36f3fbb1017f..54fd141b5934 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -29,7 +29,7 @@ env: NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # renovate: datasource=github-releases depName=nextflow-io/nextflow versioning=semver - NXF_VER: "24.10.2" + NXF_VER: "25.04.6" jobs: pytest-changes: diff --git a/.github/workflows/update-gpu-snapshot.yml b/.github/workflows/update-gpu-snapshot.yml index 8ad4a1dd672d..e439089ac488 100644 --- a/.github/workflows/update-gpu-snapshot.yml +++ b/.github/workflows/update-gpu-snapshot.yml @@ -10,7 +10,7 @@ env: NFT_VER: "0.9.2" NXF_ANSI_LOG: false # renovate: datasource=github-releases depName=nextflow/nextflow versioning=semver - NXF_VER: "24.10.2" + NXF_VER: "25.04.6" jobs: update-gpu-snapshot: diff --git a/.github/workflows/update-sentieon-snapshot.yml b/.github/workflows/update-sentieon-snapshot.yml index 41aeabaa586a..91d48cd4772f 100644 --- a/.github/workflows/update-sentieon-snapshot.yml +++ b/.github/workflows/update-sentieon-snapshot.yml @@ -10,7 +10,7 @@ env: NFT_VER: "0.9.2" NXF_ANSI_LOG: false # renovate: datasource=github-releases depName=nextflow/nextflow versioning=semver - NXF_VER: "24.10.2" + NXF_VER: "25.04.6" jobs: update-sentieon-snapshot: diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 77b17a3ca7be..bd583dc5b007 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -15,7 +15,7 @@ env: # renovate: datasource=github-releases depName=askimed/nf-test versioning=semver NFT_VER: "0.9.2" # renovate: datasource=github-releases depName=nextflow/nextflow versioning=semver - NXF_VER: "24.10.1" + NXF_VER: "25.04.6" jobs: generate-matrix: diff --git a/tests/config/nextflow.config b/tests/config/nextflow.config index e58f30886398..598bd1865b26 100644 --- a/tests/config/nextflow.config +++ b/tests/config/nextflow.config @@ -45,5 +45,5 @@ conda { createTimeout = "120 min" } includeConfig 'test_data.config' manifest { - nextflowVersion = '!>=24.10.2' + nextflowVersion = '!>=25.04.6' } diff --git a/tests/config/nf-test.config b/tests/config/nf-test.config index 8409e5707fbe..31eda2102450 100644 --- a/tests/config/nf-test.config +++ b/tests/config/nf-test.config @@ -95,5 +95,5 @@ conda { createTimeout = "120 min" } includeConfig 'test_data.config' manifest { - nextflowVersion = '!>=24.10.2' + nextflowVersion = '!>=25.04.6' } From b67e815c7f3cfbabc8d9ed07d8211feb38360135 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Thu, 25 Sep 2025 14:38:56 +0200 Subject: [PATCH 3/4] update snapshot --- .../bwamem2/index/tests/main.nf.test.snap | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/modules/nf-core/bwamem2/index/tests/main.nf.test.snap b/modules/nf-core/bwamem2/index/tests/main.nf.test.snap index 0f25aad25b2b..7407d942b730 100644 --- a/modules/nf-core/bwamem2/index/tests/main.nf.test.snap +++ b/modules/nf-core/bwamem2/index/tests/main.nf.test.snap @@ -7,11 +7,11 @@ "id": "test" }, [ - "genome.fasta.0123:md5,b02870de80106104abcb03cd9463e7d8", - "genome.fasta.amb:md5,3a68b8b2287e07dd3f5f95f4344ba76e", - "genome.fasta.ann:md5,c32e11f6c859f166c7525a9c1d583567", - "genome.fasta.bwt.2bit.64:md5,d097a1b82dee375d41a1ea69895a9216", - "genome.fasta.pac:md5,983e3d2cd6f36e2546e6d25a0da78d66" + "genome.fasta.0123:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.amb:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.ann:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.bwt.2bit.64:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.pac:md5,d41d8cd98f00b204e9800998ecf8427e" ] ] ], @@ -28,9 +28,9 @@ ], "meta": { "nf-test": "0.9.2", - "nextflow": "25.04.6" + "nextflow": "25.04.7" }, - "timestamp": "2025-09-04T08:55:53.219699135" + "timestamp": "2025-09-25T14:38:35.557107186" }, "fasta": { "content": [ @@ -40,11 +40,11 @@ "id": "test" }, [ - "genome.fasta.0123:md5,d41d8cd98f00b204e9800998ecf8427e", - "genome.fasta.amb:md5,d41d8cd98f00b204e9800998ecf8427e", - "genome.fasta.ann:md5,d41d8cd98f00b204e9800998ecf8427e", - "genome.fasta.bwt.2bit.64:md5,d41d8cd98f00b204e9800998ecf8427e", - "genome.fasta.pac:md5,d41d8cd98f00b204e9800998ecf8427e" + "genome.fasta.0123:md5,b02870de80106104abcb03cd9463e7d8", + "genome.fasta.amb:md5,3a68b8b2287e07dd3f5f95f4344ba76e", + "genome.fasta.ann:md5,c32e11f6c859f166c7525a9c1d583567", + "genome.fasta.bwt.2bit.64:md5,d097a1b82dee375d41a1ea69895a9216", + "genome.fasta.pac:md5,983e3d2cd6f36e2546e6d25a0da78d66" ] ] ], @@ -61,8 +61,8 @@ ], "meta": { "nf-test": "0.9.2", - "nextflow": "25.04.6" + "nextflow": "25.04.7" }, - "timestamp": "2025-09-04T08:55:45.007921901" + "timestamp": "2025-09-25T14:38:22.536156282" } } \ No newline at end of file From 3eeffa6ec529469be8f91a0f8999447b1dbb1b44 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Thu, 25 Sep 2025 14:45:09 +0200 Subject: [PATCH 4/4] update NF to 25.04.6 --- .github/workflows/nf-test-gpu.yml | 2 +- .github/workflows/nf-test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nf-test-gpu.yml b/.github/workflows/nf-test-gpu.yml index 434dba85069f..06878bcb351a 100644 --- a/.github/workflows/nf-test-gpu.yml +++ b/.github/workflows/nf-test-gpu.yml @@ -30,7 +30,7 @@ env: NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity # renovate: datasource=github-releases depName=nextflow-io/nextflow versioning=semver - NXF_VER: "25.04.0" + NXF_VER: "25.04.6" jobs: nf-test-changes: diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 97e55ba5c666..d27a9c6eb200 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -28,7 +28,7 @@ env: NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity # renovate: datasource=github-releases depName=nextflow/nextflow versioning=semver - NXF_VER: "25.04.0" + NXF_VER: "25.04.6" jobs: nf-test-changes: