Merge pull request #369 from nf-core/dev

Dev -> Master 2.6.0
nf-core · Jan 5, 2024 · 54f823e · 54f823e
2 parents 66c6138 + 0bc1575
commit 54f823e
Show file tree

Hide file tree

Showing 51 changed files with 950 additions and 440 deletions.
diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md
@@ -27,6 +27,9 @@ If you're not used to this workflow with git, you can start with some [docs from
 
 ## Tests
 
+You can optionally test your changes by running the pipeline locally. Then it is recommended to use the `debug` profile to
+receive warnings about process selectors and other debug info. Example: `nextflow run . -profile debug,test,docker --outdir <OUTDIR>`.
+
 When you create a pull request with changes, [GitHub Actions](https://github.com/features/actions) will run automatic tests.
 Typically, pull-requests are only fully reviewed when these tests are passing, though of course we can help out before then.
 

diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -19,6 +19,7 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/nf-core/meth
 - [ ] If necessary, also make a PR on the nf-core/methylseq _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository.
 - [ ] Make sure your code lints (`nf-core lint`).
 - [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir <OUTDIR>`).
+- [ ] Check for unexpected warnings in debug mode (`nextflow run . -profile debug,test,docker --outdir <OUTDIR>`).
 - [ ] Usage Documentation in `docs/usage.md` is updated.
 - [ ] Output Documentation in `docs/output.md` is updated.
 - [ ] `CHANGELOG.md` is updated.

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -14,97 +14,111 @@ on:
 
 env:
   NXF_ANSI_LOG: false
-  NFTEST_VER: "0.7.3"
+  NFTEST_VER: "0.8.2"
 
 concurrency:
   group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}"
   cancel-in-progress: true
 
 jobs:
-  define_nxf_versions:
-    name: Choose nextflow versions to test against depending on target branch
-    runs-on: ubuntu-latest
+  list:
+    name: List Tests
+    if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/methylseq') }}"
     outputs:
-      matrix: ${{ steps.nxf_versions.outputs.matrix }}
+      # Expose matched filters as job 'modules' output variable
+      tests: ${{ steps.list.outputs.tests }}
+    runs-on: ubuntu-latest
     steps:
-      - id: nxf_versions
+      - uses: actions/checkout@v3
+
+      - name: Setup Nextflow
+        uses: nf-core/setup-nextflow@v1
+
+      - name: Install nf-test
+        if: steps.cache-software.outputs.cache-hit != 'true'
         run: |
-          if [[ "${{ github.event_name }}" == "pull_request" && "${{ github.base_ref }}" == "dev" && "${{ matrix.NXF_VER }}" != "latest-everything" ]]; then
-            echo matrix='["latest-everything"]' | tee -a $GITHUB_OUTPUT
-          else
-            echo matrix='["latest-everything", "23.04.0"]' | tee -a $GITHUB_OUTPUT
-          fi
+          wget -qO- https://code.askimed.com/install/nf-test | bash
+          sudo mv nf-test /usr/local/bin/
+
+      - name: List nf-test
+        id: list
+        run: |
+          echo "tests=$(nf-test list --silent --format=json)" >> "$GITHUB_OUTPUT"
 
   test:
-    name: Run pipeline with test data
-    needs: define_nxf_versions
+    name: ${{ matrix.tags }} (${{ matrix.profile }}-${{ matrix.NXF_VER }})
+    # Only run on push if this is the nf-core dev branch (merged PRs)
+    if: needs.list.outputs.tests != '[]'
+    needs: [list]
     runs-on: ubuntu-latest
     strategy:
-      fail-fast: false
       matrix:
-        NXF_VER: ${{ fromJson(needs.define_nxf_versions.outputs.matrix) }}
-        test_tags:
-          - "bismark"
-          - "bismark_hisat"
-          - "bwameth"
-        profile:
-          - "docker"
-
+        NXF_VER:
+          - "23.04.0"
+          - "latest-everything"
+        profile: ["docker"] # TODO , "singularity", "conda"]
+        test: ["${{ fromJson(needs.list.outputs.tests) }}"]
+        exclude:
+          - test: "tests/subworkflows/local/grohmm/main.nf.test@3508b8c9"
+          - test: "tests/subworkflows/local/grohmm/main.nf.test@716555be"
+          - test: "tests/subworkflows/local/grohmm/main.nf.test@306b4b15"
     steps:
       - name: Check out pipeline code
-        uses: actions/checkout@v3
-
-      - name: Hash Github Workspace
-        id: hash_workspace
-        run: |
-          echo "digest=$(echo methylseq2_${{ github.workspace }} | md5sum | cut -c 1-25)" >> $GITHUB_OUTPUT
+        uses: actions/checkout@v4
 
-      - name: Cache test data
-        id: cache-testdata
+      - name: Cache Nextflow installation
+        id: cache-software
         uses: actions/cache@v3
         with:
-          path: test-datasets/
-          key: ${{ steps.hash_workspace.outputs.digest }}
-
-      - name: Check out test data
-        uses: actions/checkout@v3
-        with:
-          repository: nf-core/test-datasets
-          ref: methylseq
-          path: test-datasets/
-
-      - name: Replace remote paths in samplesheets
-        run: |
-          for f in ./test-datasets/samplesheet/*csv; do
-            sed -i 's=https://github.com/nf-core/test-datasets/raw/methylseq/=./test-datasets/=g' $f
-            sed -i 's=https://raw.githubusercontent.com/nf-core/test-datasets/methylseq/=./test-datasets/=g' $f
-            echo "========== $f ============"
-            cat $f
-            echo "========================================"
-          done;
+          path: |
+            /usr/local/bin/nf-test
+            /home/runner/.nf-test/nf-test.jar
+          key: methylseq-${{ runner.os }}-${{ matrix.NXF_VER }}
 
       - name: Install Nextflow
         uses: nf-core/setup-nextflow@v1
         with:
           version: "${{ matrix.NXF_VER }}"
 
       - name: Install nf-test
+        if: steps.cache-software.outputs.cache-hit != 'true'
         run: |
-          wget -qO- https://code.askimed.com/install/nf-test | bash -s $NFTEST_VER
+          wget -qO- https://code.askimed.com/install/nf-test | bash
           sudo mv nf-test /usr/local/bin/
 
+      - name: Set up Singularity
+        if: matrix.profile == 'singularity'
+        uses: eWaterCycle/setup-singularity@v5
+        with:
+          singularity-version: 3.7.1
+
+      - name: Set up miniconda
+        if: matrix.profile == 'conda'
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          auto-update-conda: true
+          channels: conda-forge,bioconda,defaults
+          python-version: ${{ matrix.python-version }}
+
+      - name: Conda clean
+        if: matrix.profile == 'conda'
+        run: conda clean -a
+
       - name: Run nf-test
         run: |
-          nf-test test --tag ${{ matrix.test_tags }} --profile "test,${{ matrix.profile }}" --junitxml=test.xml
+          nf-test test \
+            --profile="test,${{ matrix.profile }}" \
+            ${{ matrix.test }} \
+            --junitxml=${{ matrix.test }}-${{ matrix.profile }}.xml
+
+      - name: Publish Test Report
+        uses: mikepenz/action-junit-report@v3
+        if: success() || failure() # run this step even if previous step failed
+        with:
+          path: ${{ matrix.test }}-${{ matrix.profile }}.xml
 
       - name: Output log on failure
         if: failure()
         run: |
           sudo apt install bat > /dev/null
           batcat --decorations=always --color=always ${{ github.workspace }}/.nf-test/tests/*/output/pipeline_info/software_versions.yml
-
-      - name: Publish Test Report
-        uses: mikepenz/action-junit-report@v3
-        if: always() # always run even if the previous step fails
-        with:
-          report_paths: test.xml
diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml
@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       # Use the @nf-core-bot token to check out so we can push later
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           token: ${{ secrets.nf_core_bot_auth_token }}
 
@@ -24,7 +24,7 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }}
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
 
       - name: Install Prettier
         run: npm install -g prettier @prettier/plugin-php

diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml
@@ -14,9 +14,9 @@ jobs:
   EditorConfig:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
 
       - name: Install editorconfig-checker
         run: npm install -g editorconfig-checker
@@ -27,9 +27,9 @@ jobs:
   Prettier:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
 
       - name: Install Prettier
         run: npm install -g prettier
@@ -40,7 +40,7 @@ jobs:
   PythonBlack:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Check code lints with Black
         uses: psf/black@stable
@@ -71,7 +71,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out pipeline code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install Nextflow
         uses: nf-core/setup-nextflow@v1

diff --git a/.github/workflows/release-announcments.yml → .github/workflows/release-announcements.yml b/.github/workflows/release-announcments.yml → .github/workflows/release-announcements.yml
diff --git a/.gitignore b/.gitignore
@@ -8,4 +8,5 @@ testing*
 *.pyc
 .vscode
 .nf-test
+.nf-test.log
 test-datasets/
diff --git a/.gitpod.yml b/.gitpod.yml
@@ -4,7 +4,9 @@ tasks:
     command: |
       pre-commit install --install-hooks
       nextflow self-update
-
+  - name: unset JAVA_TOOL_OPTIONS
+    command: |
+      unset JAVA_TOOL_OPTIONS
 vscode:
   extensions: # based on nf-core.nf-core-extensionpack
     - codezombiech.gitignore # Language support for .gitignore files

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,12 @@
 # nf-core/methylseq
 
+## [v2.6.0](https://github.com/nf-core/methylseq/releases/tag/2.6.0) - 2024-01-05
+
+### Bug fixes & refactoring
+
+- 🛠 Copy methylKit-compatible files to publishDir [#357](https://github.com/nf-core/methylseq/pull/357)
+- 🐛 fix `ignore_r1` and `ignore_3prime_r1` variable expansion [#359](https://github.com/nf-core/methylseq/pull/359)
+
 ## [v2.5.0](https://github.com/nf-core/methylseq/releases/tag/2.5.0) - 2023-10-18
 
 ### Pipeline Updates

diff --git a/README.md b/README.md
@@ -42,11 +42,8 @@ Choose between workflows by using `--aligner bismark` (default, uses bowtie2 for
 
 ## Usage
 
-:::note
-If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how
-to set-up Nextflow. Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline)
-with `-profile test` before running the workflow on actual data.
-:::
+> [!NOTE]
+> If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how to set-up Nextflow. Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline) with `-profile test` before running the workflow on actual data.
 
 First, prepare a samplesheet with your input data that looks as follows:
 
@@ -68,11 +65,9 @@ Now, you can run the pipeline using:
 nextflow run nf-core/methylseq --input samplesheet.csv --outdir <OUTDIR> --genome GRCh37 -profile <docker/singularity/podman/shifter/charliecloud/conda/institute>
 ```
 
-:::warning
-Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those
-provided by the `-c` Nextflow option can be used to provide any configuration _**except for parameters**_;
-see [docs](https://nf-co.re/usage/configuration#custom-configuration-files).
-:::
+> [!WARNING]
+> Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those provided by the `-c` Nextflow option can be used to provide any configuration _**except for parameters**_;
+> see [docs](https://nf-co.re/usage/configuration#custom-configuration-files).
 
 For more details and further functionality, please refer to the [usage documentation](https://nf-co.re/methylseq/usage) and the [parameter documentation](https://nf-co.re/methylseq/parameters).
 

diff --git a/assets/multiqc_config.yml b/assets/multiqc_config.yml
@@ -1,7 +1,7 @@
 report_comment: >
-  This report has been generated by the <a href="https://github.com/nf-core/methylseq/releases/tag/2.5.0" target="_blank">nf-core/methylseq</a>
+  This report has been generated by the <a href="https://github.com/nf-core/methylseq/releases/tag/2.6.0" target="_blank">nf-core/methylseq</a>
   analysis pipeline. For information about how to interpret these results, please see the
-  <a href="https://nf-co.re/methylseq/2.5.0/docs/output" target="_blank">documentation</a>.
+  <a href="https://nf-co.re/methylseq/2.6.0/docs/output" target="_blank">documentation</a>.
 report_section_order:
   "nf-core-methylseq-methods-description":
     order: -1000

diff --git a/assets/samplesheet.csv b/assets/samplesheet.csv
@@ -1,3 +1,5 @@
 sample,fastq_1,fastq_2,genome
-SAMPLE_PAIRED_END,/path/to/fastq/files/AEG588A1_S1_L002_R1_001.fastq.gz,/path/to/fastq/files/AEG588A1_S1_L002_R2_001.fastq.gz,/path/to/fasta/files/genome.fa
-SAMPLE_SINGLE_END,/path/to/fastq/files/AEG588A4_S4_L003_R1_001.fastq.gz,,/path/to/fasta/files/genome.fa
+SRR389222_sub1,https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub1.fastq.gz,,
+SRR389222_sub2,https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub2.fastq.gz,,
+SRR389222_sub3,https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub3.fastq.gz,,
+Ecoli,https://github.com/nf-core/test-datasets/raw/methylseq/testdata/Ecoli_10K_methylated_R1.fastq.gz,https://github.com/nf-core/test-datasets/raw/methylseq/testdata/Ecoli_10K_methylated_R2.fastq.gz,
diff --git a/assets/slackreport.json b/assets/slackreport.json
@@ -3,7 +3,7 @@
         {
             "fallback": "Plain-text summary of the attachment.",
             "color": "<% if (success) { %>good<% } else { %>danger<%} %>",
-            "author_name": "nf-core/methylseq v${version} - ${runName}",
+            "author_name": "nf-core/methylseq ${version} - ${runName}",
             "author_icon": "https://www.nextflow.io/docs/latest/_static/favicon.ico",
             "text": "<% if (success) { %>Pipeline completed successfully!<% } else { %>Pipeline completed with errors<% } %>",
             "fields": [

diff --git a/conf/modules.config b/conf/modules.config
@@ -209,8 +209,8 @@ process {
             params.comprehensive   ? ' --comprehensive --merge_non_CpG' : '',
             params.meth_cutoff     ? " --cutoff ${params.meth_cutoff}" : '',
             params.nomeseq         ? '--CX' : '',
-            params.ignore_r1 > 0   ? '--ignore ${params.ignore_r1}' : '',
-            params.ignore_3prime_r1 > 0   ? '--ignore_3prime ${params.ignore_3prime_r1}' : '',
+            params.ignore_r1 > 0   ? "--ignore ${params.ignore_r1}" : '',
+            params.ignore_3prime_r1 > 0   ? "--ignore_3prime ${params.ignore_3prime_r1}" : '',
             meta.single_end ? '' : (params.no_overlap           ? ' --no_overlap'                         : '--include_overlap'),
             meta.single_end ? '' : (params.ignore_r2        > 0 ? "--ignore_r2 ${params.ignore_r2}"       : ""),
             meta.single_end ? '' : (params.ignore_3prime_r2 > 0 ? "--ignore_3prime_r2 ${params.ignore_3prime_r2}": "")
@@ -418,6 +418,11 @@ process {
                 path: { "${params.outdir}/methyldackel" },
                 mode: params.publish_dir_mode,
                 pattern: "*.bedGraph"
+            ],
+            [
+                path: { "${params.outdir}/methyldackel" },
+                mode: params.publish_dir_mode,
+                pattern: "*.methylKit"
             ]
         ]
     }
@@ -446,7 +451,7 @@ process {
     }
 
     withName: 'MULTIQC' {
-        ext.args   = params.multiqc_title ? "--title \"$params.multiqc_title\"" : ''
+        ext.args   = { params.multiqc_title ? "--title \"$params.multiqc_title\"" : '' }
         publishDir = [
             path: { "${params.outdir}/multiqc/${params.aligner}" },
             mode: params.publish_dir_mode,

diff --git a/conf/test.config b/conf/test.config
@@ -17,9 +17,11 @@ params {
     max_time   = 6.h
 
     // Input data
-    input = "${params.test_data_base}/samplesheet/samplesheet_test.csv"
+    input = "${projectDir}/assets/samplesheet.csv"
 
     // Genome references
     fasta = "${params.test_data_base}/reference/genome.fa"
     fasta_index = "${params.test_data_base}/reference/genome.fa.fai"
 }
+
+includeConfig './test_data.config'