nextflow.config

// default configs for all pipelines; overrides per-pipeline configs
import groovy.json.JsonSlurper
import java.text.SimpleDateFormat
def jsonSlurper = new JsonSlurper()
SimpleDateFormat timestamp_fmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
def currentDirPath = new File(System.getProperty("user.dir")).getCanonicalPath()
// ~~~~~~~~~~ PARAMETERS ~~~~~~~~~~ //
// configure pipeline settings
// overriden by CLI args
username = System.getProperty("user.name")
params.username = username
params.email_host = "nyumc.org"
params.email_from = "${username}@${params.email_host}"
params.email_to = "${username}@${params.email_host}"
params.cpus_num_big = 16
params.cpus_num_mid = 8
params.cpus_num_small = 4

manifest {
    author = 'Stephen Kelly, Varshini Vasudevaraja, Kelsey Zhu'
    homePage = 'https://github.com/NYU-Molecular-Pathology/NGS580-nf'
    description = 'NGS607 target exome analysis for 607 gene panel'
    mainScript = 'main.nf'
}

report {
    enabled = true
    file = "nextflow.html"
}

trace {
    enabled = true
    fields = "task_id,hash,native_id,process,tag,name,status,exit,module,container,cpus,time,disk,memory,attempt,submit,start,complete,duration,realtime,queue,%cpu,%mem,rss,vmem,peak_rss,peak_vmem,rchar,wchar,syscr,syscw,read_bytes,write_bytes"
    file = "trace.txt"
    raw = true
}

timeline {
    enabled = true
    file = "timeline.html"
}

notification {
    enabled = true
    to = "${params.email_to}"
    from = "${params.email_from}"
}

profiles {
    local { // TODO: reconfigure this
        // running on local desktop with Docker;
        params.ref_dir = "ref"
        params.ANNOVAR_DB_DIR = "annovar_db"
        docker.enabled = true
        executor {
            // tested with 6CPU, 24GB RAM Docker configuration, make sure to update Docker settings for this
            $local {
                cpus = 8
                queueSize = 1 // memory issues running parallel jobs locally
                memory = '16 GB'
            }
        }

        process {
            executor = 'local'
            cpus = 1
            memory = { 4.GB * task.cpus }
            errorStrategy = "retry"
            maxRetries = 2
            beforeScript = 'set -o pipefail'

            withName: annotate_targets {
                container = "stevekm/ngs580-nf:annovar-150617"
            }
            withName: targets_zip {
                container = "stevekm/ngs580-nf:htslib-1.7"
            }
            withName: trimmomatic {
                container = "stevekm/ngs580-nf:trimmomatic-0.36"
                cpus = 2
                beforeScript = "export NTHREADS=2; ${process.beforeScript}"
            }
            withName: alignment {
                memory = 8.GB
                cpus = 2
                beforeScript = "export NTHREADS=2; ${process.beforeScript}"
                container = "stevekm/ngs580-nf:bwa-0.7.17-sambamba-0.6.8"
            }
            withName: samtools_flagstat {
                container = "stevekm/ngs580-nf:samtools-1.7"
            }
            withName: sambamba_dedup {
                container = "stevekm/ngs580-nf:sambamba-0.6.8"
                cpus = 2
                beforeScript = "export NTHREADS=2; ${process.beforeScript}"
            }
            withName: samtools_dedup_flagstat {
                container = "stevekm/ngs580-nf:samtools-1.7"
            }
            withName: gatk_RealignerTargetCreator {
                memory = 16.GB
                container = "stevekm/ngs580-nf:variant-calling-0.0.2"
                cpus = 2
                beforeScript = "export NTHREADS=2; ${process.beforeScript}"
            }
            withName: cnvkit_pooled_reference {
                container = "stevekm/ngs580-nf:cnvkit-0.9.0"
                cpus = 2
                beforeScript = "export NTHREADS=2; ${process.beforeScript}"
            }
            withName: cnvkit {
                container = "stevekm/ngs580-nf:cnvkit-0.9.0"
                cpus = 2
                beforeScript = "export NTHREADS=2; ${process.beforeScript}"
            }
            withName: manta {
                container = "stevekm/ngs580-nf:manta-1.5.0"
                cpus = 2
                beforeScript = "export NTHREADS=2; ${process.beforeScript}"
            }
            withName: gatk_IndelRealigner {
                container = "stevekm/ngs580-nf:variant-calling-0.0.2"
                memory = 16.GB
            }
            withName: strelka {
                container = "stevekm/ngs580-nf:strelka-2.9.10"
                cpus = 2
                beforeScript = "export NTHREADS=2; ${process.beforeScript}"
            }
            withName: gatk_BaseRecalibrator {
                container = "stevekm/ngs580-nf:variant-calling-0.0.2"
                cpus = 2
                beforeScript = "export NTHREADS=2; ${process.beforeScript}"
            }
            withName: snp_pileup {
                container = "stevekm/ngs580-nf:cnv_facets-0.14.0"
            }
            withName: normalize_vcfs_pairs {
                container = "stevekm/ngs580-nf:bcftools-1.3"
            }
            withName: gatk_BaseRecalibratorBQSR {
                container = "stevekm/ngs580-nf:variant-calling-0.0.2"
                cpus = 2
                memory = { 8.GB * task.cpus }
                beforeScript = "export NTHREADS=2; ${process.beforeScript}"
            }
            withName: gatk_PrintReads {
                container = "stevekm/ngs580-nf:variant-calling-0.0.2"
                cpus = 2
                beforeScript = "export NTHREADS=2; ${process.beforeScript}"
            }
            withName: gatk_AnalyzeCovariates {
                memory = 16.GB
                container = "stevekm/ngs580-nf:variant-calling-0.0.2"
            }
            withName: qc_target_reads_gatk_genome {
                container = "stevekm/ngs580-nf:variant-calling-0.0.2"
            }
            withName: qc_target_reads_gatk_bed {
                container = "stevekm/ngs580-nf:variant-calling-0.0.2"
                memory = 16.GB
            }
            withName: qc_target_reads_gatk_pad500 {
                container = "stevekm/ngs580-nf:variant-calling-0.0.2"
                memory = 16.GB
            }
            withName: qc_target_reads_gatk_pad100 {
                container = "stevekm/ngs580-nf:variant-calling-0.0.2"
                memory = 16.GB
            }
            withName: gatk_hc {
                container = "stevekm/ngs580-nf:variant-calling-0.0.2"
                cpus = 2
                beforeScript = "export NTHREADS=2; ${process.beforeScript}"
            }
            withName: lofreq {
                container = "stevekm/ngs580-nf:lofreq-2.1.3"
                cpus = 2
                beforeScript = "export NTHREADS=2; ${process.beforeScript}"
            }
            withName: lofreq_somatic {
                container = "stevekm/ngs580-nf:lofreq-2.1.3"
                cpus = 2
                beforeScript = "export NTHREADS=2; ${process.beforeScript}"
            }
            withName: mutect2 {
                memory = 12.GB
                container = "stevekm/ngs580-nf:variant-calling-0.0.2"
            }
            withName: vcf_to_tsv_pairs {
                container = "stevekm/ngs580-nf:variant-calling-0.0.2"
            }
            withName: gatk_CallableLoci {
                memory = 16.GB
                container = "stevekm/ngs580-nf:variant-calling-0.0.2"
            }
            withName: msisensor {
                container = "stevekm/ngs580-nf:msisensor-0.2"
                cpus = 2
                beforeScript = "export NTHREADS=2; ${process.beforeScript}"
            }
            withName: snp_pileup_check_variance {
                container = "stevekm/ngs580-nf:cnv_facets-0.14.0"
            }
            withName: facets {
                container = "stevekm/ngs580-nf:cnv_facets-0.14.0"
            }
            withName: annotate {
                container = "stevekm/ngs580-nf:annovar-150617"
            }
            withName: annotate_pairs {
                container = "stevekm/ngs580-nf:annovar-150617"
            }
            withName: annotate_coverage_intervals {
                container = "stevekm/ngs580-nf:annovar-150617"
            }
            withName: vcf_to_tsv {
                container = "stevekm/ngs580-nf:variant-calling-0.0.2"
            }
            withName: igv_snapshot {
                container = "stevekm/ngs580-nf:IGV-2.4.10"
            }
            withName: custom_sample_report {
                container = "stevekm/ngs580-nf:reporting-3.4.3"
            }
            withName: custom_analysis_report {
                container = "stevekm/ngs580-nf:reporting-3.4.3"
            }
            withName: multiqc {
                container = "stevekm/ngs580-nf:multiqc-1.5"
            }
            withName: run_qc {
                container = "${params.containerDir}/python-3.8.simg"
            }
        }
    }

    bigPurple { // NYU Big Purple HPC cluster
        // reference locations
        params.ref_dir = "/gpfs/data/molecpathlab/ref"
        params.ANNOVAR_DB_DIR = "${params.ref_dir}/annovar/db" //  "/gpfs/data/molecpathlab/ref/annovar/db"

        // Singularity containers; on NYU Big Purple HPC
        params.containerDir = "/gpfs/data/molecpathlab/containers/NGS580-nf"

        // job config value
        params.cpus_num_big = 16
        params.cpus_num_mid = 8
        params.cpus_num_small = 4
        params.mem_per_cpu_small = "8G"
        params.mem_per_cpu_mid = "12G"
        params.mem_per_cpu_big = "16G"

        // SLURM & local exector configs
        process.executor = 'slurm' // default process executor
        executor {
            $slurm {
                // The number of tasks the executor will handle in a parallel manner (default: 100).
                queueSize = 50
                // Determines how often a poll occurs to check for a process termination.
                // pollInterval = '30sec'
                 // Determines how often the queue status is fetched from the cluster system. This setting is used only by grid executors (default: 1min).
                // queueStatInterval = '2min'
                // Determines how long the executor waits before return an error status when a process is terminated but the exit file does not exist or it is empty. This setting is used only by grid executors (default: 270 sec).
                exitReadTimeout = '90min'
                // Determines the number of jobs that can be killed in a single command execution (default: 100).
                // killBatchSize = 10
                // Determines the max rate of jobs that can be executed per time unit, for example '10 sec' eg. max 10 jobs per second (default: unlimited).
                submitRateLimit = '10 sec'
            }
            $local { // cpu and mem defined in Makefile 'submit'; run some tasks in 'local' to avoid SLURM overhead
                cpus = 8
                queueSize = 8
                memory = '48 GB'
            }
        }

        params.queue_default = "intellispace" // Big Purple HPC dedicated queue for NGS580
        params.queue = "" // allow to set queue from CLI
        params.queue_json = "slurm.json"
        params.queue_log = "queue.log" // keep a log of which queue was used for each task
        // check for queue
        // 0. use CLI passed arg
        // 1. check for slurm.json values; {"best_queue": "some_queue"}
        process.queue = {
            def queue_log = new File(params.queue_log)
            timestamp = timestamp_fmt.format(new Date())
            def task_hash = "${task.workDir.parent.baseName}/${task.workDir.baseName[0..5]}"

            if( params.queue != "" && params.queue != null ){
                //  condition 1; use CLI passed arg
                cond_label = "1"
                queue_log.append("[${timestamp}] [${cond_label}] [${task.process}] [${task_hash}] ${params.queue}\n")
                params.queue
            } else if( new File("${params.queue_json}").exists() ){
                // try to parse JSON contents
                try {
                    queue_json = jsonSlurper.parseText(new File("${params.queue_json}").text)
                    if( queue_json.containsKey("best_queue") && queue_json.best_queue != null ){
                        //  condition 2; read queue from JSON
                        cond_label = "2"
                        queue_log.append("[${timestamp}] [${cond_label}] [${task.process}] [${task_hash}] ${queue_json.best_queue}\n")
                        "${queue_json.best_queue}"
                    }
                } catch(Exception ex) {
                    //  condition 3; JSON invalid, use default queue
                    cond_label = "3"
                    queue_log.append("[${timestamp}] [${cond_label}] [${task.process}] [${task_hash}] ${params.queue_default}\n")
                    params.queue_default
                }
            } else {
                //  condition 4; queue JSON file does not exist, use default queue
                cond_label = "4"
                queue_log.append("[${timestamp}] [${cond_label}] [${task.process}] [${task_hash}] ${params.queue_default}\n")
                params.queue_default
            }
        }

        // execute in fresh login environment
        // keep NTHREADS env variable set from pipeline
        // slightly reduce job priority to prevent competition with parent Nextflow compute job
        // allocate memory space on same socket as CPU allocation
        params.clusterOptions = '--ntasks-per-node=1 --export=NONE --export=NTHREADS'// --mem-bind=local --nice=1
        process.clusterOptions = "${params.clusterOptions}"
        // TODO: figure out how to deal with /tmp requirements; SLURM config not currently available on system, need admin help
        // --tmp=16G

        // SLURM environment variables that I want to have printed out in every task stdout
        params.SLURM_vars='SLURM_JOB_ID SLURM_JOB_NAME SLURM_JOB_NODELIST SLURM_JOB_PARTITION SLURM_MEM_PER_CPU SLURM_MEM_PER_NODE SLURM_PRIO_PROCESS SLURM_SUBMIT_DIR SLURM_SUBMIT_HOST SLURM_TASK_PID SLURMD_NODENAME'

        // Singularity config
        process.module = "singularity/3.9.8"
        singularity.enabled = true
        singularity.autoMounts = true
        singularity.envWhitelist = "NTHREADS"
        // NOTE: Nextflow Singularity config prevents env variable passing by default, enable here

        // number of Nextflow process threads for moving files; should match Makefile 'submit' threads
        filePorter.maxThreads = executor.$local.cpus * 3

        process {
            // global process config
            // try to prevent error: module: command not found by sourcing module config, and pausing to allow environment to finish populating
            beforeScript = """
            . /etc/profile.d/modules.sh;
            sleep 1;
            printf "USER:\${USER:-none} HOSTNAME:\${HOSTNAME:-none} PWD:\$PWD NTHREADS:\${NTHREADS:-none}\n";
            for item in ${params.SLURM_vars}; do printf "\${item}: \${!item:-none}\t"; done;
            echo "";
            TIMESTART=\$(date +%s);
            env > .env.begin;
            """
            afterScript = """
            printf "elapsed time: %s\n" \$((\$(date +%s) - \${TIMESTART:-0}));
            env > .env.end;
            """
            errorStrategy = "retry" // re-submit failed processes; try to mitigate SLURM and 'module' command not found errors, etc
            maxRetries = 2 // retry a failed process up to 1 times as per ^^
            // cpus = 2 // 2 CPUs default due to cgroups on Big Purple limiting access to SLURM allocated cores only; give some
            cpus = 1
            time = '6h' // 6 hour default time limit for SLURM request
            memory = { 4.GB * task.cpus } // { 8.GB * process.cpus } // mem = 8.GB //

            // scratch = true // use node /tmp for execution; NVME SSD super fast (?) // disable beacuse it makes troubleshooting harder
            // scratch = "/gpfs/scratch/${username}" // user SSD (?) scratch space on network storage

            withName: git {
                executor = "local"
            }
            withName: fastq_merge {
                executor = "local"
            }
            withName: targets_metrics {
                executor = "local"
                container = "${params.containerDir}/bedtools-2.27.1.simg"
            }
            withName: trimmomatic {
                maxForks = 6 // gets really slow with a lot running
                container = "${params.containerDir}/trimmomatic-0.36.simg"
                cpus = params.cpus_num_mid
                beforeScript = "export NTHREADS=${params.cpus_num_mid}; ${process.beforeScript}"
            }
            withName: fastqc {
                container = "${params.containerDir}/fastqc-0.11.7.simg"
            }
            withName: alignment {
                container = "${params.containerDir}/bwa-0.7.17-sambamba-0.6.8.simg"
                cpus = params.cpus_num_mid
                beforeScript = "export NTHREADS=${params.cpus_num_mid}; ${process.beforeScript}"
            }
            withName: sambamba_dedup {
                container = "${params.containerDir}/sambamba-0.6.8.simg"
                cpus = params.cpus_num_small
                beforeScript = "export NTHREADS=${params.cpus_num_small}; ${process.beforeScript}"
            }
            withName: samtools_flagstat {
                executor = "local"
                container = "${params.containerDir}/samtools-1.7.simg"
            }
            withName: samtools_dedup_flagstat {
                executor = "local"
                container = "${params.containerDir}/samtools-1.7.simg"
            }
            withName: samtools_flagstat_table {
                executor = "local"
                container = "${params.containerDir}/R-3.4.3.simg"
            }
            withName: sambamba_dedup_log_table {
                executor = "local"
                container = "${params.containerDir}/R-3.4.3.simg"
            }
            withName: samtools_dedup_flagstat_table {
                executor = "local"
                container = "${params.containerDir}/R-3.4.3.simg"
            }
            withName: gatk_RealignerTargetCreator {
                container = "${params.containerDir}/variant-calling-0.0.2.simg"
                cpus = params.cpus_num_small
                beforeScript = "export NTHREADS=${params.cpus_num_small}; ${process.beforeScript}"
            }
            withName: gatk_IndelRealigner {
                container = "${params.containerDir}/variant-calling-0.0.2.simg"
                memory = { 16.GB * task.cpus }
            }
            withName: gatk_BaseRecalibrator {
                container = "${params.containerDir}/variant-calling-0.0.2.simg"
                cpus = params.cpus_num_mid
                beforeScript = "export NTHREADS=${params.cpus_num_mid}; ${process.beforeScript}"
            }
            withName: gatk_BaseRecalibratorBQSR {
                container = "${params.containerDir}/variant-calling-0.0.2.simg"
                cpus = params.cpus_num_mid
                memory = { 8.GB * task.cpus }
                beforeScript = "export NTHREADS=${params.cpus_num_mid}; ${process.beforeScript}"
            }
            withName: gatk_AnalyzeCovariates {
                memory = { 16.GB * task.cpus }
                container = "${params.containerDir}/variant-calling-0.0.2.simg"
            }
            withName: gatk_PrintReads {
                container = "${params.containerDir}/variant-calling-0.0.2.simg"
                cpus = params.cpus_num_mid
                beforeScript = "export NTHREADS=${params.cpus_num_mid}; ${process.beforeScript}"
            }
            withName: gatk_CollectHsMetrics {
                container = "${params.containerDir}/GATK-4.4.simg"
                cpus = params.cpus_num_mid
                memory = { 8.GB * task.cpus }
                beforeScript = "export NTHREADS=${params.cpus_num_mid}; ${process.beforeScript}"
            }
            withName: gatk_CollectInsertSizeMetrics {
                container = "${params.containerDir}/GATK-4.4.simg"
                cpus = params.cpus_num_mid
                memory = { 8.GB * task.cpus }
                beforeScript = "export NTHREADS=${params.cpus_num_mid}; ${process.beforeScript}"
            }
            withName: gatk_CallableLoci {
                memory = { 16.GB * task.cpus }
                container = "${params.containerDir}/variant-calling-0.0.2.simg"
            }
            withName: eval_pair_vcf {
                memory = { 16.GB * task.cpus }
                container = "${params.containerDir}/variant-calling-0.0.2.simg"
            }
            withName: qc_target_reads_gatk_pad100 {
                container = "${params.containerDir}/variant-calling-0.0.2.simg"
                // clusterOptions = "--mem-per-cpu=${params.mem_per_cpu_mid} ${process.clusterOptions}"
                memory = { 16.GB * task.cpus }
            }
            withName: qc_target_reads_gatk_pad500 {
                container = "${params.containerDir}/variant-calling-0.0.2.simg"
                memory = { 16.GB * task.cpus }
            }
            withName: qc_target_reads_gatk_bed {
                container = "${params.containerDir}/variant-calling-0.0.2.simg"
                memory = { 16.GB * task.cpus }
            }
            withName: qc_target_reads_gatk_genome {
                time = '8h'
                container = "${params.containerDir}/variant-calling-0.0.2.simg"
                memory = { 16.GB * task.cpus }
            }
            withName: lofreq {
                container = "${params.containerDir}/lofreq-2.1.3.simg"
                cpus = params.cpus_num_mid
                beforeScript = "export NTHREADS=${params.cpus_num_mid}; ${process.beforeScript}"
            }
            withName: lofreq_somatic {
                container = "${params.containerDir}/lofreq-2.1.3.simg"
                cpus = params.cpus_num_big
                memory = { 2.GB * task.cpus }
                beforeScript = "export NTHREADS=${params.cpus_num_big}; ${process.beforeScript}"
            }
            withName: gatk_hc {
                container = "${params.containerDir}/variant-calling-0.0.2.simg"
                cpus = params.cpus_num_mid
                beforeScript = "export NTHREADS=${params.cpus_num_mid}; ${process.beforeScript}"
            }
            withName: mutect2 {
                memory = { 12.GB * task.cpus } // max peak_rss; ~10GB
                container = "${params.containerDir}/variant-calling-0.0.2.simg"
            }
            withName: varscan_snp {
                container = "${params.containerDir}/varscan-2.4.3.simg"
            }
            withName: varscan_indel {
                container = "${params.containerDir}/varscan-2.4.3.simg"
            }
            withName: eval_sample_vcf {
                container = "${params.containerDir}/variant-calling-0.0.2.simg"
                memory = { 12.GB * task.cpus } // max peak_rss; ~10GB
            }
            withName: update_interval_tables {
                executor = "local"
                container = "${params.containerDir}/R-3.4.3.simg"
            }
            withName: coverage_intervals_to_table {
                executor = "local"
                container = "${params.containerDir}/R-3.4.3.simg"
            }
            withName: update_coverage_tables {
                executor = "local"
                container = "${params.containerDir}/R-3.4.3.simg"
            }
            withName: annotate {
                executor = "local"
                container = "${params.containerDir}/annovar-150617.simg"
            }
            withName: annotate_pairs {
                // executor = "local" // there are too many due to chunking to stick with local only for this
                time = '30m'
                container = "${params.containerDir}/annovar-150617.simg"
            }
            withName: annotate_coverage_intervals {
                executor = "local"
                container = "${params.containerDir}/annovar-150617.simg"
            }
            withName: annotate_targets {
                executor = "local"
                container = "${params.containerDir}/annovar-150617.simg"
            }
            withName: signatures_variant_filter {
                executor = "local"
                container = "${params.containerDir}/python-2.7.simg"
            }
            withName: deconstructSigs_signatures {
                executor = "local"
                container = "${params.containerDir}/deconstructSigs-1.8.0.simg"
            }
            withName: custom_sample_report {
                executor = "local"
                scratch = false
                container = "${params.containerDir}/reporting-3.4.3.simg"
            }
            withName: custom_analysis_report {
                executor = "local"
                scratch = false
                container = "${params.containerDir}/reporting-3.4.3.simg"
            }
            withName: multiqc {
                executor = "local"
                scratch = false
                container = "${params.containerDir}/multiqc-1.5.simg"
            }
            withName: merge_signatures_plots {
                executor = "local"
            }
            withName: merge_signatures_pie_plots {
                executor = "local"
            }
            withName: tmb_filter_variants {
                container = "${params.containerDir}/reporting-3.4.3.simg"
            }
            withName: msisensor {
                container = "${params.containerDir}/msisensor-0.2.simg"
                cpus = params.cpus_num_small
                beforeScript = "export NTHREADS=${params.cpus_num_small}; ${process.beforeScript}"
            }
            withName: filter_vcf {
                executor = "local"
                container = "${params.containerDir}/variant-calling-0.0.2.simg"
            }
            withName: vcf_to_tsv {
                executor = "local"
                container = "${params.containerDir}/variant-calling-0.0.2.simg"
            }
            withName: filter_vcf_pairs {
                // executor = "local" // there are too many due to chunking to stick with local only for this
                time = '30m'
                container = "${params.containerDir}/variant-calling-0.0.2.simg"
            }
            withName: vcf_to_tsv_pairs {
                // executor = "local" // there are too many due to chunking to stick with local only for this
                time = '30m'
                container = "${params.containerDir}/variant-calling-0.0.2.simg"
            }
            withName: cnvkit {
                container = "${params.containerDir}/cnvkit-0.9.0.simg"
                cpus = params.cpus_num_mid
                beforeScript = "export NTHREADS=${params.cpus_num_mid}; ${process.beforeScript}"
            }
            withName: reconCNV_plots {
                container = "${params.containerDir}/reconCNV-1.0.0.simg"
                cpus = params.cpus_num_mid
                beforeScript = "export NTHREADS=${params.cpus_num_mid}; ${process.beforeScript}"
            }
            withName: cnvkit_pooled_reference {
                container = "${params.containerDir}/cnvkit-0.9.0.simg"
                cpus = params.cpus_num_mid
                beforeScript = "export NTHREADS=${params.cpus_num_mid}; ${process.beforeScript}"
            }
            withName: cnvkit_gene_segments {
                executor = "local"
                container = "${params.containerDir}/cnvkit-0.9.0.simg"
            }
            withName: cnvkit_extract_trusted_genes {
                executor = "local"
            }
            withName: update_samtools_flagstat_table {
                executor = "local"
            }
            withName: update_sambamba_dedup_log_table {
                executor = "local"
            }
            withName: update_samtools_dedup_flagstat_table {
                executor = "local"
            }
            withName: update_updated_coverage_tables_collected {
                executor = "local"
            }
            withName: update_updated_coverage_interval_tables_collected {
                executor = "local"
            }
            withName: update_collect_annotation_tables {
                executor = "local"
            }
            withName: tmb_filter_variants {
                executor = "local"
            }
            withName: collect_annotation_tables {
                executor = "local"
            }
            withName: callable_loci_table {
                executor = "local"
            }
            withName: calculate_tmb {
                executor = "local"
            }
            withName: update_signatures_weights {
                executor = "local"
            }
            withName: update_update_signatures_weights_collected {
                executor = "local"
            }
            withName: cnvkit_extract_trusted_genes_update {
                executor = "local"
            }
            withName: update_cnvkit_extract_trusted_genes_collected {
                executor = "local"
            }
            withName: cnvkit_plotly {
                executor = "local"
                container = "${params.containerDir}/reporting-3.6.1.simg"
            }
            withName: split_annotation_table_caller {
                executor = "local"
                container = "${params.containerDir}/python-2.7.simg"
            }
            withName: split_annotation_table_paired {
                executor = "local"
                container = "${params.containerDir}/python-2.7.simg"
            }
            withName: overlap_snp_filter {
                executor = "local"
                container = "${params.containerDir}/python-2.7.simg"
            }
            withName: overlap_snps {
                executor = "local"
                container = "${params.containerDir}/reporting-3.4.3.simg"
            }
            withName: update_overlap_snp_table {
                executor = "local"
                container = "${params.containerDir}/python-2.7.simg"
            }
            withName: update_snp_overlap_collected {
                executor = "local"
                container = "${params.containerDir}/python-2.7.simg"
            }
            withName: extract_hapmap_pool_annotations {
                executor = "local"
            }
            withName: confidence_intervals_seracare {
                executor = "local"
                container = "${params.containerDir}/R-3.5.1.simg"
            }
            withName: igv_snapshot {
                container = "${params.containerDir}/IGV-2.4.10.simg"
            }
            withName: igv_filter_variant {
                time = '10m' // give these a short time limit so they get through the queue faster on SLURM
            }
            withName: igv_tsv_to_bed {
                time = '10m'
            }
            withName: igv_bed_line_chunk {
                time = '10m'
            }
            withName: aggregate_snapshots_bedChunkLabels {
                time = '10m'
            }
            withName: aggregate_snapshots_chunkLabels {
                time = '10m'
            }
            withName: manta {
                container = "${params.containerDir}/manta-1.5.0.simg"
                cpus = params.cpus_num_mid
                beforeScript = "export NTHREADS=${params.cpus_num_mid}; ${process.beforeScript}"
            }
            withName: targets_zip {
                executor = "local"
                container = "${params.containerDir}/htslib-1.7.simg"
            }
            withName: strelka {
                container = "${params.containerDir}/strelka-2.9.10.simg"
                cpus = params.cpus_num_mid
                beforeScript = "export NTHREADS=${params.cpus_num_mid}; ${process.beforeScript}"
            }
            withName: normalize_vcfs_pairs {
                // executor = "local" // there are too many due to chunking to stick with local only for this
                time = '30m'
                container = "${params.containerDir}/bcftools-1.3.simg"
            }
            withName: snp_pileup {
                container = "${params.containerDir}/cnv_facets-0.14.0.simg"
            }
            withName: snp_pileup_check_variance {
                container = "${params.containerDir}/cnv_facets-0.14.0.simg"
            }
            withName: facets {
                container = "${params.containerDir}/cnv_facets-0.14.0.simg"
            }
            withName: segment_files_updated {
                executor = "local"
            }
            withName: facets_summary {
                executor = "local"
            }
            withName: run_qc {
                executor = "local"
                container = "${params.containerDir}/python-3.8.simg"
            }
        }
    }

    hapmap_pool { // for making HapMap Pool on NYUMC Big Purple HPC
        // Singularity containers; on NYU Big Purple HPC
        params.containerDir = "/gpfs/data/molecpathlab/containers/NGS580-nf"
        singularity.enabled = true
        singularity.autoMounts = true
        singularity.envWhitelist = "NTHREADS"
        params.clusterOptions = '--ntasks-per-node=1 --export=NONE --export=NTHREADS'// --mem-bind=local --nice=1
        // SLURM environment variables that I want to have printed out in every task stdout
        params.SLURM_vars='SLURM_JOB_ID SLURM_JOB_NAME SLURM_JOB_NODELIST SLURM_JOB_PARTITION SLURM_MEM_PER_CPU SLURM_MEM_PER_NODE SLURM_PRIO_PROCESS SLURM_SUBMIT_DIR SLURM_SUBMIT_HOST SLURM_TASK_PID SLURMD_NODENAME'
        process {
            module = "singularity/3.7.1"
            executor = 'slurm'
            clusterOptions = "${params.clusterOptions}"
            cpus = 1
            time = '12h' // 6 hour default time limit for SLURM request
            memory = { 4.GB * task.cpus } // { 8.GB * process.cpus } // mem = 8.GB //

            beforeScript = """
            . /etc/profile.d/modules.sh;
            sleep 1;
            printf "USER:\${USER:-none} HOSTNAME:\${HOSTNAME:-none} PWD:\$PWD NTHREADS:\${NTHREADS:-none}\n";
            for item in ${params.SLURM_vars}; do printf "\${item}: \${!item:-none}\t"; done;
            echo "";
            TIMESTART=\$(date +%s);
            env > .env.begin;
            """

            afterScript = """
            printf "elapsed time: %s\n" \$((\$(date +%s) - \${TIMESTART:-0}));
            env > .env.end;
            """


            withName: fix_header {
                maxForks = 1
                executor = "local"
                container = "${params.containerDir}/samtools-1.7.simg"
            }
            withName: bam_merge {
                cpus = 40
                beforeScript = "export NTHREADS=40; ${process.beforeScript}"
                container = "${params.containerDir}/samtools-1.7.simg"
            }

        }
    }

    hapmap_pool_local {
        params.ref_dir = "ref"
        process.executor = 'local'
        executor.queueSize = 2
        docker.enabled = true
        process {
            executor = 'local'
            withName: fix_header {
                maxForks = 1
                container = "stevekm/ngs580-nf:samtools-1.7"
            }
            withName: bam_merge {
                container = "stevekm/ngs580-nf:samtools-1.7"
            }

        }
    }

    cnv_pool { // for making HapMap Pool on NYUMC Big Purple HPC
        params.ref_dir = "/gpfs/data/molecpathlab/ref"
        // Singularity containers; on NYU Big Purple HPC
        params.containerDir = "/gpfs/data/molecpathlab/containers/NGS580-nf"
        singularity.enabled = true
        singularity.autoMounts = true
        singularity.envWhitelist = "NTHREADS"
        params.clusterOptions = '--ntasks-per-node=1 --export=NONE --export=NTHREADS'// --mem-bind=local --nice=1
        // SLURM environment variables that I want to have printed out in every task stdout
        params.SLURM_vars='SLURM_JOB_ID SLURM_JOB_NAME SLURM_JOB_NODELIST SLURM_JOB_PARTITION SLURM_MEM_PER_CPU SLURM_MEM_PER_NODE SLURM_PRIO_PROCESS SLURM_SUBMIT_DIR SLURM_SUBMIT_HOST SLURM_TASK_PID SLURMD_NODENAME'
        process {
            module = "singularity/3.7.1"
            executor = 'slurm'
            clusterOptions = "${params.clusterOptions}"
            cpus = 1
            time = '1h'
            memory = { 4.GB * task.cpus }

            beforeScript = """
            . /etc/profile.d/modules.sh;
            sleep 1;
            printf "USER:\${USER:-none} HOSTNAME:\${HOSTNAME:-none} PWD:\$PWD NTHREADS:\${NTHREADS:-none}\n";
            for item in ${params.SLURM_vars}; do printf "\${item}: \${!item:-none}\t"; done;
            echo "";
            TIMESTART=\$(date +%s);
            env > .env.begin;
            """

            afterScript = """
            printf "elapsed time: %s\n" \$((\$(date +%s) - \${TIMESTART:-0}));
            env > .env.end;
            """


            withName: cnv_reference {
                container = "${params.containerDir}/cnvkit-0.9.0.simg"
                cpus = 8
                beforeScript = "export NTHREADS=8; ${process.beforeScript}"
            }
            withName: cnv_pooledreference {
                container = "${params.containerDir}/cnvkit-0.9.0.simg"
            }
        }
    }

    cnv_pool_local {
        params.ref_dir = "ref"
        process.executor = 'local'
        executor.queueSize = 2
        docker.enabled = true
        process {
            withName: cnv_reference {
                container = "stevekm/ngs580-nf:cnvkit-0.9.0"
            }
            withName: cnv_pooledreference {
                container = "stevekm/ngs580-nf:cnvkit-0.9.0"
            }
        }
    }

    ref { // for setting up reference data locally in the current directory
        params.ref_dir = "ref"
    }

    annovar_db { // for setting up reference data locally in the current directory
        params.ref_dir = "ref"
        params.ANNOVAR_DB_DIR = "annovar_db"
        docker.enabled = true
        executor.queueSize = 2
        process {
            withName: make_ANNOVAR_db {
                container = "stevekm/ngs580-nf:annovar-150617"
            }
        }

    }

    annovar_db_conda { // for setting up reference data locally in the current directory
        params.ref_dir = "ref"
        docker.enabled = true
        executor.queueSize = 2

        params.miniconda_env_str = "source /shared/miniconda3/bin/activate"
        params.make_ANNOVAR_db_env = "annovar-150617"
        process.$make_ANNOVAR_db.beforeScript = "${params.miniconda_env_str} ${params.make_ANNOVAR_db_env} ;"
        params.ANNOVAR_DB_DIR = "annovar_db"
    }

    annovar_db_bigpurple { // NYU Big Purple HPC
        // Singularity containers; on NYU Big Purple HPC
        params.containerDir = "/gpfs/data/molecpathlab/containers/NGS580-nf"
        params.ANNOVAR_DB_DIR = "annovar_db"
        process.module = "singularity/3.7.1"
        singularity.enabled = true
        singularity.autoMounts = true
        singularity.runOptions = "-B ${params.ANNOVAR_DB_DIR}"
        process {
            withName: make_ANNOVAR_db {
                container = "${params.containerDir}/annovar-150617.simg"
            }
        }
    }

    compare_bigpurple {
        // profile for running the output comparison workflow on Big Purple
        params.containerDir = "/gpfs/data/molecpathlab/containers/NGS580-nf"
        process.module = "singularity/3.7.1"
        singularity.enabled = true
        singularity.autoMounts = true
        process {
            withName: compare_report {
                container = "${params.containerDir}/reporting-3.4.3.simg"
            }
        }
    }
}