cloud and db handling updates

replikation · Jan 10, 2024 · d34d428 · d34d428
1 parent d09808c
commit d34d428
Show file tree

Hide file tree

Showing 26 changed files with 43 additions and 126 deletions.
diff --git a/nextflow.config b/nextflow.config
@@ -12,7 +12,6 @@ params {
     fastq = false
     dir = ''
     list = false
-    cloudDatabase = false
     filter = '1500'
     setup = ''
     all_tools = false
@@ -96,7 +95,6 @@ profiles {
                	cpus = params.max_cores
         }
         workDir = params.workdir
-        params.cloudProcess = false
         includeConfig 'configs/local.config'
     }
 
@@ -106,7 +104,6 @@ profiles {
                 queueSize = 200
         }
         workDir = params.workdir
-        params.cloudProcess = true
         process.cache = "lenient"
         includeConfig 'configs/node.config'
     }
@@ -117,7 +114,6 @@ profiles {
             queueSize = 200
         }
         workDir = params.workdir
-        params.cloudProcess = true
         process {
             cache = "lenient"
             time = '2h'
@@ -128,17 +124,18 @@ profiles {
 
     ukj_cloud { 
         workDir = params.workdir
-        params.cloudProcess = true
         docker { enabled = true } 
-        process.executor = 'google-lifesciences'
-        params.databases = 'gs://persistant-storage/What-the-Phage'
+        process { executor = 'google-batch' }
+        params { databases = 'gs://database_taxonomy/What-the-Phage' }
         bucketDir = 'gs://case-tmp-dir/What-the-Phage'
         google {  
             project = 'case-dev-302214'
-            zone = 'europe-west1-b'
-            lifeSciences.preemptible = true
-            lifeSciences.bootDiskSize = 20.GB
+            batch{ 
+                spot = true
+                bootDiskSize = 20.GB
+                zone = 'europe-west1-b'
             }
+        }
         includeConfig 'configs/container.config'
         includeConfig 'configs/node.config'
 
@@ -168,7 +165,6 @@ profiles {
     git_action {
         includeConfig 'configs/local.config'
         params {
-                cloudProcess = false
                 ma = true
                 mp = true
                 vf = false
@@ -191,7 +187,6 @@ profiles {
                 queueSize = 200
         }
         process.cache = "lenient"
-        params.cloudProcess = true
         params.workdir = "/hps/nobackup2/production/metagenomics/$USER/nextflow-work-$USER"
         workDir = params.workdir
         params.databases = "/homes/$USER/data/nextflow-databases/"

diff --git a/workflows/checkV_wf.nf b/workflows/checkV_wf.nf
@@ -8,13 +8,9 @@ workflow checkV_wf {
     main:   
             fasta = fasta_and_tool_results.map {it -> tuple(it[0],it[1])}
             // local storage via storeDir
-            if (!params.cloudProcess) { download_checkV_DB(); db = download_checkV_DB.out }
+            download_checkV_DB()
             // cloud storage via db_preload.exists()
-            if (params.cloudProcess) {
-                db_preload = file("${params.databases}/checkV/checkv-db-v0.6", type: 'dir')
-                if (db_preload.exists()) { db = db_preload }
-                else  { download_checkV_DB(); db = download_checkV_DB.out } 
-            }
+
             // tool prediction
             checkV(fasta, download_checkV_DB.out)
             //checkV_collect = checkV.out.map {it -> tuple(it[0],it[2])}

diff --git a/workflows/metaphinder_own_DB_wf.nf b/workflows/metaphinder_own_DB_wf.nf
@@ -8,23 +8,11 @@ workflow metaphinder_own_DB_wf {
     take:   fasta
     main:   if (!params.mp) {
             // download references for blast db build
-            // local storage via storeDir
-            if (!params.cloudProcess) { download_references(); db = download_references.out }
-            // cloud storage via db_preload.exists()
-            if (params.cloudProcess) {
-            db_preload = file("${params.databases}/references/phage_references.fa")
-            if (db_preload.exists()) { db = db_preload }
-            else  { download_references(); db = download_references.out } 
-            }
+            download_references()
+
             // blast db build
-            // local storage via storeDir
-            if (!params.cloudProcess) { phage_references_blastDB(download_references.out); db = phage_references_blastDB.out }
-            // cloud storage via db_preload.exists()
-            if (params.cloudProcess) {
-            db_preload = file("${params.databases}/blast_DB_phage/blast_database.tar.gz", type: 'dir')
-            if (db_preload.exists()) { db = db_preload }
-            else  { phage_references_blastDB(references); db = phage_references_blastDB.out } 
-            }
+            phage_references_blastDB(download_references.out)
+
             // tool prediction
             metaphinder_own_DB(fasta, phage_references_blastDB.out)
             // filtering

diff --git a/workflows/phage_annotation_wf.nf b/workflows/phage_annotation_wf.nf
@@ -14,30 +14,16 @@ workflow phage_annotation_wf {
                                             .view()}
             // map input for prodigal
             fasta = fasta_and_tool_results.map {it -> tuple(it[0],it[1])}
-            //Databases 
-            //Database for hmmscan
-            // local storage via storeDir
-            if (!params.cloudProcess) { pvog_DB(); dbpvog = pvog_DB.out }
-            // cloud storage via db_preload.exists()
-            if (params.cloudProcess) {
-                db_preload = file("${params.databases}/pvogs/", type: 'dir')
-                if (db_preload.exists()) { dbpvog = db_preload }
-                else  { pvog_DB(); dbpvog = pvog_DB.out } 
-            }
-            //Vog table
-            // local storage via storeDir
-            if (!params.cloudProcess) { vogtable_DB(); db = vogtable_DB.out }
-            // cloud storage via db_preload.exists()
-            if (params.cloudProcess) {
-                db_preload = file("${params.databases}/vog_table/VOGTable.txt")
-                if (db_preload.exists()) { db = db_preload }
-                else  { vogtable_DB(); db = vogtable_DB.out } 
-            }
+
+            // Database for hmmscan
+            pvog_DB()
+            vogtable_DB()
+
             //annotation-process
             prodigal(fasta)
-            if (!params.annotation_db) {hmmscan(prodigal.out, dbpvog)}
+            if (!params.annotation_db) {hmmscan(prodigal.out, pvog_DB.out)}
             else {hmmscan(prodigal.out, annotation_custom_db_ch)}         
-            chromomap_parser(fasta.join(hmmscan.out), db)
+            chromomap_parser(fasta.join(hmmscan.out), vogtable_DB.out)
             chromomap(chromomap_parser.out[0].mix(chromomap_parser.out[1]))
 
             annotationtable_markdown_input = chromomap_parser.out.annotationfile_combined_ch

diff --git a/workflows/process/checkV/download_checkV_DB.nf b/workflows/process/checkV/download_checkV_DB.nf
@@ -1,8 +1,7 @@
 
 process download_checkV_DB {
     label 'noDocker'    
-    if (params.cloudProcess) { publishDir "${params.databases}/checkV", mode: 'copy' }
-    else { storeDir "${params.databases}/checkV" }  
+    storeDir "${params.databases}/checkV"
     output:
         path("checkv-db-v*", type: 'dir')
     script:

diff --git a/workflows/process/metaphinder/download_references.nf b/workflows/process/metaphinder/download_references.nf
@@ -1,7 +1,6 @@
 process download_references {
     label 'noDocker'
-    if (params.cloudProcess) { publishDir "${params.databases}/references/", mode: 'copy', pattern: "phage_references.fa" }
-    else { storeDir "${params.databases}/references/" }    
+    storeDir "${params.databases}/references/"
     output:
         path("phage_references.fa")
     script:

diff --git a/workflows/process/metaphinder/phage_references_blastDB.nf b/workflows/process/metaphinder/phage_references_blastDB.nf
@@ -1,6 +1,5 @@
 process phage_references_blastDB {
-    if (params.cloudProcess) { publishDir "${params.databases}/blast_DB_phage", mode: 'copy', pattern: "blast_database.tar.gz" }
-    else { storeDir "${params.databases}/blast_DB_phage" }
+    storeDir "${params.databases}/blast_DB_phage"
     label 'metaphinder'
     errorStrategy = "retry"
     maxRetries = 2

diff --git a/workflows/process/metaphinder_own_DB/download_references.nf b/workflows/process/metaphinder_own_DB/download_references.nf
@@ -2,8 +2,7 @@ process download_references {
     label 'noDocker'
     errorStrategy 'retry'
     maxRetries 1   
-    if (params.cloudProcess) { publishDir "${params.databases}/references/", mode: 'copy', pattern: "phage_references.fa" }
-    else { storeDir "${params.databases}/references/" }    
+    storeDir "${params.databases}/references/"
     output:
         path("phage_references.fa")
     script:

diff --git a/workflows/process/metaphinder_own_DB/phage_references_blastDB.nf b/workflows/process/metaphinder_own_DB/phage_references_blastDB.nf
@@ -1,6 +1,5 @@
 process phage_references_blastDB {
-    if (params.cloudProcess) { publishDir "${params.databases}/blast_DB_phage", mode: 'copy', pattern: "blast_database.tar.gz" }
-    else { storeDir "${params.databases}/blast_DB_phage" }
+    storeDir "${params.databases}/blast_DB_phage"
     label 'metaphinder'
     errorStrategy = "retry"
     maxRetries = 2

diff --git a/workflows/process/phage_annotation/download_pvog_DB.nf b/workflows/process/phage_annotation/download_pvog_DB.nf
@@ -2,8 +2,7 @@ process pvog_DB {
     label 'noDocker'
     errorStrategy 'retry'
     maxRetries 1    
-    if (params.cloudProcess) { publishDir "${params.databases}/pvogs/", mode: 'copy'}
-    else { storeDir "${params.databases}/pvog" }  
+    storeDir "${params.databases}/pvog"
     output:
         path("pvogs", type: 'dir')
     script:
@@ -32,8 +31,7 @@ process vogtable_DB {
     label 'noDocker' 
     //errorStrategy 'retry'
     //maxRetries 1   
-    if (params.cloudProcess) { publishDir "${params.databases}/vog_table", mode: 'copy'}
-    else { storeDir "${params.databases}/vog_table" }  
+    storeDir "${params.databases}/vog_table" 
     output:
         path("VOGTable.txt")
     script:

diff --git a/workflows/process/phage_annotation/download_vog_DB.nf b/workflows/process/phage_annotation/download_vog_DB.nf
@@ -1,7 +1,6 @@
 process vog_DB {
     label 'noDocker'    
-    if (params.cloudProcess) { publishDir "${params.databases}/vog", mode: 'copy', pattern: "vogdb" }
-    else { storeDir "${params.databases}/vog" }  
+    storeDir "${params.databases}/vog"
     output:
         path("vogdb", type: 'dir')
     script:

diff --git a/workflows/process/phage_tax_classification/download_references.nf b/workflows/process/phage_tax_classification/download_references.nf
@@ -2,8 +2,7 @@ process download_references {
     label 'noDocker'
     errorStrategy 'retry'
     maxRetries 1   
-    if (params.cloudProcess) { publishDir "${params.databases}/references/", mode: 'copy', pattern: "phage_references.fa" }
-    else { storeDir "${params.databases}/references/" }    
+    storeDir "${params.databases}/references/"
     output:
         path("phage_references.fa")
     script:

diff --git a/workflows/process/phage_tax_classification/sourmash_download_DB.nf b/workflows/process/phage_tax_classification/sourmash_download_DB.nf
@@ -2,8 +2,7 @@ process sourmash_download_DB {
     label 'sourmash' 
     errorStrategy 'retry'
     maxRetries 1
-    if (params.cloudProcess) { publishDir "${params.databases}/sourmash/", mode: 'copy', pattern: "phages.sbt.zip" }
-    else { storeDir "${params.databases}/sourmash/" }
+    storeDir "${params.databases}/sourmash/"
     input:
         path(references)
     output:

diff --git a/workflows/process/pprmeta/ppr_download_dependencies.nf b/workflows/process/pprmeta/ppr_download_dependencies.nf
@@ -2,8 +2,7 @@ process ppr_download_dependencies {
     errorStrategy 'retry'
     maxRetries 1
     label 'noDocker'
-    if (params.cloudProcess) { publishDir "${params.databases}/pprmeta/", mode: 'copy', pattern: "PPR-Meta" }
-    else { storeDir "${params.databases}/pprmeta/" }    
+    storeDir "${params.databases}/pprmeta/"   
     output:
         path("PPR-Meta", type: 'dir')
     script:

diff --git a/workflows/process/sourmash/download_references.nf b/workflows/process/sourmash/download_references.nf
@@ -2,8 +2,7 @@ process download_references {
     label 'noDocker'
     errorStrategy 'retry'
     maxRetries 1   
-    if (params.cloudProcess) { publishDir "${params.databases}/references/", mode: 'copy', pattern: "phage_references.fa" }
-    else { storeDir "${params.databases}/references/" }    
+    storeDir "${params.databases}/references/" 
     output:
         path("phage_references.fa")
     script:

diff --git a/workflows/process/sourmash/sourmash_download_DB.nf b/workflows/process/sourmash/sourmash_download_DB.nf
@@ -2,8 +2,7 @@ process sourmash_download_DB {
     label 'sourmash' 
     errorStrategy 'retry'
     maxRetries 1
-    if (params.cloudProcess) { publishDir "${params.databases}/sourmash/", mode: 'copy', pattern: "phages.sbt.zip" }
-    else { storeDir "${params.databases}/sourmash/" }
+    storeDir "${params.databases}/sourmash/"
     input:
         path(references)
     output:

diff --git a/workflows/process/vibrant/vibrant_download_DB.nf b/workflows/process/vibrant/vibrant_download_DB.nf
@@ -2,8 +2,7 @@ process vibrant_download_DB {
     errorStrategy 'retry'
     maxRetries 1
     label 'vibrant'
-    if (params.cloudProcess) { publishDir "${params.databases}/Vibrant/", mode: 'copy', pattern: "database.tar.gz" }
-    else { storeDir "${params.databases}/Vibrant" }    
+    storeDir "${params.databases}/Vibrant"   
     output:
         path("database.tar.gz")
     script:

diff --git a/workflows/process/vibrant_virome/vibrant_download_DB.nf b/workflows/process/vibrant_virome/vibrant_download_DB.nf
@@ -2,8 +2,7 @@ process vibrant_download_DB {
     errorStrategy 'retry'
     maxRetries 1
     label 'vibrant'
-    if (params.cloudProcess) { publishDir "${params.databases}/Vibrant/", mode: 'copy', pattern: "database.tar.gz" }
-    else { storeDir "${params.databases}/Vibrant" }    
+    storeDir "${params.databases}/Vibrant"   
     output:
         path("database.tar.gz")
     script:

diff --git a/workflows/process/virsorter/virsorter_download_DB.nf b/workflows/process/virsorter/virsorter_download_DB.nf
@@ -2,8 +2,7 @@ process virsorter_download_DB {
     errorStrategy 'retry'
     maxRetries 1
     label 'noDocker'    
-    if (params.cloudProcess) { publishDir "${params.databases}/virsorter/", mode: 'copy', pattern: "virsorter-data" }
-    else { storeDir "${params.databases}/virsorter/" }
+    storeDir "${params.databases}/virsorter/"
     output:
         path("virsorter-data", type: 'dir')
     script:

diff --git a/workflows/process/virsorter2/virsorter2_download_DB.nf b/workflows/process/virsorter2/virsorter2_download_DB.nf
@@ -1,7 +1,6 @@
 process virsorter2_download_DB {
     label 'noDocker'    
-    if (params.cloudProcess) { publishDir "${params.databases}/virsorter2-db", mode: 'copy' }
-    else { storeDir "${params.databases}/virsorter2-db" }  
+    storeDir "${params.databases}/virsorter2-db"
     output:
         path('db', type: 'dir')
     script:

diff --git a/workflows/process/virsorter_virome/virsorter_download_DB.nf b/workflows/process/virsorter_virome/virsorter_download_DB.nf
@@ -2,8 +2,7 @@ process virsorter_download_DB {
     errorStrategy 'retry'
     maxRetries 1
     label 'noDocker'    
-    if (params.cloudProcess) { publishDir "${params.databases}/virsorter/", mode: 'copy', pattern: "virsorter-data" }
-    else { storeDir "${params.databases}/virsorter/" }
+    storeDir "${params.databases}/virsorter/"
     output:
         path("virsorter-data", type: 'dir')
     script:

diff --git a/workflows/vibrant_virome_wf.nf b/workflows/vibrant_virome_wf.nf
@@ -7,13 +7,7 @@ workflow vibrant_virome_wf {
     take:   fasta
     main:   if (!params.vb && !params.virome) {
                     // local storage via storeDir
-                    if (!params.cloudProcess) { vibrant_download_DB(); db = vibrant_download_DB.out }
-                    //cloud storage via db_preload.exists()
-                    if (params.cloudProcess) {
-                    db_preload = file("${params.databases}/Vibrant/database.tar.gz")
-                    if (db_preload.exists()) { db = db_preload }
-                    else  { vibrant_download_DB(); db = vibrant_download_DB.out } 
-                    }
+                    vibrant_download_DB()
                     // tool prediction
                     vibrant_virome(fasta, vibrant_download_DB.out)
                     // filtering

diff --git a/workflows/vibrant_wf.nf b/workflows/vibrant_wf.nf
@@ -7,13 +7,7 @@ workflow vibrant_wf {
     take:   fasta       
     main:    if (!params.vb) {
                     // local storage via storeDir
-                    if (!params.cloudProcess) { vibrant_download_DB(); db = vibrant_download_DB.out }
-                    //cloud storage via db_preload.exists()
-                    if (params.cloudProcess) {
-                    db_preload = file("${params.databases}/Vibrant/database.tar.gz")
-                    if (db_preload.exists()) { db = db_preload }
-                    else  { vibrant_download_DB(); db = vibrant_download_DB.out } 
-                    }
+                    vibrant_download_DB()
                     // tool prediction
                     vibrant(fasta, vibrant_download_DB.out)
                     // filtering

diff --git a/workflows/virsorter2_wf.nf b/workflows/virsorter2_wf.nf
@@ -10,13 +10,7 @@ workflow virsorter2_wf {
     main:   if (!params.vs2) { 
 
                     // local storage via storeDir
-                    if (!params.cloudProcess) { virsorter2_download_DB(); db = virsorter2_download_DB.out }
-                    // cloud storage via db_preload.exists()
-                    if (params.cloudProcess) {
-                    db_preload = file("${params.databases}/virsorter2-db", type: 'dir')
-                    if (db_preload.exists()) { db = db_preload }
-                    else  { virsorter2_download_DB(); db = virsorter2_download_DB.out } 
-                    }
+                    virsorter2_download_DB()
                     // tool prediction
                     virsorter2(fasta, virsorter2_download_DB.out)
                     // filtering

diff --git a/workflows/virsorter_virome_wf.nf b/workflows/virsorter_virome_wf.nf
@@ -8,13 +8,7 @@ workflow virsorter_virome_wf {
     take:   fasta
     main:   if (!params.vs && !params.virome) {
                 // local storage via storeDir
-                if (!params.cloudProcess) { virsorter_download_DB(); db = virsorter_download_DB.out }
-                // cloud storage via db_preload.exists()
-                if (params.cloudProcess) {
-                db_preload = file("${params.databases}/virsorter/virsorter-data", type: 'dir')
-                if (db_preload.exists()) { db = db_preload }
-                else  { virsorter_download_DB(); db = virsorter_download_DB.out } 
-                }
+                virsorter_download_DB()
                 // tool prediction
                 virsorter_virome(fasta, virsorter_download_DB.out)
                 // filtering