add nextflow d30e48d
This commit is contained in:
119
nextflow/modules/nextflow/build.gradle
Normal file
119
nextflow/modules/nextflow/build.gradle
Normal file
@@ -0,0 +1,119 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
plugins {
|
||||
id "com.gradleup.shadow" version "9.3.1"
|
||||
}
|
||||
apply plugin: 'groovy'
|
||||
apply plugin: 'application'
|
||||
|
||||
sourceSets {
|
||||
main.java.srcDirs = []
|
||||
main.groovy.srcDirs = ['src/main/java', 'src/main/groovy']
|
||||
main.resources.srcDirs = ['src/main/resources']
|
||||
test.java.srcDirs = []
|
||||
test.groovy.srcDirs = ['src/test/groovy']
|
||||
test.resources.srcDirs = ['src/test/resources']
|
||||
}
|
||||
|
||||
compileGroovy {
|
||||
options.compilerArgs = ['-XDignore.symbol.file']
|
||||
}
|
||||
|
||||
configurations {
|
||||
lineageImplementation
|
||||
}
|
||||
|
||||
dependencies {
|
||||
api(project(':nf-commons'))
|
||||
api(project(':nf-httpfs'))
|
||||
api(project(':nf-lang'))
|
||||
api "org.apache.groovy:groovy:4.0.31"
|
||||
api "org.apache.groovy:groovy-nio:4.0.31"
|
||||
api "org.apache.groovy:groovy-xml:4.0.31"
|
||||
api "org.apache.groovy:groovy-json:4.0.31"
|
||||
api "org.apache.groovy:groovy-templates:4.0.31"
|
||||
api "org.apache.groovy:groovy-yaml:4.0.31"
|
||||
api "org.slf4j:jcl-over-slf4j:2.0.17"
|
||||
api "org.slf4j:jul-to-slf4j:2.0.17"
|
||||
api "org.slf4j:log4j-over-slf4j:2.0.17"
|
||||
api "ch.qos.logback:logback-classic:1.5.26"
|
||||
api "ch.qos.logback:logback-core:1.5.26"
|
||||
api "org.codehaus.gpars:gpars:1.2.1"
|
||||
api("ch.artecat.grengine:grengine:3.0.2") { exclude group: 'org.codehaus.groovy' }
|
||||
api "org.apache.commons:commons-lang3:3.18.0"
|
||||
api "commons-codec:commons-codec:1.15"
|
||||
api "commons-io:commons-io:2.15.1"
|
||||
api "com.beust:jcommander:1.35"
|
||||
api("com.esotericsoftware.kryo:kryo:2.24.0") { exclude group: 'com.esotericsoftware.minlog', module: 'minlog' }
|
||||
api('org.iq80.leveldb:leveldb:0.12')
|
||||
api('org.eclipse.jgit:org.eclipse.jgit:7.1.1.202505221757-r')
|
||||
api ('javax.activation:activation:1.1.1')
|
||||
api ('javax.mail:mail:1.4.7')
|
||||
api ('org.yaml:snakeyaml:2.2')
|
||||
api ('org.jsoup:jsoup:1.15.4')
|
||||
api 'jline:jline:2.9'
|
||||
api 'org.pf4j:pf4j:3.14.1'
|
||||
api 'dev.failsafe:failsafe:3.1.0'
|
||||
api 'io.seqera:lib-trace:0.1.0'
|
||||
api 'com.fasterxml.woodstox:woodstox-core:7.1.1'
|
||||
api 'org.apache.commons:commons-compress:1.27.1' // For tar.gz extraction
|
||||
api 'io.seqera:npr-api:0.22.0'
|
||||
api 'io.seqera:npr-client:0.22.0'
|
||||
|
||||
testImplementation 'org.subethamail:subethasmtp:3.1.7'
|
||||
testImplementation (project(':nf-lineage'))
|
||||
testImplementation 'org.wiremock:wiremock:3.13.1'
|
||||
// test configuration
|
||||
testFixturesApi ("org.apache.groovy:groovy-test:4.0.31") { exclude group: 'org.apache.groovy' }
|
||||
testFixturesApi ("org.objenesis:objenesis:3.4")
|
||||
testFixturesApi ("net.bytebuddy:byte-buddy:1.14.17")
|
||||
testFixturesApi ("org.spockframework:spock-core:2.4-groovy-4.0") { exclude group: 'org.apache.groovy' }
|
||||
testFixturesApi ('org.spockframework:spock-junit4:2.4-groovy-4.0') { exclude group: 'org.apache.groovy' }
|
||||
testFixturesApi 'com.google.jimfs:jimfs:1.2'
|
||||
// note: declare as separate dependency to avoid a circular dependency
|
||||
lineageImplementation (project(':nf-lineage'))
|
||||
}
|
||||
|
||||
|
||||
test {
|
||||
minHeapSize = "512m"
|
||||
maxHeapSize = "4096m"
|
||||
}
|
||||
|
||||
application {
|
||||
mainClass = 'nextflow.cli.Launcher'
|
||||
}
|
||||
|
||||
run{
|
||||
args( (project.hasProperty("runCmd") ? project.findProperty("runCmd") : "set a cmd to run").split(' ') )
|
||||
}
|
||||
|
||||
shadowJar {
|
||||
// add 'lineage' because it cannot be added to this project
|
||||
// explicitly otherwise it will result into a circular dependency
|
||||
configurations = [project.configurations.runtimeClasspath, project.configurations.lineageImplementation]
|
||||
archiveClassifier='one'
|
||||
manifest {
|
||||
attributes 'Main-Class': application.mainClass.get()
|
||||
}
|
||||
mergeServiceFiles()
|
||||
mergeGroovyExtensionModules()
|
||||
transform(com.github.jengelman.gradle.plugins.shadow.transformers.AppendingTransformer) {
|
||||
resource = 'META-INF/extensions.idx'
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,653 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow
|
||||
|
||||
import static nextflow.util.CheckHelper.*
|
||||
|
||||
import java.nio.file.FileSystem
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
import java.util.concurrent.CompletableFuture
|
||||
import java.util.regex.Pattern
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.PackageScope
|
||||
import groovy.util.logging.Slf4j
|
||||
import groovyx.gpars.dataflow.DataflowQueue
|
||||
import groovyx.gpars.dataflow.DataflowReadChannel
|
||||
import groovyx.gpars.dataflow.DataflowVariable
|
||||
import groovyx.gpars.dataflow.DataflowWriteChannel
|
||||
import groovyx.gpars.dataflow.operator.ControlMessage
|
||||
import nextflow.dag.NodeMarker
|
||||
import nextflow.datasource.SraExplorer
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.extension.CH
|
||||
import nextflow.extension.GroupTupleOp
|
||||
import nextflow.extension.LinExtension
|
||||
import nextflow.extension.MapOp
|
||||
import nextflow.file.DirListener
|
||||
import nextflow.file.DirWatcher
|
||||
import nextflow.file.DirWatcherV2
|
||||
import nextflow.file.FileHelper
|
||||
import nextflow.file.FilePatternSplitter
|
||||
import nextflow.file.PathVisitor
|
||||
import nextflow.plugin.Plugins
|
||||
import nextflow.plugin.extension.PluginExtensionProvider
|
||||
import nextflow.util.Duration
|
||||
import nextflow.util.TestOnly
|
||||
import org.codehaus.groovy.runtime.InvokerHelper
|
||||
import org.codehaus.groovy.runtime.NullObject
|
||||
/**
|
||||
* Channel factory object
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class Channel {
|
||||
|
||||
static public ControlMessage STOP = CH.stop()
|
||||
|
||||
static public NullObject VOID = NullObject.getNullObject()
|
||||
|
||||
@TestOnly
|
||||
private static CompletableFuture fromPath0Future
|
||||
|
||||
static private Session getSession() { Global.session as Session }
|
||||
|
||||
/**
|
||||
* Allow the dynamic loading of plugin provided channel extension methods
|
||||
*
|
||||
* @param name The name of the method
|
||||
* @param args The method arguments
|
||||
* @return The method return value
|
||||
*/
|
||||
static def $static_methodMissing(String name, Object args) {
|
||||
PluginExtensionProvider.INSTANCE().invokeFactoryExtensionMethod(name, InvokerHelper.asArray(args))
|
||||
}
|
||||
|
||||
static Object[] array0(Object value) {
|
||||
if( value instanceof Object[] )
|
||||
return (Object[])value
|
||||
if( value instanceof Collection )
|
||||
return value.toArray()
|
||||
else
|
||||
return new Object[] {value}
|
||||
}
|
||||
|
||||
static private checkNoChannels(String name, Object value) {
|
||||
if( value==null )
|
||||
return
|
||||
Object[] items = array0(value)
|
||||
if( items.size()==1 && CH.isChannel(items[0]))
|
||||
throw new IllegalArgumentException("Argument of '$name' method cannot be a channel object — Likely you can replace the use of '$name' with the channel object itself")
|
||||
for( int i=0; i<items.size(); i++ ) {
|
||||
if( CH.isChannel(items[i]) )
|
||||
throw new IllegalArgumentException("Argument ${nth(i+1)} of method '$name' cannot be a channel object")
|
||||
}
|
||||
}
|
||||
|
||||
static private String nth(int i) {
|
||||
if( i==1 ) return '1-st'
|
||||
if( i==2 ) return '2-nd'
|
||||
if( i==3 ) return '3-rd'
|
||||
return "$i-th"
|
||||
}
|
||||
|
||||
static DataflowWriteChannel topic(String name) {
|
||||
return CH.topic(name)
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a empty channel i.e. only emits a STOP signal
|
||||
*
|
||||
* @return The channel instance
|
||||
*/
|
||||
static DataflowWriteChannel empty() {
|
||||
final result = CH.emit(CH.queue(), STOP)
|
||||
NodeMarker.addSourceNode('channel.empty', result)
|
||||
return result
|
||||
}
|
||||
|
||||
static DataflowWriteChannel of(Object ... items) {
|
||||
checkNoChannels('channel.of', items)
|
||||
final result = CH.create()
|
||||
final values = new ArrayList()
|
||||
if( items == null ) {
|
||||
values.add(null)
|
||||
}
|
||||
else {
|
||||
for( int i=0; i<items.size(); i++ )
|
||||
addToList0(values, items[i])
|
||||
}
|
||||
values.add(STOP)
|
||||
CH.emitValues(result, values)
|
||||
NodeMarker.addSourceNode('channel.of', result)
|
||||
return result
|
||||
}
|
||||
|
||||
static private void addToList0(List list, obj) {
|
||||
if( obj instanceof Range ) {
|
||||
for( def x : obj ) {
|
||||
list.add(x)
|
||||
}
|
||||
}
|
||||
else {
|
||||
list.add(obj)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a channel sending the items in the collection over it
|
||||
*
|
||||
* @param items
|
||||
* @return
|
||||
*/
|
||||
@Deprecated
|
||||
static DataflowWriteChannel from( Collection items ) {
|
||||
final result = from0(items)
|
||||
NodeMarker.addSourceNode('channel.from', result)
|
||||
return result
|
||||
}
|
||||
|
||||
static private DataflowWriteChannel from0( Collection items ) {
|
||||
final result = CH.create()
|
||||
if( items != null )
|
||||
CH.emitAndClose(result, items)
|
||||
return result
|
||||
}
|
||||
|
||||
static DataflowWriteChannel fromList( Collection items ) {
|
||||
final result = CH.create()
|
||||
CH.emitAndClose(result, items as List)
|
||||
NodeMarker.addSourceNode('channel.fromList', result)
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a channel sending the items in the collection over it
|
||||
*
|
||||
* @param items
|
||||
* @return
|
||||
*/
|
||||
@Deprecated
|
||||
static DataflowWriteChannel from( Object... items ) {
|
||||
checkNoChannels('channel.from', items)
|
||||
for( Object it : items ) if(CH.isChannel(it))
|
||||
throw new IllegalArgumentException("channel.from argument is already a channel object")
|
||||
|
||||
final result = from0(items as List)
|
||||
NodeMarker.addSourceNode('channel.from', result)
|
||||
return result
|
||||
}
|
||||
|
||||
static DataflowVariable value( obj = null ) {
|
||||
checkNoChannels('channel.value', obj)
|
||||
obj != null ? CH.value(obj) : CH.value()
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a channel emitting a sequence of integers spaced by a given time interval
|
||||
*
|
||||
* @param duration
|
||||
* @return
|
||||
*/
|
||||
static DataflowWriteChannel interval(String duration) {
|
||||
final result = interval0( duration, { index -> index })
|
||||
NodeMarker.addSourceNode('channel.interval', result)
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a channel emitting a sequence of value given by the closure and spaced by a given time interval.
|
||||
*
|
||||
* To stop the interval return the special value {@code #STOP}
|
||||
*
|
||||
* @param duration
|
||||
* @return
|
||||
*/
|
||||
|
||||
static DataflowWriteChannel interval(String duration, Closure closure ) {
|
||||
final result = interval0(duration, closure)
|
||||
NodeMarker.addSourceNode('channel.interval', result)
|
||||
return result
|
||||
}
|
||||
|
||||
static private DataflowWriteChannel interval0(String duration, Closure closure) {
|
||||
def millis = Duration.of(duration).toMillis()
|
||||
def timer = new Timer()
|
||||
def result = CH.create()
|
||||
long index = 0
|
||||
|
||||
def task = {
|
||||
def value = closure.call(index++)
|
||||
result << value
|
||||
if( value == STOP ) {
|
||||
timer.cancel()
|
||||
}
|
||||
}
|
||||
|
||||
session.addIgniter {
|
||||
timer.schedule( task as TimerTask, 0, millis )
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
/*
|
||||
* valid parameters for fromPath operator
|
||||
*/
|
||||
static private Map VALID_FROM_PATH_PARAMS = [
|
||||
type:['file','dir','any'],
|
||||
followLinks: Boolean,
|
||||
hidden: Boolean,
|
||||
maxDepth: Integer,
|
||||
checkIfExists: Boolean,
|
||||
glob: Boolean,
|
||||
relative: Boolean
|
||||
]
|
||||
|
||||
/**
|
||||
* Implements {@code Channel.fromPath} factory method
|
||||
*
|
||||
* @param opts
|
||||
* A map object holding the optional method parameters
|
||||
* @param pattern
|
||||
* A file path or a glob pattern matching the required files.
|
||||
* Multiple paths or patterns can be using a list object
|
||||
* @return
|
||||
* A channel emitting the matching files
|
||||
*/
|
||||
static DataflowWriteChannel<Path> fromPath( Map opts = null, pattern ) {
|
||||
if( !pattern ) throw new AbortOperationException("Missing `fromPath` parameter")
|
||||
checkNoChannels('channel.fromPath', pattern)
|
||||
|
||||
// verify that the 'type' parameter has a valid value
|
||||
checkParams( 'path', opts, VALID_FROM_PATH_PARAMS )
|
||||
|
||||
final result = fromPath0(opts, pattern instanceof List ? pattern : [pattern])
|
||||
NodeMarker.addSourceNode('channel.fromPath', result)
|
||||
return result
|
||||
}
|
||||
|
||||
private static DataflowWriteChannel<Path> fromPath0( Map opts, List allPatterns ) {
|
||||
|
||||
final result = CH.create()
|
||||
session.addIgniter {
|
||||
pumpFiles0(result, opts, allPatterns)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
private static void pumpFiles0(DataflowWriteChannel result, Map opts, List allPatterns) {
|
||||
|
||||
def future = CompletableFuture.completedFuture(null)
|
||||
for( int index=0; index<allPatterns.size(); index++ ) {
|
||||
def factory = new PathVisitor(target: result, opts: opts)
|
||||
factory.closeChannelOnComplete = index==allPatterns.size()-1
|
||||
future = factory.applyAsync(future, allPatterns[index])
|
||||
}
|
||||
|
||||
// abort the execution when an exception is raised
|
||||
fromPath0Future = future.exceptionally(Channel.&handleException)
|
||||
}
|
||||
|
||||
static private void handleException(Throwable e) {
|
||||
final error = e.cause ?: e
|
||||
log.error(error.message, error)
|
||||
session?.abort(error)
|
||||
}
|
||||
|
||||
static private DataflowWriteChannel watchImpl( String syntax, String folder, String pattern, boolean skipHidden, String events, FileSystem fs ) {
|
||||
|
||||
final result = CH.create()
|
||||
final legacy = System.getenv('NXF_DIRWATCHER_LEGACY')
|
||||
final DirListener watcher = legacy=='true'
|
||||
? new DirWatcher(syntax,folder,pattern,skipHidden,events,fs)
|
||||
: new DirWatcherV2(syntax,folder,pattern,skipHidden,events,fs)
|
||||
watcher.onComplete { result.bind(STOP) }
|
||||
Global.onCleanup(it -> watcher.terminate())
|
||||
|
||||
session.addIgniter {
|
||||
watcher.apply { Path file -> result.bind(file.toAbsolutePath()) }
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Watch the a folder for the specified events emitting the files that matches
|
||||
* the specified regular expression.
|
||||
*
|
||||
*
|
||||
* @param filePattern
|
||||
* The file pattern to match e.g. /*.fasta/
|
||||
*
|
||||
* @param events
|
||||
* The events to watch, a comma separated string of the following values:
|
||||
* {@code create}, {@code modify}, {@code delete}
|
||||
*
|
||||
* @return A dataflow channel that will emit the matching files
|
||||
*
|
||||
*/
|
||||
static DataflowWriteChannel watchPath( Pattern filePattern, String events = 'create' ) {
|
||||
assert filePattern
|
||||
// split the folder and the pattern
|
||||
final splitter = FilePatternSplitter.regex().parse(filePattern.toString())
|
||||
def fs = FileHelper.fileSystemForScheme(splitter.scheme)
|
||||
def result = watchImpl( 'regex', splitter.parent, splitter.fileName, false, events, fs )
|
||||
|
||||
NodeMarker.addSourceNode('channel.watchPath', result)
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Watch the a folder for the specified events emitting the files that matches
|
||||
* the specified {@code glob} pattern.
|
||||
*
|
||||
* @link http://docs.oracle.com/javase/tutorial/essential/io/fileOps.html#glob
|
||||
*
|
||||
* @param filePattern
|
||||
* The file pattern to match e.g. /some/path/*.fasta
|
||||
*
|
||||
* @param events
|
||||
* The events to watch, a comma separated string of the following values:
|
||||
* {@code create}, {@code modify}, {@code delete}
|
||||
*
|
||||
* @return A dataflow channel that will emit the matching files
|
||||
*
|
||||
*/
|
||||
static DataflowWriteChannel watchPath( String filePattern, String events = 'create' ) {
|
||||
|
||||
if( filePattern.endsWith('/') )
|
||||
filePattern += '*'
|
||||
else if(Files.isDirectory(Paths.get(filePattern)))
|
||||
filePattern += '/*'
|
||||
|
||||
final splitter = FilePatternSplitter.glob().parse(filePattern)
|
||||
final fs = FileHelper.fileSystemForScheme(splitter.scheme)
|
||||
final folder = splitter.parent
|
||||
final pattern = splitter.fileName
|
||||
def result = watchImpl('glob', folder, pattern, pattern.startsWith('*'), events, fs)
|
||||
|
||||
NodeMarker.addSourceNode('channel.watchPath', result)
|
||||
return result
|
||||
}
|
||||
|
||||
static DataflowWriteChannel watchPath( Path path, String events = 'create' ) {
|
||||
final fs = path.getFileSystem()
|
||||
final splitter = FilePatternSplitter.glob().parse(path.toString())
|
||||
final folder = splitter.parent
|
||||
final pattern = splitter.fileName
|
||||
final result = watchImpl('glob', folder, pattern, pattern.startsWith('*'), events, fs)
|
||||
|
||||
NodeMarker.addSourceNode('channel.watchPath', result)
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Implements the `fromFilePairs` channel factory method
|
||||
*
|
||||
* @param options
|
||||
* A {@link Map} holding the optional parameters
|
||||
* - type: either `file`, `dir` or `any`
|
||||
* - followLinks: Boolean
|
||||
* - hidden: Boolean
|
||||
* - maxDepth: Integer
|
||||
* - glob: Boolean
|
||||
* - relative: Boolean
|
||||
* @param pattern
|
||||
* One or more path patterns eg. `/some/path/*_{1,2}.fastq`
|
||||
* @return
|
||||
* A channel emitting the file pairs matching the specified pattern(s)
|
||||
*/
|
||||
static DataflowWriteChannel fromFilePairs(Map options = null, pattern) {
|
||||
checkNoChannels('channel.fromFilePairs', pattern)
|
||||
final allPatterns = pattern instanceof List ? pattern : [pattern]
|
||||
final allGrouping = new ArrayList(allPatterns.size())
|
||||
for( int i=0; i<allPatterns.size(); i++ ) {
|
||||
final template = allPatterns[i]
|
||||
allGrouping[i] = { Path path -> readPrefix(path,template) }
|
||||
}
|
||||
|
||||
final result = fromFilePairs0(options, allPatterns, allGrouping)
|
||||
NodeMarker.addSourceNode('channel.fromFilePairs', result)
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Implements the `fromFilePairs` channel factory method
|
||||
*
|
||||
* @param options
|
||||
* A {@link Map} holding the optional parameters
|
||||
* - type: either `file`, `dir` or `any`
|
||||
* - followLinks: Boolean
|
||||
* - hidden: Boolean
|
||||
* - maxDepth: Integer
|
||||
* - glob: Boolean
|
||||
* - relative: Boolean
|
||||
* @param pattern
|
||||
* One or more path patterns eg. `/some/path/*_{1,2}.fastq`
|
||||
* @param grouping
|
||||
* A closure implementing a pair grouping rule for the specified
|
||||
* file patterns
|
||||
* @return
|
||||
* A channel emitting the file pairs matching the specified pattern(s)
|
||||
*/
|
||||
static DataflowWriteChannel fromFilePairs(Map options = null, pattern, Closure grouping) {
|
||||
checkNoChannels('channel.fromFilePairs', pattern)
|
||||
final allPatterns = pattern instanceof List ? pattern : [pattern]
|
||||
final allGrouping = new ArrayList(allPatterns.size())
|
||||
for( int i=0; i<allPatterns.size(); i++ ) {
|
||||
allGrouping[i] = grouping
|
||||
}
|
||||
|
||||
final result = fromFilePairs0(options, allPatterns, allGrouping)
|
||||
NodeMarker.addSourceNode('channel.fromFilePairs', result)
|
||||
return result
|
||||
}
|
||||
|
||||
private static DataflowWriteChannel fromFilePairs0(Map options, List allPatterns, List<Closure> grouping) {
|
||||
assert allPatterns.size() == grouping.size()
|
||||
if( !allPatterns ) throw new AbortOperationException("Missing `fromFilePairs` parameter")
|
||||
if( !grouping ) throw new AbortOperationException("Missing `fromFilePairs` grouping parameter")
|
||||
|
||||
// -- a channel from the path
|
||||
final fromOpts = fetchParams0(VALID_FROM_PATH_PARAMS, options)
|
||||
final files = new DataflowQueue()
|
||||
session.addIgniter {
|
||||
pumpFilePairs0(files, fromOpts, allPatterns)
|
||||
}
|
||||
|
||||
// -- map the files to a tuple like ( ID, filePath )
|
||||
final mapper = { path, int index ->
|
||||
def prefix = grouping[index].call(path)
|
||||
return [ prefix, path ]
|
||||
}
|
||||
final mapChannel = (DataflowReadChannel)new MapOp(files, mapper)
|
||||
.setTarget(new DataflowQueue())
|
||||
.apply()
|
||||
|
||||
boolean anyPattern=false
|
||||
for( int index=0; index<allPatterns.size(); index++ ) {
|
||||
anyPattern |= FilePatternSplitter.isMatchingPattern(allPatterns.get(index))
|
||||
}
|
||||
|
||||
// -- result the files having the same ID
|
||||
def DEF_SIZE = anyPattern ? 2 : 1
|
||||
def size = (options?.size ?: DEF_SIZE)
|
||||
def isFlat = options?.flat == true
|
||||
def groupOpts = [sort: true, size: size]
|
||||
def groupChannel = isFlat ? new DataflowQueue<>() : CH.create()
|
||||
|
||||
new GroupTupleOp(groupOpts, mapChannel)
|
||||
.setTarget(groupChannel)
|
||||
.apply()
|
||||
|
||||
// -- flat the group resulting tuples
|
||||
DataflowWriteChannel result
|
||||
if( isFlat ) {
|
||||
def makeFlat = { id, List items ->
|
||||
def tuple = new ArrayList(items.size()+1);
|
||||
tuple.add(id)
|
||||
tuple.addAll(items)
|
||||
return tuple
|
||||
}
|
||||
result = new MapOp((DataflowReadChannel)groupChannel,makeFlat).apply()
|
||||
}
|
||||
else {
|
||||
result = groupChannel
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
static private void pumpFilePairs0(DataflowWriteChannel files, Map fromOpts, List allPatterns) {
|
||||
def future = CompletableFuture.completedFuture(null)
|
||||
for( int index=0; index<allPatterns.size(); index++ ) {
|
||||
def factory = new PathVisitor(opts: fromOpts, target: files)
|
||||
factory.bindPayload = index
|
||||
factory.closeChannelOnComplete = index == allPatterns.size()-1
|
||||
future = factory.applyAsync( future, allPatterns.get(index) )
|
||||
}
|
||||
// abort the execution when an exception is raised
|
||||
fromPath0Future = future.exceptionally(Channel.&handleException)
|
||||
}
|
||||
|
||||
static private Map fetchParams0(Map valid, Map actual ) {
|
||||
if( actual==null ) return null
|
||||
def result = [:]
|
||||
def keys = valid.keySet()
|
||||
keys.each {
|
||||
if( actual.containsKey(it) ) result.put(it, actual.get(it))
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
/*
|
||||
* Helper function, given a file Path
|
||||
* returns the file name region matching a specified glob pattern
|
||||
* starting from the beginning of the name up to last matching group.
|
||||
*
|
||||
* For example:
|
||||
* readPrefix('/some/data/file_alpha_1.fa', 'file*_1.fa' )
|
||||
*
|
||||
* Returns:
|
||||
* 'file_alpha'
|
||||
*/
|
||||
@PackageScope
|
||||
static String readPrefix( Path actual, template ) {
|
||||
|
||||
final fileName = actual.getFileName().toString()
|
||||
|
||||
def filePattern = template.toString()
|
||||
int p = filePattern.lastIndexOf('/')
|
||||
if( p != -1 )
|
||||
filePattern = filePattern.substring(p+1)
|
||||
|
||||
final indexOfWildcards = filePattern.findIndexOf { it=='*' || it=='?' }
|
||||
final indexOfBrackets = filePattern.findIndexOf { it=='{' || it=='[' }
|
||||
if( indexOfWildcards==-1 && indexOfBrackets==-1 ) {
|
||||
// when the pattern does not contain any glob characters
|
||||
// then it can only have been used only to filter files with exactly
|
||||
// that name, therefore if the name is matching return it (without suffix)
|
||||
if( fileName == filePattern )
|
||||
return actual.getSimpleName()
|
||||
throw new IllegalArgumentException("Not a valid file pair globbing pattern: pattern=$filePattern file=$fileName")
|
||||
}
|
||||
|
||||
// count the `*` and `?` wildcard before any {} and [] glob pattern
|
||||
int groupCount = 0
|
||||
for( int i=0; i<filePattern.size(); i++ ) {
|
||||
def ch = filePattern[i]
|
||||
if( ch=='?' || ch=='*' )
|
||||
groupCount++
|
||||
else if( ch=='{' || ch=='[' )
|
||||
break
|
||||
}
|
||||
|
||||
def regex = filePattern
|
||||
.replace('.','\\.')
|
||||
.replace('*','(.*)')
|
||||
.replace('?','(.?)')
|
||||
.replace('{','(?:')
|
||||
.replace('}',')')
|
||||
.replace(',','|')
|
||||
|
||||
def matcher = (fileName =~ /$regex/)
|
||||
if( matcher.matches() ) {
|
||||
def c=Math.min(groupCount, matcher.groupCount())
|
||||
def end = c ? matcher.end(c) : ( indexOfBrackets != -1 ? indexOfBrackets : fileName.size() )
|
||||
def prefix = fileName.substring(0,end)
|
||||
while(prefix.endsWith('-') || prefix.endsWith('_') || prefix.endsWith('.') )
|
||||
prefix=prefix[0..-2]
|
||||
|
||||
return prefix
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
static DataflowWriteChannel fromSRA(query) {
|
||||
checkNoChannels('channel.fromSRA', query)
|
||||
fromSRA( Collections.emptyMap(), query )
|
||||
}
|
||||
|
||||
static DataflowWriteChannel fromSRA(Map opts, query) {
|
||||
checkParams('fromSRA', opts, SraExplorer.PARAMS)
|
||||
checkNoChannels('channel.fromSRA', query)
|
||||
|
||||
def target = new DataflowQueue()
|
||||
def explorer = new SraExplorer(target, opts).setQuery(query)
|
||||
session.addIgniter {
|
||||
fetchSraFiles0(explorer)
|
||||
}
|
||||
|
||||
NodeMarker.addSourceNode('channel.fromSRA', target)
|
||||
return target
|
||||
}
|
||||
|
||||
static private void fetchSraFiles0(SraExplorer explorer) {
|
||||
def future = CompletableFuture.runAsync(() -> {
|
||||
explorer.apply()
|
||||
})
|
||||
fromPath0Future = future.exceptionally(Channel.&handleException)
|
||||
}
|
||||
|
||||
static DataflowWriteChannel fromLineage(Map<String,?> params) {
|
||||
checkParams('fromLineage', params, LinExtension.PARAMS)
|
||||
final result = CH.create()
|
||||
session.addIgniter {
|
||||
fromLineage0(result, params)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
private static void fromLineage0(DataflowWriteChannel channel, Map<String,?> params) {
|
||||
final linExt = Plugins.getExtension(LinExtension)
|
||||
if( !linExt )
|
||||
throw new IllegalStateException("Unable to load lineage extensions.")
|
||||
final session = Global.session as Session
|
||||
final future = CompletableFuture.runAsync(() -> {
|
||||
linExt.fromLineage(session, channel, params)
|
||||
})
|
||||
future.exceptionally(Channel.&handleException)
|
||||
}
|
||||
}
|
||||
68
nextflow/modules/nextflow/src/main/groovy/nextflow/NF.groovy
Normal file
68
nextflow/modules/nextflow/src/main/groovy/nextflow/NF.groovy
Normal file
@@ -0,0 +1,68 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow
|
||||
|
||||
import groovy.runtime.metaclass.NextflowDelegatingMetaClass
|
||||
import nextflow.extension.CH
|
||||
import nextflow.plugin.extension.PluginExtensionProvider
|
||||
import nextflow.script.ExecutionStack
|
||||
import nextflow.script.WorkflowBinding
|
||||
/**
|
||||
* Helper class to shortcut common api
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class NF {
|
||||
|
||||
static String getSyntaxParserVersion() {
|
||||
return SysEnv.get('NXF_SYNTAX_PARSER', 'v2')
|
||||
}
|
||||
|
||||
static boolean isSyntaxParserV2() {
|
||||
return getSyntaxParserVersion() == 'v2'
|
||||
}
|
||||
|
||||
static void init() {
|
||||
NextflowDelegatingMetaClass.provider = PluginExtensionProvider.INSTANCE()
|
||||
CH.init()
|
||||
WorkflowBinding.init()
|
||||
}
|
||||
|
||||
static boolean hasOperator(String name) {
|
||||
NextflowDelegatingMetaClass.provider.operatorNames().contains(name)
|
||||
}
|
||||
|
||||
static Binding getBinding() {
|
||||
ExecutionStack.binding()
|
||||
}
|
||||
|
||||
static String lookupVariable(value) {
|
||||
return WorkflowBinding.lookup(value)
|
||||
}
|
||||
|
||||
static boolean isStrictMode() {
|
||||
NextflowMeta.instance.isStrictModeEnabled()
|
||||
}
|
||||
|
||||
static boolean isModuleBinariesEnabled() {
|
||||
NextflowMeta.instance.isModuleBinariesEnabled()
|
||||
}
|
||||
|
||||
static boolean isRecurseEnabled() {
|
||||
NextflowMeta.instance.preview.recursion
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,433 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow
|
||||
|
||||
import static nextflow.file.FileHelper.*
|
||||
|
||||
import java.nio.file.FileSystem
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.NoSuchFileException
|
||||
import java.nio.file.Path
|
||||
|
||||
import groovyx.gpars.dataflow.DataflowReadChannel
|
||||
import nextflow.exception.StopSplitIterationException
|
||||
import nextflow.exception.WorkflowScriptErrorException
|
||||
import nextflow.extension.GroupKey
|
||||
import nextflow.extension.OperatorImpl
|
||||
import nextflow.file.FileHelper
|
||||
import nextflow.file.FilePatternSplitter
|
||||
import nextflow.mail.Mailer
|
||||
import nextflow.script.TokenBranchDef
|
||||
import nextflow.script.TokenMultiMapDef
|
||||
import nextflow.splitter.FastaSplitter
|
||||
import nextflow.splitter.FastqSplitter
|
||||
import nextflow.util.ArrayTuple
|
||||
import nextflow.util.CacheHelper
|
||||
import nextflow.util.RecordMap
|
||||
import org.slf4j.Logger
|
||||
import org.slf4j.LoggerFactory
|
||||
/**
|
||||
* Defines the main methods imported by default in the script scope
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class Nextflow {
|
||||
|
||||
static private Session getSession() { Global.session as Session }
|
||||
|
||||
// note: groovy `Slf4j` annotation causes a bizarre issue
|
||||
// https://issues.apache.org/jira/browse/GROOVY-7371
|
||||
// declare public so that can be accessed from the user script
|
||||
public static final Logger log = LoggerFactory.getLogger(Nextflow)
|
||||
|
||||
private static final Random random = new Random()
|
||||
|
||||
/**
|
||||
* Get the value of an environment variable from the launch environment.
|
||||
*
|
||||
* @param name
|
||||
* The environment variable name to be referenced
|
||||
* @return
|
||||
* The value associate with the specified variable name or {@code null} if the variable does not exist.
|
||||
*/
|
||||
static String env(String name) {
|
||||
return SysEnv.get(name)
|
||||
}
|
||||
|
||||
private static List<Path> fileNamePattern( FilePatternSplitter splitter, Map opts ) {
|
||||
final scheme = splitter.scheme
|
||||
final target = scheme ? "$scheme://$splitter.parent" : splitter.parent
|
||||
final folder = toCanonicalPath(target)
|
||||
final pattern = splitter.fileName
|
||||
|
||||
if( opts == null ) opts = [:]
|
||||
if( !opts.type ) opts.type = 'file'
|
||||
|
||||
def result = new LinkedList<Path>()
|
||||
try {
|
||||
FileHelper.visitFiles(opts, folder, pattern) { Path it -> result.add(it) }
|
||||
}
|
||||
catch (NoSuchFileException e) {
|
||||
log.debug "No such file or directory: $folder -- Skipping visit"
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
static private String str0(value) {
|
||||
if( value==null )
|
||||
return null
|
||||
if( value instanceof CharSequence )
|
||||
return value.toString()
|
||||
if( value instanceof File )
|
||||
return value.toString()
|
||||
if( value instanceof Path )
|
||||
return value.toUriString()
|
||||
throw new IllegalArgumentException("Invalid file path type - offending value: $value [${value.getClass().getName()}]")
|
||||
}
|
||||
|
||||
/**
|
||||
* Get one or more file object given the specified path or glob pattern.
|
||||
*
|
||||
* @param options
|
||||
* A map holding one or more of the following options:
|
||||
* - type: Type of paths returned, either `file`, `dir` or `any` (default: `file`)
|
||||
* - hidden: When `true` includes hidden files in the resulting paths (default: `false`)
|
||||
* - maxDepth: Maximum number of directory levels to visit (default: no limit)
|
||||
* - followLinks: When `true` it follows symbolic links during directories tree traversal, otherwise they are managed as files (default: `true`)
|
||||
*
|
||||
* @param path A file path eventually including a glob pattern e.g. /some/path/file*.txt
|
||||
* @return An instance of {@link Path} when a single file is matched or a list of {@link Path}s
|
||||
*/
|
||||
static file(Map options = null, def filePattern) {
|
||||
if( !filePattern )
|
||||
throw new IllegalArgumentException("Argument of `file()` function cannot be ${filePattern==null?'null':'empty'}")
|
||||
final result = file0(options, filePattern)
|
||||
if( result instanceof Collection && result.size() != 1 && NF.isSyntaxParserV2() )
|
||||
log.warn "The `file()` function was called with a glob pattern that matched a collection of files -- use `files()` instead."
|
||||
return result
|
||||
}
|
||||
|
||||
private static file0( Map options = null, def filePattern ) {
|
||||
final path = filePattern as Path
|
||||
final glob = options?.containsKey('glob') ? options.glob as boolean : isGlobAllowed(path)
|
||||
if( !glob ) {
|
||||
return checkIfExists(path, options)
|
||||
}
|
||||
|
||||
// if it isn't a glob pattern simply return it a normalized absolute Path object
|
||||
final strPattern = str0(filePattern)
|
||||
final splitter = FilePatternSplitter.glob().parse(strPattern)
|
||||
if( !splitter.isPattern() ) {
|
||||
final normalised = splitter.strip(strPattern)
|
||||
return checkIfExists(asPath(normalised), options)
|
||||
}
|
||||
|
||||
// revolve the glob pattern returning all matches
|
||||
final result = fileNamePattern(splitter, options)
|
||||
if( glob && options?.checkIfExists && result.isEmpty() )
|
||||
throw new NoSuchFileException(path.toString())
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
static Collection<Path> files(Map options=null, def filePattern) {
|
||||
if( !filePattern )
|
||||
throw new IllegalArgumentException("Argument of `files()` function cannot be ${filePattern==null?'null':'empty'}")
|
||||
final result = file0(options, filePattern)
|
||||
return result instanceof Collection ? result : [result]
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a {@link RecordMap} from the given named arguments.
|
||||
*
|
||||
* @param props
|
||||
*/
|
||||
static RecordMap record(Map<String,?> props) {
|
||||
return new RecordMap(props)
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a {@link ArrayTuple} object with the given open array items
|
||||
*
|
||||
* @param args The items used to created the tuple object
|
||||
* @return An instance of {@link ArrayTuple} populated with the given argument(s)
|
||||
*/
|
||||
static ArrayTuple tuple( def value ) {
|
||||
if( !value )
|
||||
return new ArrayTuple()
|
||||
|
||||
new ArrayTuple( value instanceof Collection ? (Collection)value : [value] )
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a {@link ArrayTuple} object with the given open array items
|
||||
*
|
||||
* @param args The items used to created the tuple object
|
||||
* @return An instance of {@link ArrayTuple} populated with the given argument(s)
|
||||
*/
|
||||
static ArrayTuple tuple( Object ... args ) {
|
||||
tuple( args as List )
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a FASTQ splitter handler for the given object
|
||||
*
|
||||
* @param obj The object to be managed as a FASTQ
|
||||
* @return An instance of {@link FastqSplitter
|
||||
*/
|
||||
@Deprecated
|
||||
static FastqSplitter fastq( obj ) {
|
||||
(FastqSplitter)new FastqSplitter('fastq').target(obj)
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a FASTA splitter handler for the given object
|
||||
*
|
||||
* @param obj The object to be managed as a FASTA
|
||||
* @return An instance of {@link FastqSplitter
|
||||
*/
|
||||
@Deprecated
|
||||
static FastaSplitter fasta( obj ) {
|
||||
(FastaSplitter)new FastaSplitter('fasta').target(obj)
|
||||
}
|
||||
|
||||
/**
|
||||
* Interrupts the iteration when using a split operators
|
||||
*/
|
||||
static void stop() {
|
||||
throw new StopSplitIterationException()
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Stop the current execution returning an error code and message
|
||||
*
|
||||
* @param exitCode The exit code to be returned
|
||||
* @param message The message that will be reported in the log file (optional)
|
||||
*/
|
||||
@Deprecated
|
||||
static void exit(int exitCode, String message = null) {
|
||||
if( session.aborted ) {
|
||||
log.debug "Ignoring exit because execution is already aborted -- message=$message"
|
||||
return
|
||||
}
|
||||
|
||||
if ( exitCode && message ) {
|
||||
log.error message
|
||||
}
|
||||
else if ( message ) {
|
||||
log.info message
|
||||
}
|
||||
System.exit(exitCode)
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the current execution returning a 0 error code and the specified message
|
||||
*
|
||||
* @param message The message that will be reported in the log file
|
||||
*/
|
||||
@Deprecated
|
||||
static void exit( String message ) {
|
||||
exit(0, message)
|
||||
}
|
||||
|
||||
/**
|
||||
* Throws a script runtime error
|
||||
* @param message An optional error message
|
||||
*/
|
||||
static void error( String message = null ) {
|
||||
throw message ? new WorkflowScriptErrorException(message) : new WorkflowScriptErrorException()
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a folder for the given key. It guarantees to return the same folder name
|
||||
* the same provided object key.
|
||||
*
|
||||
* @param key An object to be used as cache-key creating the folder, it can be any object
|
||||
* or an array or objects to use multi-objects key
|
||||
*
|
||||
* @return The {@code Path} to the cached directory or a newly created folder for the specified key
|
||||
*/
|
||||
@Deprecated
|
||||
static Path cacheableDir( Object key ) {
|
||||
assert key, "Please specify the 'key' argument on 'cacheableDir' method"
|
||||
|
||||
final session = (Session)Global.session
|
||||
if( !session )
|
||||
throw new IllegalStateException("Invalid access to `cacheableDir` method -- Session object not yet defined")
|
||||
|
||||
def hash = CacheHelper.hasher([ session.uniqueId, key, session.cacheable ? 0 : random.nextInt() ]).hash()
|
||||
|
||||
def file = FileHelper.getWorkFolder(session.workDir, hash)
|
||||
if( !file.exists() && !file.mkdirs() ) {
|
||||
throw new IOException("Unable to create directory: $file -- Check file system permissions" )
|
||||
}
|
||||
|
||||
return file
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a file for the given key. It guarantees to return the same file name
|
||||
* the same provided object key.
|
||||
*
|
||||
* @param key
|
||||
* @param name
|
||||
* @return
|
||||
*/
|
||||
@Deprecated
|
||||
static Path cacheableFile( Object key, String name = null ) {
|
||||
|
||||
// the cacheability is guaranteed by the folder
|
||||
def folder = cacheableDir(key)
|
||||
|
||||
if( !name ) {
|
||||
if( key instanceof File ) {
|
||||
name = key.getName()
|
||||
}
|
||||
else if( key instanceof Path ) {
|
||||
name = key.getName()
|
||||
}
|
||||
else {
|
||||
name = key.toString()
|
||||
}
|
||||
}
|
||||
|
||||
return folder.resolve(name)
|
||||
}
|
||||
|
||||
/**
|
||||
* This method is exposed as a public API to script, it should be removed
|
||||
*
|
||||
* @return Create a temporary directory
|
||||
*/
|
||||
@Deprecated
|
||||
static Path tempDir( String name = null, boolean create = true ) {
|
||||
final session = (ISession)Global.session
|
||||
if( !session )
|
||||
throw new IllegalStateException("Invalid access to `tempDir` method -- Session object not yet defined")
|
||||
|
||||
def path = FileHelper.createTempFolder(session.workDir)
|
||||
if( name )
|
||||
path = path.resolve(name)
|
||||
|
||||
if( !path.exists() && create && !path.mkdirs() )
|
||||
throw new IOException("Unable to create directory: $path -- Check file system permissions" )
|
||||
|
||||
return path
|
||||
}
|
||||
|
||||
/**
|
||||
* This method is exposed as a public API to script, it should be removed
|
||||
*
|
||||
* @return Create a temporary file
|
||||
*/
|
||||
@Deprecated
|
||||
static Path tempFile( String name = null, boolean create = false ) {
|
||||
|
||||
if( !name )
|
||||
name = 'file.tmp'
|
||||
|
||||
def folder = tempDir()
|
||||
def result = folder.resolve(name)
|
||||
if( create )
|
||||
Files.createFile(result)
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Implements built-in send mail functionality
|
||||
*
|
||||
* @param params
|
||||
* A map object holding the mail parameters
|
||||
* - to: String or a List of strings representing the mail recipients
|
||||
* - cc: String or a List of strings representing the mail recipients in carbon copy
|
||||
* - from: String representing the mail sender address
|
||||
* - subject: The mail subject
|
||||
* - content: The mail content
|
||||
* - attach: One or more list attachment
|
||||
*/
|
||||
static void sendMail( Map params ) {
|
||||
final opts = Global.session.config.mail as Map ?: Collections.emptyMap()
|
||||
new Mailer(opts).send(params)
|
||||
}
|
||||
|
||||
/**
|
||||
* Implements built-in send mail functionality
|
||||
*
|
||||
* @param params
|
||||
* A closure representing the mail message to send eg
|
||||
* <code>
|
||||
* sendMail {
|
||||
* to 'me@dot.com'
|
||||
* from 'your@name.com'
|
||||
* attach '/some/file/path'
|
||||
* subject 'Hello'
|
||||
* content '''
|
||||
* Hi there,
|
||||
* Hope this email find you well
|
||||
* '''
|
||||
* }
|
||||
* <code>
|
||||
*/
|
||||
static void sendMail( Closure params ) {
|
||||
final opts = Global.session.config.mail as Map ?: Collections.emptyMap()
|
||||
new Mailer(opts).send(params)
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a groupTuple dynamic key
|
||||
*
|
||||
* @param key
|
||||
* @param size
|
||||
* @return
|
||||
*/
|
||||
static GroupKey groupKey(key, int size) {
|
||||
new GroupKey(key,size)
|
||||
}
|
||||
|
||||
/**
|
||||
* Marker method to create a closure to be passed to {@link OperatorImpl#branch(DataflowReadChannel, groovy.lang.Closure)}
|
||||
* operator.
|
||||
*
|
||||
* Despite apparently is doing nothing, this method is needed as marker to apply the {@link OpXform} AST
|
||||
* transformation required to interpret the closure content as required for the branch evaluation.
|
||||
*
|
||||
* @see OperatorImpl#branch(DataflowReadChannel, Closure)
|
||||
* @see OpXformImpl
|
||||
*
|
||||
* @param closure
|
||||
* @return
|
||||
*/
|
||||
static Closure<TokenBranchDef> branchCriteria(Closure<TokenBranchDef> closure) { closure }
|
||||
|
||||
/**
|
||||
* Marker method to create a closure to be passed to {@link OperatorImpl#fork(DataflowReadChannel, Closure)}
|
||||
* operator.
|
||||
*
|
||||
* Despite apparently is doing nothing, this method is needed as marker to apply the {@link OpXform} AST
|
||||
* transformation required to interpret the closure content as required for the branch evaluation.
|
||||
*
|
||||
* @see OperatorImpl#multiMap(groovyx.gpars.dataflow.DataflowReadChannel, groovy.lang.Closure) (DataflowReadChannel, Closure)
|
||||
* @see OpXformImpl
|
||||
*
|
||||
* @param closure
|
||||
* @return
|
||||
*/
|
||||
static Closure<TokenMultiMapDef> multiMapCriteria(Closure<TokenBranchDef> closure) { closure }
|
||||
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow
|
||||
|
||||
import static nextflow.extension.Bolts.*
|
||||
|
||||
import java.text.SimpleDateFormat
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import groovy.transform.ToString
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.util.VersionNumber
|
||||
/**
|
||||
* Models nextflow script properties and metadata
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
@Singleton(strict = false)
|
||||
@ToString(includeNames = true)
|
||||
@EqualsAndHashCode
|
||||
class NextflowMeta {
|
||||
|
||||
static trait Flags {
|
||||
abstract float dsl
|
||||
abstract boolean strict
|
||||
abstract boolean moduleBinaries
|
||||
}
|
||||
|
||||
@Slf4j
|
||||
static class Preview implements Flags {
|
||||
@Deprecated volatile float dsl
|
||||
@Deprecated boolean strict
|
||||
boolean recursion
|
||||
boolean moduleBinaries
|
||||
|
||||
void setRecursion(Boolean recursion) {
|
||||
if( recursion )
|
||||
log.warn "NEXTFLOW RECURSION IS A PREVIEW FEATURE - SYNTAX AND FUNCTIONALITY CAN CHANGE IN FUTURE RELEASES"
|
||||
this.recursion = recursion
|
||||
}
|
||||
}
|
||||
|
||||
static class Features implements Flags {
|
||||
volatile float dsl
|
||||
boolean strict
|
||||
boolean moduleBinaries
|
||||
}
|
||||
|
||||
final VersionNumber version
|
||||
final int build
|
||||
|
||||
/*
|
||||
* Timestamp as dd-MM-yyyy HH:mm UTC formatted string
|
||||
*/
|
||||
final String timestamp
|
||||
|
||||
final Preview preview = new Preview()
|
||||
|
||||
final Features enable = new Features()
|
||||
|
||||
private NextflowMeta() {
|
||||
version = new VersionNumber(BuildInfo.version)
|
||||
build = BuildInfo.buildNum as int
|
||||
timestamp = BuildInfo.timestampUTC
|
||||
}
|
||||
|
||||
protected NextflowMeta(String ver, int build, String timestamp ) {
|
||||
this.version = new VersionNumber(ver)
|
||||
this.build = build
|
||||
this.timestamp = timestamp
|
||||
}
|
||||
|
||||
Map featuresMap() {
|
||||
final result = new LinkedHashMap()
|
||||
if( isStrictModeEnabled() )
|
||||
result.strict = true
|
||||
if( isModuleBinariesEnabled() )
|
||||
result.moduleBinaries = true
|
||||
return result
|
||||
}
|
||||
|
||||
Map toJsonMap() {
|
||||
final result = new LinkedHashMap<>(5)
|
||||
result.version = version.toString()
|
||||
result.build = build
|
||||
result.timestamp = parseDateStr(timestamp)
|
||||
result.enable = featuresMap()
|
||||
return result
|
||||
}
|
||||
|
||||
private Date parseDateStr(String str) {
|
||||
def fmt = new SimpleDateFormat(DATETIME_FORMAT + ' Z')
|
||||
fmt.parse(str)
|
||||
}
|
||||
|
||||
boolean isStrictModeEnabled() {
|
||||
return enable.strict
|
||||
}
|
||||
|
||||
void strictMode(boolean mode) {
|
||||
enable.strict = mode
|
||||
}
|
||||
|
||||
boolean isModuleBinariesEnabled() {
|
||||
return enable.moduleBinaries
|
||||
}
|
||||
|
||||
void moduleBinaries(boolean mode) {
|
||||
enable.moduleBinaries = mode
|
||||
}
|
||||
|
||||
}
|
||||
1320
nextflow/modules/nextflow/src/main/groovy/nextflow/Session.groovy
Normal file
1320
nextflow/modules/nextflow/src/main/groovy/nextflow/Session.groovy
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,152 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.ast
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import org.codehaus.groovy.ast.ASTNode
|
||||
import org.codehaus.groovy.ast.ClassNode
|
||||
import org.codehaus.groovy.ast.expr.ArgumentListExpression
|
||||
import org.codehaus.groovy.ast.expr.BinaryExpression
|
||||
import org.codehaus.groovy.ast.expr.CastExpression
|
||||
import org.codehaus.groovy.ast.expr.ClosureExpression
|
||||
import org.codehaus.groovy.ast.expr.ConstantExpression
|
||||
import org.codehaus.groovy.ast.expr.ConstructorCallExpression
|
||||
import org.codehaus.groovy.ast.expr.DeclarationExpression
|
||||
import org.codehaus.groovy.ast.expr.Expression
|
||||
import org.codehaus.groovy.ast.expr.MapExpression
|
||||
import org.codehaus.groovy.ast.expr.MethodCallExpression
|
||||
import org.codehaus.groovy.ast.expr.TupleExpression
|
||||
import org.codehaus.groovy.ast.expr.VariableExpression
|
||||
import org.codehaus.groovy.ast.stmt.BlockStatement
|
||||
import org.codehaus.groovy.ast.stmt.ExpressionStatement
|
||||
import org.codehaus.groovy.ast.stmt.ReturnStatement
|
||||
import org.codehaus.groovy.ast.stmt.Statement
|
||||
import org.codehaus.groovy.ast.tools.GeneralUtils
|
||||
import org.codehaus.groovy.control.SourceUnit
|
||||
import org.codehaus.groovy.syntax.SyntaxException
|
||||
import org.codehaus.groovy.syntax.Types
|
||||
/**
|
||||
* Helper methods to handle AST xforms
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class ASTHelpers {
|
||||
|
||||
static void syntaxError(String message, ASTNode node, SourceUnit unit) {
|
||||
int line = node.lineNumber
|
||||
int coln = node.columnNumber
|
||||
final ex = new SyntaxException(message,line,coln)
|
||||
if( unit )
|
||||
unit.addError(ex)
|
||||
else
|
||||
throw ex
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new {@code ConstructorCallExpression} for the specified class and arguments
|
||||
*
|
||||
* @param clazz The {@code Class} for which the create a constructor call expression
|
||||
* @param args The arguments to be passed to the constructor
|
||||
* @return The instance for the constructor call
|
||||
*/
|
||||
static Expression createX(Class clazz, TupleExpression args ) {
|
||||
def type = new ClassNode(clazz)
|
||||
return new ConstructorCallExpression(type,args)
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new {@code ConstructorCallExpression} for the specified class and arguments
|
||||
* specified using an open array.
|
||||
*
|
||||
* @param clazz The {@code Class} for which the create a constructor call expression
|
||||
* @param args The arguments to be passed to the constructor, they will be wrapped by in a {@code ArgumentListExpression}
|
||||
* @return The instance for the constructor call
|
||||
*/
|
||||
static Expression createX(Class clazz, Expression... params ) {
|
||||
def type = new ClassNode(clazz)
|
||||
def args = new ArgumentListExpression(params as List<Expression>)
|
||||
return new ConstructorCallExpression(type,args)
|
||||
}
|
||||
|
||||
static Expression createX(Class clazz, List<Expression> params ) {
|
||||
def type = new ClassNode(clazz)
|
||||
def args = new ArgumentListExpression(params as List<Expression>)
|
||||
return new ConstructorCallExpression(type,args)
|
||||
}
|
||||
|
||||
static Expression declX(Expression left, Expression right) {
|
||||
new DeclarationExpression(left, GeneralUtils.ASSIGN, right)
|
||||
}
|
||||
|
||||
static MethodCallExpression isMethodCallX(Expression expr) {
|
||||
return expr instanceof MethodCallExpression ? expr : null
|
||||
}
|
||||
|
||||
static VariableExpression isVariableX(Expression expr) {
|
||||
return expr instanceof VariableExpression ? expr : null
|
||||
}
|
||||
|
||||
static CastExpression isCastX(Expression expr) {
|
||||
return expr instanceof CastExpression ? expr : null
|
||||
}
|
||||
|
||||
static VariableExpression isThisX(Expression expr) {
|
||||
isVariableX(expr)?.name == 'this' ? (VariableExpression)expr : null
|
||||
}
|
||||
|
||||
static BinaryExpression isBinaryX(Expression expr) {
|
||||
expr instanceof BinaryExpression ? expr : null
|
||||
}
|
||||
|
||||
static ArgumentListExpression isArgsX(Expression expr) {
|
||||
expr instanceof ArgumentListExpression ? expr : null
|
||||
}
|
||||
|
||||
static TupleExpression isTupleX(Expression expr) {
|
||||
expr instanceof TupleExpression ? expr : null
|
||||
}
|
||||
|
||||
static MapExpression isMapX(Expression exp) {
|
||||
exp instanceof MapExpression ? exp : null
|
||||
}
|
||||
|
||||
static ConstantExpression isConstX(Expression exp) {
|
||||
exp instanceof ConstantExpression ? exp : null
|
||||
}
|
||||
|
||||
static BinaryExpression isAssignX(Expression expr) {
|
||||
isBinaryX(expr)?.operation?.type == Types.ASSIGN ? (BinaryExpression)expr : null
|
||||
}
|
||||
|
||||
static ClosureExpression isClosureX(Expression exp) {
|
||||
exp instanceof ClosureExpression ? exp : null
|
||||
}
|
||||
|
||||
static BlockStatement isBlockStmt(Statement stm) {
|
||||
stm instanceof BlockStatement ? stm : null
|
||||
}
|
||||
|
||||
static ExpressionStatement isStmtX(Statement stm) {
|
||||
stm instanceof ExpressionStatement ? stm : null
|
||||
}
|
||||
|
||||
static ReturnStatement isReturnS(Statement stmt) {
|
||||
stmt instanceof ReturnStatement ? stmt : null
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.ast
|
||||
|
||||
import java.lang.annotation.ElementType
|
||||
import java.lang.annotation.Retention
|
||||
import java.lang.annotation.RetentionPolicy
|
||||
import java.lang.annotation.Target
|
||||
|
||||
import org.codehaus.groovy.transform.GroovyASTTransformationClass
|
||||
|
||||
/**
|
||||
* Marker interface which to apply AST transformation to {@code process} declaration
|
||||
*/
|
||||
@Retention(RetentionPolicy.SOURCE)
|
||||
@Target(ElementType.METHOD)
|
||||
@GroovyASTTransformationClass(classes = [NextflowDSLImpl])
|
||||
@interface NextflowDSL {}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,53 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.ast
|
||||
|
||||
import java.lang.annotation.ElementType
|
||||
import java.lang.annotation.Retention
|
||||
import java.lang.annotation.RetentionPolicy
|
||||
import java.lang.annotation.Target
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.script.control.PathCompareVisitor
|
||||
import org.codehaus.groovy.ast.ASTNode
|
||||
import org.codehaus.groovy.ast.ClassNode
|
||||
import org.codehaus.groovy.control.CompilePhase
|
||||
import org.codehaus.groovy.control.SourceUnit
|
||||
import org.codehaus.groovy.transform.ASTTransformation
|
||||
import org.codehaus.groovy.transform.GroovyASTTransformation
|
||||
import org.codehaus.groovy.transform.GroovyASTTransformationClass
|
||||
|
||||
/**
|
||||
* Replace path comparisons with explicit method calls.
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Retention(RetentionPolicy.SOURCE)
|
||||
@Target(ElementType.METHOD)
|
||||
@GroovyASTTransformationClass(classes = [NextflowXformImpl])
|
||||
@interface NextflowXform {
|
||||
|
||||
@CompileStatic
|
||||
@GroovyASTTransformation(phase = CompilePhase.CONVERSION)
|
||||
class NextflowXformImpl implements ASTTransformation {
|
||||
|
||||
@Override
|
||||
void visit(ASTNode[] nodes, SourceUnit source) {
|
||||
new PathCompareVisitor(source).visitClass((ClassNode)nodes[1])
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.ast
|
||||
|
||||
import java.lang.annotation.ElementType
|
||||
import java.lang.annotation.Retention
|
||||
import java.lang.annotation.RetentionPolicy
|
||||
import java.lang.annotation.Target
|
||||
|
||||
import nextflow.script.control.OpCriteriaVisitor
|
||||
import groovy.transform.CompileStatic
|
||||
import org.codehaus.groovy.ast.ASTNode
|
||||
import org.codehaus.groovy.ast.ClassNode
|
||||
import org.codehaus.groovy.control.CompilePhase
|
||||
import org.codehaus.groovy.control.SourceUnit
|
||||
import org.codehaus.groovy.transform.ASTTransformation
|
||||
import org.codehaus.groovy.transform.GroovyASTTransformation
|
||||
import org.codehaus.groovy.transform.GroovyASTTransformationClass
|
||||
|
||||
/**
|
||||
* Transform closure arguments for branch and multiMap
|
||||
* into the appropriate criteria objects.
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Retention(RetentionPolicy.SOURCE)
|
||||
@Target(ElementType.METHOD)
|
||||
@GroovyASTTransformationClass(classes = [OpXformImpl])
|
||||
@interface OpXform {
|
||||
|
||||
@CompileStatic
|
||||
@GroovyASTTransformation(phase = CompilePhase.CONVERSION)
|
||||
class OpXformImpl implements ASTTransformation {
|
||||
|
||||
@Override
|
||||
void visit(ASTNode[] nodes, SourceUnit source) {
|
||||
new OpCriteriaVisitor(source).visitClass((ClassNode)nodes[1])
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.ast
|
||||
|
||||
import java.lang.annotation.ElementType
|
||||
import java.lang.annotation.Retention
|
||||
import java.lang.annotation.RetentionPolicy
|
||||
import java.lang.annotation.Target
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.script.control.TaskCmdXformVisitor
|
||||
import org.codehaus.groovy.ast.ASTNode
|
||||
import org.codehaus.groovy.ast.ClassNode
|
||||
import org.codehaus.groovy.control.CompilePhase
|
||||
import org.codehaus.groovy.control.SourceUnit
|
||||
import org.codehaus.groovy.transform.ASTTransformation
|
||||
import org.codehaus.groovy.transform.GroovyASTTransformation
|
||||
import org.codehaus.groovy.transform.GroovyASTTransformationClass
|
||||
|
||||
/**
|
||||
* Declares an AST xform to escape and manipulate task command
|
||||
* special value
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Retention(RetentionPolicy.SOURCE)
|
||||
@Target(ElementType.METHOD)
|
||||
@GroovyASTTransformationClass(classes = [TaskCmdXformImpl])
|
||||
@interface TaskCmdXform {
|
||||
|
||||
//--- === implementation === ---
|
||||
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
@GroovyASTTransformation(phase = CompilePhase.CONVERSION)
|
||||
class TaskCmdXformImpl implements ASTTransformation {
|
||||
@Override
|
||||
void visit(ASTNode[] nodes, SourceUnit source) {
|
||||
new TaskCmdXformVisitor(source).visitClass((ClassNode)nodes[1])
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.ast
|
||||
|
||||
import java.lang.annotation.ElementType
|
||||
import java.lang.annotation.Retention
|
||||
import java.lang.annotation.RetentionPolicy
|
||||
import java.lang.annotation.Target
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import org.codehaus.groovy.ast.ASTNode
|
||||
import org.codehaus.groovy.ast.ClassNode
|
||||
import org.codehaus.groovy.control.CompilePhase
|
||||
import org.codehaus.groovy.control.SourceUnit
|
||||
import org.codehaus.groovy.transform.ASTTransformation
|
||||
import org.codehaus.groovy.transform.GroovyASTTransformation
|
||||
import org.codehaus.groovy.transform.GroovyASTTransformationClass
|
||||
|
||||
/**
|
||||
* Capture variables that are declared in a task template.
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Retention(RetentionPolicy.SOURCE)
|
||||
@Target(ElementType.METHOD)
|
||||
@GroovyASTTransformationClass(classes = [TaskTemplateVarsXformImpl])
|
||||
@interface TaskTemplateVarsXform {
|
||||
|
||||
@CompileStatic
|
||||
@GroovyASTTransformation(phase = CompilePhase.CONVERSION)
|
||||
class TaskTemplateVarsXformImpl implements ASTTransformation {
|
||||
|
||||
@Override
|
||||
void visit(ASTNode[] nodes, SourceUnit source) {
|
||||
new TaskTemplateVisitor(source).visitClass((ClassNode)nodes[1])
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.ast
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import org.codehaus.groovy.ast.ClassNode
|
||||
import org.codehaus.groovy.ast.ConstructorNode
|
||||
import org.codehaus.groovy.ast.expr.ArgumentListExpression
|
||||
import org.codehaus.groovy.ast.expr.ConstantExpression
|
||||
import org.codehaus.groovy.ast.expr.GStringExpression
|
||||
import org.codehaus.groovy.ast.expr.ListExpression
|
||||
import org.codehaus.groovy.ast.expr.MethodCallExpression
|
||||
import org.codehaus.groovy.ast.expr.VariableExpression
|
||||
import org.codehaus.groovy.ast.stmt.BlockStatement
|
||||
import org.codehaus.groovy.ast.stmt.ExpressionStatement
|
||||
import org.codehaus.groovy.ast.stmt.Statement
|
||||
import org.codehaus.groovy.control.SourceUnit
|
||||
/**
|
||||
* Inject string var names in the script binding object
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class TaskTemplateVisitor extends VariableVisitor {
|
||||
|
||||
int withinGString
|
||||
|
||||
TaskTemplateVisitor(SourceUnit unit) {
|
||||
super(unit)
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean scopeEnabled() {
|
||||
return withinGString>0
|
||||
}
|
||||
|
||||
@Override
|
||||
void visitGStringExpression(GStringExpression expression) {
|
||||
try {
|
||||
withinGString++
|
||||
super.visitGStringExpression(expression)
|
||||
}
|
||||
finally {
|
||||
withinGString--
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void visitObjectInitializerStatements(ClassNode node) {
|
||||
injectMetadata(node)
|
||||
super.visitObjectInitializerStatements(node)
|
||||
}
|
||||
|
||||
protected void injectMetadata(ClassNode node) {
|
||||
// inject the invocation of a custom method
|
||||
// as last statement of the constructor invocation
|
||||
for( ConstructorNode constructor : node.getDeclaredConstructors() ) {
|
||||
def code = constructor.getCode()
|
||||
if( code instanceof BlockStatement ) {
|
||||
code.addStatement(makeSetVariableNamesStm())
|
||||
}
|
||||
else if( code instanceof ExpressionStatement ) {
|
||||
def expr = code
|
||||
def block = new BlockStatement()
|
||||
block.addStatement(expr)
|
||||
block.addStatement(makeSetVariableNamesStm())
|
||||
constructor.setCode(block)
|
||||
}
|
||||
else
|
||||
throw new IllegalStateException("Invalid constructor expression: $code")
|
||||
}
|
||||
}
|
||||
|
||||
protected Statement makeSetVariableNamesStm() {
|
||||
final names = new ListExpression()
|
||||
for( String it: getAllVariableNames() ) {
|
||||
names.addExpression(new ConstantExpression(it))
|
||||
}
|
||||
|
||||
|
||||
final varArgs = new ArgumentListExpression()
|
||||
varArgs.addExpression( new ConstantExpression('__$$_template_vars') )
|
||||
varArgs.addExpression( names )
|
||||
|
||||
// some magic code
|
||||
// this generates the invocation of the method:
|
||||
// this.getBinding().setVariable('__$$_template_vars', [variable names])
|
||||
final thiz = new VariableExpression('this')
|
||||
final bind = new MethodCallExpression( thiz, 'getBinding', ArgumentListExpression.EMPTY_ARGUMENTS )
|
||||
final call = new MethodCallExpression( bind, 'setVariable', varArgs )
|
||||
final stm = new ExpressionStatement(call)
|
||||
return stm
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.ast
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.script.TokenValRef
|
||||
import org.codehaus.groovy.ast.ClassCodeVisitorSupport
|
||||
import org.codehaus.groovy.ast.expr.ClosureExpression
|
||||
import org.codehaus.groovy.ast.expr.ConstantExpression
|
||||
import org.codehaus.groovy.ast.expr.DeclarationExpression
|
||||
import org.codehaus.groovy.ast.expr.PropertyExpression
|
||||
import org.codehaus.groovy.ast.expr.VariableExpression
|
||||
import org.codehaus.groovy.control.SourceUnit
|
||||
import org.codehaus.groovy.syntax.SyntaxException
|
||||
/**
|
||||
* Visit a closure and collect all referenced variable names
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class VariableVisitor extends ClassCodeVisitorSupport {
|
||||
|
||||
final Map<String, TokenValRef> fAllVariables = [:]
|
||||
|
||||
final Set<String> localDef = []
|
||||
|
||||
final SourceUnit sourceUnit
|
||||
|
||||
private boolean declaration
|
||||
|
||||
private int deep
|
||||
|
||||
VariableVisitor( SourceUnit unit ) {
|
||||
this.sourceUnit = unit
|
||||
}
|
||||
|
||||
protected boolean scopeEnabled() { true }
|
||||
|
||||
protected boolean isNormalized(PropertyExpression expr) {
|
||||
if( !(expr.getProperty() instanceof ConstantExpression) )
|
||||
return false
|
||||
|
||||
def target = expr.getObjectExpression()
|
||||
while( target instanceof PropertyExpression) {
|
||||
target = (target as PropertyExpression).getObjectExpression()
|
||||
}
|
||||
|
||||
return target instanceof VariableExpression
|
||||
}
|
||||
|
||||
@Override
|
||||
void visitClosureExpression(ClosureExpression expression) {
|
||||
if( deep++ == 0 )
|
||||
super.visitClosureExpression(expression)
|
||||
}
|
||||
|
||||
@Override
|
||||
void visitDeclarationExpression(DeclarationExpression expr) {
|
||||
declaration = true
|
||||
try {
|
||||
visit(expr.getLeftExpression())
|
||||
}
|
||||
finally {
|
||||
declaration = false
|
||||
}
|
||||
|
||||
visit(expr.getRightExpression())
|
||||
}
|
||||
|
||||
@Override
|
||||
void visitPropertyExpression(PropertyExpression expr) {
|
||||
|
||||
if( isNormalized(expr)) {
|
||||
final name = expr.text.replace('?','')
|
||||
final line = expr.lineNumber
|
||||
final coln = expr.columnNumber
|
||||
|
||||
if( !name.startsWith('this.') && !fAllVariables.containsKey(name) && scopeEnabled() ) {
|
||||
fAllVariables[name] = new TokenValRef(name,line,coln)
|
||||
}
|
||||
}
|
||||
else
|
||||
super.visitPropertyExpression(expr)
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
void visitVariableExpression(VariableExpression var) {
|
||||
final name = var.name
|
||||
final line = var.lineNumber
|
||||
final coln = var.columnNumber
|
||||
|
||||
if( name == 'this' )
|
||||
return
|
||||
|
||||
if( declaration ) {
|
||||
if( fAllVariables.containsKey(name) )
|
||||
sourceUnit.addError( new SyntaxException("Variable `$name` already declared in the process scope", line, coln))
|
||||
else
|
||||
localDef.add(name)
|
||||
}
|
||||
|
||||
// Note: variable declared in the process scope are not added
|
||||
// to the set of referenced variables. Only global ones are tracked
|
||||
else if( !localDef.contains(name) && !fAllVariables.containsKey(name) && scopeEnabled() ) {
|
||||
fAllVariables[name] = new TokenValRef(name,line,coln)
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected SourceUnit getSourceUnit() {
|
||||
return sourceUnit
|
||||
}
|
||||
|
||||
/**
|
||||
* @return The set of all variables referenced in the script.
|
||||
* NOTE: it includes properties in the form {@code object.propertyName}
|
||||
*/
|
||||
Set<TokenValRef> getAllVariables() {
|
||||
new HashSet<TokenValRef>(fAllVariables.values())
|
||||
}
|
||||
|
||||
Set<String> getAllVariableNames() {
|
||||
def all = getAllVariables()
|
||||
def result = new HashSet(all.size())
|
||||
for( TokenValRef ref : all )
|
||||
result.add(ref.name)
|
||||
return result
|
||||
}
|
||||
}
|
||||
268
nextflow/modules/nextflow/src/main/groovy/nextflow/cache/CacheDB.groovy
vendored
Normal file
268
nextflow/modules/nextflow/src/main/groovy/nextflow/cache/CacheDB.groovy
vendored
Normal file
@@ -0,0 +1,268 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cache
|
||||
|
||||
|
||||
import com.google.common.hash.HashCode
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.PackageScope
|
||||
import groovy.transform.stc.ClosureParams
|
||||
import groovy.transform.stc.SimpleType
|
||||
import groovy.util.logging.Slf4j
|
||||
import groovyx.gpars.agent.Agent
|
||||
import nextflow.processor.TaskContext
|
||||
import nextflow.processor.TaskEntry
|
||||
import nextflow.processor.TaskHandler
|
||||
import nextflow.processor.TaskProcessor
|
||||
import nextflow.trace.TraceRecord
|
||||
import nextflow.util.KryoHelper
|
||||
/**
|
||||
* Manages nextflow cache DB
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class CacheDB implements Closeable {
|
||||
|
||||
/** An agent used to apply asynchronously DB write operations */
|
||||
private Agent writer
|
||||
|
||||
private CacheStore store
|
||||
|
||||
CacheDB(CacheStore store) {
|
||||
this.store = store
|
||||
this.writer = new Agent()
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialise the database structure on the underlying file system
|
||||
*
|
||||
* @return The {@link CacheDB} instance itself
|
||||
*/
|
||||
CacheDB open() {
|
||||
store.open()
|
||||
return this
|
||||
}
|
||||
|
||||
/**
|
||||
* Open the database in read mode
|
||||
*
|
||||
* @return The {@link CacheDB} instance itself
|
||||
*/
|
||||
CacheDB openForRead() {
|
||||
store.openForRead()
|
||||
return this
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve a task runtime information from the cache DB
|
||||
*
|
||||
* @param taskHash The {@link HashCode} of the task to retrieve
|
||||
* @param processor The {@link TaskProcessor} instance to be assigned to the retrieved task
|
||||
* @return A {link TaskEntry} instance or {@code null} if a task for the given hash does not exist
|
||||
*/
|
||||
TaskEntry getTaskEntry(HashCode taskHash, TaskProcessor processor) {
|
||||
|
||||
final payload = store.getEntry(taskHash)
|
||||
if( !payload )
|
||||
return null
|
||||
|
||||
final record = (List)KryoHelper.deserialize(payload)
|
||||
TraceRecord trace = TraceRecord.deserialize( (byte[])record[0] )
|
||||
TaskContext ctx = record[1]!=null && processor!=null ? TaskContext.deserialize(processor, (byte[])record[1]) : null
|
||||
|
||||
return new TaskEntry(trace,ctx)
|
||||
}
|
||||
|
||||
void incTaskEntry( HashCode hash ) {
|
||||
final payload = store.getEntry(hash)
|
||||
if( !payload ) {
|
||||
log.debug "Can't increment reference for cached task with key: $hash"
|
||||
return
|
||||
}
|
||||
|
||||
final record = (List)KryoHelper.deserialize(payload)
|
||||
// third record contains the reference count for this record
|
||||
record[2] = ((Integer)record[2]) +1
|
||||
// save it again
|
||||
store.putEntry(hash, KryoHelper.serialize(record))
|
||||
|
||||
}
|
||||
|
||||
boolean removeTaskEntry( HashCode hash ) {
|
||||
final payload = store.getEntry(hash)
|
||||
if( !payload ) {
|
||||
log.debug "Can't increment reference for cached task with key: $hash"
|
||||
return false
|
||||
}
|
||||
|
||||
final record = (List)KryoHelper.deserialize(payload)
|
||||
// third record contains the reference count for this record
|
||||
def count = record[2] = ((Integer)record[2]) -1
|
||||
// save or delete
|
||||
if( count > 0 ) {
|
||||
store.putEntry(hash, KryoHelper.serialize(record))
|
||||
return false
|
||||
}
|
||||
else {
|
||||
store.deleteEntry(hash)
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Save task runtime information to th cache DB
|
||||
*
|
||||
* @param handler A {@link TaskHandler} instance
|
||||
*/
|
||||
@PackageScope
|
||||
void writeTaskEntry0( TaskHandler handler, TraceRecord trace ) {
|
||||
|
||||
final task = handler.task
|
||||
final proc = task.processor
|
||||
final key = task.hash
|
||||
|
||||
// save the context map for caching purpose
|
||||
// only the 'cache' is active and
|
||||
TaskContext ctx = proc.isCacheable() && task.hasCacheableValues() ? task.context : null
|
||||
|
||||
def record = new ArrayList(3)
|
||||
record[0] = trace.serialize()
|
||||
record[1] = ctx != null ? ctx.serialize() : null
|
||||
record[2] = 1
|
||||
|
||||
// -- save in the db
|
||||
store.putEntry( key, KryoHelper.serialize(record) )
|
||||
|
||||
}
|
||||
|
||||
void putTaskAsync( TaskHandler handler, TraceRecord trace ) {
|
||||
writer.send { writeTaskEntry0(handler, trace) }
|
||||
}
|
||||
|
||||
void cacheTaskAsync( TaskHandler handler ) {
|
||||
writer.send {
|
||||
writeTaskIndex0(handler,true)
|
||||
incTaskEntry(handler.task.hash)
|
||||
}
|
||||
}
|
||||
|
||||
void putIndexAsync(TaskHandler handler ) {
|
||||
writer.send { writeTaskIndex0(handler) }
|
||||
}
|
||||
|
||||
@PackageScope
|
||||
void writeTaskIndex0( TaskHandler handler, boolean cached = false ) {
|
||||
store.writeIndex(handler.task.hash, cached)
|
||||
}
|
||||
|
||||
void deleteIndex() {
|
||||
store.deleteIndex()
|
||||
}
|
||||
|
||||
void drop() {
|
||||
store.drop()
|
||||
}
|
||||
|
||||
/**
|
||||
* Iterate the tasks cache using the index file
|
||||
* @param closure The operation to applied
|
||||
* @return The {@link CacheDB} instance itself
|
||||
*/
|
||||
CacheDB eachRecord( Closure closure ) {
|
||||
assert closure
|
||||
|
||||
final itr = store.iterateIndex()
|
||||
while( itr.hasNext() ) {
|
||||
final index = itr.next()
|
||||
|
||||
final payload = store.getEntry(index.key)
|
||||
if( !payload ) {
|
||||
log.trace "Unable to retrieve cache record for key: ${-> index.key}"
|
||||
continue
|
||||
}
|
||||
|
||||
final record = (List<byte[]>)KryoHelper.deserialize(payload)
|
||||
TraceRecord trace = TraceRecord.deserialize(record[0])
|
||||
trace.setCached(index.cached)
|
||||
|
||||
final refCount = record[2] as Integer
|
||||
|
||||
final len=closure.maximumNumberOfParameters
|
||||
if( len==1 )
|
||||
closure.call(trace)
|
||||
|
||||
else if( len==2 )
|
||||
closure.call(index.key, trace)
|
||||
|
||||
else if( len==3 )
|
||||
closure.call(index.key, trace, refCount)
|
||||
|
||||
else
|
||||
throw new IllegalArgumentException("Invalid closure signature -- Too many parameters")
|
||||
|
||||
}
|
||||
|
||||
return this
|
||||
}
|
||||
|
||||
TraceRecord getTraceRecord( HashCode hashCode ) {
|
||||
final result = getTaskEntry(hashCode, null)
|
||||
return result ? result.trace : null
|
||||
}
|
||||
|
||||
TraceRecord findTraceRecord( @ClosureParams(value = SimpleType.class, options = "nextflow.trace.TraceRecord") Closure<Boolean> criteria ) {
|
||||
|
||||
final itr = store.iterateIndex()
|
||||
while( itr.hasNext() ) {
|
||||
final index = itr.next()
|
||||
|
||||
final payload = store.getEntry(index.key)
|
||||
if( !payload ) {
|
||||
log.trace "Unable to retrieve cache record for key: ${-> index.key}"
|
||||
continue
|
||||
}
|
||||
|
||||
final record = (List<byte[]>)KryoHelper.deserialize(payload)
|
||||
TraceRecord trace = TraceRecord.deserialize(record[0])
|
||||
trace.setCached(index.cached)
|
||||
|
||||
final len=criteria.maximumNumberOfParameters
|
||||
if( len!=1 ) {
|
||||
throw new IllegalArgumentException("Invalid criteria signature -- Too many parameters")
|
||||
}
|
||||
if( criteria.call(trace) )
|
||||
return trace
|
||||
}
|
||||
// no matches
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the underlying database and index file
|
||||
*/
|
||||
@Override
|
||||
void close() {
|
||||
log.trace "Closing CacheDB.."
|
||||
writer.await()
|
||||
log.trace "Closing CacheDB index"
|
||||
store.close()
|
||||
log.debug "Closing CacheDB done"
|
||||
}
|
||||
}
|
||||
46
nextflow/modules/nextflow/src/main/groovy/nextflow/cache/CacheFactory.groovy
vendored
Normal file
46
nextflow/modules/nextflow/src/main/groovy/nextflow/cache/CacheFactory.groovy
vendored
Normal file
@@ -0,0 +1,46 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cache
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.plugin.Plugins
|
||||
import org.pf4j.ExtensionPoint
|
||||
|
||||
/**
|
||||
* Factory class that create an instance of the {@link CacheDB}
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
abstract class CacheFactory implements ExtensionPoint {
|
||||
|
||||
protected abstract CacheDB newInstance(UUID uniqueId, String runName, Path home=null)
|
||||
|
||||
static CacheDB create(UUID uniqueId, String runName, Path home=null) {
|
||||
final all = Plugins.getPriorityExtensions(CacheFactory)
|
||||
if( !all )
|
||||
throw new IllegalStateException("Unable to find Nextflow cache factory")
|
||||
final factory = all.first()
|
||||
log.debug "Using Nextflow cache factory: ${factory.getClass().getName()}"
|
||||
return factory.newInstance(uniqueId, runName, home)
|
||||
}
|
||||
|
||||
}
|
||||
48
nextflow/modules/nextflow/src/main/groovy/nextflow/cache/CacheStore.groovy
vendored
Normal file
48
nextflow/modules/nextflow/src/main/groovy/nextflow/cache/CacheStore.groovy
vendored
Normal file
@@ -0,0 +1,48 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cache
|
||||
|
||||
import com.google.common.hash.HashCode
|
||||
import groovy.transform.TupleConstructor
|
||||
|
||||
/**
|
||||
* Defines the contract for a pluggable cache storage
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
interface CacheStore {
|
||||
|
||||
@TupleConstructor
|
||||
static class Index {
|
||||
final HashCode key
|
||||
final boolean cached
|
||||
}
|
||||
|
||||
CacheStore open()
|
||||
CacheStore openForRead()
|
||||
void close()
|
||||
void drop()
|
||||
|
||||
byte[] getEntry(HashCode key)
|
||||
void putEntry(HashCode key, byte[] value)
|
||||
void deleteEntry(HashCode key)
|
||||
|
||||
void writeIndex(HashCode key, boolean cached)
|
||||
Iterator<Index> iterateIndex()
|
||||
void deleteIndex()
|
||||
|
||||
}
|
||||
44
nextflow/modules/nextflow/src/main/groovy/nextflow/cache/DefaultCacheFactory.groovy
vendored
Normal file
44
nextflow/modules/nextflow/src/main/groovy/nextflow/cache/DefaultCacheFactory.groovy
vendored
Normal file
@@ -0,0 +1,44 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cache
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.plugin.Priority
|
||||
|
||||
/**
|
||||
* Implements the default cache factory
|
||||
*
|
||||
* @see DefaultCacheStore
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
@Priority(0)
|
||||
class DefaultCacheFactory extends CacheFactory {
|
||||
|
||||
@Override
|
||||
protected CacheDB newInstance(UUID uniqueId, String runName, Path home) {
|
||||
if( !uniqueId ) throw new AbortOperationException("Missing cache `uuid`")
|
||||
if( !runName ) throw new AbortOperationException("Missing cache `runName`")
|
||||
final store = new DefaultCacheStore(uniqueId, runName, home)
|
||||
return new CacheDB(store)
|
||||
}
|
||||
|
||||
}
|
||||
188
nextflow/modules/nextflow/src/main/groovy/nextflow/cache/DefaultCacheStore.groovy
vendored
Normal file
188
nextflow/modules/nextflow/src/main/groovy/nextflow/cache/DefaultCacheStore.groovy
vendored
Normal file
@@ -0,0 +1,188 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cache
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
import com.google.common.hash.HashCode
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.Const
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.util.CacheHelper
|
||||
import org.iq80.leveldb.DB
|
||||
import org.iq80.leveldb.Options
|
||||
import org.iq80.leveldb.impl.Iq80DBFactory
|
||||
/**
|
||||
* Implement the default nextflow cache store that save the cache data
|
||||
* into the local directory using an embedded LevelDVB instance
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class DefaultCacheStore implements CacheStore {
|
||||
|
||||
/** The underlying Level DB instance */
|
||||
private DB db
|
||||
|
||||
/** The session UUID */
|
||||
private UUID uniqueId
|
||||
|
||||
/** The unique run name associated to this cache instance */
|
||||
private String runName
|
||||
|
||||
/** The base folder against which the cache is located. Default: current working directory */
|
||||
private Path baseDir
|
||||
|
||||
/** The actual path where DB and index file are located */
|
||||
private Path dataDir
|
||||
|
||||
private final int KEY_SIZE
|
||||
|
||||
/** The path to the index file */
|
||||
private Path indexFile
|
||||
|
||||
/** Index file read/write handle */
|
||||
private RandomAccessFile indexHandle
|
||||
|
||||
|
||||
DefaultCacheStore(UUID uniqueId, String runName, Path home=null) {
|
||||
this.KEY_SIZE = CacheHelper.hasher('x').hash().asBytes().size()
|
||||
this.uniqueId = uniqueId
|
||||
this.runName = runName
|
||||
this.baseDir = home ?: Const.appCacheDir.toAbsolutePath()
|
||||
this.dataDir = baseDir.resolve("cache/$uniqueId")
|
||||
this.indexFile = dataDir.resolve("index.$runName")
|
||||
}
|
||||
|
||||
private void openDb() {
|
||||
// make sure the db path exists
|
||||
dataDir.mkdirs()
|
||||
// open a LevelDB instance
|
||||
final file=dataDir.resolve('db').toFile()
|
||||
try {
|
||||
db = Iq80DBFactory.@factory.open(file, new Options().createIfMissing(true))
|
||||
}
|
||||
catch( Exception e ) {
|
||||
String msg
|
||||
if( e.message?.startsWith('Unable to acquire lock') ) {
|
||||
msg = "Unable to acquire lock on session with ID $uniqueId"
|
||||
msg += "\n\n"
|
||||
msg += "Common reasons for this error are:"
|
||||
msg += "\n - You are trying to resume the execution of an already running pipeline"
|
||||
msg += "\n - A previous execution was abruptly interrupted, leaving the session open"
|
||||
msg += '\n'
|
||||
msg += '\nYou can see which process is holding the lock file by using the following command:'
|
||||
msg += "\n - lsof $file/LOCK"
|
||||
throw new IOException(msg)
|
||||
}
|
||||
else {
|
||||
msg = "Can't open cache DB: $file"
|
||||
msg += '\n\n'
|
||||
msg += "Nextflow needs to be executed in a shared file system that supports file locks.\n"
|
||||
msg += "Alternatively, you can run it in a local directory and specify the shared work\n"
|
||||
msg += "directory by using the `-w` command line option."
|
||||
throw new IOException(msg, e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
DefaultCacheStore open() {
|
||||
openDb()
|
||||
//
|
||||
indexFile.delete()
|
||||
indexHandle = new RandomAccessFile(indexFile.toFile(), 'rw')
|
||||
return this
|
||||
}
|
||||
|
||||
@Override
|
||||
DefaultCacheStore openForRead() {
|
||||
openDb()
|
||||
if( !indexFile.exists() )
|
||||
throw new AbortOperationException("Missing cache index file: $indexFile")
|
||||
indexHandle = new RandomAccessFile(indexFile.toFile(), 'r')
|
||||
return this
|
||||
}
|
||||
|
||||
@Override
|
||||
void drop() {
|
||||
dataDir.deleteDir()
|
||||
}
|
||||
|
||||
@Override
|
||||
void close() {
|
||||
indexHandle.closeQuietly()
|
||||
db.closeQuietly()
|
||||
}
|
||||
|
||||
@Override
|
||||
void writeIndex(HashCode key, boolean cached) {
|
||||
indexHandle.write(key.asBytes())
|
||||
indexHandle.writeBoolean(cached)
|
||||
}
|
||||
|
||||
@Override
|
||||
void deleteIndex() {
|
||||
indexFile.delete()
|
||||
}
|
||||
|
||||
@Override
|
||||
Iterator<Index> iterateIndex() {
|
||||
return new Iterator<Index>() {
|
||||
private Index next
|
||||
|
||||
{
|
||||
next = fetch()
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean hasNext() {
|
||||
return next != null
|
||||
}
|
||||
|
||||
@Override
|
||||
Index next() {
|
||||
final result = next
|
||||
next = fetch()
|
||||
return result
|
||||
}
|
||||
|
||||
private Index fetch() {
|
||||
byte[] key = new byte[KEY_SIZE]
|
||||
if( indexHandle.read(key) == -1 )
|
||||
return null
|
||||
final cached = indexHandle.readBoolean()
|
||||
return new Index (HashCode.fromBytes(key), cached)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
byte[] getEntry(HashCode key) {
|
||||
return db.get(key.asBytes())
|
||||
}
|
||||
|
||||
@Override
|
||||
void putEntry(HashCode key, byte[] value) {
|
||||
db.put(key.asBytes(), value)
|
||||
}
|
||||
|
||||
@Override
|
||||
void deleteEntry(HashCode key) {
|
||||
db.delete(key.asBytes())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
import java.nio.file.Path
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.PackageScope
|
||||
import nextflow.cache.CacheDB
|
||||
import nextflow.cache.CacheFactory
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.util.HistoryFile
|
||||
|
||||
/**
|
||||
* Common cache operations shared by {@link CmdLog} and {@link CmdClean}
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
trait CacheBase {
|
||||
|
||||
@PackageScope
|
||||
Path basePath
|
||||
|
||||
@PackageScope
|
||||
HistoryFile history
|
||||
|
||||
abstract String getBut()
|
||||
|
||||
abstract String getBefore()
|
||||
|
||||
abstract String getAfter()
|
||||
|
||||
abstract List<String> getArgs()
|
||||
|
||||
void init() {
|
||||
|
||||
if( !history ) {
|
||||
history = !basePath ? HistoryFile.DEFAULT : new HistoryFile(basePath.resolve(HistoryFile.defaultFileName()))
|
||||
}
|
||||
|
||||
if( !history.exists() || history.empty() )
|
||||
throw new AbortOperationException("It looks like no pipeline was executed in this folder (or execution history is empty)")
|
||||
|
||||
if( after && before )
|
||||
throw new AbortOperationException("Options `after` and `before` cannot be used in the same command")
|
||||
|
||||
if( after && but )
|
||||
throw new AbortOperationException("Options `after` and `but` cannot be used in the same command")
|
||||
|
||||
if( before && but )
|
||||
throw new AbortOperationException("Options `before` and `but` cannot be used in the same command")
|
||||
|
||||
}
|
||||
|
||||
CacheDB cacheFor(HistoryFile.Record entry) {
|
||||
CacheFactory.create(entry.sessionId, entry.runName, basePath)
|
||||
}
|
||||
|
||||
List<HistoryFile.Record> listIds() {
|
||||
|
||||
if( but ) {
|
||||
return history.findBut(but)
|
||||
}
|
||||
|
||||
if( before ) {
|
||||
return history.findBefore(before)
|
||||
}
|
||||
|
||||
else if( after ) {
|
||||
return history.findAfter(after)
|
||||
}
|
||||
|
||||
// -- get the session ID from the command line if specified or retrieve from
|
||||
if( !args )
|
||||
return history.findByIdOrName('last')
|
||||
|
||||
List<HistoryFile.Record> result = []
|
||||
for( String name : args ) {
|
||||
result.addAll(history.findByIdOrName(name))
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
|
||||
import com.beust.jcommander.DynamicParameter
|
||||
import com.beust.jcommander.Parameter
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.SysEnv
|
||||
import nextflow.exception.AbortOperationException
|
||||
import org.fusesource.jansi.Ansi
|
||||
|
||||
/**
|
||||
* Main application command line options
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class CliOptions {
|
||||
|
||||
/**
|
||||
* The packages to debug
|
||||
*/
|
||||
@Parameter(hidden = true, names='-debug')
|
||||
List<String> debug
|
||||
|
||||
@Parameter(names=['-log'], description = 'Set nextflow log file path')
|
||||
String logFile
|
||||
|
||||
@Parameter(names=['-c','-config'], description = 'Add the specified file to configuration set')
|
||||
List<String> userConfig
|
||||
|
||||
@Parameter(names=['-config-ignore-includes'], description = 'Disable the parsing of config includes')
|
||||
boolean ignoreConfigIncludes
|
||||
|
||||
@Parameter(names=['-C'], description = 'Use the specified configuration file(s) overriding any defaults')
|
||||
List<String> config
|
||||
|
||||
/**
|
||||
* the packages to trace
|
||||
*/
|
||||
@Parameter(names='-trace', description = 'Enable trace level logging for the specified package name - multiple packages can be provided separating them with a comma e.g. \'-trace nextflow,io.seqera\'')
|
||||
List<String> trace
|
||||
|
||||
/**
|
||||
* Enable syslog appender
|
||||
*/
|
||||
@Parameter(names = ['-syslog'], description = 'Send logs to syslog server (eg. localhost:514)' )
|
||||
String syslog
|
||||
|
||||
/**
|
||||
* Print out the version number and exit
|
||||
*/
|
||||
@Parameter(names = ['-v','-version'], description = 'Print the program version')
|
||||
boolean version
|
||||
|
||||
/**
|
||||
* Print out the 'help' and exit
|
||||
*/
|
||||
@Parameter(names = ['-h'], description = 'Print this help', help = true)
|
||||
boolean help
|
||||
|
||||
@Parameter(names = ['-q','-quiet'], description = 'Do not print information messages' )
|
||||
boolean quiet
|
||||
|
||||
@Parameter(names = ['-bg'], description = 'Execute nextflow in background', arity = 0)
|
||||
boolean background
|
||||
|
||||
@DynamicParameter(names = ['-D'], description = 'Set JVM properties' )
|
||||
Map<String,String> jvmOpts = [:]
|
||||
|
||||
@Parameter(names = ['-self-update'], description = 'Update nextflow to the latest version', arity = 0, hidden = true)
|
||||
boolean selfUpdate
|
||||
|
||||
@Parameter(names=['-remote-debug'], description = "Enable JVM interactive remote debugging (experimental)")
|
||||
boolean remoteDebug
|
||||
|
||||
Boolean ansiLog
|
||||
|
||||
boolean getAnsiLog() {
|
||||
if( ansiLog && quiet )
|
||||
throw new AbortOperationException("Command line options `quiet` and `ansi-log` cannot be used together")
|
||||
|
||||
if( ansiLog != null )
|
||||
return ansiLog
|
||||
|
||||
if( background )
|
||||
return ansiLog = false
|
||||
|
||||
if( quiet )
|
||||
return ansiLog = false
|
||||
|
||||
final env = SysEnv.get('NXF_ANSI_LOG')
|
||||
if( env ) try {
|
||||
return Boolean.parseBoolean(env)
|
||||
}
|
||||
catch (Exception e) {
|
||||
log.warn "Invalid boolean value for variable NXF_ANSI_LOG: $env -- it must be 'true' or 'false'"
|
||||
}
|
||||
|
||||
// Check NO_COLOR environment variable (https://no-color.org/)
|
||||
final noColor = SysEnv.get('NO_COLOR')
|
||||
if( noColor ) {
|
||||
return ansiLog = false
|
||||
}
|
||||
|
||||
// Disable ANSI log in agent mode for plain, parseable output
|
||||
if( SysEnv.isAgentMode() ) {
|
||||
return ansiLog = false
|
||||
}
|
||||
|
||||
return Ansi.isEnabled()
|
||||
}
|
||||
|
||||
boolean hasAnsiLogFlag() {
|
||||
ansiLog==true || SysEnv.get('NXF_ANSI_LOG')=='true'
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,498 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.Parameters
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.Const
|
||||
import nextflow.SysEnv
|
||||
import nextflow.config.ConfigBuilder
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.platform.PlatformHelper
|
||||
import nextflow.plugin.Plugins
|
||||
import org.pf4j.ExtensionPoint
|
||||
|
||||
/**
|
||||
* Command-line interface for managing Seqera Platform authentication and configuration.
|
||||
*
|
||||
* <p>This class provides the {@code nextflow auth} command with four sub-commands for managing
|
||||
* authentication credentials and platform settings:
|
||||
*
|
||||
* <ul>
|
||||
* <li>{@code login} - Authenticate with Seqera Platform using OAuth2 (Cloud) or PAT (Enterprise)
|
||||
* <li>{@code logout} - Revoke access token and remove authentication from local config
|
||||
* <li>{@code config} - Configure platform settings (workspace, monitoring, compute environment)
|
||||
* <li>{@code status} - Display current authentication status and configuration sources
|
||||
* </ul>
|
||||
*
|
||||
* <h2>Authentication Flow</h2>
|
||||
* <p>The authentication mechanism varies based on the platform type:
|
||||
* <ul>
|
||||
* <li><b>Seqera Platform Cloud</b>: Uses OAuth2 device authorization flow with Auth0
|
||||
* <li><b>Seqera Platform Enterprise</b>: Prompts for Personal Access Token (PAT)
|
||||
* </ul>
|
||||
*
|
||||
* <h2>Configuration Management</h2>
|
||||
* <p>Authentication credentials and settings are stored in the user's home directory:
|
||||
* <pre>
|
||||
* ~/.nextflow/config (includes seqera-auth.config)
|
||||
* ~/.nextflow/seqera-auth.config (contains tower.* settings)
|
||||
* </pre>
|
||||
*
|
||||
* <p>Configuration values can be sourced from:
|
||||
* <ol>
|
||||
* <li>Nextflow config files (highest priority)
|
||||
* <li>Environment variables (TOWER_ACCESS_TOKEN, TOWER_API_ENDPOINT, etc.)
|
||||
* <li>Default values (lowest priority)
|
||||
* </ol>
|
||||
*
|
||||
* <h2>Plugin Architecture</h2>
|
||||
* <p>The actual authentication implementation is provided by the {@code nf-tower} plugin through
|
||||
* the {@link AuthCommand} extension point. This allows the authentication logic to be maintained
|
||||
* separately from the CLI interface.
|
||||
*
|
||||
* @author Phil Ewels <phil.ewels@seqera.io>
|
||||
* @see AuthCommand
|
||||
* @see nextflow.platform.PlatformHelper
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
@Parameters(commandDescription = "Manage Seqera Platform authentication")
|
||||
class CmdAuth extends CmdBase implements UsageAware {
|
||||
|
||||
/**
|
||||
* Interface for auth sub-commands that defines the contract for command execution and help text.
|
||||
*
|
||||
* <p>Each sub-command (login, logout, config, status) implements this interface to provide:
|
||||
* <ul>
|
||||
* <li>Command name identification
|
||||
* <li>Argument processing and execution logic
|
||||
* <li>Usage/help text generation
|
||||
* </ul>
|
||||
*/
|
||||
interface SubCmd {
|
||||
/**
|
||||
* @return the name of this sub-command (e.g., "login", "logout")
|
||||
*/
|
||||
String getName()
|
||||
|
||||
/**
|
||||
* Executes the sub-command with the provided arguments.
|
||||
*
|
||||
* @param result the command-line arguments passed to this sub-command
|
||||
*/
|
||||
void apply(List<String> result)
|
||||
|
||||
/**
|
||||
* Generates usage/help text for this sub-command.
|
||||
*
|
||||
* @param result the list to which usage text lines should be added
|
||||
*/
|
||||
void usage(List<String> result)
|
||||
}
|
||||
|
||||
/**
|
||||
* Extension point interface for authentication command implementations.
|
||||
*
|
||||
* <p>This interface is implemented by the {@code nf-tower} plugin to provide the actual
|
||||
* authentication logic while keeping the CLI interface in the core Nextflow codebase.
|
||||
* The plugin system uses PF4J to discover and load implementations at runtime.
|
||||
*
|
||||
* <p>The implementation handles:
|
||||
* <ul>
|
||||
* <li>OAuth2 device flow for Seqera Platform Cloud
|
||||
* <li>PAT (Personal Access Token) authentication for Enterprise
|
||||
* <li>Token storage and management in config files
|
||||
* <li>Workspace and compute environment configuration
|
||||
* </ul>
|
||||
*
|
||||
* @see io.seqera.tower.plugin.auth.AuthCommandImpl
|
||||
*/
|
||||
interface AuthCommand extends ExtensionPoint {
|
||||
/**
|
||||
* Authenticates with Seqera Platform and saves credentials to config.
|
||||
*
|
||||
* @param url the Seqera Platform API endpoint URL (null for default)
|
||||
*/
|
||||
void login(String url)
|
||||
|
||||
/**
|
||||
* Revokes access token and removes authentication from local config.
|
||||
*/
|
||||
void logout()
|
||||
|
||||
/**
|
||||
* Configures Seqera Platform settings (workspace, monitoring, compute environment).
|
||||
*/
|
||||
void config()
|
||||
|
||||
/**
|
||||
* Displays current authentication status and configuration sources.
|
||||
*/
|
||||
void status()
|
||||
}
|
||||
|
||||
static public final String NAME = 'auth'
|
||||
|
||||
private List<SubCmd> commands = []
|
||||
|
||||
private AuthCommand operation
|
||||
|
||||
String getName() {
|
||||
return NAME
|
||||
}
|
||||
|
||||
@Parameter(hidden = true)
|
||||
List<String> args
|
||||
|
||||
@Parameter(names = ['-u', '-url'], description = 'Seqera Platform API endpoint')
|
||||
String apiUrl
|
||||
|
||||
CmdAuth() {
|
||||
commands.add(new LoginCmd())
|
||||
commands.add(new LogoutCmd())
|
||||
commands.add(new ConfigCmd())
|
||||
commands.add(new StatusCmd())
|
||||
}
|
||||
|
||||
void usage() {
|
||||
usage(args)
|
||||
}
|
||||
|
||||
void usage(List<String> args) {
|
||||
List<String> result = []
|
||||
if (!args) {
|
||||
result << this.getClass().getAnnotation(Parameters).commandDescription()
|
||||
result << 'Usage: nextflow auth <sub-command> [options]'
|
||||
result << ''
|
||||
result << 'Commands:'
|
||||
result << ' login Authenticate with Seqera Platform'
|
||||
result << ' logout Remove authentication and revoke access token'
|
||||
result << ' status Show current authentication status and configuration'
|
||||
result << ' config Configure Seqera Platform settings'
|
||||
result << ''
|
||||
} else {
|
||||
def sub = commands.find { it.name == args[0] }
|
||||
if (sub)
|
||||
sub.usage(result)
|
||||
else {
|
||||
throw new AbortOperationException("Unknown auth sub-command: ${args[0]}")
|
||||
}
|
||||
}
|
||||
println result.join('\n').toString()
|
||||
}
|
||||
|
||||
@Override
|
||||
void run() {
|
||||
if (!args) {
|
||||
usage()
|
||||
return
|
||||
}
|
||||
// load the Auth command implementation
|
||||
this.operation = loadOperation()
|
||||
if( !operation )
|
||||
throw new IllegalStateException("Unable to load auth extensions.")
|
||||
// consume the first argument
|
||||
getCmd(args).apply(args.drop(1))
|
||||
}
|
||||
|
||||
protected AuthCommand loadOperation(){
|
||||
// setup the plugins system and load the secrets provider
|
||||
Plugins.init()
|
||||
// load the config
|
||||
Plugins.start('nf-tower')
|
||||
// get Auth command operations implementation from plugins
|
||||
return Plugins.getExtension(AuthCommand)
|
||||
}
|
||||
|
||||
protected SubCmd getCmd(List<String> args) {
|
||||
def cmd = commands.find { it.name == args[0] }
|
||||
if (cmd) {
|
||||
return cmd
|
||||
}
|
||||
|
||||
def matches = commands.collect { it.name }.closest(args[0])
|
||||
def msg = "Unknown auth sub-command: ${args[0]}"
|
||||
if (matches)
|
||||
msg += " -- Did you mean one of these?\n" + matches.collect { " $it" }.join('\n')
|
||||
throw new AbortOperationException(msg)
|
||||
}
|
||||
|
||||
/**
|
||||
* Implements the {@code nextflow auth login} sub-command for authenticating with Seqera Platform.
|
||||
*
|
||||
* <p>This command initiates the authentication process, which varies based on the platform type:
|
||||
* <ul>
|
||||
* <li><b>Seqera Platform Cloud</b>: Launches OAuth2 device authorization flow
|
||||
* <ol>
|
||||
* <li>Requests device code from Auth0
|
||||
* <li>Displays verification URL and code to user
|
||||
* <li>Opens browser for authentication
|
||||
* <li>Polls for token completion
|
||||
* <li>Generates Personal Access Token from OAuth token
|
||||
* </ol>
|
||||
* </li>
|
||||
* <li><b>Seqera Platform Enterprise</b>: Prompts for manual PAT entry
|
||||
* <ol>
|
||||
* <li>Displays URL for token creation
|
||||
* <li>Prompts user to paste PAT
|
||||
* <li>Validates token with API
|
||||
* </ol>
|
||||
* </li>
|
||||
* </ul>
|
||||
*
|
||||
* <p>Upon successful authentication, the command:
|
||||
* <ul>
|
||||
* <li>Saves {@code tower.accessToken} to {@code ~/.nextflow/seqera-auth.config}
|
||||
* <li>Configures {@code tower.endpoint} with the API URL
|
||||
* <li>Enables {@code tower.enabled} for automatic monitoring
|
||||
* <li>Adds {@code includeConfig 'seqera-auth.config'} to main config
|
||||
* </ul>
|
||||
*
|
||||
* <p><b>Usage:</b> {@code nextflow auth login [-u <endpoint>]}
|
||||
*/
|
||||
class LoginCmd implements SubCmd {
|
||||
|
||||
@Override
|
||||
String getName() { 'login' }
|
||||
|
||||
@Override
|
||||
void apply(List<String> args) {
|
||||
if (args.size() > 0) {
|
||||
throw new AbortOperationException("Too many arguments for ${name} command")
|
||||
}
|
||||
operation.login(apiUrl)
|
||||
}
|
||||
|
||||
@Override
|
||||
void usage(List<String> result) {
|
||||
// Read config to get the actual resolved endpoint value
|
||||
final builder = new ConfigBuilder().setHomeDir(Const.APP_HOME_DIR).setCurrentDir(Const.APP_HOME_DIR)
|
||||
final config = builder.buildConfigObject().flatten()
|
||||
final towerConfig = config.findAll { it.key.toString().startsWith('tower.') }
|
||||
.collectEntries { k, v -> [(k.toString().substring(6)): v] }
|
||||
def defaultEndpoint = PlatformHelper.getEndpoint(towerConfig, SysEnv.get())
|
||||
|
||||
result << 'Authenticate with Seqera Platform'
|
||||
result << "Usage: nextflow auth $name [-u <endpoint>]".toString()
|
||||
result << ''
|
||||
result << 'Options:'
|
||||
result << " -u, -url <endpoint> Seqera Platform API endpoint (default: ${defaultEndpoint})".toString()
|
||||
result << ''
|
||||
result << 'This command will:'
|
||||
result << ' 1. Display a URL and device code for OAuth2 authentication (Cloud) or prompt for PAT (Enterprise)'
|
||||
result << ' 2. Wait for user to complete authentication in web browser'
|
||||
result << ' 3. Generate and save access token to home-directory Nextflow config'
|
||||
result << ' 4. Configure tower.accessToken, tower.endpoint, and tower.enabled settings'
|
||||
result << ''
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Implements the {@code nextflow auth logout} sub-command for removing authentication credentials.
|
||||
*
|
||||
* <p>This command performs a safe logout by:
|
||||
* <ol>
|
||||
* <li>Checking if {@code tower.accessToken} is configured in {@code ~/.nextflow/seqera-auth.config}
|
||||
* <li>Validating the token with the Seqera Platform API
|
||||
* <li>Displaying current configuration details to the user
|
||||
* <li>Prompting for confirmation before proceeding
|
||||
* <li>Deleting the Personal Access Token from Platform (Cloud only)
|
||||
* <li>Removing {@code seqera-auth.config} file
|
||||
* <li>Removing {@code includeConfig} line from main config
|
||||
* </ol>
|
||||
*
|
||||
* <p><b>Important behavior differences:</b>
|
||||
* <ul>
|
||||
* <li><b>Seqera Platform Cloud</b>: Deletes the PAT from the platform via API before removing local config
|
||||
* <li><b>Seqera Platform Enterprise</b>: Only removes local config; does not delete PAT from platform
|
||||
* </ul>
|
||||
*
|
||||
* <p><b>Note:</b> This command only removes credentials from Nextflow config files. If
|
||||
* {@code TOWER_ACCESS_TOKEN} environment variable is set, it will remain unchanged and
|
||||
* the user will be warned about this.
|
||||
*
|
||||
* <p><b>Usage:</b> {@code nextflow auth logout}
|
||||
*/
|
||||
class LogoutCmd implements SubCmd {
|
||||
|
||||
@Override
|
||||
String getName() { 'logout' }
|
||||
|
||||
@Override
|
||||
void apply(List<String> args) {
|
||||
if (args.size() > 0) {
|
||||
throw new AbortOperationException("Too many arguments for ${name} command")
|
||||
}
|
||||
operation.logout()
|
||||
}
|
||||
|
||||
@Override
|
||||
void usage(List<String> result) {
|
||||
result << 'Log out and remove Seqera Platform authentication'
|
||||
result << "Usage: nextflow auth $name".toString()
|
||||
result << ''
|
||||
result << 'This command will:'
|
||||
result << ' 1. Check if tower.accessToken is configured'
|
||||
result << ' 2. Validate the token with Seqera Platform'
|
||||
result << ' 3. Delete the PAT from Platform (only if Seqera Platform Cloud)'
|
||||
result << ' 4. Remove the authentication from Nextflow config'
|
||||
result << ''
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Implements the {@code nextflow auth config} sub-command for configuring Seqera Platform settings.
|
||||
*
|
||||
* <p>This interactive command guides users through configuring their Seqera Platform integration.
|
||||
* It requires an existing authentication (see {@link LoginCmd}) and allows configuration of:
|
||||
*
|
||||
* <h3>Workflow Monitoring</h3>
|
||||
* <ul>
|
||||
* <li>Configures {@code tower.enabled} setting
|
||||
* <li>When enabled: all workflow runs are automatically monitored
|
||||
* <li>When disabled: monitoring requires {@code -with-tower} flag per run
|
||||
* </ul>
|
||||
*
|
||||
* <h3>Default Workspace</h3>
|
||||
* <ul>
|
||||
* <li>Configures {@code tower.workspaceId} setting
|
||||
* <li>Lists all accessible workspaces organized by organization
|
||||
* <li>For many workspaces: uses two-stage selection (org → workspace)
|
||||
* <li>For few workspaces: displays all at once
|
||||
* <li>Option to use Personal workspace (no workspace ID)
|
||||
* </ul>
|
||||
*
|
||||
* <h3>Primary Compute Environment</h3>
|
||||
* <ul>
|
||||
* <li>Displays compute environments available in selected workspace
|
||||
* <li>Allows setting a primary compute environment
|
||||
* <li>Primary compute environment is used by default for pipeline execution
|
||||
* </ul>
|
||||
*
|
||||
* <p>All configuration changes are saved to {@code ~/.nextflow/seqera-auth.config}.
|
||||
*
|
||||
* <p><b>Usage:</b> {@code nextflow auth config}
|
||||
*/
|
||||
class ConfigCmd implements SubCmd {
|
||||
|
||||
@Override
|
||||
String getName() { 'config' }
|
||||
|
||||
@Override
|
||||
void apply(List<String> args) {
|
||||
if (args.size() > 0) {
|
||||
throw new AbortOperationException("Too many arguments for ${name} command")
|
||||
}
|
||||
|
||||
operation.config()
|
||||
}
|
||||
|
||||
@Override
|
||||
void usage(List<String> result) {
|
||||
result << 'Configure Seqera Platform settings'
|
||||
result << "Usage: nextflow auth $name".toString()
|
||||
result << ''
|
||||
result << 'This command will:'
|
||||
result << ' 1. Check authentication status'
|
||||
result << ' 2. Configure tower.enabled setting for workflow monitoring'
|
||||
result << ' 3. Configure default workspace (tower.workspaceId)'
|
||||
result << ''
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Implements the {@code nextflow auth status} sub-command for displaying authentication status.
|
||||
*
|
||||
* <p>This command provides a comprehensive overview of the current Seqera Platform configuration,
|
||||
* displaying all settings in a formatted table with their values and sources.
|
||||
*
|
||||
* <h3>Information Displayed</h3>
|
||||
* <table border="1">
|
||||
* <tr><th>Setting</th><th>Description</th><th>Possible Sources</th></tr>
|
||||
* <tr>
|
||||
* <td>API endpoint</td>
|
||||
* <td>Seqera Platform API URL</td>
|
||||
* <td>nextflow config, env var $TOWER_API_ENDPOINT, default</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>API connection</td>
|
||||
* <td>Whether the API is reachable</td>
|
||||
* <td>Health check result (✔ OK or ERROR)</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>Authentication</td>
|
||||
* <td>Token status and username</td>
|
||||
* <td>nextflow config, env var $TOWER_ACCESS_TOKEN, not set</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>Workflow monitoring</td>
|
||||
* <td>Whether monitoring is enabled</td>
|
||||
* <td>nextflow config, default (No)</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>Default workspace</td>
|
||||
* <td>Configured workspace ID and name</td>
|
||||
* <td>nextflow config, env var $TOWER_WORKSPACE_ID, default (Personal)</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>Primary compute env</td>
|
||||
* <td>Primary compute environment name</td>
|
||||
* <td>Workspace setting</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>Default work dir</td>
|
||||
* <td>Working directory from compute env</td>
|
||||
* <td>Compute environment setting</td>
|
||||
* </tr>
|
||||
* </table>
|
||||
*
|
||||
* <p>The source column shows where each setting's value comes from, making it easy to
|
||||
* understand configuration precedence and troubleshoot issues.
|
||||
*
|
||||
* <p><b>Usage:</b> {@code nextflow auth status}
|
||||
*/
|
||||
class StatusCmd implements SubCmd {
|
||||
|
||||
@Override
|
||||
String getName() { 'status' }
|
||||
|
||||
@Override
|
||||
void apply(List<String> args) {
|
||||
if (args.size() > 0) {
|
||||
throw new AbortOperationException("Too many arguments for ${name} command")
|
||||
}
|
||||
operation.status()
|
||||
}
|
||||
|
||||
@Override
|
||||
void usage(List<String> result) {
|
||||
result << 'Show authentication status and configuration'
|
||||
result << "Usage: nextflow auth $name".toString()
|
||||
result << ''
|
||||
result << 'This command shows:'
|
||||
result << ' - Authentication status (yes/no) and source'
|
||||
result << ' - API endpoint and source'
|
||||
result << ' - Monitoring enabled status and source'
|
||||
result << ' - Default workspace and source'
|
||||
result << ' - System health status (API connection and authentication)'
|
||||
result << ''
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
|
||||
import com.beust.jcommander.Parameter
|
||||
|
||||
/**
|
||||
* Implement command shared methods
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
abstract class CmdBase implements Runnable {
|
||||
|
||||
private Launcher launcher
|
||||
private List<String> unknownOptions
|
||||
|
||||
abstract String getName()
|
||||
|
||||
List<String> getAliases() { Collections.emptyList() }
|
||||
|
||||
protected List<String> getUnknownOptions(){ return this.unknownOptions }
|
||||
|
||||
void setUnknownOptions(List<String> options){ this.unknownOptions = options }
|
||||
|
||||
Launcher getLauncher() { launcher }
|
||||
|
||||
void setLauncher( Launcher value ) { this.launcher = value }
|
||||
|
||||
@Parameter(names=['-h','-help'], description = 'Print the command usage', arity = 0, help = true)
|
||||
boolean help
|
||||
}
|
||||
@@ -0,0 +1,281 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
import java.nio.file.FileVisitResult
|
||||
import java.nio.file.FileVisitor
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.NoSuchFileException
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
import java.nio.file.attribute.BasicFileAttributes
|
||||
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.Parameters
|
||||
import com.google.common.hash.HashCode
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.cache.CacheDB
|
||||
import nextflow.Global
|
||||
import nextflow.ISession
|
||||
import nextflow.Session
|
||||
import nextflow.config.ConfigBuilder
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.file.FileHelper
|
||||
import nextflow.plugin.Plugins
|
||||
import nextflow.trace.TraceRecord
|
||||
import nextflow.util.HistoryFile.Record
|
||||
|
||||
/**
|
||||
* Implements cache clean up command
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
* @author Lorenz Gerber <lorenzottogerber@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
@Parameters(commandDescription = "Clean up project cache and work directories")
|
||||
class CmdClean extends CmdBase implements CacheBase {
|
||||
|
||||
static final public NAME = 'clean'
|
||||
|
||||
@Parameter(names=['-q', '-quiet'], description = 'Do not print names of files removed', arity = 0)
|
||||
boolean quiet
|
||||
|
||||
@Parameter(names=['-f', '-force'], description = 'Force clean command', arity = 0)
|
||||
boolean force
|
||||
|
||||
@Parameter(names=['-n', '-dry-run'], description = 'Print names of file to be removed without deleting them' , arity = 0)
|
||||
boolean dryRun
|
||||
|
||||
@Parameter(names='-after', description = 'Clean up runs executed after the specified one')
|
||||
String after
|
||||
|
||||
@Parameter(names='-before', description = 'Clean up runs executed before the specified one')
|
||||
String before
|
||||
|
||||
@Parameter(names='-but', description = 'Clean up all runs except the specified one')
|
||||
String but
|
||||
|
||||
@Parameter(names=['-k', '-keep-logs'], description = 'Removes only temporary files but retains execution log entries and metadata')
|
||||
boolean keepLogs
|
||||
|
||||
@Parameter
|
||||
List<String> args
|
||||
|
||||
private CacheDB currentCacheDb
|
||||
|
||||
private Map<HashCode, Short> dryHash = new HashMap<>()
|
||||
|
||||
/**
|
||||
* @return The name of this command {@code clean}
|
||||
*/
|
||||
@Override
|
||||
String getName() {
|
||||
return NAME
|
||||
}
|
||||
|
||||
/**
|
||||
* Command entry method
|
||||
*/
|
||||
@Override
|
||||
void run() {
|
||||
init()
|
||||
validateOptions()
|
||||
createSession()
|
||||
Plugins.init()
|
||||
listIds().each { entry -> cleanup(entry) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Create the NF session which can be required to access cloud file store
|
||||
*/
|
||||
private void createSession() {
|
||||
Global.setLazySession {
|
||||
final builder = new ConfigBuilder()
|
||||
.setShowClosures(true)
|
||||
.showMissingVariables(true)
|
||||
.setOptions(launcher.options)
|
||||
.setBaseDir(Paths.get('.'))
|
||||
|
||||
final config = builder.buildConfigObject()
|
||||
return (ISession) new Session(config)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extra CLI option validation
|
||||
*/
|
||||
private void validateOptions() {
|
||||
|
||||
if( !dryRun && !force )
|
||||
throw new AbortOperationException("Neither -f or -n specified -- refused to clean")
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a history entry clean up execution cache, deleting
|
||||
* task work directories and cache DB records
|
||||
*
|
||||
* @param entry
|
||||
* A {@link Record} object representing a row in the history log file
|
||||
*/
|
||||
private void cleanup(Record entry) {
|
||||
currentCacheDb = cacheFor(entry).openForRead()
|
||||
// -- remove each entry and work dir
|
||||
currentCacheDb.eachRecord(this.&removeRecord)
|
||||
// -- close the cache
|
||||
currentCacheDb.close()
|
||||
|
||||
// -- STOP HERE !
|
||||
if( dryRun || keepLogs ) return
|
||||
|
||||
// -- remove the index file
|
||||
currentCacheDb.deleteIndex()
|
||||
// -- remove the session from the history file
|
||||
history.deleteEntry(entry)
|
||||
// -- check if exists another history entry for the same session
|
||||
if( !history.checkExistsById(entry.sessionId)) {
|
||||
currentCacheDb.drop()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a tasks can be removed during a dry-run simulation.
|
||||
*
|
||||
* @param hash
|
||||
* The task unique hash code
|
||||
* @param refCount
|
||||
* The number of times the task cache is references by other run instances
|
||||
* @return
|
||||
* {@code true} when task will be removed by the clean command, {@code false} otherwise i.e.
|
||||
* entry cannot be deleted because is referenced by other run instances
|
||||
*/
|
||||
private boolean wouldRemove(HashCode hash, Integer refCount) {
|
||||
|
||||
if( dryHash.containsKey(hash) ) {
|
||||
refCount = dryHash.get(hash)-1
|
||||
}
|
||||
|
||||
if( refCount == 1 ) {
|
||||
dryHash.remove(hash)
|
||||
return true
|
||||
}
|
||||
else {
|
||||
dryHash.put(hash, (short)refCount)
|
||||
return false
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete task cache entry
|
||||
*
|
||||
* @param hash The task unique hash code
|
||||
* @param record The task {@link TraceRecord}
|
||||
* @param refCount The number of times the task cache is references by other run instances
|
||||
*/
|
||||
private void removeRecord(HashCode hash, TraceRecord record, int refCount) {
|
||||
if( dryRun ) {
|
||||
if( wouldRemove(hash,refCount) )
|
||||
printMessage(record.workDir,true)
|
||||
return
|
||||
}
|
||||
|
||||
// decrement the ref count in the db
|
||||
def proceed = keepLogs || currentCacheDb.removeTaskEntry(hash)
|
||||
if( proceed ) {
|
||||
// delete folder
|
||||
if( deleteFolder(FileHelper.asPath(record.workDir), keepLogs)) {
|
||||
if(!quiet) printMessage(record.workDir,false)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
private printMessage(String path, boolean dryRun) {
|
||||
if( dryRun ) {
|
||||
println keepLogs ? "Would remove temp files from ${path}" : "Would remove ${path}"
|
||||
}
|
||||
else {
|
||||
println keepLogs ? "Removed temp files from ${path}" : "Removed ${path}"
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Traverse a directory structure and delete all the content
|
||||
*
|
||||
* @param folder
|
||||
* The directory to delete
|
||||
* @return
|
||||
* {@code true} in the directory was removed, {@code false} otherwise
|
||||
*/
|
||||
private boolean deleteFolder( Path folder, boolean keepLogs ) {
|
||||
|
||||
def result = true
|
||||
Files.walkFileTree(folder, new FileVisitor<Path>() {
|
||||
|
||||
@Override
|
||||
FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
|
||||
result ? FileVisitResult.CONTINUE : FileVisitResult.TERMINATE
|
||||
}
|
||||
|
||||
@Override
|
||||
FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
|
||||
|
||||
final canDelete = !keepLogs || ( keepLogs && !(file.name.startsWith('.command.') || file.name == '.exitcode'))
|
||||
if( canDelete && !delete0(file,false) ) {
|
||||
result = false
|
||||
if(!quiet) System.err.println "Failed to remove ${file.toUriString()}"
|
||||
}
|
||||
|
||||
result ? FileVisitResult.CONTINUE : FileVisitResult.TERMINATE
|
||||
}
|
||||
|
||||
@Override
|
||||
FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
|
||||
FileVisitResult.CONTINUE
|
||||
}
|
||||
|
||||
@Override
|
||||
FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
|
||||
if( !keepLogs && result && !delete0(dir,true) ) {
|
||||
result = false
|
||||
if(!quiet) System.err.println "Failed to remove ${dir.toUriString()}"
|
||||
}
|
||||
|
||||
result ? FileVisitResult.CONTINUE : FileVisitResult.TERMINATE
|
||||
}
|
||||
})
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
private static delete0(Path path, boolean dir) {
|
||||
try {
|
||||
log.trace "Deleting path [dir=$dir]: ${path.toUriString()}"
|
||||
Files.delete(path)
|
||||
return true
|
||||
}
|
||||
catch( IOException e ) {
|
||||
// kind of hack: directory deletion
|
||||
if( dir && path.scheme=='gs' && e instanceof NoSuchFileException )
|
||||
return true
|
||||
log.debug("Failed to remove path: ${path.toUriString()}", e)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.Parameters
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.plugin.Plugins
|
||||
import nextflow.scm.AssetManager
|
||||
/**
|
||||
* CLI sub-command clone
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
@Parameters(commandDescription = "Clone a project into a folder")
|
||||
class CmdClone extends CmdBase implements HubOptions {
|
||||
|
||||
static final public NAME = 'clone'
|
||||
|
||||
@Parameter(required=true, description = 'name of the project to clone')
|
||||
List<String> args
|
||||
|
||||
@Parameter(names='-r', description = 'Revision of the project to clone (either a git branch, tag or commit SHA number)')
|
||||
String revision
|
||||
|
||||
@Parameter(names=['-d','-deep'], description = 'Create a shallow clone of the specified depth')
|
||||
Integer deep
|
||||
|
||||
@Override
|
||||
final String getName() { NAME }
|
||||
|
||||
@Override
|
||||
void run() {
|
||||
// init plugin system
|
||||
Plugins.init()
|
||||
// the pipeline name
|
||||
String pipeline = args[0]
|
||||
try (final manager = new AssetManager(pipeline, this)) {
|
||||
// the target directory is the second parameter
|
||||
// otherwise default the current pipeline name
|
||||
def target = new File(args.size()> 1 ? args[1] : manager.getBaseName())
|
||||
if( target.exists() ) {
|
||||
if( target.isFile() )
|
||||
throw new AbortOperationException("A file with the same name already exists: $target")
|
||||
if( !target.empty() )
|
||||
throw new AbortOperationException("Clone target directory must be empty: $target")
|
||||
}
|
||||
else if( !target.mkdirs() ) {
|
||||
throw new AbortOperationException("Cannot create clone target directory: $target")
|
||||
}
|
||||
|
||||
manager.checkValidRemoteRepo()
|
||||
print "Cloning ${manager.getProjectWithRevision()} ..."
|
||||
manager.clone(target, revision, deep)
|
||||
print "\r"
|
||||
println "${manager.getProjectWithRevision()} cloned to: $target"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,245 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.Parameters
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.PackageScope
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.NF
|
||||
import nextflow.config.ConfigBuilder
|
||||
import nextflow.config.ConfigValidator
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.plugin.Plugins
|
||||
import nextflow.scm.AssetManager
|
||||
import nextflow.util.ConfigHelper
|
||||
/**
|
||||
* Prints the pipeline configuration
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
@Parameters(commandDescription = "Print a project configuration")
|
||||
class CmdConfig extends CmdBase {
|
||||
|
||||
static final public NAME = 'config'
|
||||
|
||||
static final List<String> FORMATS = ['flat','properties','canonical','json','yaml']
|
||||
|
||||
@Parameter(description = 'project name')
|
||||
List<String> args = []
|
||||
|
||||
@Parameter(names=['-r','-revision'], description = 'Revision of the project (either a git branch, tag or commit SHA number)')
|
||||
String revision
|
||||
|
||||
@Parameter(names=['-a','-show-profiles'], description = 'Show all configuration profiles')
|
||||
boolean showAllProfiles
|
||||
|
||||
@Parameter(names=['-profile'], description = 'Choose a configuration profile')
|
||||
String profile
|
||||
|
||||
@Deprecated
|
||||
@Parameter(names = '-properties', description = 'Prints config using Java properties notation (deprecated: use `-o properties` instead)')
|
||||
boolean printProperties
|
||||
|
||||
@Deprecated
|
||||
@Parameter(names = '-flat', description = 'Print config using flat notation (deprecated: use `-o flat` instead)')
|
||||
boolean printFlatten
|
||||
|
||||
@Parameter(names = '-sort', description = 'Sort config attributes')
|
||||
boolean sort
|
||||
|
||||
@Parameter(names = '-value', description = 'Print the value of a config option, or fail if the option is not defined')
|
||||
String printValue
|
||||
|
||||
@Parameter(names = ['-o','-output'], description = 'Print the config using the specified format: canonical,properties,flat,json,yaml')
|
||||
String outputFormat
|
||||
|
||||
@Override
|
||||
String getName() { NAME }
|
||||
|
||||
private OutputStream stdout = System.out
|
||||
|
||||
@Override
|
||||
void run() {
|
||||
Plugins.init()
|
||||
Path base = null
|
||||
if( args ) base = getBaseDir(args[0])
|
||||
if( !base ) base = Paths.get('.')
|
||||
|
||||
// -- validate command line options
|
||||
if( profile && showAllProfiles ) {
|
||||
throw new AbortOperationException("Option `-profile` conflicts with option `-show-profiles`")
|
||||
}
|
||||
|
||||
if( printProperties && printFlatten )
|
||||
throw new AbortOperationException("Option `-flat` and `-properties` conflicts each other")
|
||||
|
||||
if ( printValue && printFlatten )
|
||||
throw new AbortOperationException("Option `-value` and `-flat` conflicts each other")
|
||||
|
||||
if ( printValue && printProperties )
|
||||
throw new AbortOperationException("Option `-value` and `-properties` conflicts each other")
|
||||
|
||||
if( printValue && outputFormat )
|
||||
throw new AbortOperationException("Option `-value` and `-output` conflicts each other")
|
||||
|
||||
if( printFlatten )
|
||||
outputFormat = 'flat'
|
||||
|
||||
if( printProperties )
|
||||
outputFormat = 'properties'
|
||||
|
||||
// -- build the config
|
||||
final builder = new ConfigBuilder()
|
||||
.setShowClosures(true)
|
||||
.setStripSecrets(true)
|
||||
.showMissingVariables(true)
|
||||
.setOptions(launcher.options)
|
||||
.setBaseDir(base)
|
||||
.setCmdConfig(this)
|
||||
|
||||
final config = builder.buildConfigObject()
|
||||
|
||||
// -- validate config options
|
||||
if( NF.isSyntaxParserV2() ) {
|
||||
Plugins.load(config)
|
||||
new ConfigValidator().validate(config)
|
||||
}
|
||||
|
||||
// -- print config options
|
||||
if( printValue ) {
|
||||
printValue0(config, printValue, stdout)
|
||||
}
|
||||
else if( outputFormat=='properties' ) {
|
||||
printProperties0(config, stdout)
|
||||
}
|
||||
else if( outputFormat=='flat' ) {
|
||||
printFlatten0(config, stdout)
|
||||
}
|
||||
else if( outputFormat=='yaml' ) {
|
||||
printYaml0(config, stdout)
|
||||
}
|
||||
else if( outputFormat=='json') {
|
||||
printJson0(config, stdout)
|
||||
}
|
||||
else if( !outputFormat || outputFormat=='canonical' ) {
|
||||
printCanonical0(config, stdout)
|
||||
}
|
||||
else {
|
||||
def msg = "Unknown output format: $outputFormat"
|
||||
def suggest = FORMATS.closest(outputFormat)
|
||||
if( suggest )
|
||||
msg += " - did you mean '${suggest.first()}' instead?"
|
||||
throw new AbortOperationException(msg)
|
||||
}
|
||||
|
||||
for( String msg : builder.warnings )
|
||||
log.warn(msg)
|
||||
}
|
||||
|
||||
/**
|
||||
* Prints a {@link ConfigObject} using Java {@link Properties} in canonical format
|
||||
* ie. any nested config object is printed within curly brackets
|
||||
*
|
||||
* @param config The {@link ConfigObject} representing the parsed workflow configuration
|
||||
* @param output The stream where output the formatted configuration notation
|
||||
*/
|
||||
@PackageScope void printCanonical0(ConfigObject config, OutputStream output) {
|
||||
output << ConfigHelper.toCanonicalString(config, sort)
|
||||
}
|
||||
|
||||
/**
|
||||
* Prints a {@link ConfigObject} using Java {@link Properties} format
|
||||
*
|
||||
* @param config The {@link ConfigObject} representing the parsed workflow configuration
|
||||
* @param output The stream where output the formatted configuration notation
|
||||
*/
|
||||
@PackageScope void printProperties0(ConfigObject config, OutputStream output) {
|
||||
output << ConfigHelper.toPropertiesString(config, sort)
|
||||
}
|
||||
|
||||
/**
|
||||
* Prints a property of a {@link ConfigObject}.
|
||||
*
|
||||
* @param config The {@link ConfigObject} representing the parsed workflow configuration
|
||||
* @param name The {@link String} representing the property name using dot notation
|
||||
* @param output The stream where output the formatted configuration notation
|
||||
*/
|
||||
@PackageScope void printValue0(ConfigObject config, String name, OutputStream output) {
|
||||
final map = config.flatten()
|
||||
if( !map.containsKey(name) )
|
||||
throw new AbortOperationException("Configuration option '$name' not found")
|
||||
|
||||
output << map.get(name).toString() << '\n'
|
||||
}
|
||||
|
||||
/**
|
||||
* Prints a {@link ConfigObject} using properties dot notation.
|
||||
* String values are enclosed in single quote characters.
|
||||
*
|
||||
* @param config The {@link ConfigObject} representing the parsed workflow configuration
|
||||
* @param output The stream where output the formatted configuration notation
|
||||
*/
|
||||
@PackageScope void printFlatten0(ConfigObject config, OutputStream output) {
|
||||
output << ConfigHelper.toFlattenString(config, sort)
|
||||
}
|
||||
|
||||
/**
|
||||
* Prints the {@link ConfigObject} configuration object using the default notation
|
||||
*
|
||||
* @param config The {@link ConfigObject} representing the parsed workflow configuration
|
||||
* @param output The stream where output the formatted configuration notation
|
||||
*/
|
||||
@PackageScope void printDefault0(ConfigObject config, OutputStream output) {
|
||||
def writer = new PrintWriter(output,true)
|
||||
config.writeTo( writer )
|
||||
}
|
||||
|
||||
@PackageScope void printJson0(ConfigObject config, OutputStream output) {
|
||||
output << ConfigHelper.toJsonString(config, sort) << '\n'
|
||||
}
|
||||
|
||||
@PackageScope void printYaml0(ConfigObject config, OutputStream output) {
|
||||
output << ConfigHelper.toYamlString(config, sort)
|
||||
}
|
||||
|
||||
Path getBaseDir(String path) {
|
||||
|
||||
def file = Paths.get(path)
|
||||
if( file.isDirectory() )
|
||||
return file
|
||||
|
||||
if( file.exists() ) {
|
||||
return file.parent ?: Paths.get('/')
|
||||
}
|
||||
|
||||
try (final manager = new AssetManager(path, revision)) {
|
||||
if( revision && manager.isUsingLegacyStrategy() ){
|
||||
log.warn("The local asset for ${path} does not support multi-revision - 'revision' option is ignored\n" +
|
||||
"Consider updating the project using 'nextflow pull ${path} -r $revision -migrate'")
|
||||
}
|
||||
return manager.isLocal() ? manager.localPath.toPath() : manager.configFile?.parent
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.Parameters
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.plugin.Plugins
|
||||
import nextflow.ui.console.ConsoleExtension
|
||||
|
||||
/**
|
||||
* Launch the Nextflow Console plugin
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
@Parameters(commandDescription = "Launch Nextflow interactive console")
|
||||
class CmdConsole extends CmdBase {
|
||||
|
||||
@Parameter(description = 'Nextflow console arguments')
|
||||
List<String> args
|
||||
|
||||
String getName() { 'console' }
|
||||
|
||||
void run() {
|
||||
Plugins.init()
|
||||
Plugins.start('nf-console')
|
||||
final console = Plugins.getExtension(ConsoleExtension)
|
||||
if( !console )
|
||||
throw new IllegalStateException("Failed to find Nextflow Console extension")
|
||||
// normalise the console args prepending the `console` command itself
|
||||
if( args == null )
|
||||
args = []
|
||||
args.add(0, 'console')
|
||||
// go !
|
||||
console.run(args as String[])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.Parameters
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.plugin.Plugins
|
||||
import nextflow.scm.AssetManager
|
||||
|
||||
/**
|
||||
* CLI sub-command DROP
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
@Parameters(commandDescription = "Delete the local copy of a project")
|
||||
class CmdDrop extends CmdBase {
|
||||
|
||||
static final public NAME = 'drop'
|
||||
|
||||
@Parameter(required=true, description = 'name of the project to drop')
|
||||
List<String> args
|
||||
|
||||
@Parameter(names=['-r','-revision'], description = 'Revision of the project to drop (either a git branch, tag or commit SHA number)')
|
||||
String revision
|
||||
|
||||
@Parameter(names='-f', description = 'Delete the repository without taking care of local changes')
|
||||
boolean force
|
||||
|
||||
@Override
|
||||
final String getName() { NAME }
|
||||
|
||||
@Override
|
||||
void run() {
|
||||
Plugins.init()
|
||||
try (final manager = new AssetManager(args[0])) {
|
||||
manager.drop(revision, force)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,324 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
|
||||
import static nextflow.file.FileHelper.toCanonicalPath
|
||||
|
||||
import java.nio.charset.Charset
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
import java.nio.file.attribute.BasicFileAttributes
|
||||
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.Parameters
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.Global
|
||||
import nextflow.Session
|
||||
import nextflow.config.ConfigBuilder
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.extension.FilesEx
|
||||
import nextflow.file.FileHelper
|
||||
import nextflow.file.FilePatternSplitter
|
||||
import nextflow.plugin.Plugins
|
||||
/**
|
||||
* Implements `fs` command
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
@Slf4j
|
||||
@Parameters(commandDescription = "Perform filesystem operations")
|
||||
class CmdFs extends CmdBase implements UsageAware {
|
||||
|
||||
static final public NAME = 'fs'
|
||||
|
||||
static final List<SubCmd> commands = new ArrayList<>()
|
||||
|
||||
static {
|
||||
commands << new CmdCopy()
|
||||
commands << new CmdMove()
|
||||
commands << new CmdList()
|
||||
commands << new CmdCat()
|
||||
commands << new CmdRemove()
|
||||
commands << new CmdStat()
|
||||
}
|
||||
|
||||
trait SubCmd {
|
||||
abstract int getArity()
|
||||
|
||||
abstract String getName()
|
||||
|
||||
abstract String getDescription()
|
||||
|
||||
abstract void apply(Path source, Path target)
|
||||
|
||||
String usage() {
|
||||
"Usage: nextflow fs ${name} " + (arity==1 ? "<path>" : "source_file target_file")
|
||||
}
|
||||
}
|
||||
|
||||
static class CmdCopy implements SubCmd {
|
||||
|
||||
@Override
|
||||
int getArity() { 2 }
|
||||
|
||||
@Override
|
||||
String getName() { 'cp' }
|
||||
|
||||
String getDescription() { 'Copy a file' }
|
||||
|
||||
@Override
|
||||
void apply(Path source, Path target) {
|
||||
FilesEx.copyTo(source, target)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
static class CmdMove implements SubCmd {
|
||||
|
||||
@Override
|
||||
int getArity() { 2 }
|
||||
|
||||
@Override
|
||||
String getName() { 'mv' }
|
||||
|
||||
String getDescription() { 'Move a file' }
|
||||
|
||||
@Override
|
||||
void apply(Path source, Path target) {
|
||||
FilesEx.moveTo(source, target)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
static class CmdList implements SubCmd {
|
||||
|
||||
@Override
|
||||
int getArity() { 1 }
|
||||
|
||||
String getDescription() { 'List the content of a folder' }
|
||||
|
||||
@Override
|
||||
String getName() { 'ls' }
|
||||
|
||||
@Override
|
||||
void apply(Path source, Path target) {
|
||||
println source.name
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
static class CmdCat implements SubCmd {
|
||||
|
||||
@Override
|
||||
int getArity() { 1 }
|
||||
|
||||
@Override
|
||||
String getName() { 'cat' }
|
||||
|
||||
@Override
|
||||
String getDescription() { 'Print a file to the stdout' }
|
||||
|
||||
@Override
|
||||
void apply(Path source, Path target) {
|
||||
String line
|
||||
def reader = Files.newBufferedReader(source, Charset.defaultCharset())
|
||||
while( line = reader.readLine() )
|
||||
println line
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
static class CmdStat implements SubCmd {
|
||||
@Override
|
||||
int getArity() { 1 }
|
||||
|
||||
@Override
|
||||
String getName() { 'stat' }
|
||||
|
||||
@Override
|
||||
String getDescription() { 'Print file to meta info' }
|
||||
|
||||
@Override
|
||||
void apply(Path source, Path target) {
|
||||
try {
|
||||
final attr = Files.readAttributes(source, BasicFileAttributes)
|
||||
print """\
|
||||
name : ${source.name}
|
||||
size : ${attr.size()}
|
||||
is directory : ${attr.isDirectory()}
|
||||
last modified : ${attr.lastModifiedTime() ?: '-'}
|
||||
creation time : ${attr.creationTime() ?: '-'}
|
||||
""".stripIndent()
|
||||
}
|
||||
catch (IOException e) {
|
||||
log.warn "Unable to read attributes for file: ${source.toUriString()} - cause: $e.message", e
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static class CmdRemove implements SubCmd {
|
||||
|
||||
@Override
|
||||
int getArity() { 1 }
|
||||
|
||||
@Override
|
||||
String getName() { 'rm' }
|
||||
|
||||
@Override
|
||||
String getDescription() { 'Remove a file' }
|
||||
|
||||
@Override
|
||||
void apply(Path source, Path target) {
|
||||
Files.isDirectory(source) ? FilesEx.deleteDir(source) : FilesEx.delete(source)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Parameter
|
||||
List<String> args
|
||||
|
||||
@Override
|
||||
String getName() {
|
||||
return NAME
|
||||
}
|
||||
|
||||
private Session createSession() {
|
||||
// create the config
|
||||
final config = new ConfigBuilder()
|
||||
.setOptions(getLauncher().getOptions())
|
||||
.setBaseDir(Paths.get('.'))
|
||||
.build()
|
||||
|
||||
return new Session(config)
|
||||
}
|
||||
|
||||
@Override
|
||||
void run() {
|
||||
if( !args ) {
|
||||
usage()
|
||||
return
|
||||
}
|
||||
|
||||
Plugins.init()
|
||||
final session = createSession()
|
||||
try {
|
||||
run0()
|
||||
}
|
||||
finally {
|
||||
try {
|
||||
session.destroy()
|
||||
Global.cleanUp()
|
||||
Plugins.stop()
|
||||
} catch (Throwable t) {
|
||||
log.warn "Unexpected error while destroying the session object - cause: ${t.message}"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void run0() {
|
||||
final cmd = findCmd(args[0])
|
||||
if( !cmd ) {
|
||||
throw new AbortOperationException("Unknown fs sub-command: `$cmd`")
|
||||
}
|
||||
|
||||
Path target
|
||||
String source
|
||||
if( cmd.arity==1 ) {
|
||||
if( args.size() < 2 )
|
||||
throw new AbortOperationException(cmd.usage())
|
||||
source = args[1]
|
||||
target = null
|
||||
}
|
||||
else {
|
||||
if( args.size() < 3 )
|
||||
throw new AbortOperationException(cmd.usage())
|
||||
source = args[1]
|
||||
target = args[2] as Path
|
||||
}
|
||||
|
||||
traverse(source) { Path path -> cmd.apply(path, target) }
|
||||
}
|
||||
|
||||
private SubCmd findCmd( String name ) {
|
||||
commands.find { it.name == name }
|
||||
}
|
||||
|
||||
private void traverse( String source, Closure op ) {
|
||||
|
||||
// if it isn't a glob pattern simply return it a normalized absolute Path object
|
||||
def splitter = FilePatternSplitter.glob().parse(source)
|
||||
if( splitter.isPattern() ) {
|
||||
final scheme = splitter.scheme
|
||||
final target = scheme ? "$scheme://$splitter.parent" : splitter.parent
|
||||
final folder = toCanonicalPath(target)
|
||||
final pattern = splitter.fileName
|
||||
|
||||
def opts = [:]
|
||||
opts.type = 'any'
|
||||
|
||||
FileHelper.visitFiles(opts, folder, pattern, op)
|
||||
}
|
||||
else {
|
||||
def normalised = splitter.strip(source)
|
||||
op.call(FileHelper.asPath(normalised))
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Print the command usage help
|
||||
*/
|
||||
@Override
|
||||
void usage() {
|
||||
usage(args)
|
||||
}
|
||||
|
||||
/**
|
||||
* Print the command usage help
|
||||
*
|
||||
* @param args The arguments as entered by the user
|
||||
*/
|
||||
@Override
|
||||
void usage(List<String> args) {
|
||||
|
||||
def result = []
|
||||
if( !args ) {
|
||||
result << 'Usage: nextflow fs <command> [arg]'
|
||||
result << ''
|
||||
result << 'Commands:'
|
||||
commands.each {
|
||||
result << " ${it.name}\t${it.description}"
|
||||
}
|
||||
result << ''
|
||||
println result.join('\n').toString()
|
||||
}
|
||||
else {
|
||||
def sub = findCmd(args[0])
|
||||
if( sub )
|
||||
println sub.usage()
|
||||
else {
|
||||
throw new AbortOperationException("Unknown fs sub-command: ${args[0]}")
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.Parameters
|
||||
import groovy.transform.CompileStatic
|
||||
/**
|
||||
* CLI sub-command HELP
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
@Parameters(commandDescription = "Print the usage help for a command")
|
||||
class CmdHelp extends CmdBase {
|
||||
|
||||
static final public NAME = 'help'
|
||||
|
||||
@Override
|
||||
final String getName() { NAME }
|
||||
|
||||
@Parameter(description = 'command name', arity = 1)
|
||||
List<String> args
|
||||
|
||||
private UsageAware getUsage( List<String> args ) {
|
||||
def result = args ? launcher.findCommand(args[0]) : null
|
||||
result instanceof UsageAware ? result as UsageAware: null
|
||||
}
|
||||
|
||||
@Override
|
||||
void run() {
|
||||
def cmd = getUsage(args)
|
||||
if( cmd ) {
|
||||
cmd.usage(args.size()>1 ? args[1..-1] : Collections.<String>emptyList())
|
||||
}
|
||||
else {
|
||||
launcher.usage(args ? args[0] : null)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
|
||||
/**
|
||||
* Cmd CLI helper methods
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class CmdHelper {
|
||||
|
||||
static private String operators = '<>!='
|
||||
|
||||
/**
|
||||
* Expand single `=` to `==` in filter string
|
||||
*
|
||||
* @param filter A filter string e.g. `cpus = 4`
|
||||
* @return A filter in which `=` char is expanded to `==` operator
|
||||
*/
|
||||
static String fixEqualsOp( String filter ) {
|
||||
if( !filter ) return filter
|
||||
|
||||
def result = new StringBuilder()
|
||||
int i=0
|
||||
int len=filter.length()
|
||||
|
||||
while( i<len ) {
|
||||
def ch = filter[i++]
|
||||
result << ch
|
||||
if( i<len-1 && filter[i]=='=' && filter[i+1]!='=' && !operators.contains(ch)) {
|
||||
result.append('=')
|
||||
}
|
||||
}
|
||||
return result.toString()
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,356 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
|
||||
import java.lang.management.ManagementFactory
|
||||
import java.nio.file.spi.FileSystemProvider
|
||||
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.Parameters
|
||||
import com.sun.management.OperatingSystemMXBean
|
||||
import groovy.json.JsonOutput
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.BuildInfo
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.plugin.DefaultPlugins
|
||||
import nextflow.plugin.Plugins
|
||||
import nextflow.scm.AssetManager
|
||||
import nextflow.util.MemoryUnit
|
||||
import nextflow.util.Threads
|
||||
import org.yaml.snakeyaml.Yaml
|
||||
/**
|
||||
* CLI sub-command INFO
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
@Parameters(commandDescription = "Print project and system runtime information")
|
||||
class CmdInfo extends CmdBase {
|
||||
|
||||
static final public NAME = 'info'
|
||||
|
||||
private PrintStream out = System.out
|
||||
|
||||
@Parameter(description = 'project name')
|
||||
List<String> args
|
||||
|
||||
@Parameter(names='-d',description = 'Show detailed information', arity = 0)
|
||||
boolean detailed
|
||||
|
||||
@Parameter(names='-dd', hidden = true, arity = 0)
|
||||
boolean moreDetailed
|
||||
|
||||
@Parameter(names='-o', description = 'Output format, either: text (default), json, yaml')
|
||||
String format
|
||||
|
||||
@Parameter(names=['-u','-check-updates'], description = 'Check for remote updates')
|
||||
boolean checkForUpdates
|
||||
|
||||
@Override
|
||||
final String getName() { NAME }
|
||||
|
||||
@Override
|
||||
void run() {
|
||||
|
||||
int level = moreDetailed ? 2 : ( detailed ? 1 : 0 )
|
||||
if( !args ) {
|
||||
println getInfo(level)
|
||||
return
|
||||
}
|
||||
|
||||
Plugins.init()
|
||||
try (final manager = new AssetManager(args[0])) {
|
||||
if( manager.isNotInitialized() ) {
|
||||
throw new AbortOperationException("Unknown project `${args[0]}`")
|
||||
}
|
||||
|
||||
if( !format || format == 'text' ) {
|
||||
printText(manager,level)
|
||||
return
|
||||
}
|
||||
|
||||
def map = createMap(manager)
|
||||
if( format == 'json' ) {
|
||||
printJson(map)
|
||||
}
|
||||
else if( format == 'yaml' ) {
|
||||
printYaml(map)
|
||||
}
|
||||
else
|
||||
throw new AbortOperationException("Unknown output format: $format");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
protected printText(AssetManager manager, int level) {
|
||||
final manifest = manager.getManifest()
|
||||
|
||||
out.println " project name: ${manager.project}"
|
||||
out.println " repository : ${manager.repositoryUrl}"
|
||||
out.println " local path : ${manager.projectPath}"
|
||||
out.println " main script : ${manager.mainScriptName}"
|
||||
if( manager.homePage && manager.homePage != manager.repositoryUrl )
|
||||
out.println " home page : ${manager.homePage}"
|
||||
if( manifest.description )
|
||||
out.println " description : ${manifest.description}"
|
||||
if( manifest.author )
|
||||
out.println " author : ${manifest.author}"
|
||||
|
||||
def revs = manager.getRevisions(level)
|
||||
if( revs.size() == 1 )
|
||||
out.println " revision : ${revs[0]}"
|
||||
else {
|
||||
out.println " revisions : "
|
||||
revs.each { out.println " $it" }
|
||||
}
|
||||
|
||||
if( !checkForUpdates )
|
||||
return
|
||||
|
||||
def updates = manager.getUpdates(level)
|
||||
if( updates ) {
|
||||
if( updates.size() == 1 && revs.size() == 1 )
|
||||
out.println " updates : ${updates[0]}"
|
||||
else {
|
||||
out.println " updates : "
|
||||
updates.each { out.println " $it" }
|
||||
}
|
||||
}
|
||||
|
||||
out.flush()
|
||||
}
|
||||
|
||||
protected Map createMap(AssetManager manager) {
|
||||
def result = [:]
|
||||
result.projectName = manager.project
|
||||
result.repository = manager.repositoryUrl
|
||||
result.localPath = manager.projectPath?.toString()
|
||||
result.manifest = manager.manifest.toMap()
|
||||
result.revisions = manager.getBranchesAndTags(checkForUpdates)
|
||||
return result
|
||||
}
|
||||
|
||||
protected printJson(Map map) {
|
||||
out.println JsonOutput.prettyPrint(JsonOutput.toJson(map))
|
||||
out.flush()
|
||||
}
|
||||
|
||||
protected printYaml(Map map) {
|
||||
out.println new Yaml().dump(map).toString()
|
||||
out.flush()
|
||||
}
|
||||
|
||||
final static private BLANK = ' '
|
||||
final static private NEWLINE = '\n'
|
||||
|
||||
static String status(boolean detailed = false) {
|
||||
getInfo( detailed ? 1 : 0, true )
|
||||
}
|
||||
|
||||
static private String totMem(OperatingSystemMXBean os) {
|
||||
try {
|
||||
new MemoryUnit(os.totalPhysicalMemorySize).toString()
|
||||
}
|
||||
catch (Throwable t) {
|
||||
log.debug "Unable to fetch totalPhysicalMemorySize - ${t.message ?: t}"
|
||||
return '-'
|
||||
}
|
||||
}
|
||||
|
||||
static private String freeMem(OperatingSystemMXBean os) {
|
||||
try {
|
||||
return new MemoryUnit(os.freePhysicalMemorySize).toString()
|
||||
}
|
||||
catch (Throwable t) {
|
||||
log.debug "Unable to fetch freePhysicalMemorySize - ${t.message ?: t}"
|
||||
return '-'
|
||||
}
|
||||
}
|
||||
|
||||
static private String totSwap(OperatingSystemMXBean os) {
|
||||
try {
|
||||
new MemoryUnit(os.totalSwapSpaceSize).toString()
|
||||
}
|
||||
catch (Throwable t) {
|
||||
log.debug "Unable to fetch totalSwapSpaceSize - ${t.message ?: t}"
|
||||
return '-'
|
||||
}
|
||||
}
|
||||
|
||||
static private String freeSwap(OperatingSystemMXBean os) {
|
||||
try {
|
||||
return new MemoryUnit(os.freeSwapSpaceSize).toString()
|
||||
}
|
||||
catch (Throwable t) {
|
||||
log.debug "Unable to fetch freeSwapSpaceSize - ${t.message ?: t}"
|
||||
return '-'
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return A string containing some system runtime information
|
||||
*/
|
||||
static String getInfo(int level, boolean printProc=false) {
|
||||
|
||||
def props = System.getProperties()
|
||||
def result = new StringBuilder()
|
||||
result << BLANK << "Version: ${BuildInfo.version} build ${BuildInfo.buildNum}" << NEWLINE
|
||||
result << BLANK << "Created: ${BuildInfo.timestampUTC} ${BuildInfo.timestampDelta}" << NEWLINE
|
||||
result << BLANK << "System: ${props['os.name']} ${props['os.version']}" << NEWLINE
|
||||
result << BLANK << "Runtime: Groovy ${GroovySystem.getVersion()} on ${System.getProperty('java.vm.name')} ${props['java.runtime.version']}" << NEWLINE
|
||||
result << BLANK << "Encoding: ${System.getProperty('file.encoding')} (${System.getProperty('sun.jnu.encoding')})" << NEWLINE
|
||||
|
||||
if( printProc ) {
|
||||
final os = (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean()
|
||||
result << BLANK << "Process: ${ManagementFactory.getRuntimeMXBean().getName()} " << getLocalAddress() << NEWLINE
|
||||
result << BLANK << "CPUs: ${os.availableProcessors} - Mem: ${totMem(os)} (${freeMem(os)}) - Swap: ${totSwap(os)} (${freeSwap(os)})"
|
||||
if( Threads.useVirtual() )
|
||||
result << " - Virtual threads ON"
|
||||
}
|
||||
|
||||
if( level == 0 )
|
||||
return result.toString()
|
||||
|
||||
List<String> capsule = []
|
||||
List<String> args = []
|
||||
ManagementFactory
|
||||
.getRuntimeMXBean()
|
||||
.getInputArguments()
|
||||
.each { String it ->
|
||||
if( it.startsWith('-Dcapsule.'))
|
||||
capsule << it.substring(2)
|
||||
else
|
||||
args << it
|
||||
}
|
||||
|
||||
// file system
|
||||
result << BLANK << "File systems: "
|
||||
result << FileSystemProvider.installedProviders().collect { it.scheme }.join(', ')
|
||||
result << NEWLINE
|
||||
|
||||
// plugins
|
||||
result << BLANK << "Core plugins: "
|
||||
result << DefaultPlugins.INSTANCE.toSortedString(', ')
|
||||
result << NEWLINE
|
||||
|
||||
// JVM options
|
||||
result << BLANK << "JVM opts:" << NEWLINE
|
||||
for( String entry : args ) {
|
||||
int p = entry.indexOf('=')
|
||||
if( p!=-1 ) {
|
||||
def key = entry.substring(0,p)
|
||||
def value = entry.substring(p+1)
|
||||
dump(key,value,2, result)
|
||||
}
|
||||
else {
|
||||
dump(entry,null,2, result)
|
||||
}
|
||||
}
|
||||
|
||||
// Capsule options
|
||||
dump("Capsule", capsule, 1, result)
|
||||
|
||||
// Env
|
||||
result << BLANK << "Environment:" << NEWLINE
|
||||
def entries = System.getenv().keySet().sort()
|
||||
for( def key : entries ) {
|
||||
if( key.startsWith('NXF_') || level>1 ) {
|
||||
dump( key, System.getenv().get(key), 2, result )
|
||||
}
|
||||
}
|
||||
|
||||
// java properties
|
||||
if( level>1 ) {
|
||||
result << BLANK << "Properties:" << NEWLINE
|
||||
entries = System.getProperties().keySet().sort()
|
||||
for( String key : entries ) {
|
||||
if( key == 'java.class.path' ) continue
|
||||
dump( key, System.getProperties().get(key), 2, result )
|
||||
}
|
||||
}
|
||||
|
||||
// Class path
|
||||
dump("Class-path" , System.getProperty('java.class.path'), 1, result)
|
||||
|
||||
// final string
|
||||
return result.toString()
|
||||
}
|
||||
|
||||
static private void dump(String key, def value, int indent, StringBuilder result) {
|
||||
|
||||
if( value instanceof String && value.count(':') && (key=~/.*PATH$/ || key=='PERL5LIB' || key.contains('.path') || key.contains('-path') || key.contains('.dir')) ) {
|
||||
value = value.split(":") as Collection
|
||||
}
|
||||
else if( value?.class?.isArray() ) {
|
||||
value = value as Collection
|
||||
}
|
||||
|
||||
if ( value instanceof Collection ) {
|
||||
blanks(indent, result)
|
||||
result << key << (indent==1 ? ':' : '=')
|
||||
for( String item : value ) {
|
||||
result << NEWLINE
|
||||
blanks(indent+1, result)
|
||||
result << item
|
||||
}
|
||||
}
|
||||
else if( value ) {
|
||||
blanks(indent, result)
|
||||
result << key << (indent==1 ? ':' : '=')
|
||||
if( value == '\n' ) {
|
||||
result << '\\n'
|
||||
}
|
||||
else if( value == '\r' ) {
|
||||
result << '\\r'
|
||||
}
|
||||
else if( value == '\n\r' ) {
|
||||
result << '\\n\\r'
|
||||
}
|
||||
else if( value == '\r\n' ) {
|
||||
result << '\\r\\n'
|
||||
}
|
||||
else {
|
||||
result << value
|
||||
}
|
||||
}
|
||||
else if( key ) {
|
||||
blanks(indent, result)
|
||||
result << key
|
||||
}
|
||||
result << NEWLINE
|
||||
}
|
||||
|
||||
static private blanks( int n, StringBuilder result ) {
|
||||
for( int i=0; i<n; i++ ) {
|
||||
result << BLANK
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return A string holding the local host name and address used for logging
|
||||
*/
|
||||
static private getLocalAddress() {
|
||||
try {
|
||||
return "[${InetAddress.getLocalHost().getHostAddress()}]"
|
||||
}
|
||||
catch(Exception e) {
|
||||
return null
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
|
||||
import com.beust.jcommander.DynamicParameter
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.Parameters
|
||||
import groovy.transform.CompileDynamic
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.Session
|
||||
import nextflow.container.inspect.ContainerInspectMode
|
||||
import nextflow.container.inspect.ContainersInspector
|
||||
import nextflow.util.LoggerHelper
|
||||
/**
|
||||
* Implement `inspect` command
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
@Parameters(commandDescription = "Inspect process settings in a pipeline project")
|
||||
class CmdInspect extends CmdBase {
|
||||
|
||||
@Override
|
||||
String getName() {
|
||||
return 'inspect'
|
||||
}
|
||||
|
||||
@Parameter(names=['-concretize'], description = "Build the container images resolved by the inspect command")
|
||||
boolean concretize
|
||||
|
||||
@Parameter(names=['-c','-config'], hidden = true)
|
||||
List<String> runConfig
|
||||
|
||||
@Parameter(names=['-format'], description = "Inspect output format. Can be 'json' or 'config'")
|
||||
String format = 'json'
|
||||
|
||||
@Parameter(names=['-i','-ignore-errors'], description = 'Ignore errors while inspecting the pipeline')
|
||||
boolean ignoreErrors
|
||||
|
||||
@DynamicParameter(names = '--', hidden = true)
|
||||
Map<String,String> params = new LinkedHashMap<>()
|
||||
|
||||
@Parameter(names='-params-file', description = 'Load script parameters from a JSON/YAML file')
|
||||
String paramsFile
|
||||
|
||||
@Parameter(names=['-profile'], description = 'Use the given configuration profile(s)')
|
||||
String profile
|
||||
|
||||
@Parameter(names=['-r','-revision'], description = 'Revision of the project to inspect (either a git branch, tag or commit SHA number)')
|
||||
String revision
|
||||
|
||||
@Parameter(description = 'Project name or repository url')
|
||||
List<String> args
|
||||
|
||||
@Override
|
||||
void run() {
|
||||
ContainerInspectMode.activate(!concretize)
|
||||
// configure quiet mode
|
||||
LoggerHelper.setQuiet(true)
|
||||
// setup the target run command
|
||||
final target = new CmdRun()
|
||||
target.launcher = this.launcher
|
||||
target.args = args
|
||||
target.profile = this.profile
|
||||
target.revision = this.revision
|
||||
target.runConfig = this.runConfig
|
||||
target.params = this.params
|
||||
target.paramsFile = this.paramsFile
|
||||
target.preview = true
|
||||
target.previewAction = this.&applyInspect
|
||||
target.ansiLog = false
|
||||
target.skipHistoryFile = true
|
||||
// run it
|
||||
target.run()
|
||||
}
|
||||
|
||||
protected void applyInspect(Session session) {
|
||||
// slow down max rate when concretize is specified
|
||||
if( concretize ) {
|
||||
configureMaxRate(session.config)
|
||||
}
|
||||
// run the inspector
|
||||
new ContainersInspector(concretize)
|
||||
.withFormat(format)
|
||||
.withIgnoreErrors(ignoreErrors)
|
||||
.printContainers()
|
||||
}
|
||||
|
||||
@CompileDynamic
|
||||
protected void configureMaxRate(Map config) {
|
||||
if( config.wave == null )
|
||||
config.wave = new HashMap()
|
||||
if( config.wave.httpClient == null )
|
||||
config.wave.httpClient = new HashMap()
|
||||
config.wave.httpClient.maxRate = '5/30sec'
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.Parameters
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.plugin.Plugins
|
||||
import org.pf4j.ExtensionPoint
|
||||
/**
|
||||
* Extends `run` command to support Kubernetes deployment
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
@Parameters(commandDescription = "Execute a workflow in a Kubernetes cluster (experimental)")
|
||||
class CmdKubeRun extends CmdRun {
|
||||
|
||||
interface KubeCommand extends ExtensionPoint {
|
||||
int run(CmdKubeRun cmd, String pipeline, List<String> args)
|
||||
}
|
||||
|
||||
static private String POD_NAME = /[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/
|
||||
|
||||
/**
|
||||
* One or more volume claims mounts
|
||||
*/
|
||||
@Parameter(names = ['-v','-volume-mount'], description = 'Volume claim mounts eg. my-pvc:/mnt/path')
|
||||
List<String> volMounts
|
||||
|
||||
@Parameter(names = ['-n','-namespace'], description = 'Specify the K8s namespace to use')
|
||||
String namespace
|
||||
|
||||
@Parameter(names = '-head-image', description = 'Specify the container image for the Nextflow driver pod')
|
||||
String headImage
|
||||
|
||||
@Parameter(names = '-pod-image', description = 'Alias for -head-image (deprecated)')
|
||||
String podImage
|
||||
|
||||
@Parameter(names = '-head-cpus', description = 'Specify number of CPUs requested for the Nextflow driver pod')
|
||||
int headCpus
|
||||
|
||||
@Parameter(names = '-head-memory', description = 'Specify amount of memory requested for the Nextflow driver pod')
|
||||
String headMemory
|
||||
|
||||
@Parameter(names = '-head-prescript', description = 'Specify script to be run before nextflow run starts')
|
||||
String headPreScript
|
||||
|
||||
@Parameter(names= '-remoteConfig', description = 'Add the specified file from the K8s cluster to configuration set', hidden = true )
|
||||
List<String> runRemoteConfig
|
||||
|
||||
@Parameter(names=['-remoteProfile'], description = 'Choose a configuration profile in the remoteConfig')
|
||||
String remoteProfile
|
||||
|
||||
|
||||
@Override
|
||||
String getName() { 'kuberun' }
|
||||
|
||||
@Override
|
||||
protected void checkRunName() {
|
||||
if( runName && !runName.matches(POD_NAME) )
|
||||
throw new AbortOperationException("Not a valid K8s pod name -- It can only contain lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character")
|
||||
super.checkRunName()
|
||||
runName = runName.replace('_','-')
|
||||
}
|
||||
|
||||
boolean background() { launcher.options.background }
|
||||
|
||||
protected hasAnsiLogFlag() { launcher.options.hasAnsiLogFlag() }
|
||||
|
||||
@Override
|
||||
void run() {
|
||||
final scriptArgs = (args?.size()>1 ? args[1..-1] : []) as List<String>
|
||||
final pipeline = stdin ? '-' : ( args ? args[0] : null )
|
||||
if( !pipeline )
|
||||
throw new AbortOperationException("No project name was specified")
|
||||
if( hasAnsiLogFlag() )
|
||||
log.warn "Ansi logging not supported by kuberun command"
|
||||
if( podImage ) {
|
||||
log.warn "-pod-image is deprecated (use -head-image instead)"
|
||||
headImage = podImage
|
||||
}
|
||||
checkRunName()
|
||||
Plugins.init()
|
||||
Plugins.start('nf-k8s')
|
||||
// load the command operations
|
||||
final command = Plugins.getExtension(KubeCommand)
|
||||
final status = command.run(this, pipeline, scriptArgs)
|
||||
System.exit(status)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
|
||||
import com.beust.jcommander.DynamicParameter
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.Parameters
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.plugin.Plugins
|
||||
import org.pf4j.ExtensionPoint
|
||||
|
||||
/**
|
||||
* Implements the 'nextflow launch' command
|
||||
*
|
||||
* @author Phil Ewels <phil.ewels@seqera.io>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
@Parameters(commandDescription = "Launch a workflow in Seqera Platform")
|
||||
class CmdLaunch extends CmdBase implements UsageAware {
|
||||
|
||||
interface LaunchCommand extends ExtensionPoint {
|
||||
void launch(LaunchOptions options)
|
||||
}
|
||||
|
||||
static final public String NAME = 'launch'
|
||||
|
||||
@Override
|
||||
String getName() { NAME }
|
||||
|
||||
@Parameter(description = 'Pipeline repository URL')
|
||||
List<String> args
|
||||
|
||||
@Parameter(names = ['-workspace'], description = 'Workspace name')
|
||||
String workspace
|
||||
|
||||
@Parameter(names = ['-compute-env'], description = 'Compute environment name')
|
||||
String computeEnv
|
||||
|
||||
@Parameter(names = ['-name'], description = 'Assign a mnemonic name to the pipeline run')
|
||||
String runName
|
||||
|
||||
@Parameter(names = ['-w', '-work-dir'], description = 'Directory where intermediate result files are stored')
|
||||
String workDir
|
||||
|
||||
@Parameter(names = ['-r', '-revision'], description = 'Revision of the project to run (either a git branch, tag or commit SHA number)')
|
||||
String revision
|
||||
|
||||
@Parameter(names = ['-profile'], description = 'Choose a configuration profile')
|
||||
String profile
|
||||
|
||||
@Parameter(names = ['-c', '-config'], description = 'Add the specified file to configuration set')
|
||||
List<String> configFiles
|
||||
|
||||
@Parameter(names = ['-params-file'], description = 'Load script parameters from a JSON/YAML file')
|
||||
String paramsFile
|
||||
|
||||
@Parameter(names = ['-entry'], description = 'Entry workflow name to be executed')
|
||||
String entryName
|
||||
|
||||
@Parameter(names = ['-resume'], description = 'Execute the script using the cached results')
|
||||
String resume
|
||||
|
||||
@Parameter(names = ['-latest'], description = 'Pull latest changes before run')
|
||||
boolean latest
|
||||
|
||||
@Parameter(names = ['-stub-run', '-stub'], description = 'Execute the workflow replacing process scripts with command stubs')
|
||||
boolean stubRun
|
||||
|
||||
@Parameter(names = ['-main-script'], description = 'The script file to be executed when launching a project directory or repository')
|
||||
String mainScript
|
||||
|
||||
@Parameter(names = ['-user-secret'], description = 'Specify a user secret name to use in the pipeline (can be specified multiple times)')
|
||||
List<String> userSecrets = []
|
||||
|
||||
@Parameter(names = ['-workspace-secret'], description = 'Specify a workspace secret name to use in the pipeline (can be specified multiple times)')
|
||||
List<String> workspaceSecrets = []
|
||||
|
||||
/**
|
||||
* Defines the parameters to be passed to the pipeline script
|
||||
*/
|
||||
@DynamicParameter(names = '--', description = 'Set a parameter used by the pipeline', hidden = true)
|
||||
Map<String, String> params = new LinkedHashMap<>()
|
||||
|
||||
private LaunchCommand operation
|
||||
|
||||
@Override
|
||||
void run() {
|
||||
// Validate required parameters
|
||||
if (!args || args.isEmpty()) {
|
||||
log.debug "No pipeline repository URL provided"
|
||||
throw new AbortOperationException("Pipeline repository URL is required")
|
||||
}
|
||||
|
||||
// Load the Launch command implementation
|
||||
this.operation = loadOperation()
|
||||
if (!operation)
|
||||
throw new IllegalStateException("Unable to load launch extension.")
|
||||
|
||||
// Create options object
|
||||
final options = new LaunchOptions(
|
||||
pipeline: args[0],
|
||||
workspace: workspace,
|
||||
computeEnv: computeEnv,
|
||||
runName: runName,
|
||||
workDir: workDir,
|
||||
revision: revision,
|
||||
profile: profile,
|
||||
configFiles: configFiles,
|
||||
paramsFile: paramsFile,
|
||||
entryName: entryName,
|
||||
resume: resume,
|
||||
latest: latest,
|
||||
stubRun: stubRun,
|
||||
mainScript: mainScript,
|
||||
params: params,
|
||||
launcher: launcher,
|
||||
userSecrets: userSecrets,
|
||||
workspaceSecrets: workspaceSecrets
|
||||
)
|
||||
|
||||
// Execute launch
|
||||
operation.launch(options)
|
||||
}
|
||||
|
||||
@Override
|
||||
void usage() {
|
||||
usage(null)
|
||||
}
|
||||
|
||||
@Override
|
||||
void usage(List<String> args) {
|
||||
def result = []
|
||||
result << this.getClass().getAnnotation(Parameters).commandDescription()
|
||||
result << 'Usage: nextflow launch <pipeline> [options]'
|
||||
result << ''
|
||||
result << 'Options:'
|
||||
result << ' -workspace <name> Workspace name'
|
||||
result << ' -compute-env <name> Compute environment name (default: primary)'
|
||||
result << ' -name <name> Assign a mnemonic name to the pipeline run'
|
||||
result << ' -w, -work-dir <path> Directory where intermediate result files are stored'
|
||||
result << ' -r, -revision <revision> Revision of the project to run (git branch, tag or commit SHA)'
|
||||
result << ' -profile <profile> Choose a configuration profile'
|
||||
result << ' -c, -config <file> Add the specified file to configuration set'
|
||||
result << ' -params-file <file> Load script parameters from a JSON/YAML file'
|
||||
result << ' -entry <name> Entry workflow name to be executed'
|
||||
result << ' -resume [session] Execute the script using the cached results'
|
||||
result << ' -latest Pull latest changes before run'
|
||||
result << ' -stub-run, -stub Execute the workflow replacing process scripts with command stubs'
|
||||
result << ' -main-script <file> The script file to be executed when launching a project'
|
||||
result << ' -user-secret <name> User secret name to use in the pipeline (can be specified multiple times)'
|
||||
result << ' -workspace-secret <name> Workspace secret name to use in the pipeline (can be specified multiple times)'
|
||||
result << ' --<param>=<value> Set a parameter used by the pipeline'
|
||||
result << ''
|
||||
println result.join('\n').toString()
|
||||
}
|
||||
|
||||
protected LaunchCommand loadOperation() {
|
||||
// Setup the plugins system and load the launch provider
|
||||
Plugins.init()
|
||||
// Load the config
|
||||
Plugins.start('nf-tower')
|
||||
// Get Launch command operations implementation from plugins
|
||||
return Plugins.getExtension(LaunchCommand)
|
||||
}
|
||||
|
||||
/**
|
||||
* Data class to hold launch options
|
||||
*/
|
||||
@CompileStatic
|
||||
static class LaunchOptions {
|
||||
String pipeline
|
||||
String workspace
|
||||
String computeEnv
|
||||
String runName
|
||||
String workDir
|
||||
String revision
|
||||
String profile
|
||||
List<String> configFiles
|
||||
String paramsFile
|
||||
String entryName
|
||||
String resume
|
||||
boolean latest
|
||||
boolean stubRun
|
||||
String mainScript
|
||||
Map<String, String> params
|
||||
Launcher launcher
|
||||
List<String> userSecrets
|
||||
List<String> workspaceSecrets
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,313 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
|
||||
import java.nio.file.Paths
|
||||
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.Parameters
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.config.ConfigBuilder
|
||||
import nextflow.config.ConfigMap
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.plugin.Plugins
|
||||
import org.pf4j.ExtensionPoint
|
||||
|
||||
/**
|
||||
* CID command line interface
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
@Parameters(commandDescription = "Explore workflows lineage metadata", commandNames = ['li'])
|
||||
class CmdLineage extends CmdBase implements UsageAware {
|
||||
|
||||
private static final String NAME = 'lineage'
|
||||
|
||||
interface LinCommand extends ExtensionPoint {
|
||||
void list(ConfigMap config)
|
||||
void view(ConfigMap config, List<String> args)
|
||||
void render(ConfigMap config, List<String> args)
|
||||
void diff(ConfigMap config, List<String> args)
|
||||
void find(ConfigMap config, List<String> args)
|
||||
void check(ConfigMap config, List<String> args)
|
||||
}
|
||||
|
||||
interface SubCmd {
|
||||
String getName()
|
||||
String getDescription()
|
||||
void apply(List<String> args)
|
||||
void usage()
|
||||
}
|
||||
|
||||
private List<SubCmd> commands = new ArrayList<>()
|
||||
|
||||
private LinCommand operation
|
||||
|
||||
private ConfigMap config
|
||||
|
||||
CmdLineage() {
|
||||
commands << new CmdList()
|
||||
commands << new CmdView()
|
||||
commands << new CmdRender()
|
||||
commands << new CmdDiff()
|
||||
commands << new CmdFind()
|
||||
commands << new CmdCheck()
|
||||
}
|
||||
|
||||
@Parameter(hidden = true)
|
||||
List<String> args
|
||||
|
||||
@Override
|
||||
String getName() {
|
||||
return NAME
|
||||
}
|
||||
|
||||
@Override
|
||||
void run() {
|
||||
if( !args ) {
|
||||
usage(List.of())
|
||||
return
|
||||
}
|
||||
// setup the plugins system and load the secrets provider
|
||||
Plugins.init()
|
||||
// load the config
|
||||
this.config = new ConfigBuilder()
|
||||
.setOptions(launcher.options)
|
||||
.setBaseDir(Paths.get('.'))
|
||||
.build()
|
||||
// init plugins
|
||||
Plugins.load(config)
|
||||
// load the command operations
|
||||
this.operation = Plugins.getExtension(LinCommand)
|
||||
if( !operation )
|
||||
throw new IllegalStateException("Unable to load lineage extensions.")
|
||||
// consume the first argument
|
||||
getCmd(args).apply(args.drop(1))
|
||||
}
|
||||
|
||||
/**
|
||||
* Print the command usage help
|
||||
*/
|
||||
void usage() {
|
||||
usage(args)
|
||||
}
|
||||
|
||||
/**
|
||||
* Print the command usage help
|
||||
*
|
||||
* @param args The arguments as entered by the user
|
||||
*/
|
||||
void usage(List<String> args) {
|
||||
if( !args ) {
|
||||
List<String> result = []
|
||||
result << this.getClass().getAnnotation(Parameters).commandDescription()
|
||||
result << "Usage: nextflow $NAME <sub-command> [options]".toString()
|
||||
result << ''
|
||||
result << 'Commands:'
|
||||
int len = 0
|
||||
commands.forEach {len = it.name.size() > len ? it.name.size() : len }
|
||||
commands.sort(){it.name}.each { result << " ${it.name.padRight(len)}\t${it.description}".toString() }
|
||||
result << ''
|
||||
println result.join('\n').toString()
|
||||
}
|
||||
else {
|
||||
def sub = commands.find { it.name == args[0] }
|
||||
if( sub )
|
||||
sub.usage()
|
||||
else {
|
||||
throw new AbortOperationException("Unknown $NAME sub-command: ${args[0]}")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected SubCmd getCmd(List<String> args) {
|
||||
|
||||
def cmd = commands.find { it.name == args[0] }
|
||||
if( cmd ) {
|
||||
return cmd
|
||||
}
|
||||
|
||||
def matches = commands.collect{ it.name }.closest(args[0])
|
||||
def msg = "Unknown cloud sub-command: ${args[0]}"
|
||||
if( matches )
|
||||
msg += " -- Did you mean one of these?\n" + matches.collect { " $it"}.join('\n')
|
||||
throw new AbortOperationException(msg)
|
||||
}
|
||||
|
||||
class CmdList implements SubCmd {
|
||||
|
||||
@Override
|
||||
String getName() {
|
||||
return 'list'
|
||||
}
|
||||
|
||||
@Override
|
||||
String getDescription() {
|
||||
return 'List the executions with lineage enabled'
|
||||
}
|
||||
|
||||
@Override
|
||||
void apply(List<String> args) {
|
||||
if (args.size() != 0) {
|
||||
println("ERROR: Incorrect number of parameters")
|
||||
return
|
||||
}
|
||||
operation.list(config)
|
||||
}
|
||||
|
||||
@Override
|
||||
void usage() {
|
||||
println description
|
||||
println "Usage: nextflow $NAME $name"
|
||||
}
|
||||
}
|
||||
|
||||
class CmdView implements SubCmd{
|
||||
|
||||
@Override
|
||||
String getName() {
|
||||
return 'view'
|
||||
}
|
||||
|
||||
@Override
|
||||
String getDescription() {
|
||||
return 'Print the description of a Lineage ID (lid)'
|
||||
}
|
||||
|
||||
void apply(List<String> args) {
|
||||
if (args.size() != 1) {
|
||||
println("ERROR: Incorrect number of parameters")
|
||||
return
|
||||
}
|
||||
|
||||
operation.view(config, args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void usage() {
|
||||
println description
|
||||
println "Usage: nextflow $NAME $name <lid> "
|
||||
}
|
||||
}
|
||||
|
||||
class CmdRender implements SubCmd {
|
||||
|
||||
@Override
|
||||
String getName() { 'render' }
|
||||
|
||||
@Override
|
||||
String getDescription() {
|
||||
return 'Render the lineage graph for a workflow output'
|
||||
}
|
||||
|
||||
void apply(List<String> args) {
|
||||
if (args.size() < 1 || args.size() > 2) {
|
||||
println("ERROR: Incorrect number of parameters")
|
||||
return
|
||||
}
|
||||
|
||||
operation.render(config, args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void usage() {
|
||||
println description
|
||||
println "Usage: nextflow $NAME $name <workflow output lid> [<html output file>]"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
class CmdDiff implements SubCmd {
|
||||
|
||||
@Override
|
||||
String getName() { 'diff' }
|
||||
|
||||
@Override
|
||||
String getDescription() {
|
||||
return 'Show differences between two lineage descriptions'
|
||||
}
|
||||
|
||||
void apply(List<String> args) {
|
||||
if (args.size() != 2) {
|
||||
println("ERROR: Incorrect number of parameters")
|
||||
return
|
||||
}
|
||||
operation.diff(config, args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void usage() {
|
||||
println description
|
||||
println "Usage: nextflow $NAME $name <lid-1> <lid-2>"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
class CmdFind implements SubCmd {
|
||||
|
||||
@Override
|
||||
String getName() { 'find' }
|
||||
|
||||
@Override
|
||||
String getDescription() {
|
||||
return 'Find lineage metadata descriptions matching with a query'
|
||||
}
|
||||
|
||||
void apply(List<String> args) {
|
||||
if (args.size() < 1) {
|
||||
println("ERROR: Incorrect number of parameters")
|
||||
return
|
||||
}
|
||||
operation.find(config, args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void usage() {
|
||||
println description
|
||||
println "Usage: nextflow $NAME $name <query>"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
class CmdCheck implements SubCmd {
|
||||
|
||||
@Override
|
||||
String getName() { 'check' }
|
||||
|
||||
@Override
|
||||
String getDescription() {
|
||||
return 'Checks the integrity of an lineage file path'
|
||||
}
|
||||
|
||||
void apply(List<String> args) {
|
||||
if (args.size() != 1) {
|
||||
println("ERROR: Incorrect number of parameters")
|
||||
return
|
||||
}
|
||||
operation.check(config, args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void usage() {
|
||||
println description
|
||||
println "Usage: nextflow $NAME $name <lid-file>"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,541 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
|
||||
import java.nio.file.Path
|
||||
import java.time.Instant
|
||||
import java.time.ZoneOffset
|
||||
import java.time.format.DateTimeFormatter
|
||||
|
||||
import com.beust.jcommander.IParameterValidator
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.Parameters
|
||||
import com.beust.jcommander.ParameterException
|
||||
import groovy.json.JsonOutput
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.BuildInfo
|
||||
import nextflow.config.control.ConfigParser
|
||||
import nextflow.config.formatter.ConfigFormattingVisitor
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.script.control.Compiler
|
||||
import nextflow.script.control.ParanoidWarning
|
||||
import nextflow.script.control.ScriptParser
|
||||
import nextflow.script.formatter.FormattingOptions
|
||||
import nextflow.script.formatter.ScriptFormattingVisitor
|
||||
import nextflow.script.parser.v2.ErrorListener
|
||||
import nextflow.script.parser.v2.ErrorSummary
|
||||
import nextflow.script.parser.v2.StandardErrorListener
|
||||
import nextflow.util.ClassLoaderFactory
|
||||
import nextflow.util.PathUtils
|
||||
import org.codehaus.groovy.control.SourceUnit
|
||||
import org.codehaus.groovy.control.messages.SyntaxErrorMessage
|
||||
import org.codehaus.groovy.control.messages.WarningMessage
|
||||
import org.codehaus.groovy.syntax.SyntaxException
|
||||
/**
|
||||
* CLI sub-command LINT
|
||||
*
|
||||
* @author Ben Sherman <bentshermann@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
@Parameters(commandDescription = "Lint Nextflow scripts and config files")
|
||||
class CmdLint extends CmdBase {
|
||||
|
||||
@Parameter(description = 'List of paths to lint')
|
||||
List<String> args = []
|
||||
|
||||
@Parameter(
|
||||
names = ['-exclude'],
|
||||
description = 'File pattern to exclude from error checking (can be specified multiple times)'
|
||||
)
|
||||
List<String> excludePatterns = ['.git', '.lineage', '.nextflow', '.nf-test', 'nf-test.config', 'work']
|
||||
|
||||
@Parameter(
|
||||
names = ['-files-from'],
|
||||
description = 'Read list of paths to lint from a text file (one per line, use - for stdin)'
|
||||
)
|
||||
String filesFrom
|
||||
|
||||
@Parameter(
|
||||
names = ['-o', '-output'],
|
||||
description = 'Output mode for reporting errors: full, extended, concise, json, markdown',
|
||||
validateWith = OutputModeValidator
|
||||
)
|
||||
String outputMode = 'full'
|
||||
|
||||
static class OutputModeValidator implements IParameterValidator {
|
||||
|
||||
private static final List<String> MODES = List.of('full', 'extended', 'concise', 'json', 'markdown')
|
||||
|
||||
@Override
|
||||
void validate(String name, String value) {
|
||||
if( !MODES.contains(value) )
|
||||
throw new ParameterException("Output mode must be one of $MODES (found: $value)")
|
||||
}
|
||||
}
|
||||
|
||||
@Parameter(
|
||||
names = ['-project-dir'],
|
||||
description = 'Path to project directory (default: .)'
|
||||
)
|
||||
String projectDir = '.'
|
||||
|
||||
@Parameter(names = ['-format'], description = 'Format scripts and config files that have no errors')
|
||||
boolean formatting
|
||||
|
||||
@Parameter(names = ['-harshil-alignment'], description = 'Use Harshil alignment')
|
||||
boolean harhsilAlignment
|
||||
|
||||
@Parameter(names = ['-sort-declarations'], description = 'Sort script declarations in Nextflow scripts')
|
||||
boolean sortDeclarations
|
||||
|
||||
@Parameter(names=['-spaces'], description = 'Number of spaces to indent')
|
||||
int spaces
|
||||
|
||||
@Parameter(names = ['-tabs'], description = 'Indent with tabs')
|
||||
boolean tabs
|
||||
|
||||
private ScriptParser scriptParser
|
||||
|
||||
private ConfigParser configParser
|
||||
|
||||
private ErrorListener errorListener
|
||||
|
||||
private FormattingOptions formattingOptions
|
||||
|
||||
private ErrorSummary summary = new ErrorSummary()
|
||||
|
||||
@Override
|
||||
String getName() { 'lint' }
|
||||
|
||||
@Override
|
||||
void run() {
|
||||
// read input files from positional args and -files-from option
|
||||
final inputs = getInputs(args, filesFrom)
|
||||
|
||||
if( !inputs )
|
||||
throw new AbortOperationException("Error: No input files were specified")
|
||||
|
||||
if( spaces && tabs )
|
||||
throw new AbortOperationException("Error: Cannot specify both `-spaces` and `-tabs`")
|
||||
|
||||
if( !spaces && !tabs )
|
||||
spaces = 4
|
||||
|
||||
final baseDir = Path.of(projectDir)
|
||||
final libDir = baseDir.resolve('lib')
|
||||
final classLoader = ClassLoaderFactory.create([ libDir ])
|
||||
|
||||
scriptParser = new ScriptParser(baseDir, classLoader)
|
||||
configParser = new ConfigParser()
|
||||
errorListener = switch( outputMode ) {
|
||||
case 'json' -> new JsonErrorListener()
|
||||
case 'markdown' -> new MarkdownErrorListener()
|
||||
default -> new StandardErrorListener(outputMode, launcher.options.ansiLog, launcher.options.quiet)
|
||||
}
|
||||
formattingOptions = new FormattingOptions(spaces, !tabs, harhsilAlignment, false, sortDeclarations)
|
||||
|
||||
errorListener.beforeAll()
|
||||
|
||||
// collect files to lint
|
||||
final List<File> files = []
|
||||
|
||||
for( final input : inputs ) {
|
||||
PathUtils.visitFiles(
|
||||
Path.of(input),
|
||||
(path) -> !PathUtils.isExcluded(path, excludePatterns),
|
||||
(path) -> files.add(path.toFile()))
|
||||
}
|
||||
|
||||
// parse and analyze files
|
||||
for( final file : files )
|
||||
parse(file)
|
||||
|
||||
scriptParser.analyze()
|
||||
configParser.analyze()
|
||||
|
||||
// report errors
|
||||
checkErrors(scriptParser.compiler())
|
||||
checkErrors(configParser.compiler())
|
||||
|
||||
// format files if specified
|
||||
if( formatting ) {
|
||||
for( final file : files )
|
||||
format(file)
|
||||
}
|
||||
|
||||
// print summary
|
||||
errorListener.afterAll(summary)
|
||||
|
||||
// If there were errors, throw an exception to return a non-zero exit code
|
||||
if( summary.errors > 0 )
|
||||
throw new AbortOperationException()
|
||||
}
|
||||
|
||||
private static List<String> getInputs(List<String> args, String filesFrom) {
|
||||
final List<String> result = []
|
||||
result.addAll(args)
|
||||
|
||||
if( filesFrom ) {
|
||||
final lines = filesFrom == '-'
|
||||
? System.in.readLines()
|
||||
: Path.of(filesFrom).readLines()
|
||||
for( final line : lines ) {
|
||||
final trimmed = line.trim()
|
||||
if( trimmed )
|
||||
result.add(trimmed)
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
private void parse(File file) {
|
||||
final name = file.getName()
|
||||
if( name.endsWith('.nf') )
|
||||
parseScript(file)
|
||||
else if( name.endsWith('.config') )
|
||||
parseConfig(file)
|
||||
}
|
||||
|
||||
private void parseScript(File file) {
|
||||
log.debug "Linting script ${file}"
|
||||
errorListener.beforeFile(file)
|
||||
scriptParser.parse(file)
|
||||
}
|
||||
|
||||
private void parseConfig(File file) {
|
||||
log.debug "Linting config ${file}"
|
||||
errorListener.beforeFile(file)
|
||||
configParser.parse(file)
|
||||
}
|
||||
|
||||
private void checkErrors(Compiler compiler) {
|
||||
compiler.getSources()
|
||||
.values()
|
||||
.stream()
|
||||
.sorted(Comparator.comparing((SourceUnit source) -> source.getSource().getURI()))
|
||||
.forEach((source) -> {
|
||||
final errorCollector = source.getErrorCollector()
|
||||
final hasWarnings = (errorCollector.getWarnings() ?: []).stream()
|
||||
.anyMatch(warning -> warning !instanceof ParanoidWarning)
|
||||
if( errorCollector.hasErrors() || hasWarnings )
|
||||
printErrors(source)
|
||||
if( errorCollector.hasErrors() )
|
||||
summary.filesWithErrors += 1
|
||||
else
|
||||
summary.filesWithoutErrors += 1
|
||||
if( hasWarnings )
|
||||
summary.filesWithWarnings += 1
|
||||
else
|
||||
summary.filesWithoutWarnings += 1
|
||||
})
|
||||
}
|
||||
|
||||
private void printErrors(SourceUnit source) {
|
||||
errorListener.beforeErrors()
|
||||
|
||||
final errors = source.getErrorCollector().getErrors() ?: []
|
||||
errors.stream()
|
||||
.filter(message -> message instanceof SyntaxErrorMessage)
|
||||
.map(message -> ((SyntaxErrorMessage) message).getCause())
|
||||
.sorted(ERROR_COMPARATOR)
|
||||
.forEach((cause) -> {
|
||||
errorListener.onError(cause, source.getName(), source)
|
||||
summary.errors += 1
|
||||
})
|
||||
|
||||
final warnings = source.getErrorCollector().getWarnings() ?: []
|
||||
warnings.stream()
|
||||
.filter(warning -> warning !instanceof ParanoidWarning)
|
||||
.sorted(WARNING_COMPARATOR)
|
||||
.forEach((warning) -> {
|
||||
errorListener.onWarning(warning, source.getName(), source)
|
||||
summary.warnings += 1
|
||||
})
|
||||
|
||||
errorListener.afterErrors()
|
||||
}
|
||||
|
||||
private static final Comparator<SyntaxException> ERROR_COMPARATOR = (SyntaxException a, SyntaxException b) -> {
|
||||
return a.getStartLine() != b.getStartLine()
|
||||
? a.getStartLine() - b.getStartLine()
|
||||
: a.getStartColumn() - b.getStartColumn()
|
||||
}
|
||||
|
||||
private static final Comparator<WarningMessage> WARNING_COMPARATOR = (WarningMessage w1, WarningMessage w2) -> {
|
||||
final a = w1.getContext()
|
||||
final b = w2.getContext()
|
||||
return a.getStartLine() != b.getStartLine()
|
||||
? a.getStartLine() - b.getStartLine()
|
||||
: a.getStartColumn() - b.getStartColumn()
|
||||
}
|
||||
|
||||
private void format(File file) {
|
||||
final name = file.getName()
|
||||
final result =
|
||||
name.endsWith('.nf') ? formatScript(file) :
|
||||
name.endsWith('.config') ? formatConfig(file) :
|
||||
null
|
||||
|
||||
if( result != null && file.text != result ) {
|
||||
summary.filesFormatted += 1
|
||||
file.text = result
|
||||
}
|
||||
}
|
||||
|
||||
private String formatScript(File file) {
|
||||
final source = scriptParser.compiler().getSource(file.toURI())
|
||||
if( source.getErrorCollector().hasErrors() ) {
|
||||
printErrors(source)
|
||||
return null
|
||||
}
|
||||
|
||||
log.debug "Formatting script ${file}"
|
||||
errorListener.beforeFormat(file)
|
||||
|
||||
final formatter = new ScriptFormattingVisitor(source, formattingOptions)
|
||||
formatter.visit()
|
||||
return formatter.toString()
|
||||
}
|
||||
|
||||
private String formatConfig(File file) {
|
||||
final source = configParser.compiler().getSource(file.toURI())
|
||||
if( source.getErrorCollector().hasErrors() ) {
|
||||
printErrors(source)
|
||||
return null
|
||||
}
|
||||
|
||||
log.debug "Formatting config ${file}"
|
||||
errorListener.beforeFormat(file)
|
||||
|
||||
final formatter = new ConfigFormattingVisitor(source, formattingOptions)
|
||||
formatter.visit()
|
||||
return formatter.toString()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@CompileStatic
|
||||
class JsonErrorListener implements ErrorListener {
|
||||
|
||||
private List<Map> errors = []
|
||||
|
||||
private List<Map> warnings = []
|
||||
|
||||
@Override
|
||||
void beforeAll() {
|
||||
}
|
||||
|
||||
@Override
|
||||
void beforeFile(File file) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void beforeErrors() {
|
||||
}
|
||||
|
||||
@Override
|
||||
void onError(SyntaxException error, String filename, SourceUnit source) {
|
||||
errors.add([
|
||||
filename: filename,
|
||||
startLine: error.getStartLine(),
|
||||
startColumn: error.getStartColumn(),
|
||||
message: error.getOriginalMessage()
|
||||
])
|
||||
}
|
||||
|
||||
@Override
|
||||
void onWarning(WarningMessage warning, String filename, SourceUnit source) {
|
||||
final token = warning.getContext().getRoot()
|
||||
warnings.add([
|
||||
filename: filename,
|
||||
startLine: token.getStartLine(),
|
||||
startColumn: token.getStartColumn(),
|
||||
message: warning.getMessage()
|
||||
])
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterErrors() {
|
||||
}
|
||||
|
||||
@Override
|
||||
void beforeFormat(File file) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterAll(ErrorSummary summary) {
|
||||
final result = [
|
||||
date: Instant.now().toString(),
|
||||
summary: summary,
|
||||
errors: errors,
|
||||
warnings: warnings
|
||||
]
|
||||
println JsonOutput.prettyPrint(JsonOutput.toJson(result))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@CompileStatic
|
||||
class MarkdownErrorListener implements ErrorListener {
|
||||
|
||||
private static class LintEntry {
|
||||
String filename
|
||||
int startLine
|
||||
int startColumn
|
||||
int endLine
|
||||
int endColumn
|
||||
String message
|
||||
SourceUnit source
|
||||
}
|
||||
|
||||
private List<LintEntry> errors = []
|
||||
|
||||
private List<LintEntry> warnings = []
|
||||
|
||||
@Override
|
||||
void beforeAll() {
|
||||
}
|
||||
|
||||
@Override
|
||||
void beforeFile(File file) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void beforeErrors() {
|
||||
}
|
||||
|
||||
@Override
|
||||
void onError(SyntaxException error, String filename, SourceUnit source) {
|
||||
errors.add(new LintEntry(
|
||||
filename: filename,
|
||||
startLine: error.getStartLine(),
|
||||
startColumn: error.getStartColumn(),
|
||||
endLine: error.getEndLine(),
|
||||
endColumn: error.getEndColumn(),
|
||||
message: error.getOriginalMessage(),
|
||||
source: source
|
||||
))
|
||||
}
|
||||
|
||||
@Override
|
||||
void onWarning(WarningMessage warning, String filename, SourceUnit source) {
|
||||
final token = warning.getContext().getRoot()
|
||||
warnings.add(new LintEntry(
|
||||
filename: filename,
|
||||
startLine: token.getStartLine(),
|
||||
startColumn: token.getStartColumn(),
|
||||
endLine: token.getStartLine(),
|
||||
endColumn: token.getStartColumn() + token.getText().length(),
|
||||
message: warning.getMessage(),
|
||||
source: source
|
||||
))
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterErrors() {
|
||||
}
|
||||
|
||||
@Override
|
||||
void beforeFormat(File file) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterAll(ErrorSummary summary) {
|
||||
final sb = new StringBuilder()
|
||||
|
||||
// Header
|
||||
sb.append('# Nextflow lint results\n\n')
|
||||
|
||||
// Metadata
|
||||
final timestamp = DateTimeFormatter.ISO_INSTANT.format(Instant.now().atOffset(ZoneOffset.UTC))
|
||||
sb.append("- Generated: ${timestamp}\n")
|
||||
sb.append("- Nextflow version: ${BuildInfo.version}\n")
|
||||
|
||||
// Summary line
|
||||
final parts = []
|
||||
if( summary.errors > 0 )
|
||||
parts.add("${summary.errors} error${summary.errors == 1 ? '' : 's'}")
|
||||
if( summary.warnings > 0 )
|
||||
parts.add("${summary.warnings} warning${summary.warnings == 1 ? '' : 's'}")
|
||||
if( parts.size() > 0 )
|
||||
sb.append("- Summary: ${parts.join(', ')}\n")
|
||||
else
|
||||
sb.append("- Summary: No issues found\n")
|
||||
|
||||
// Sort entries by filename then position
|
||||
final sortedErrors = errors.sort { a, b ->
|
||||
final cmp = a.filename <=> b.filename
|
||||
if( cmp != 0 ) return cmp
|
||||
final lineCmp = a.startLine <=> b.startLine
|
||||
if( lineCmp != 0 ) return lineCmp
|
||||
return a.startColumn <=> b.startColumn
|
||||
}
|
||||
|
||||
final sortedWarnings = warnings.sort { a, b ->
|
||||
final cmp = a.filename <=> b.filename
|
||||
if( cmp != 0 ) return cmp
|
||||
final lineCmp = a.startLine <=> b.startLine
|
||||
if( lineCmp != 0 ) return lineCmp
|
||||
return a.startColumn <=> b.startColumn
|
||||
}
|
||||
|
||||
// Errors section
|
||||
if( sortedErrors.size() > 0 ) {
|
||||
sb.append('\n## :x: Errors\n\n')
|
||||
for( final entry : sortedErrors ) {
|
||||
sb.append(formatEntry('Error', entry))
|
||||
}
|
||||
}
|
||||
|
||||
// Warnings section
|
||||
if( sortedWarnings.size() > 0 ) {
|
||||
sb.append('\n## :warning: Warnings\n\n')
|
||||
for( final entry : sortedWarnings ) {
|
||||
sb.append(formatEntry('Warning', entry))
|
||||
}
|
||||
}
|
||||
|
||||
println sb.toString().trim()
|
||||
}
|
||||
|
||||
private String formatEntry(String type, LintEntry entry) {
|
||||
final sb = new StringBuilder()
|
||||
sb.append("- ${type}: `${entry.filename}:${entry.startLine}:${entry.startColumn}`: ${entry.message}\n\n")
|
||||
|
||||
// Add code context
|
||||
final lines = entry.source.getSource().getReader().readLines()
|
||||
if( entry.startLine > 0 && entry.startLine <= lines.size() ) {
|
||||
final line = lines[entry.startLine - 1]
|
||||
final startCol = Math.max(0, entry.startColumn - 1)
|
||||
final endCol = Math.min(line.length(), Math.max(startCol + 1, entry.endColumn - 1))
|
||||
|
||||
sb.append(" ```nextflow\n")
|
||||
sb.append(" ${line}\n")
|
||||
|
||||
// Add caret markers
|
||||
final caretCount = Math.max(1, endCol - startCol)
|
||||
sb.append(" ${' ' * startCol}${'^' * caretCount}\n")
|
||||
sb.append(" ```\n\n")
|
||||
}
|
||||
|
||||
return sb.toString()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
|
||||
import com.beust.jcommander.Parameters
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.scm.AssetManager
|
||||
|
||||
/**
|
||||
* CLI sub-command LIST. Prints a list of locally installed pipelines
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
@Parameters(commandDescription = "List all downloaded projects")
|
||||
class CmdList extends CmdBase {
|
||||
|
||||
static final public NAME = 'list'
|
||||
|
||||
@Override
|
||||
final String getName() { NAME }
|
||||
|
||||
@Override
|
||||
void run() {
|
||||
|
||||
def all = AssetManager.list()
|
||||
if( !all ) {
|
||||
log.info '(none)'
|
||||
return
|
||||
}
|
||||
|
||||
all.each { println it }
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,316 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
import java.nio.file.Path
|
||||
|
||||
import ch.artecat.grengine.Grengine
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.Parameters
|
||||
import com.google.common.hash.HashCode
|
||||
import groovy.text.Template
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.file.FileHelper
|
||||
import nextflow.plugin.Plugins
|
||||
import nextflow.processor.TaskRun
|
||||
import nextflow.processor.TaskTemplateEngine
|
||||
import nextflow.trace.TraceRecord
|
||||
import nextflow.ui.TableBuilder
|
||||
|
||||
import static nextflow.cli.CmdHelper.fixEqualsOp
|
||||
|
||||
/**
|
||||
* Implements the `log` command to print tasks runtime information of an execute pipeline
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
@Parameters(commandDescription = "Print executions log and runtime info")
|
||||
class CmdLog extends CmdBase implements CacheBase {
|
||||
|
||||
static private List<String> ALL_FIELDS
|
||||
|
||||
static private DEFAULT_FIELDS = 'workdir'
|
||||
|
||||
static {
|
||||
ALL_FIELDS = []
|
||||
ALL_FIELDS.addAll( TraceRecord.FIELDS.keySet().collect { it.startsWith('%') ? 'p'+it.substring(1) : it } )
|
||||
ALL_FIELDS << 'stdout'
|
||||
ALL_FIELDS << 'stderr'
|
||||
ALL_FIELDS << 'log'
|
||||
ALL_FIELDS.sort(true)
|
||||
}
|
||||
|
||||
static final public NAME = 'log'
|
||||
|
||||
@Parameter(names = ['-s'], description='Character used to separate column values')
|
||||
String sep = '\\t'
|
||||
|
||||
@Parameter(names=['-f','-fields'], description = 'Comma separated list of fields to include in the printed log -- Use the `-l` option to show the list of available fields')
|
||||
String fields
|
||||
|
||||
@Parameter(names = ['-t','-template'], description = 'Text template used to each record in the log ')
|
||||
String templateStr
|
||||
|
||||
@Parameter(names=['-l','-list-fields'], description = 'Show all available fields', arity = 0)
|
||||
boolean listFields
|
||||
|
||||
@Parameter(names=['-F','-filter'], description = "Filter log entries by a custom expression e.g. process =~ /foo.*/ && status == 'COMPLETED'")
|
||||
String filterStr
|
||||
|
||||
@Parameter(names='-after', description = 'Show log entries for runs executed after the specified one')
|
||||
String after
|
||||
|
||||
@Parameter(names='-before', description = 'Show log entries for runs executed before the specified one')
|
||||
String before
|
||||
|
||||
@Parameter(names='-but', description = 'Show log entries of all runs except the specified one')
|
||||
String but
|
||||
|
||||
@Parameter(names=['-q','-quiet'], description = 'Show only run names', arity = 0)
|
||||
boolean quiet
|
||||
|
||||
@Parameter(description = 'Run name or session id')
|
||||
List<String> args
|
||||
|
||||
private Script filterScript
|
||||
|
||||
private boolean showHistory
|
||||
|
||||
private Template templateScript
|
||||
|
||||
private Map<HashCode,Boolean> printed = new HashMap<>()
|
||||
|
||||
@Override
|
||||
final String getName() { NAME }
|
||||
|
||||
|
||||
void init() {
|
||||
CacheBase.super.init()
|
||||
|
||||
//
|
||||
// validate input options
|
||||
//
|
||||
if( fields && templateStr )
|
||||
throw new AbortOperationException("Options `-f` and `-t` cannot be used in the same command")
|
||||
|
||||
//
|
||||
// when no CLI options have been specified, just show the history log
|
||||
showHistory = !args && !before && !after && !but
|
||||
|
||||
//
|
||||
// initialise filter engine
|
||||
//
|
||||
if( filterStr ) {
|
||||
filterScript = new Grengine().create("{ it -> ${fixEqualsOp(filterStr)} }")
|
||||
}
|
||||
|
||||
//
|
||||
// initialize the template engine
|
||||
//
|
||||
if( !templateStr ) {
|
||||
if( !fields ) fields = DEFAULT_FIELDS
|
||||
templateStr = fields.tokenize(', \n').collect { '$'+it } .join(sep)
|
||||
}
|
||||
else if( new File(templateStr).exists() ) {
|
||||
templateStr = new File(templateStr).text
|
||||
}
|
||||
|
||||
templateScript = new TaskTemplateEngine().createTemplate(templateStr)
|
||||
}
|
||||
|
||||
/**
|
||||
* Implements the `log` command
|
||||
*/
|
||||
@Override
|
||||
void run() {
|
||||
Plugins.init()
|
||||
init()
|
||||
|
||||
// -- show the list of expected fields and exit
|
||||
if( listFields ) {
|
||||
ALL_FIELDS.each { println " $it" }
|
||||
return
|
||||
}
|
||||
|
||||
// -- show the current history and exit
|
||||
if( showHistory ) {
|
||||
quiet ? printQuiet() : printHistory()
|
||||
return
|
||||
}
|
||||
|
||||
// -- main
|
||||
listIds().each { entry ->
|
||||
|
||||
cacheFor(entry)
|
||||
.openForRead()
|
||||
.eachRecord(this.&printRecord)
|
||||
.close()
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Print a log {@link TraceRecord} the the standard output by using the specified {@link #templateStr}
|
||||
*
|
||||
* @param record A {@link TraceRecord} instance representing a task runtime information
|
||||
*/
|
||||
protected void printRecord(HashCode hash, TraceRecord record) {
|
||||
|
||||
if( printed.containsKey(hash) )
|
||||
return
|
||||
else
|
||||
printed.put(hash,Boolean.TRUE)
|
||||
|
||||
final adaptor = new TraceAdaptor(record)
|
||||
|
||||
if( filterScript ) {
|
||||
filterScript.setBinding(adaptor)
|
||||
// dynamic execution of the filter statement
|
||||
// the `run` method interprets the statement groovy closure
|
||||
// then the `call` method invokes the closure which returns a bool value
|
||||
// if `false` skip this record
|
||||
if( !((Closure)filterScript.run()).call() ) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
println templateScript.make(adaptor).toString()
|
||||
}
|
||||
|
||||
private void printHistory() {
|
||||
def table = new TableBuilder(cellSeparator: '\t')
|
||||
.head('TIMESTAMP')
|
||||
.head('DURATION')
|
||||
.head('RUN NAME')
|
||||
.head('STATUS')
|
||||
.head('REVISION ID')
|
||||
.head('SESSION ID')
|
||||
.head('COMMAND')
|
||||
|
||||
history.eachRow { List<String> row ->
|
||||
row[4] = row[4].size()>10 ? row[4].substring(0,10) : row[4]
|
||||
table.append(row)
|
||||
}
|
||||
|
||||
println table.toString()
|
||||
}
|
||||
|
||||
private void printQuiet() {
|
||||
history.eachRow { List row -> println(row[2]) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap a {@link TraceRecord} instance as a {@link Map} or a {@link Binding} object
|
||||
*/
|
||||
private static class TraceAdaptor extends Binding {
|
||||
|
||||
static private int MAX_LINES = 100
|
||||
|
||||
private TraceRecord record
|
||||
|
||||
@Delegate
|
||||
private Map<String,Object> delegate = [:]
|
||||
|
||||
TraceAdaptor(TraceRecord record) {
|
||||
this.record = record
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean containsKey(Object key) {
|
||||
delegate.containsKey(key.toString()) || record.containsKey(key.toString())
|
||||
}
|
||||
|
||||
@Override
|
||||
Object get(Object key) {
|
||||
if( delegate.containsKey(key) ) {
|
||||
return delegate.get(key)
|
||||
}
|
||||
|
||||
if( key == 'stdout' ) {
|
||||
return fetch(getWorkDir().resolve(TaskRun.CMD_OUTFILE))
|
||||
}
|
||||
|
||||
if( key == 'stderr' ) {
|
||||
return fetch(getWorkDir().resolve(TaskRun.CMD_ERRFILE))
|
||||
}
|
||||
|
||||
if( key == 'log' ) {
|
||||
return fetch(getWorkDir().resolve(TaskRun.CMD_LOG))
|
||||
}
|
||||
|
||||
if( key == 'pcpu' )
|
||||
return record.getFmtStr('%cpu')
|
||||
|
||||
if( key == 'pmem' )
|
||||
return record.getFmtStr('%mem')
|
||||
|
||||
return record.getFmtStr(normaliseKey(key))
|
||||
}
|
||||
|
||||
|
||||
String normaliseKey(key) {
|
||||
key .toString() .toLowerCase()
|
||||
}
|
||||
|
||||
Object getVariable(String name) {
|
||||
|
||||
if( name == 'pcpu' )
|
||||
return record.store.get('%cpu')
|
||||
|
||||
if( name == 'pmem' )
|
||||
return record.store.get('%mem')
|
||||
|
||||
if( record.containsKey(name) )
|
||||
return record.store.get(name)
|
||||
|
||||
throw new MissingPropertyException(name)
|
||||
}
|
||||
|
||||
Map getVariables() {
|
||||
new HashMap(record.store)
|
||||
}
|
||||
|
||||
private Path getWorkDir() {
|
||||
def folder = (String)record.get('workdir')
|
||||
folder ? FileHelper.asPath(folder) : null
|
||||
}
|
||||
|
||||
private String fetch(Path path) {
|
||||
try {
|
||||
int c=0
|
||||
def result = new StringBuilder()
|
||||
path.withReader { reader ->
|
||||
String line
|
||||
while( (line=reader.readLine()) && c++<MAX_LINES ) {
|
||||
result << line << '\n'
|
||||
}
|
||||
}
|
||||
|
||||
result.toString() ?: TraceRecord.NA
|
||||
|
||||
}
|
||||
catch( IOError e ) {
|
||||
log.debug "Failed to fetch content for file: $path -- Cause: ${e.message ?: e}"
|
||||
return TraceRecord.NA
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
|
||||
import com.beust.jcommander.JCommander
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.ParameterException
|
||||
import com.beust.jcommander.Parameters
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.cli.module.CmdModuleCreate
|
||||
import nextflow.cli.module.CmdModuleView
|
||||
import nextflow.cli.module.CmdModuleInstall
|
||||
import nextflow.cli.module.CmdModuleList
|
||||
import nextflow.cli.module.CmdModulePublish
|
||||
import nextflow.cli.module.CmdModuleRemove
|
||||
import nextflow.cli.module.CmdModuleRun
|
||||
import nextflow.cli.module.CmdModuleSearch
|
||||
import nextflow.cli.module.CmdModuleSpec
|
||||
import nextflow.cli.module.CmdModuleValidate
|
||||
import nextflow.exception.AbortOperationException
|
||||
|
||||
/**
|
||||
* Implements `module` command
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
@CompileStatic
|
||||
@Slf4j
|
||||
@Parameters(commandDescription = "Manage Nextflow modules")
|
||||
class CmdModule extends CmdBase implements UsageAware {
|
||||
|
||||
static final public String NAME = 'module'
|
||||
|
||||
private JCommander jCommander
|
||||
|
||||
static final List<CmdBase> commands = new ArrayList<>()
|
||||
|
||||
static {
|
||||
commands << new CmdModuleCreate()
|
||||
commands << new CmdModuleInstall()
|
||||
commands << new CmdModuleRun()
|
||||
commands << new CmdModuleList()
|
||||
commands << new CmdModuleRemove()
|
||||
commands << new CmdModuleSearch()
|
||||
commands << new CmdModuleView()
|
||||
commands << new CmdModulePublish()
|
||||
commands << new CmdModuleSpec()
|
||||
commands << new CmdModuleValidate()
|
||||
}
|
||||
|
||||
protected JCommander commander() {
|
||||
if( !this.jCommander ) {
|
||||
this.jCommander = new JCommander(this)
|
||||
this.jCommander.setProgramName('nextflow module')
|
||||
// Register all subcommands
|
||||
commands.each { cmd ->
|
||||
cmd.launcher = this.launcher
|
||||
this.jCommander.addCommand(cmd.getName(), cmd, cmd.getAliases() as String[])
|
||||
}
|
||||
}
|
||||
return jCommander
|
||||
}
|
||||
|
||||
@Parameter
|
||||
List<String> args
|
||||
|
||||
@Override
|
||||
String getName() {
|
||||
return NAME
|
||||
}
|
||||
|
||||
@Override
|
||||
void run() {
|
||||
|
||||
|
||||
try {
|
||||
if( !args ) {
|
||||
usage()
|
||||
return
|
||||
}
|
||||
final jc = commander()
|
||||
final moduleArgs = args + unknownOptions
|
||||
jc.parse(moduleArgs as String[])
|
||||
|
||||
final parsedCommand = jc.getParsedCommand()
|
||||
if( !parsedCommand ) {
|
||||
jc.usage()
|
||||
return
|
||||
}
|
||||
|
||||
// Get the parsed subcommand instance
|
||||
final subcommand = jc.getCommands()
|
||||
.get(parsedCommand)
|
||||
.getObjects()[0] as CmdBase
|
||||
|
||||
// Execute with fields already populated by JCommander
|
||||
subcommand.run()
|
||||
|
||||
} catch( ParameterException e ) {
|
||||
throw new AbortOperationException("${e.getMessage()} -- Check the available commands and options and syntax with 'nextflow module -h'")
|
||||
}
|
||||
}
|
||||
|
||||
private CmdBase findCmd(String name) {
|
||||
commands.find { it.name == name || name in it.aliases }
|
||||
}
|
||||
|
||||
/**
|
||||
* Print the command usage help
|
||||
*/
|
||||
@Override
|
||||
void usage() {
|
||||
usage(args)
|
||||
}
|
||||
|
||||
/**
|
||||
* Print the command usage help
|
||||
*
|
||||
* @param args The arguments as entered by the user
|
||||
*/
|
||||
@Override
|
||||
void usage(List<String> args) {
|
||||
def result = []
|
||||
if( !args ) {
|
||||
result << 'Usage: nextflow module <command> [options]'
|
||||
result << ''
|
||||
result << 'Commands:'
|
||||
commands.each {
|
||||
def description = it.getClass().getAnnotation(Parameters)?.commandDescription()
|
||||
result << " ${it.name.padRight(12)}${description}"
|
||||
}
|
||||
result << ''
|
||||
println result.join('\n').toString()
|
||||
} else {
|
||||
final sub = findCmd(args[0])
|
||||
if( sub ) {
|
||||
commander().usage(args[0])
|
||||
} else {
|
||||
throw new AbortOperationException("Unknown module sub-command: ${args[0]}")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
import com.beust.jcommander.DynamicParameter
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.Parameters
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.config.ConfigBuilder
|
||||
import nextflow.daemon.DaemonLauncher
|
||||
import nextflow.plugin.Plugins
|
||||
import nextflow.util.ServiceName
|
||||
import nextflow.util.ServiceDiscover
|
||||
/**
|
||||
* CLI-command NODE
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
@Parameters
|
||||
class CmdNode extends CmdBase {
|
||||
|
||||
static final public NAME = 'node'
|
||||
|
||||
@Override
|
||||
final String getName() { NAME }
|
||||
|
||||
@DynamicParameter(names ='-cluster.', description='Define cluster config options')
|
||||
Map<String,String> clusterOptions = [:]
|
||||
|
||||
@Parameter(names = ['-bg'], arity = 0, description = 'Start the cluster node daemon in background')
|
||||
void setBackground(boolean value) {
|
||||
launcher.options.background = value
|
||||
}
|
||||
|
||||
@Parameter
|
||||
List<String> provider
|
||||
|
||||
@Override
|
||||
void run() {
|
||||
System.setProperty('nxf.node.daemon', 'true')
|
||||
launchDaemon(provider ? provider[0] : null)
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Launch the daemon service
|
||||
*
|
||||
* @param config The nextflow configuration map
|
||||
*/
|
||||
protected launchDaemon(String name = null) {
|
||||
|
||||
// create the config object
|
||||
def config = new ConfigBuilder()
|
||||
.setOptions(launcher.options)
|
||||
.setCmdNode(this)
|
||||
.build()
|
||||
|
||||
DaemonLauncher instance
|
||||
if( name ) {
|
||||
if( name.contains('.') ) {
|
||||
instance = loadDaemonByClass(name)
|
||||
}
|
||||
else {
|
||||
instance = loadDaemonByName(name)
|
||||
}
|
||||
}
|
||||
else {
|
||||
instance = loadDaemonFirst()
|
||||
}
|
||||
|
||||
|
||||
// launch it
|
||||
instance.launch(config)
|
||||
}
|
||||
|
||||
/**
|
||||
* Load a {@code DaemonLauncher} instance of the its *friendly* name i.e. the name provided
|
||||
* by using the {@code ServiceName} annotation on the daemon class definition
|
||||
*
|
||||
* @param name The executor name e.g. {@code gridgain}
|
||||
* @return The daemon launcher instance
|
||||
* @throws IllegalStateException if the class does not exist or it cannot be instantiated
|
||||
*/
|
||||
static DaemonLauncher loadDaemonByName( String name ) {
|
||||
|
||||
Class<DaemonLauncher> clazz = null
|
||||
for( Class<DaemonLauncher> item : ServiceDiscover.load(DaemonLauncher) ) {
|
||||
log.debug "Discovered daemon class: ${item.name}"
|
||||
ServiceName annotation = item.getAnnotation(ServiceName)
|
||||
if( annotation && annotation.value() == name ) {
|
||||
clazz = item
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if( !clazz )
|
||||
throw new IllegalStateException("Unknown daemon name: $name")
|
||||
|
||||
try {
|
||||
clazz.newInstance()
|
||||
}
|
||||
catch( Exception e ) {
|
||||
throw new IllegalStateException("Unable to launch executor: $name", e)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load a class implementing the {@code DaemonLauncher} interface by the specified class name
|
||||
*
|
||||
* @param name The fully qualified class name e.g. {@code nextflow.executor.local.LocalExecutor}
|
||||
* @return The daemon launcher instance
|
||||
* @throws IllegalStateException if the class does not exist or it cannot be instantiated
|
||||
*/
|
||||
static DaemonLauncher loadDaemonByClass( String name ) {
|
||||
try {
|
||||
return (DaemonLauncher)Class.forName(name).newInstance()
|
||||
}
|
||||
catch( Exception e ) {
|
||||
throw new IllegalStateException("Cannot load daemon: ${name}")
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return The first available instance of a class implementing {@code DaemonLauncher}
|
||||
* @throws IllegalStateException when no class implementing {@code DaemonLauncher} is available
|
||||
*/
|
||||
static DaemonLauncher loadDaemonFirst() {
|
||||
Plugins.init()
|
||||
final loader = Plugins.getExtension(DaemonLauncher)
|
||||
if( !loader )
|
||||
throw new IllegalStateException("No cluster services are available -- Cannot launch Nextflow in cluster mode")
|
||||
|
||||
return loader
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
|
||||
import static nextflow.cli.PluginExecAware.CMD_SEP
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
import com.beust.jcommander.DynamicParameter
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.Parameters
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.plugin.Plugins
|
||||
import nextflow.plugin.util.PluginRefactor
|
||||
import org.eclipse.jgit.api.Git
|
||||
/**
|
||||
* Plugin manager command
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
@Parameters(commandDescription = "Execute plugin-specific commands")
|
||||
class CmdPlugin extends CmdBase {
|
||||
|
||||
@Override
|
||||
String getName() {
|
||||
return 'plugin'
|
||||
}
|
||||
|
||||
@DynamicParameter(names = "--", description = "Custom plugin parameters go here", hidden = true)
|
||||
private Map<String, String> params = new HashMap<>();
|
||||
|
||||
@Parameter(hidden = true)
|
||||
List<String> args
|
||||
|
||||
@Parameter(names = ['-template'], description = 'Plugin template version to use', hidden = true)
|
||||
String templateVersion = 'v0.3.0'
|
||||
|
||||
@Override
|
||||
void run() {
|
||||
if( !args )
|
||||
throw new AbortOperationException("Missing plugin command - usage: nextflow plugin install <pluginId,..>")
|
||||
// setup plugins system
|
||||
Plugins.init()
|
||||
Runtime.addShutdownHook((it)-> Plugins.stop())
|
||||
|
||||
// check for the plugins install
|
||||
if( args[0] == 'install' ) {
|
||||
if( args.size()!=2 )
|
||||
throw new AbortOperationException("Missing plugin install target - usage: nextflow plugin install <pluginId,..>")
|
||||
Plugins.pull(args[1].tokenize(','))
|
||||
}
|
||||
else if( args[0] == 'create' ) {
|
||||
createPlugin(args, templateVersion)
|
||||
}
|
||||
// plugin run command
|
||||
else if( args[0].contains(CMD_SEP) ) {
|
||||
final head = args.pop()
|
||||
final items = head.tokenize(CMD_SEP)
|
||||
final target = items[0]
|
||||
final cmd = items[1] ? items[1..-1].join(CMD_SEP) : null
|
||||
|
||||
// push back the command as the first item
|
||||
Plugins.start(target)
|
||||
final wrapper = Plugins.manager.getPlugin(target)
|
||||
if( !wrapper )
|
||||
throw new AbortOperationException("Cannot find target plugin: $target")
|
||||
final plugin = wrapper.getPlugin()
|
||||
if( plugin instanceof PluginExecAware ) {
|
||||
def mapped = [] as List<String>
|
||||
params.entrySet().each{
|
||||
mapped << "--$it.key".toString()
|
||||
mapped << "$it.value".toString()
|
||||
}
|
||||
args.addAll(mapped)
|
||||
final ret = plugin.exec(getLauncher(), target, cmd, args)
|
||||
// use explicit exit to invoke the system shutdown hooks
|
||||
System.exit(ret)
|
||||
}
|
||||
else
|
||||
throw new AbortOperationException("Invalid target plugin: $target")
|
||||
}
|
||||
else {
|
||||
throw new AbortOperationException("Invalid plugin command: ${args[0]}")
|
||||
}
|
||||
}
|
||||
|
||||
static createPlugin(List<String> args, String templateVersion) {
|
||||
if( args != ['create'] && (args[0] != 'create' || !(args.size() in [3, 4])) )
|
||||
throw new AbortOperationException("Invalid create parameters - usage: nextflow plugin create <Plugin name> <Provider name>")
|
||||
|
||||
final refactor = new PluginRefactor()
|
||||
if( args.size()>1 ) {
|
||||
refactor.withPluginName(args[1])
|
||||
refactor.withProviderName(args[2])
|
||||
refactor.withPluginDir(Path.of(args[3] ?: refactor.pluginName).toFile())
|
||||
}
|
||||
else {
|
||||
// Prompt for plugin name
|
||||
print "Enter plugin name: "
|
||||
refactor.withPluginName(readLine())
|
||||
|
||||
// Prompt for provider name
|
||||
print "Enter provider name: "
|
||||
refactor.withProviderName(readLine())
|
||||
|
||||
// Prompt for project path (default to the normalised plugin name)
|
||||
print "Enter project path [${refactor.pluginName}]: "
|
||||
refactor.withPluginDir(Path.of(readLine() ?: refactor.pluginName).toFile())
|
||||
|
||||
// confirm and proceed
|
||||
print "All good, are you OK to continue [y/N]? "
|
||||
final confirm = readLine()
|
||||
if( confirm?.toLowerCase()!='y' ) {
|
||||
println "Plugin creation aborted."
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// the final directory where the plugin is created
|
||||
final File targetDir = refactor.getPluginDir()
|
||||
|
||||
// clone the template repo
|
||||
clonePluginTemplate(targetDir, templateVersion)
|
||||
// now refactor the template code
|
||||
refactor.apply()
|
||||
// remove git plat
|
||||
cleanup(targetDir)
|
||||
// done
|
||||
println "Plugin created successfully at path: $targetDir"
|
||||
}
|
||||
|
||||
static private String readLine() {
|
||||
final console = System.console()
|
||||
return console != null
|
||||
? console.readLine()
|
||||
: new BufferedReader(new InputStreamReader(System.in)).readLine()
|
||||
}
|
||||
|
||||
static private void clonePluginTemplate(File targetDir, String templateVersion) {
|
||||
final templateUri = "https://github.com/nextflow-io/nf-plugin-template.git"
|
||||
final isTag = templateVersion.startsWith('v')
|
||||
final refSpec = isTag ? "refs/tags/$templateVersion".toString() : templateVersion
|
||||
|
||||
try {
|
||||
final gitCmd = Git.cloneRepository()
|
||||
.setURI(templateUri)
|
||||
.setDirectory(targetDir)
|
||||
|
||||
if (isTag) {
|
||||
gitCmd.setBranchesToClone([refSpec])
|
||||
gitCmd.setBranch(refSpec)
|
||||
} else {
|
||||
// For branches, let Git handle the default behavior
|
||||
gitCmd.setBranch(templateVersion)
|
||||
}
|
||||
|
||||
gitCmd.call()
|
||||
}
|
||||
catch (Exception e) {
|
||||
throw new AbortOperationException("Unable to clone pluging template repository - cause: ${e.message}")
|
||||
}
|
||||
}
|
||||
|
||||
static private void cleanup(File targetDir) {
|
||||
new File(targetDir, '.git').deleteDir()
|
||||
new File(targetDir, '.github').deleteDir()
|
||||
new File(targetDir, 'validation').deleteDir()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.Parameters
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.plugin.Plugins
|
||||
import nextflow.scm.AssetManager
|
||||
import nextflow.util.TestOnly
|
||||
/**
|
||||
* CLI sub-command PULL
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
@Parameters(commandDescription = "Download or update a project")
|
||||
class CmdPull extends CmdBase implements HubOptions {
|
||||
|
||||
static final public NAME = 'pull'
|
||||
|
||||
@Parameter(description = 'project name or repository url to pull', arity = 1)
|
||||
List<String> args
|
||||
|
||||
@Parameter(names=['-a','-all'], description = 'Update all downloaded projects', arity = 0)
|
||||
boolean all
|
||||
|
||||
@Parameter(names=['-r','-revision'], description = 'Revision of the project to pull (either a git branch, tag or commit SHA number)')
|
||||
String revision
|
||||
|
||||
@Parameter(names=['-d','-deep'], description = 'Create a shallow clone of the specified depth')
|
||||
Integer deep
|
||||
|
||||
@Parameter(names=['-m','-migrate'], description = 'Migrate projects to multi-revision strategy', arity = 0)
|
||||
boolean migrate
|
||||
|
||||
@Override
|
||||
final String getName() { NAME }
|
||||
|
||||
@TestOnly
|
||||
protected File root
|
||||
|
||||
@Override
|
||||
void run() {
|
||||
|
||||
if( !all && !args )
|
||||
throw new AbortOperationException('Missing argument')
|
||||
|
||||
if( all && args )
|
||||
throw new AbortOperationException('Option `all` requires no arguments')
|
||||
|
||||
if( all && revision )
|
||||
throw new AbortOperationException('Option `all` is not compatible with `revision`')
|
||||
|
||||
def list = all ? AssetManager.list() : args.toList()
|
||||
if( !list ) {
|
||||
log.info "(nothing to do)"
|
||||
return
|
||||
}
|
||||
|
||||
if( root ) {
|
||||
AssetManager.root = root
|
||||
}
|
||||
|
||||
// init plugin system
|
||||
Plugins.init()
|
||||
|
||||
for( String proj : list ) {
|
||||
if( all ) {
|
||||
try (def mgr = new AssetManager(proj)) {
|
||||
def branches = mgr.getBranchesAndTags(false).pulled as List<String>
|
||||
branches.each { rev -> pullProjectRevision(proj, rev) }
|
||||
}
|
||||
} else {
|
||||
pullProjectRevision(proj, revision)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private pullProjectRevision(String project, String revision) {
|
||||
try (final manager = new AssetManager(project, this)) {
|
||||
if( manager.isUsingLegacyStrategy() ) {
|
||||
if( migrate ) {
|
||||
log.info "Migrating ${project} revision ${revision} to multi-revision strategy"
|
||||
manager.setStrategyType(AssetManager.RepositoryStrategyType.MULTI_REVISION)
|
||||
} else {
|
||||
log.warn "The local asset for ${project} does not support multi-revision - Pulling with legacy strategy\n" +
|
||||
"Consider updating the project ${project} using '-migrate' option"
|
||||
}
|
||||
}
|
||||
|
||||
if( revision )
|
||||
manager.setRevision(revision)
|
||||
|
||||
log.info "Checking ${manager.getProjectWithRevision()} ..."
|
||||
|
||||
def result = manager.download(revision, deep)
|
||||
manager.updateModules()
|
||||
|
||||
def scriptFile = manager.getScriptFile()
|
||||
String message = !result ? " done" : " $result"
|
||||
message += " - revision: ${scriptFile.revisionInfo}"
|
||||
log.info message
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,864 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
|
||||
import static org.fusesource.jansi.Ansi.*
|
||||
|
||||
import java.nio.file.NoSuchFileException
|
||||
import java.nio.file.Path
|
||||
import java.util.regex.Pattern
|
||||
|
||||
import com.beust.jcommander.DynamicParameter
|
||||
import com.beust.jcommander.IStringConverter
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.Parameters
|
||||
import groovy.json.JsonSlurper
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.Memoized
|
||||
import groovy.util.logging.Slf4j
|
||||
import groovyx.gpars.GParsConfig
|
||||
import nextflow.BuildInfo
|
||||
import nextflow.NF
|
||||
import nextflow.NextflowMeta
|
||||
import nextflow.SysEnv
|
||||
import nextflow.config.ConfigBuilder
|
||||
import nextflow.config.ConfigMap
|
||||
import nextflow.config.ConfigValidator
|
||||
import nextflow.config.Manifest
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.file.FileHelper
|
||||
import nextflow.plugin.Plugins
|
||||
import nextflow.scm.AssetManager
|
||||
import nextflow.script.ScriptFile
|
||||
import nextflow.script.ScriptRunner
|
||||
import nextflow.secret.EmptySecretProvider
|
||||
import nextflow.secret.SecretsLoader
|
||||
import nextflow.util.CustomPoolFactory
|
||||
import nextflow.util.Duration
|
||||
import nextflow.util.HistoryFile
|
||||
import nextflow.util.VersionNumber
|
||||
import org.apache.commons.lang3.StringUtils
|
||||
import org.fusesource.jansi.AnsiConsole
|
||||
import org.yaml.snakeyaml.Yaml
|
||||
/**
|
||||
* CLI sub-command RUN
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
@Parameters(commandDescription = "Execute a pipeline project")
|
||||
class CmdRun extends CmdBase implements HubOptions {
|
||||
|
||||
static final public Pattern RUN_NAME_PATTERN = Pattern.compile(/^[a-z](?:[a-z\d]|[-_](?=[a-z\d])){0,79}$/, Pattern.CASE_INSENSITIVE)
|
||||
|
||||
static final public List<String> VALID_PARAMS_FILE = ['json', 'yml', 'yaml']
|
||||
|
||||
static final public String DSL2 = '2'
|
||||
static final public String DSL1 = '1'
|
||||
|
||||
static {
|
||||
// install the custom pool factory for GPars threads
|
||||
GParsConfig.poolFactory = new CustomPoolFactory()
|
||||
}
|
||||
|
||||
static class DurationConverter implements IStringConverter<Long> {
|
||||
@Override
|
||||
Long convert(String value) {
|
||||
if( !value ) throw new IllegalArgumentException()
|
||||
if( value.isLong() ) { return value.toLong() }
|
||||
return Duration.of(value).toMillis()
|
||||
}
|
||||
}
|
||||
|
||||
static final public String NAME = 'run'
|
||||
|
||||
private Map<String,String> sysEnv = System.getenv()
|
||||
|
||||
@Parameter(names=['-name'], description = 'Assign a mnemonic name to the a pipeline run')
|
||||
String runName
|
||||
|
||||
@Parameter(names=['-lib'], description = 'Library extension path')
|
||||
String libPath
|
||||
|
||||
@Parameter(names=['-cache'], description = 'Enable/disable processes caching', arity = 1)
|
||||
Boolean cacheable
|
||||
|
||||
@Parameter(names=['-resume'], description = 'Execute the script using the cached results, useful to continue executions that was stopped by an error')
|
||||
String resume
|
||||
|
||||
@Parameter(names=['-ps','-pool-size'], description = 'Number of threads in the execution pool', hidden = true)
|
||||
Integer poolSize
|
||||
|
||||
@Parameter(names=['-pi','-poll-interval'], description = 'Executor poll interval (duration string ending with ms|s|m)', converter = DurationConverter, hidden = true)
|
||||
long pollInterval
|
||||
|
||||
@Parameter(names=['-qs','-queue-size'], description = 'Max number of processes that can be executed in parallel by each executor')
|
||||
Integer queueSize
|
||||
|
||||
@Parameter(names=['-test'], description = 'Test a script function with the name specified')
|
||||
String test
|
||||
|
||||
@Parameter(names=['-o', '-output-dir'], description = 'Directory where workflow outputs are stored')
|
||||
String outputDir
|
||||
|
||||
@Parameter(names=['-output-format'], description = 'Output format for printing workflow outputs. Options: `text` (default), `json`, `none`')
|
||||
String outputFormat = 'text'
|
||||
|
||||
@Parameter(names=['-w', '-work-dir'], description = 'Directory where intermediate result files are stored')
|
||||
String workDir
|
||||
|
||||
@Parameter(names=['-bucket-dir'], description = 'Remote bucket where intermediate result files are stored')
|
||||
String bucketDir
|
||||
|
||||
@Parameter(names=['-with-cloudcache'], description = 'Enable the use of object storage bucket as storage for cache meta-data')
|
||||
String cloudCachePath
|
||||
|
||||
/**
|
||||
* Defines the parameters to be passed to the pipeline script
|
||||
*/
|
||||
@DynamicParameter(names = '--', description = 'Set a parameter used by the pipeline', hidden = true)
|
||||
Map<String,String> params = new LinkedHashMap<>()
|
||||
|
||||
@Parameter(names='-params-file', description = 'Load script parameters from a JSON/YAML file')
|
||||
String paramsFile
|
||||
|
||||
@DynamicParameter(names = ['-process.'], description = 'Set process options' )
|
||||
Map<String,String> process = [:]
|
||||
|
||||
@DynamicParameter(names = ['-e.'], description = 'Add the specified variable to execution environment')
|
||||
Map<String,String> env = [:]
|
||||
|
||||
@Parameter(names = ['-E'], description = 'Exports all current system environment')
|
||||
boolean exportSysEnv
|
||||
|
||||
@DynamicParameter(names = ['-executor.'], description = 'Set executor options', hidden = true )
|
||||
Map<String,String> executorOptions = [:]
|
||||
|
||||
@Parameter(description = 'Project name or repository url')
|
||||
List<String> args
|
||||
|
||||
@Parameter(names=['-r','-revision'], description = 'Revision of the project to run (either a git branch, tag or commit SHA number)')
|
||||
String revision
|
||||
|
||||
@Parameter(names=['-d','-deep'], description = 'Create a shallow clone of the specified depth')
|
||||
Integer deep
|
||||
|
||||
@Parameter(names=['-latest'], description = 'Pull latest changes before run')
|
||||
boolean latest
|
||||
|
||||
@Parameter(names='-stdin', hidden = true)
|
||||
boolean stdin
|
||||
|
||||
@Parameter(names = ['-ansi'], hidden = true, arity = 0)
|
||||
void setAnsi(boolean value) {
|
||||
launcher.options.ansiLog = value
|
||||
}
|
||||
|
||||
@Parameter(names = ['-ansi-log'], description = 'Enable/disable ANSI console logging', arity = 1)
|
||||
void setAnsiLog(boolean value) {
|
||||
launcher.options.ansiLog = value
|
||||
}
|
||||
|
||||
@Parameter(names = ['-with-tower'], description = 'Monitor workflow execution with Seqera Platform (formerly Tower Cloud)')
|
||||
String withTower
|
||||
|
||||
@Parameter(names = ['-with-wave'], hidden = true)
|
||||
String withWave
|
||||
|
||||
@Parameter(names = ['-with-fusion'], hidden = true)
|
||||
String withFusion
|
||||
|
||||
@Parameter(names = ['-with-weblog'], description = 'Send workflow status messages via HTTP to target URL')
|
||||
String withWebLog
|
||||
|
||||
@Parameter(names = ['-with-trace'], description = 'Create processes execution tracing file')
|
||||
String withTrace
|
||||
|
||||
@Parameter(names = ['-with-report'], description = 'Create processes execution html report')
|
||||
String withReport
|
||||
|
||||
@Parameter(names = ['-with-timeline'], description = 'Create processes execution timeline file')
|
||||
String withTimeline
|
||||
|
||||
@Parameter(names = '-with-charliecloud', description = 'Enable process execution in a Charliecloud container runtime')
|
||||
def withCharliecloud
|
||||
|
||||
@Parameter(names = '-with-singularity', description = 'Enable process execution in a Singularity container')
|
||||
def withSingularity
|
||||
|
||||
@Parameter(names = '-with-apptainer', description = 'Enable process execution in a Apptainer container')
|
||||
def withApptainer
|
||||
|
||||
@Parameter(names = '-with-podman', description = 'Enable process execution in a Podman container')
|
||||
def withPodman
|
||||
|
||||
@Parameter(names = '-without-podman', description = 'Disable process execution in a Podman container')
|
||||
def withoutPodman
|
||||
|
||||
@Parameter(names = '-with-docker', description = 'Enable process execution in a Docker container')
|
||||
def withDocker
|
||||
|
||||
@Parameter(names = '-without-docker', description = 'Disable process execution with Docker', arity = 0)
|
||||
boolean withoutDocker
|
||||
|
||||
@Parameter(names = '-with-mpi', hidden = true)
|
||||
boolean withMpi
|
||||
|
||||
@Parameter(names = '-with-dag', description = 'Create pipeline DAG file')
|
||||
String withDag
|
||||
|
||||
@Parameter(names = ['-bg'], arity = 0, hidden = true)
|
||||
void setBackground(boolean value) {
|
||||
launcher.options.background = value
|
||||
}
|
||||
|
||||
@Parameter(names=['-c','-config'], hidden = true )
|
||||
List<String> runConfig
|
||||
|
||||
@DynamicParameter(names = ['-cluster.'], description = 'Set cluster options', hidden = true )
|
||||
Map<String,String> clusterOptions = [:]
|
||||
|
||||
@Parameter(names=['-profile'], description = 'Choose a configuration profile')
|
||||
String profile
|
||||
|
||||
@Parameter(names=['-dump-hashes'], description = 'Dump task hash keys for debugging purpose')
|
||||
String dumpHashes
|
||||
|
||||
@Parameter(names=['-dump-channels'], description = 'Dump channels for debugging purpose')
|
||||
String dumpChannels
|
||||
|
||||
@Parameter(names=['-N','-with-notification'], description = 'Send a notification email on workflow completion to the specified recipients')
|
||||
String withNotification
|
||||
|
||||
@Parameter(names=['-with-conda'], description = 'Use the specified Conda environment package or file (must end with .yml|.yaml suffix)')
|
||||
String withConda
|
||||
|
||||
@Parameter(names=['-without-conda'], description = 'Disable the use of Conda environments')
|
||||
Boolean withoutConda
|
||||
|
||||
@Parameter(names=['-with-spack'], description = 'Use the specified Spack environment package or file (must end with .yaml suffix)')
|
||||
String withSpack
|
||||
|
||||
@Parameter(names=['-without-spack'], description = 'Disable the use of Spack environments')
|
||||
Boolean withoutSpack
|
||||
|
||||
@Parameter(names=['-offline'], description = 'Do not check for remote project updates')
|
||||
boolean offline = System.getenv('NXF_OFFLINE')=='true'
|
||||
|
||||
@Parameter(names=['-entry'], description = 'Entry workflow name to be executed', arity = 1)
|
||||
String entryName
|
||||
|
||||
@Parameter(names=['-main-script'], description = 'The script file to be executed when launching a project directory or repository' )
|
||||
String mainScript
|
||||
|
||||
@Parameter(names=['-stub-run','-stub'], description = 'Execute the workflow replacing process scripts with command stubs')
|
||||
boolean stubRun
|
||||
|
||||
@Parameter(names=['-preview'], description = "Run the workflow script skipping the execution of all processes")
|
||||
boolean preview
|
||||
|
||||
@Parameter(names=['-plugins'], description = 'Specify the plugins to be applied for this run e.g. nf-amazon,nf-tower')
|
||||
String plugins
|
||||
|
||||
@Parameter(names=['-disable-jobs-cancellation'], description = 'Prevent the cancellation of child jobs on execution termination')
|
||||
Boolean disableJobsCancellation
|
||||
|
||||
Boolean skipHistoryFile
|
||||
|
||||
Boolean getDisableJobsCancellation() {
|
||||
return disableJobsCancellation!=null
|
||||
? disableJobsCancellation
|
||||
: sysEnv.get('NXF_DISABLE_JOBS_CANCELLATION') as boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Optional closure modelling an action to be invoked when the preview mode is enabled
|
||||
*/
|
||||
Closure<Void> previewAction
|
||||
|
||||
@Override
|
||||
String getName() { NAME }
|
||||
|
||||
String getParamsFile() {
|
||||
return paramsFile ?: sysEnv.get('NXF_PARAMS_FILE')
|
||||
}
|
||||
|
||||
boolean hasParams() {
|
||||
return params || getParamsFile()
|
||||
}
|
||||
|
||||
@Override
|
||||
void run() {
|
||||
final scriptArgs = (args?.size()>1 ? args[1..-1] : []) as List<String>
|
||||
final pipeline = stdin ? '-' : ( args ? args[0] : null )
|
||||
if( !pipeline )
|
||||
throw new AbortOperationException("No project name was specified")
|
||||
|
||||
if( withPodman && withoutPodman )
|
||||
throw new AbortOperationException("Command line options `-with-podman` and `-without-podman` cannot be specified at the same time")
|
||||
|
||||
if( withDocker && withoutDocker )
|
||||
throw new AbortOperationException("Command line options `-with-docker` and `-without-docker` cannot be specified at the same time")
|
||||
|
||||
if( withConda && withoutConda )
|
||||
throw new AbortOperationException("Command line options `-with-conda` and `-without-conda` cannot be specified at the same time")
|
||||
|
||||
if( withSpack && withoutSpack )
|
||||
throw new AbortOperationException("Command line options `-with-spack` and `-without-spack` cannot be specified at the same time")
|
||||
|
||||
if( offline && latest )
|
||||
throw new AbortOperationException("Command line options `-latest` and `-offline` cannot be specified at the same time")
|
||||
|
||||
if( outputFormat !in ['text', 'json', 'none'] )
|
||||
throw new AbortOperationException("Command line option `-output-format` should be either `text`, `json`, or `none`")
|
||||
|
||||
checkRunName()
|
||||
|
||||
printBanner()
|
||||
|
||||
Plugins.init()
|
||||
|
||||
// -- resolve main script
|
||||
final scriptFile = getScriptFile(pipeline)
|
||||
|
||||
// -- load command line params
|
||||
final baseDir = scriptFile.parent
|
||||
final cliParams = parsedParams(ConfigBuilder.getConfigVars(baseDir, null))
|
||||
|
||||
/*
|
||||
* 2-PHASE CONFIGURATION LOADING STRATEGY
|
||||
*
|
||||
* Problem: Configuration files may reference secrets provided by plugins (e.g., AWS secrets),
|
||||
* but plugins are loaded AFTER configuration parsing. This creates a chicken-and-egg problem:
|
||||
* - Config parsing needs secret values to complete
|
||||
* - Plugin loading needs config to determine which plugins to load
|
||||
* - Secret providers are registered by plugins
|
||||
*
|
||||
* Solution: Parse configuration twice when secrets are referenced
|
||||
*
|
||||
* PHASE 1: Parse config with EmptySecretProvider (returns "" for all secrets)
|
||||
* - Configuration must use defensive patterns: secrets.FOO ? "value-${secrets.FOO}" : "fallback"
|
||||
* - Config parses successfully with fallback values
|
||||
* - EmptySecretProvider tracks if ANY secrets were accessed
|
||||
*/
|
||||
|
||||
// -- PHASE 1: Load config with mock secrets provider
|
||||
final secretsProvider = new EmptySecretProvider()
|
||||
ConfigBuilder builder = new ConfigBuilder()
|
||||
.setOptions(launcher.options)
|
||||
.setCmdRun(this)
|
||||
.setBaseDir(scriptFile.parent)
|
||||
.setCliParams(cliParams)
|
||||
.setSecretsProvider(secretsProvider) // Mock provider returns empty strings
|
||||
ConfigMap config = builder.build()
|
||||
Map configParams = builder.getConfigParams()
|
||||
|
||||
// -- Check Nextflow version
|
||||
checkVersion(config)
|
||||
|
||||
// -- Load plugins (may register secret providers)
|
||||
Plugins.load(config)
|
||||
|
||||
// -- Initialize real secrets system
|
||||
SecretsLoader.getInstance().load()
|
||||
|
||||
/*
|
||||
* PHASE 2: Conditionally reload config with real secrets
|
||||
* - Only reload if Phase 1 actually accessed any secrets
|
||||
* - This time, real secret providers are available (including plugin-provided ones)
|
||||
* - Same config expressions now resolve with actual secret values
|
||||
*/
|
||||
|
||||
// -- PHASE 2: Reload config if secrets were used in Phase 1
|
||||
if( secretsProvider.usedSecrets() ) {
|
||||
log.debug "Config file used secrets -- reloading config with secrets provider"
|
||||
builder = new ConfigBuilder()
|
||||
.setOptions(launcher.options)
|
||||
.setCmdRun(this)
|
||||
.setBaseDir(scriptFile.parent)
|
||||
.setCliParams(cliParams)
|
||||
// No .setSecretsProvider() - uses real secrets system now
|
||||
config = builder.build()
|
||||
configParams = builder.getConfigParams()
|
||||
}
|
||||
|
||||
// check DSL syntax in the config
|
||||
launchInfo(config, scriptFile)
|
||||
|
||||
// -- validate config options
|
||||
if( NF.isSyntaxParserV2() )
|
||||
new ConfigValidator().validate(config)
|
||||
|
||||
// -- create a new runner instance
|
||||
final runner = new ScriptRunner(config)
|
||||
runner.setScript(scriptFile)
|
||||
runner.setPreview(this.preview, previewAction)
|
||||
runner.session.profile = profile
|
||||
runner.session.commandLine = launcher.cliString
|
||||
runner.session.ansiLog = launcher.options.ansiLog && !SysEnv.isAgentMode()
|
||||
runner.session.agentLog = SysEnv.isAgentMode()
|
||||
runner.session.debug = launcher.options.remoteDebug
|
||||
runner.session.disableJobsCancellation = getDisableJobsCancellation()
|
||||
|
||||
final isTowerEnabled = config.navigate('tower.enabled') as Boolean
|
||||
final isDataEnabled = config.navigate("lineage.enabled") as Boolean
|
||||
if( isTowerEnabled || isDataEnabled || log.isTraceEnabled() )
|
||||
runner.session.resolvedConfig = ConfigBuilder.resolveConfig(scriptFile.parent, this, cliParams)
|
||||
// note config files are collected during the build process
|
||||
// this line should be after `ConfigBuilder#build`
|
||||
runner.session.configFiles = builder.parsedConfigFiles
|
||||
// set the commit id (if any)
|
||||
runner.session.commitId = scriptFile.commitId
|
||||
if( this.test ) {
|
||||
runner.test(this.test, scriptArgs)
|
||||
return
|
||||
}
|
||||
|
||||
def info = CmdInfo.status( log.isTraceEnabled() )
|
||||
log.debug( '\n'+info )
|
||||
|
||||
// -- add this run to the local history
|
||||
if( !skipHistoryFile ) {
|
||||
runner.verifyAndTrackHistory(launcher.cliString, runName)
|
||||
}
|
||||
|
||||
// -- run it!
|
||||
runner.execute(scriptArgs, cliParams, configParams, this.entryName)
|
||||
}
|
||||
|
||||
protected void printBanner() {
|
||||
// Suppress banner in agent mode for minimal output
|
||||
if( SysEnv.isAgentMode() ) {
|
||||
return
|
||||
}
|
||||
|
||||
if( launcher.options.ansiLog ){
|
||||
// Plain header for verbose log
|
||||
log.debug "N E X T F L O W ~ version ${BuildInfo.version}"
|
||||
|
||||
// Fancy coloured header for the ANSI console output
|
||||
def fmt = ansi()
|
||||
fmt.a("\n")
|
||||
// Use exact colour codes so that they render the same on every terminal,
|
||||
// irrespective of terminal colour scheme.
|
||||
// Nextflow green RGB (13, 192, 157) and exact black text (0,0,0),
|
||||
// Apple Terminal only supports 256 colours, so use the closest match:
|
||||
// light sea green | #20B2AA | 38;5;0
|
||||
// Don't use black for text as terminals mess with this in their colour schemes.
|
||||
// Use very dark grey, which is more reliable.
|
||||
// Jansi library bundled in Jline can't do exact RGBs,
|
||||
// so just do the ANSI codes manually
|
||||
final BACKGROUND = "\033[1m\033[38;5;232m\033[48;5;43m"
|
||||
fmt.a("$BACKGROUND N E X T F L O W ").reset()
|
||||
|
||||
// Show Nextflow version
|
||||
fmt.a(Attribute.INTENSITY_FAINT).a(" ~ ").reset().a("version " + BuildInfo.version).reset()
|
||||
fmt.a("\n")
|
||||
AnsiConsole.out.println(fmt.eraseLine())
|
||||
}
|
||||
else {
|
||||
// Plain header to the console if ANSI is disabled
|
||||
log.info "N E X T F L O W ~ version ${BuildInfo.version}"
|
||||
}
|
||||
}
|
||||
|
||||
protected void launchInfo(ConfigMap config, ScriptFile scriptFile) {
|
||||
// -- determine strict mode
|
||||
detectStrictFeature(config, sysEnv)
|
||||
// -- determine moduleBinary
|
||||
detectModuleBinaryFeature(config)
|
||||
// -- show launch info
|
||||
final repo = scriptFile.repository ?: scriptFile.source.toString()
|
||||
final head = preview ? "* PREVIEW * $scriptFile.repository" : "Launching `$repo`"
|
||||
final revision = scriptFile.repository
|
||||
? scriptFile.revisionInfo.toString()
|
||||
: scriptFile.getScriptId()?.substring(0,10)
|
||||
printLaunchInfo(repo, head, revision)
|
||||
}
|
||||
|
||||
static void detectModuleBinaryFeature(ConfigMap config) {
|
||||
final moduleBinaries = config.navigate('nextflow.enable.moduleBinaries', false)
|
||||
if( moduleBinaries ) {
|
||||
log.debug "Enabling module binaries"
|
||||
NextflowMeta.instance.moduleBinaries(true)
|
||||
}
|
||||
}
|
||||
|
||||
static void detectStrictFeature(ConfigMap config, Map sysEnv) {
|
||||
if( NF.isSyntaxParserV2() )
|
||||
return
|
||||
final defStrict = sysEnv.get('NXF_ENABLE_STRICT') ?: false
|
||||
final strictMode = config.navigate('nextflow.enable.strict', defStrict)
|
||||
if( strictMode ) {
|
||||
log.debug "Enabling nextflow strict mode"
|
||||
NextflowMeta.instance.strictMode(true)
|
||||
}
|
||||
}
|
||||
|
||||
protected void printLaunchInfo(String repo, String head, String revision) {
|
||||
// Agent mode output is handled by AgentLogObserver
|
||||
if( SysEnv.isAgentMode() ) {
|
||||
return
|
||||
}
|
||||
|
||||
if( launcher.options.ansiLog ){
|
||||
log.debug "${head} [$runName] - revision: ${revision}"
|
||||
|
||||
def fmt = ansi()
|
||||
fmt.a("Launching").fg(Color.MAGENTA).a(" `$repo` ").reset()
|
||||
fmt.a(Attribute.INTENSITY_FAINT).a("[").reset()
|
||||
fmt.bold().fg(Color.CYAN).a(runName).reset()
|
||||
fmt.a(Attribute.INTENSITY_FAINT).a("] ").reset()
|
||||
fmt.fg(Color.CYAN).a("revision: ").reset()
|
||||
fmt.fg(Color.CYAN).a(revision).reset()
|
||||
fmt.a("\n")
|
||||
AnsiConsole.out().println(fmt.eraseLine())
|
||||
}
|
||||
else {
|
||||
log.info "${head} [$runName] - revision: ${revision}"
|
||||
}
|
||||
}
|
||||
|
||||
protected void checkRunName() {
|
||||
if( runName == 'last' )
|
||||
throw new AbortOperationException("Not a valid run name: `last`")
|
||||
if( runName && !matchRunName(runName) )
|
||||
throw new AbortOperationException("Not a valid run name: `$runName` -- It must match the pattern $RUN_NAME_PATTERN")
|
||||
|
||||
if( !runName ) {
|
||||
if( HistoryFile.disabled() )
|
||||
throw new AbortOperationException("Missing workflow run name")
|
||||
// -- make sure the generated name does not exist already
|
||||
runName = HistoryFile.DEFAULT.generateNextName()
|
||||
}
|
||||
|
||||
else if( !HistoryFile.disabled() && HistoryFile.DEFAULT.checkExistsByName(runName) )
|
||||
throw new AbortOperationException("Run name `$runName` has been already used -- Specify a different one")
|
||||
}
|
||||
|
||||
static protected boolean matchRunName(String name) {
|
||||
RUN_NAME_PATTERN.matcher(name).matches()
|
||||
}
|
||||
|
||||
protected ScriptFile getScriptFile(String pipelineName) {
|
||||
try {
|
||||
getScriptFile0(pipelineName)
|
||||
}
|
||||
catch (IllegalArgumentException | AbortOperationException e) {
|
||||
if( e.message.startsWith("Not a valid project name:") && !guessIsRepo(pipelineName)) {
|
||||
throw new AbortOperationException("Cannot find script file: $pipelineName")
|
||||
}
|
||||
else
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
static protected boolean guessIsRepo(String name) {
|
||||
if( FileHelper.getUrlProtocol(name) != null )
|
||||
return true
|
||||
if( name.startsWith('/') )
|
||||
return false
|
||||
if( name.startsWith('./') || name.startsWith('../') )
|
||||
return false
|
||||
if( name.endsWith('.nf') )
|
||||
return false
|
||||
if( name.count('/') != 1 )
|
||||
return false
|
||||
return true
|
||||
}
|
||||
|
||||
protected ScriptFile getScriptFile0(String pipelineName) {
|
||||
assert pipelineName
|
||||
|
||||
/*
|
||||
* read from the stdin
|
||||
*/
|
||||
if( pipelineName == '-' ) {
|
||||
final file = tryReadFromStdin()
|
||||
if( !file )
|
||||
throw new AbortOperationException("Cannot access `stdin` stream")
|
||||
|
||||
if( revision )
|
||||
throw new AbortOperationException("Revision option cannot be used when running a script from stdin")
|
||||
|
||||
return new ScriptFile(file)
|
||||
}
|
||||
|
||||
/*
|
||||
* look for a file with the specified pipeline name
|
||||
*/
|
||||
def script = new File(pipelineName)
|
||||
if( script.isDirectory() ) {
|
||||
script = mainScript ? new File(mainScript) : new AssetManager().setLocalPath(script).getMainScriptFile()
|
||||
}
|
||||
|
||||
if( script.exists() ) {
|
||||
if( revision )
|
||||
throw new AbortOperationException("Revision option cannot be used when running a local script")
|
||||
return new ScriptFile(script)
|
||||
}
|
||||
|
||||
/*
|
||||
* try to look for a pipeline in the repository
|
||||
*/
|
||||
try( final manager = new AssetManager(pipelineName, revision, mainScript, this) ) {
|
||||
final repo = manager.getProjectWithRevision()
|
||||
final remoteSource = !manager.isLocalScmSource()
|
||||
|
||||
boolean checkForUpdate = true
|
||||
if( !manager.isRunnable() || latest ) {
|
||||
if( offline && remoteSource )
|
||||
throw new AbortOperationException("Unknown project `$repo` -- NOTE: automatic download from remote repositories is disabled")
|
||||
log.info "Pulling $repo ..."
|
||||
final result = manager.download(revision,deep)
|
||||
if( result )
|
||||
log.info " $result"
|
||||
checkForUpdate = false
|
||||
}
|
||||
// Warn if using legacy
|
||||
if( manager.isUsingLegacyStrategy() ) {
|
||||
log.warn1 "This Nextflow version supports a new Multi-revision strategy for managing the SCM repositories, " +
|
||||
"but '${repo}' is single-revision legacy strategy - Please consider to update the repository with the 'nextflow pull -migrate' command."
|
||||
}
|
||||
// post download operations
|
||||
try {
|
||||
manager.checkout(revision)
|
||||
manager.updateModules()
|
||||
final scriptFile = manager.getScriptFile(mainScript)
|
||||
if( checkForUpdate && !(offline && remoteSource) )
|
||||
manager.checkRemoteStatus(scriptFile.revisionInfo)
|
||||
// return the script file
|
||||
return scriptFile
|
||||
}
|
||||
catch( AbortOperationException e ) {
|
||||
throw e
|
||||
}
|
||||
catch( Exception e ) {
|
||||
throw new AbortOperationException("Unknown error accessing project `$repo` -- Repository may be corrupted: ${manager.localPath}", e)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
static protected File tryReadFromStdin() {
|
||||
if( !System.in.available() )
|
||||
return null
|
||||
|
||||
getScriptFromStream(System.in)
|
||||
}
|
||||
|
||||
static protected File getScriptFromStream( InputStream input, String name = 'nextflow' ) {
|
||||
input != null
|
||||
File result = File.createTempFile(name, null)
|
||||
result.deleteOnExit()
|
||||
input.withReader { Reader reader -> result << reader }
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Check the Nextflow version against the version required by
|
||||
* the pipeline via `manifest.nextflowVersion`.
|
||||
*
|
||||
* When the version spec is prefixed with '!', the run will fail
|
||||
* if the Nextflow version does not match.
|
||||
*
|
||||
* @param config
|
||||
*/
|
||||
protected void checkVersion(Map config) {
|
||||
final manifest = new Manifest(config.manifest as Map ?: Collections.emptyMap())
|
||||
String version = manifest.nextflowVersion?.trim()
|
||||
if( !version )
|
||||
return
|
||||
|
||||
final important = version.startsWith('!')
|
||||
if( important )
|
||||
version = version.substring(1).trim()
|
||||
|
||||
if( !getCurrentVersion().matches(version) ) {
|
||||
if( important )
|
||||
showVersionError(version)
|
||||
else
|
||||
showVersionWarning(version)
|
||||
}
|
||||
}
|
||||
|
||||
protected VersionNumber getCurrentVersion() {
|
||||
return new VersionNumber(BuildInfo.version)
|
||||
}
|
||||
|
||||
protected void showVersionError(String ver) {
|
||||
throw new AbortOperationException("Nextflow version ${BuildInfo.version} does not match version required by pipeline: ${ver}")
|
||||
}
|
||||
|
||||
protected void showVersionWarning(String ver) {
|
||||
log.warn "Nextflow version ${BuildInfo.version} does not match version required by pipeline: ${ver} -- execution will continue, but things might break!"
|
||||
}
|
||||
|
||||
@Memoized // <-- avoid parse multiple times the same file and params
|
||||
Map parsedParams(Map configVars) {
|
||||
|
||||
final result = [:]
|
||||
|
||||
// apply params file
|
||||
final file = getParamsFile()
|
||||
if( file ) {
|
||||
def path = validateParamsFile(file)
|
||||
def type = path.extension.toLowerCase() ?: null
|
||||
if( type == 'json' )
|
||||
readJsonFile(path, configVars, result)
|
||||
else if( type == 'yml' || type == 'yaml' )
|
||||
readYamlFile(path, configVars, result)
|
||||
}
|
||||
|
||||
// apply CLI params
|
||||
if( !params )
|
||||
return result
|
||||
|
||||
for( Map.Entry<String,String> entry : params ) {
|
||||
addParam( result, entry.key, entry.value )
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
|
||||
static final private Pattern DOT_ESCAPED = ~/\\\./
|
||||
static final private Pattern DOT_NOT_ESCAPED = ~/(?<!\\)\./
|
||||
|
||||
static protected void addParam(Map params, String key, String value, List path=[], String fullKey=null) {
|
||||
if( !fullKey )
|
||||
fullKey = key
|
||||
final m = DOT_NOT_ESCAPED.matcher(key)
|
||||
if( m.find() ) {
|
||||
final p = m.start()
|
||||
final root = key.substring(0, p)
|
||||
if( !root ) throw new AbortOperationException("Invalid parameter name: $fullKey")
|
||||
path.add(root)
|
||||
def nested = params.get(root)
|
||||
if( nested == null ) {
|
||||
nested = new LinkedHashMap<>()
|
||||
params.put(root, nested)
|
||||
}
|
||||
else if( nested !instanceof Map ) {
|
||||
log.warn "Command line parameter --${path.join('.')} is overwritten by --${fullKey}"
|
||||
nested = new LinkedHashMap<>()
|
||||
params.put(root, nested)
|
||||
}
|
||||
addParam((Map)nested, key.substring(p+1), value, path, fullKey)
|
||||
}
|
||||
else {
|
||||
addParam0(params, key.replaceAll(DOT_ESCAPED,'.'), parseParamValue(value))
|
||||
}
|
||||
}
|
||||
|
||||
static protected void addParam0(Map params, String key, Object value) {
|
||||
if( key.contains('-') )
|
||||
key = kebabToCamelCase(key)
|
||||
params.put(key, value)
|
||||
}
|
||||
|
||||
static protected String kebabToCamelCase(String str) {
|
||||
final result = new StringBuilder()
|
||||
str.split('-').eachWithIndex { String entry, int i ->
|
||||
result << (i>0 ? StringUtils.capitalize(entry) : entry )
|
||||
}
|
||||
return result.toString()
|
||||
}
|
||||
|
||||
static protected parseParamValue(String str) {
|
||||
if ( SysEnv.get('NXF_DISABLE_PARAMS_TYPE_DETECTION') || NF.isSyntaxParserV2() )
|
||||
return str
|
||||
|
||||
if ( str == null ) return null
|
||||
|
||||
if ( str.toLowerCase() == 'true') return Boolean.TRUE
|
||||
if ( str.toLowerCase() == 'false' ) return Boolean.FALSE
|
||||
|
||||
if ( str==~/-?\d+(\.\d+)?/ && str.isInteger() ) return str.toInteger()
|
||||
if ( str==~/-?\d+(\.\d+)?/ && str.isLong() ) return str.toLong()
|
||||
if ( str==~/-?\d+(\.\d+)?/ && str.isDouble() ) return str.toDouble()
|
||||
|
||||
return str
|
||||
}
|
||||
|
||||
private Path validateParamsFile(String file) {
|
||||
|
||||
def result = FileHelper.asPath(file)
|
||||
def ext = result.getExtension()
|
||||
if( !VALID_PARAMS_FILE.contains(ext) )
|
||||
throw new AbortOperationException("Not a valid params file extension: $file -- It must be one of the following: ${VALID_PARAMS_FILE.join(',')}")
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
static private Pattern PARAMS_VAR = ~/(?m)\$\{(\p{javaJavaIdentifierStart}\p{javaJavaIdentifierPart}*)}/
|
||||
|
||||
protected String replaceVars0(String content, Map binding) {
|
||||
content.replaceAll(PARAMS_VAR) { List<String> matcher ->
|
||||
// - the regex matcher is represented as list
|
||||
// - the first element is the matching string ie. `${something}`
|
||||
// - the second element is the group content ie. `something`
|
||||
// - make sure the regex contains at least a group otherwise the closure
|
||||
// parameter is a string instead of a list of the call fail
|
||||
final placeholder = matcher.get(0)
|
||||
final key = matcher.get(1)
|
||||
|
||||
if( !binding.containsKey(key) ) {
|
||||
final msg = "Missing params file variable: $placeholder"
|
||||
if(NF.strictMode)
|
||||
throw new AbortOperationException(msg)
|
||||
log.warn msg
|
||||
return placeholder
|
||||
}
|
||||
|
||||
return binding.get(key)
|
||||
}
|
||||
}
|
||||
|
||||
private void readJsonFile(Path file, Map configVars, Map result) {
|
||||
try {
|
||||
def text = configVars ? replaceVars0(file.text, configVars) : file.text
|
||||
def json = (Map<String,Object>) new JsonSlurper().parseText(text)
|
||||
json.forEach((name, value) -> {
|
||||
addParam0(result, name, value)
|
||||
})
|
||||
}
|
||||
catch (NoSuchFileException | FileNotFoundException e) {
|
||||
throw new AbortOperationException("Specified params file does not exist: ${file.toUriString()}")
|
||||
}
|
||||
catch( Exception e ) {
|
||||
throw new AbortOperationException("Cannot parse params file: ${file.toUriString()} - Cause: ${e.message}", e)
|
||||
}
|
||||
}
|
||||
|
||||
private void readYamlFile(Path file, Map configVars, Map result) {
|
||||
try {
|
||||
def text = configVars ? replaceVars0(file.text, configVars) : file.text
|
||||
def yaml = (Map<String,Object>) new Yaml().load(text)
|
||||
yaml.forEach((name, value) -> {
|
||||
addParam0(result, name, value)
|
||||
})
|
||||
}
|
||||
catch (NoSuchFileException | FileNotFoundException e) {
|
||||
throw new AbortOperationException("Specified params file does not exist: ${file.toUriString()}")
|
||||
}
|
||||
catch( Exception e ) {
|
||||
throw new AbortOperationException("Cannot parse params file: ${file.toUriString()}", e)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.Parameters
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.plugin.Plugins
|
||||
import nextflow.secret.SecretsLoader
|
||||
import nextflow.secret.SecretsProvider
|
||||
/**
|
||||
* Implements the {@code secret} command
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
@Parameters(commandDescription = "Manage pipeline secrets")
|
||||
class CmdSecret extends CmdBase implements UsageAware {
|
||||
|
||||
interface SubCmd {
|
||||
String getName()
|
||||
void apply(List<String> result)
|
||||
void usage(List<String> result)
|
||||
}
|
||||
|
||||
static public final String NAME = 'secrets'
|
||||
|
||||
private List<SubCmd> commands = []
|
||||
|
||||
String getName() {
|
||||
return NAME
|
||||
}
|
||||
|
||||
@Parameter(hidden = true)
|
||||
List<String> args
|
||||
|
||||
private SecretsProvider provider
|
||||
|
||||
CmdSecret() {
|
||||
commands.add( new GetCmd() )
|
||||
commands.add( new SetCmd() )
|
||||
commands.add( new ListCmd() )
|
||||
commands.add( new DeleteCmd() )
|
||||
}
|
||||
|
||||
/**
|
||||
* Print the command usage help
|
||||
*/
|
||||
void usage() {
|
||||
usage(args)
|
||||
}
|
||||
|
||||
/**
|
||||
* Print the command usage help
|
||||
*
|
||||
* @param args The arguments as entered by the user
|
||||
*/
|
||||
void usage(List<String> args) {
|
||||
|
||||
List<String> result = []
|
||||
if( !args ) {
|
||||
result << this.getClass().getAnnotation(Parameters).commandDescription()
|
||||
result << 'Usage: nextflow secrets <sub-command> [options]'
|
||||
result << ''
|
||||
result << 'Commands:'
|
||||
commands.collect{ it.name }.sort().each { result << " $it".toString() }
|
||||
result << ''
|
||||
}
|
||||
else {
|
||||
def sub = commands.find { it.name == args[0] }
|
||||
if( sub )
|
||||
sub.usage(result)
|
||||
else {
|
||||
throw new AbortOperationException("Unknown secrets sub-command: ${args[0]}")
|
||||
}
|
||||
}
|
||||
|
||||
println result.join('\n').toString()
|
||||
}
|
||||
|
||||
/**
|
||||
* Main command entry point
|
||||
*/
|
||||
@Override
|
||||
void run() {
|
||||
if( !args ) {
|
||||
usage()
|
||||
return
|
||||
}
|
||||
|
||||
// setup the plugins system and load the secrets provider
|
||||
Plugins.init()
|
||||
provider = SecretsLoader.instance.load()
|
||||
|
||||
// run the command
|
||||
try {
|
||||
getCmd(args).apply(args.drop(1))
|
||||
}
|
||||
finally {
|
||||
// close the provider
|
||||
provider?.close()
|
||||
}
|
||||
}
|
||||
|
||||
protected SubCmd getCmd(List<String> args) {
|
||||
|
||||
def cmd = commands.find { it.name == args[0] }
|
||||
if( cmd ) {
|
||||
return cmd
|
||||
}
|
||||
|
||||
def matches = commands.collect{ it.name }.closest(args[0])
|
||||
def msg = "Unknown cloud sub-command: ${args[0]}"
|
||||
if( matches )
|
||||
msg += " -- Did you mean one of these?\n" + matches.collect { " $it"}.join('\n')
|
||||
throw new AbortOperationException(msg)
|
||||
}
|
||||
|
||||
private void addOption(String fieldName, List<String> result) {
|
||||
def annot = this.class.getDeclaredField(fieldName)?.getAnnotation(Parameter)
|
||||
if( annot ) {
|
||||
result << ' ' + annot.names().join(', ')
|
||||
result << ' ' + annot.description()
|
||||
}
|
||||
else {
|
||||
log.debug "Unknown help field: $fieldName"
|
||||
}
|
||||
}
|
||||
|
||||
class SetCmd implements SubCmd {
|
||||
|
||||
@Override
|
||||
String getName() { 'set' }
|
||||
|
||||
@Override
|
||||
void apply(List<String> result) {
|
||||
if( result.size() < 1 )
|
||||
throw new AbortOperationException("Missing secret name")
|
||||
if( result.size() < 2 )
|
||||
throw new AbortOperationException("Missing secret value")
|
||||
|
||||
String secretName = result.first()
|
||||
String secretValue = result.last()
|
||||
provider.putSecret(secretName, secretValue)
|
||||
}
|
||||
|
||||
@Override
|
||||
void usage(List<String> result) {
|
||||
result << 'Set a key-pair in the secrets store'
|
||||
result << "Usage: nextflow secrets $name <NAME> <VALUE>".toString()
|
||||
result << ''
|
||||
result << ''
|
||||
}
|
||||
}
|
||||
|
||||
class GetCmd implements SubCmd {
|
||||
|
||||
@Override
|
||||
String getName() { 'get' }
|
||||
|
||||
@Override
|
||||
void apply(List<String> result) {
|
||||
if( result.size() != 1 )
|
||||
throw new AbortOperationException("Wrong number of arguments")
|
||||
|
||||
String secretName = result.first()
|
||||
if( !secretName )
|
||||
throw new AbortOperationException("Missing secret name")
|
||||
println provider.getSecret(secretName)?.value
|
||||
}
|
||||
|
||||
@Override
|
||||
void usage(List<String> result) {
|
||||
result << 'Get a secret value with the name'
|
||||
result << "Usage: nextflow secrets $name <NAME>".toString()
|
||||
result << ''
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Implements the secret `list` sub-command
|
||||
*/
|
||||
class ListCmd implements SubCmd {
|
||||
@Override
|
||||
String getName() { 'list' }
|
||||
|
||||
@Override
|
||||
void apply(List<String> result) {
|
||||
if( result.size() )
|
||||
throw new AbortOperationException("Wrong number of arguments")
|
||||
|
||||
final names = new ArrayList(provider.listSecretsNames()).sort()
|
||||
if( names ) {
|
||||
for( String it : names ) {
|
||||
println it
|
||||
}
|
||||
}
|
||||
else {
|
||||
println "no secrets available"
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
void usage(List<String> result) {
|
||||
result << 'List all names in the secrets store'
|
||||
result << "Usage: nextflow secrets $name".toString()
|
||||
result << ''
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Implements the secret `remove` sub-command
|
||||
*/
|
||||
class DeleteCmd implements SubCmd {
|
||||
@Override
|
||||
String getName() { 'delete' }
|
||||
|
||||
@Override
|
||||
void apply(List<String> result) {
|
||||
if( result.size() != 1 )
|
||||
throw new AbortOperationException("Wrong number of arguments")
|
||||
|
||||
String secretName = result.first()
|
||||
|
||||
if( !secretName )
|
||||
throw new AbortOperationException("Missing secret name")
|
||||
provider.removeSecret(secretName)
|
||||
}
|
||||
|
||||
@Override
|
||||
void usage(List<String> result) {
|
||||
result << 'Delete an entry from the secrets store'
|
||||
result << "Usage: nextflow secrets $name".toString()
|
||||
result << ''
|
||||
addOption('secretName', result)
|
||||
result << ''
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
|
||||
import com.beust.jcommander.Parameters
|
||||
|
||||
/**
|
||||
* Self-update command
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Parameters(commandDescription = "Update nextflow runtime to the latest available version")
|
||||
class CmdSelfUpdate extends CmdBase {
|
||||
@Override
|
||||
String getName() { 'self-update' }
|
||||
|
||||
@Override
|
||||
void run() {
|
||||
// actually it's doing nothing, the update process is managed by the external launcher script
|
||||
// this class in only necessary to print the command line the usage output
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.Parameters
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.plugin.Plugins
|
||||
import nextflow.scm.AssetManager
|
||||
|
||||
/**
|
||||
* CLI sub-command VIEW -- Print a pipeline script to console
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
@Parameters(commandDescription = "View project script file(s)")
|
||||
class CmdView extends CmdBase {
|
||||
|
||||
static final public NAME = 'view'
|
||||
|
||||
@Override
|
||||
String getName() { NAME }
|
||||
|
||||
@Parameter(description = 'project name', required = true)
|
||||
List<String> args = []
|
||||
|
||||
@Parameter(names=['-r','-revision'], description = 'Revision of the project (either a git branch, tag or commit SHA number)')
|
||||
String revision
|
||||
|
||||
@Parameter(names = '-q', description = 'Hide header line', arity = 0)
|
||||
boolean quiet
|
||||
|
||||
@Parameter(names = '-l', description = 'List repository content', arity = 0)
|
||||
boolean all
|
||||
|
||||
@Override
|
||||
void run() {
|
||||
Plugins.init()
|
||||
try (final manager = new AssetManager(args[0], revision)) {
|
||||
if( !manager.isLocal() )
|
||||
throw new AbortOperationException("Unknown project `${manager.getProjectWithRevision()}`")
|
||||
if( revision && manager.isUsingLegacyStrategy()){
|
||||
log.warn("The local asset ${args[0]} does not support multi-revision - 'revision' option is ignored\n" +
|
||||
"Consider updating the asset using 'nextflow pull ${args[0]} -r $revision -migrate'")
|
||||
}
|
||||
if( all ) {
|
||||
if( !quiet )
|
||||
println "== content of path: ${manager.localPath}"
|
||||
|
||||
manager.localPath.eachFile { File it ->
|
||||
println it.name
|
||||
}
|
||||
}
|
||||
|
||||
else {
|
||||
/*
|
||||
* prints the script main file
|
||||
*/
|
||||
final script = manager.getMainScriptFile()
|
||||
if( !script.exists() )
|
||||
throw new AbortOperationException("Missing script file: '${script}'")
|
||||
|
||||
if( !quiet )
|
||||
println "== content of file: $script"
|
||||
|
||||
script.readLines().each { println it }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
|
||||
import com.beust.jcommander.Parameter
|
||||
import groovy.transform.CompileStatic
|
||||
/**
|
||||
* Defines the command line parameters for command that need to interact with a pipeline service hub i.e. GitHub or BitBucket
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
|
||||
@CompileStatic
|
||||
trait HubOptions {
|
||||
|
||||
@Parameter(names=['-hub'], description = "Service hub where the project is hosted")
|
||||
String hubProvider
|
||||
|
||||
@Parameter(names='-user', description = 'Private repository user name')
|
||||
String hubUser
|
||||
|
||||
/**
|
||||
* Return the password provided on the command line or stop allowing the user to enter it on the console
|
||||
*
|
||||
* @return The password entered or {@code null} if no user has been entered
|
||||
*/
|
||||
String getHubPassword() {
|
||||
|
||||
if( !hubUser )
|
||||
return null
|
||||
|
||||
def p = hubUser.indexOf(':')
|
||||
if( p != -1 )
|
||||
return hubUser.substring(p+1)
|
||||
|
||||
def console = System.console()
|
||||
if( !console )
|
||||
return null
|
||||
|
||||
print "Enter your $hubProvider password: "
|
||||
char[] pwd = console.readPassword()
|
||||
new String(pwd)
|
||||
}
|
||||
|
||||
String getHubUser() {
|
||||
if(!hubUser) {
|
||||
return hubUser
|
||||
}
|
||||
|
||||
def p = hubUser.indexOf(':')
|
||||
return p != -1 ? hubUser.substring(0,p) : hubUser
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,724 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
|
||||
import static nextflow.Const.*
|
||||
|
||||
import java.lang.reflect.Field
|
||||
|
||||
import com.beust.jcommander.DynamicParameter
|
||||
import com.beust.jcommander.JCommander
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.ParameterException
|
||||
import com.beust.jcommander.Parameters
|
||||
import groovy.transform.CompileDynamic
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.PackageScope
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.BuildInfo
|
||||
import nextflow.NF
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.exception.AbortRunException
|
||||
import nextflow.exception.ConfigParseException
|
||||
import nextflow.exception.ScriptCompilationException
|
||||
import nextflow.exception.ScriptRuntimeException
|
||||
import nextflow.secret.SecretsLoader
|
||||
import nextflow.util.Escape
|
||||
import nextflow.util.LoggerHelper
|
||||
import nextflow.util.ProxyConfig
|
||||
import nextflow.util.SpuriousDeps
|
||||
import org.eclipse.jgit.api.errors.GitAPIException
|
||||
|
||||
import static nextflow.util.SysHelper.dumpThreads
|
||||
|
||||
/**
|
||||
* Main application entry point. It parses the command line and
|
||||
* launch the pipeline execution.
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class Launcher {
|
||||
|
||||
/**
|
||||
* Create the application command line parser
|
||||
*
|
||||
* @return An instance of {@code CliBuilder}
|
||||
*/
|
||||
|
||||
private JCommander jcommander
|
||||
|
||||
private CliOptions options
|
||||
|
||||
private boolean fullVersion
|
||||
|
||||
private CmdBase command
|
||||
|
||||
private String cliString
|
||||
|
||||
private List<CmdBase> allCommands
|
||||
|
||||
private List<String> normalizedArgs
|
||||
|
||||
private boolean daemonMode
|
||||
|
||||
private String colsString
|
||||
|
||||
/**
|
||||
* Create a launcher object and parse the command line parameters
|
||||
*
|
||||
* @param args The command line arguments provided by the user
|
||||
*/
|
||||
Launcher() {
|
||||
init()
|
||||
}
|
||||
|
||||
protected void init() {
|
||||
allCommands = (List<CmdBase>)[
|
||||
new CmdAuth(),
|
||||
new CmdClean(),
|
||||
new CmdClone(),
|
||||
new CmdConsole(),
|
||||
new CmdFs(),
|
||||
new CmdInfo(),
|
||||
new CmdLaunch(),
|
||||
new CmdList(),
|
||||
new CmdLog(),
|
||||
new CmdPull(),
|
||||
new CmdRun(),
|
||||
new CmdKubeRun(),
|
||||
new CmdDrop(),
|
||||
new CmdConfig(),
|
||||
new CmdNode(),
|
||||
new CmdView(),
|
||||
new CmdHelp(),
|
||||
new CmdSelfUpdate(),
|
||||
new CmdPlugin(),
|
||||
new CmdInspect(),
|
||||
new CmdLint(),
|
||||
new CmdLineage(),
|
||||
new CmdModule()
|
||||
]
|
||||
|
||||
if(SecretsLoader.isEnabled())
|
||||
allCommands.add(new CmdSecret())
|
||||
|
||||
// legacy command
|
||||
final cmdCloud = SpuriousDeps.cmdCloud()
|
||||
if( cmdCloud )
|
||||
allCommands.add(cmdCloud)
|
||||
|
||||
options = new CliOptions()
|
||||
jcommander = new JCommander(options)
|
||||
for( CmdBase cmd : allCommands ) {
|
||||
cmd.launcher = this;
|
||||
jcommander.addCommand(cmd.name, cmd, aliases(cmd))
|
||||
}
|
||||
jcommander.setProgramName( APP_NAME )
|
||||
|
||||
//Allow unknown options for module command
|
||||
jcommander.getCommands().get(CmdModule.NAME)?.setAcceptUnknownOptions(true)
|
||||
}
|
||||
|
||||
private static final String[] EMPTY = new String[0]
|
||||
|
||||
private static String[] aliases(CmdBase cmd) {
|
||||
final aliases = cmd.getClass().getAnnotation(Parameters)?.commandNames()
|
||||
return aliases ?: EMPTY
|
||||
}
|
||||
|
||||
/**
|
||||
* Create the Jcommander 'interpreter' and parse the command line arguments
|
||||
*/
|
||||
@PackageScope
|
||||
Launcher parseMainArgs(String... args) {
|
||||
this.cliString = makeCli(System.getenv('NXF_CLI'), args)
|
||||
this.colsString = System.getenv('COLUMNS')
|
||||
|
||||
def cols = getColumns()
|
||||
if( cols )
|
||||
jcommander.setColumnSize(cols)
|
||||
|
||||
normalizedArgs = normalizeArgs(args)
|
||||
jcommander.parse( normalizedArgs as String[] )
|
||||
fullVersion = '-version' in normalizedArgs
|
||||
command = allCommands.find { it.name == jcommander.getParsedCommand() }
|
||||
//Attach unknown options to command in case of needed
|
||||
if (command) {
|
||||
final unknownOptions = jcommander.commands.get(jcommander.getParsedCommand())?.getUnknownOptions() ?: []
|
||||
command.setUnknownOptions(unknownOptions)
|
||||
}
|
||||
// whether is running a daemon
|
||||
daemonMode = command instanceof CmdNode
|
||||
// set the log file name
|
||||
checkLogFileName()
|
||||
|
||||
return this
|
||||
}
|
||||
|
||||
protected String makeCli(String cli, String... args) {
|
||||
if( !cli )
|
||||
cli = 'nextflow'
|
||||
if( !args )
|
||||
return cli
|
||||
def cmd = ' ' + args[0]
|
||||
int p = cli.indexOf(cmd)
|
||||
if( p!=-1 )
|
||||
cli = cli.substring(0,p)
|
||||
if( cli.endsWith('nextflow') )
|
||||
cli = 'nextflow'
|
||||
cli += ' ' + Escape.cli(args)
|
||||
return cli
|
||||
}
|
||||
|
||||
private void checkLogFileName() {
|
||||
if( !options.logFile ) {
|
||||
if( isDaemon() )
|
||||
options.logFile = System.getenv('NXF_LOG_FILE') ?: '.node-nextflow.log'
|
||||
else if( command instanceof CmdModule && (command as CmdModule).args?.first() == 'run' )
|
||||
options.logFile = System.getenv('NXF_LOG_FILE') ?: ".module.log"
|
||||
else if( command instanceof CmdRun || command instanceof CmdLaunch || command instanceof CmdAuth || options.debug || options.trace )
|
||||
options.logFile = System.getenv('NXF_LOG_FILE') ?: ".nextflow.log"
|
||||
}
|
||||
}
|
||||
|
||||
private short getColumns() {
|
||||
if( !colsString ) {
|
||||
return 0
|
||||
}
|
||||
|
||||
try {
|
||||
colsString.toShort()
|
||||
}
|
||||
catch( Exception e ) {
|
||||
log.debug "Unexpected terminal \$COLUMNS value: $colsString"
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
CliOptions getOptions() { options }
|
||||
|
||||
List<String> getNormalizedArgs() { normalizedArgs }
|
||||
|
||||
String getCliString() { cliString }
|
||||
|
||||
boolean isDaemon() { daemonMode }
|
||||
|
||||
/**
|
||||
* normalize the command line arguments to handle some corner cases
|
||||
*/
|
||||
@PackageScope
|
||||
List<String> normalizeArgs( String ... args ) {
|
||||
|
||||
List<String> normalized = []
|
||||
int i=0
|
||||
while( true ) {
|
||||
if( i==args.size() ) { break }
|
||||
|
||||
def current = args[i++]
|
||||
normalized << current
|
||||
|
||||
// when the first argument is a file, it's supposed to be a script to be executed
|
||||
if( i==1 && !allCommands.find { it.name == current } && new File(current).isFile() ) {
|
||||
normalized.add(0,CmdRun.NAME)
|
||||
}
|
||||
|
||||
else if( current == '-resume' ) {
|
||||
if( i<args.size() && !args[i].startsWith('-') && (args[i]=='last' || args[i] =~~ /[0-9a-f]{8}\-[0-9a-f]{4}\-[0-9a-f]{4}\-[0-9a-f]{4}\-[0-9a-f]{8}/) ) {
|
||||
normalized << args[i++]
|
||||
}
|
||||
else {
|
||||
normalized << 'last'
|
||||
}
|
||||
}
|
||||
else if( current == '-test' && (i==args.size() || args[i].startsWith('-'))) {
|
||||
normalized << '%all'
|
||||
}
|
||||
|
||||
else if( current == '-dump-hashes' && (i==args.size() || args[i].startsWith('-'))) {
|
||||
normalized << '-'
|
||||
}
|
||||
|
||||
else if( current == '-with-cloudcache' && (i==args.size() || args[i].startsWith('-'))) {
|
||||
normalized << '-'
|
||||
}
|
||||
|
||||
else if( current == '-with-trace' && (i==args.size() || args[i].startsWith('-'))) {
|
||||
normalized << '-'
|
||||
}
|
||||
|
||||
else if( current == '-with-report' && (i==args.size() || args[i].startsWith('-'))) {
|
||||
normalized << '-'
|
||||
}
|
||||
|
||||
else if( current == '-with-timeline' && (i==args.size() || args[i].startsWith('-'))) {
|
||||
normalized << '-'
|
||||
}
|
||||
|
||||
else if( current == '-with-dag' && (i==args.size() || args[i].startsWith('-'))) {
|
||||
normalized << '-'
|
||||
}
|
||||
|
||||
else if( current == '-with-docker' && (i==args.size() || args[i].startsWith('-'))) {
|
||||
normalized << '-'
|
||||
}
|
||||
|
||||
else if( current == '-with-podman' && (i==args.size() || args[i].startsWith('-'))) {
|
||||
normalized << '-'
|
||||
}
|
||||
|
||||
else if( current == '-with-singularity' && (i==args.size() || args[i].startsWith('-'))) {
|
||||
normalized << '-'
|
||||
}
|
||||
|
||||
else if( current == '-with-apptainer' && (i==args.size() || args[i].startsWith('-'))) {
|
||||
normalized << '-'
|
||||
}
|
||||
|
||||
else if( current == '-with-charliecloud' && (i==args.size() || args[i].startsWith('-'))) {
|
||||
normalized << '-'
|
||||
}
|
||||
|
||||
else if( current == '-with-conda' && (i==args.size() || args[i].startsWith('-'))) {
|
||||
normalized << '-'
|
||||
}
|
||||
|
||||
else if( current == '-with-spack' && (i==args.size() || args[i].startsWith('-'))) {
|
||||
normalized << '-'
|
||||
}
|
||||
|
||||
else if( current == '-with-weblog' && (i==args.size() || args[i].startsWith('-'))) {
|
||||
normalized << '-'
|
||||
}
|
||||
|
||||
else if( current == '-with-tower' && (i==args.size() || args[i].startsWith('-'))) {
|
||||
normalized << '-'
|
||||
}
|
||||
|
||||
else if( current == '-with-wave' && (i==args.size() || args[i].startsWith('-'))) {
|
||||
normalized << '-'
|
||||
}
|
||||
|
||||
else if( current == '-ansi-log' && (i==args.size() || args[i].startsWith('-'))) {
|
||||
normalized << 'true'
|
||||
}
|
||||
|
||||
else if( (current == '-stub' || current == '-stub-run') && (i==args.size() || args[i].startsWith('-'))) {
|
||||
normalized << 'true'
|
||||
}
|
||||
|
||||
else if( (current == '-N' || current == '-with-notification') && (i==args.size() || args[i].startsWith('-'))) {
|
||||
normalized << 'true'
|
||||
}
|
||||
|
||||
else if( current == '-with-fusion' && (i==args.size() || args[i].startsWith('-'))) {
|
||||
normalized << 'true'
|
||||
}
|
||||
|
||||
else if( current == '-syslog' && (i==args.size() || args[i].startsWith('-') || allCommands.find { it.name == args[i] } )) {
|
||||
normalized << 'localhost'
|
||||
}
|
||||
|
||||
else if( current == '-dump-channels' && (i==args.size() || args[i].startsWith('-'))) {
|
||||
normalized << '*'
|
||||
}
|
||||
|
||||
else if( current ==~ /^\-\-[a-zA-Z\d].*/ && !current.contains('=') ) {
|
||||
current += '='
|
||||
current += ( i<args.size() && isValue(args[i]) ? args[i++] : 'true' )
|
||||
normalized[-1] = current
|
||||
}
|
||||
|
||||
else if( current ==~ /^\-process\..+/ && !current.contains('=')) {
|
||||
current += '='
|
||||
current += ( i<args.size() && isValue(args[i]) ? args[i++] : 'true' )
|
||||
normalized[-1] = current
|
||||
}
|
||||
|
||||
else if( current ==~ /^\-cluster\..+/ && !current.contains('=')) {
|
||||
current += '='
|
||||
current += ( i<args.size() && isValue(args[i]) ? args[i++] : 'true' )
|
||||
normalized[-1] = current
|
||||
}
|
||||
|
||||
else if( current ==~ /^\-executor\..+/ && !current.contains('=')) {
|
||||
current += '='
|
||||
current += ( i<args.size() && isValue(args[i]) ? args[i++] : 'true' )
|
||||
normalized[-1] = current
|
||||
}
|
||||
|
||||
else if( current == 'run' && i<args.size() && args[i] == '-' ) {
|
||||
i++
|
||||
normalized << '-stdin'
|
||||
}
|
||||
}
|
||||
|
||||
return normalized
|
||||
}
|
||||
|
||||
static private boolean isValue( String x ) {
|
||||
if( !x ) return false // an empty string -> not a value
|
||||
if( x.size() == 1 ) return true // a single char is not an option -> value true
|
||||
!x.startsWith('-') || x.isNumber() || x.contains(' ')
|
||||
}
|
||||
|
||||
CmdBase findCommand( String cmdName ) {
|
||||
allCommands.find { it.name == cmdName }
|
||||
}
|
||||
|
||||
/**
|
||||
* Print the usage string for the given command - or -
|
||||
* the main program usage string if not command is specified
|
||||
*
|
||||
* @param command The command for which get help or {@code null}
|
||||
* @return The usage string
|
||||
*/
|
||||
void usage(String command = null ) {
|
||||
|
||||
if( command ) {
|
||||
def exists = allCommands.find { it.name == command } != null
|
||||
if( !exists ) {
|
||||
println "Asking help for unknown command: $command"
|
||||
return
|
||||
}
|
||||
|
||||
jcommander.usage(command)
|
||||
return
|
||||
}
|
||||
|
||||
println "Usage: nextflow [options] COMMAND [arg...]\n"
|
||||
printOptions(CliOptions)
|
||||
printCommands(allCommands)
|
||||
}
|
||||
|
||||
@CompileDynamic
|
||||
protected void printOptions(Class clazz) {
|
||||
List params = []
|
||||
for( Field f : clazz.getDeclaredFields() ) {
|
||||
def p = f.getAnnotation(Parameter)
|
||||
if(!p)
|
||||
p = f.getAnnotation(DynamicParameter)
|
||||
|
||||
if( p && !p.hidden() && p.description() && p.names() )
|
||||
params.add(p)
|
||||
|
||||
}
|
||||
|
||||
params.sort(true) { it -> it.names()[0] }
|
||||
|
||||
println "Options:"
|
||||
for( def p : params ) {
|
||||
println " ${p.names().join(', ')}"
|
||||
println " ${p.description()}"
|
||||
}
|
||||
}
|
||||
|
||||
protected void printCommands(List<CmdBase> commands) {
|
||||
println "\nCommands:"
|
||||
|
||||
int len = 0
|
||||
def all = new TreeMap<String,String>()
|
||||
new ArrayList<CmdBase>(commands).each {
|
||||
def description = it.getClass().getAnnotation(Parameters)?.commandDescription()
|
||||
if( description ) {
|
||||
all[it.name] = description
|
||||
if( it.name.size()>len ) len = it.name.size()
|
||||
}
|
||||
}
|
||||
|
||||
all.each { String name, String desc ->
|
||||
print ' '
|
||||
print name.padRight(len)
|
||||
print ' '
|
||||
println desc
|
||||
}
|
||||
println ''
|
||||
}
|
||||
|
||||
Launcher command( String[] args ) {
|
||||
/*
|
||||
* CLI argument parsing
|
||||
*/
|
||||
try {
|
||||
parseMainArgs(args)
|
||||
LoggerHelper.configureLogger(this)
|
||||
}
|
||||
catch( ParameterException e ) {
|
||||
// print command line parsing errors
|
||||
// note: use system.err.println since if an exception is raised
|
||||
// parsing the cli params the logging is not configured
|
||||
System.err.println "${e.getMessage()} -- Check the available commands and options and syntax with 'help'"
|
||||
System.exit(1)
|
||||
|
||||
}
|
||||
catch ( AbortOperationException e ) {
|
||||
final msg = e.message ?: "Unknown abort reason"
|
||||
System.err.println(LoggerHelper.formatErrMessage(msg, e))
|
||||
System.exit(1)
|
||||
}
|
||||
catch( Throwable e ) {
|
||||
e.printStackTrace(System.err)
|
||||
System.exit(1)
|
||||
}
|
||||
return this
|
||||
}
|
||||
|
||||
protected void checkForHelp() {
|
||||
if( options.help || !command || command.help ) {
|
||||
if( command instanceof UsageAware ) {
|
||||
(command as UsageAware).usage()
|
||||
// reset command to null to skip default execution
|
||||
command = null
|
||||
return
|
||||
}
|
||||
|
||||
// replace the current command with the `help` command
|
||||
def target = command?.name
|
||||
command = allCommands.find { it instanceof CmdHelp }
|
||||
if( target ) {
|
||||
(command as CmdHelp).args = [target]
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Launch the pipeline execution
|
||||
*/
|
||||
int run() {
|
||||
|
||||
/*
|
||||
* setup environment
|
||||
*/
|
||||
setupEnvironment()
|
||||
|
||||
/*
|
||||
* Real execution starts here
|
||||
*/
|
||||
try {
|
||||
log.debug '$> ' + cliString
|
||||
|
||||
// -- print out the version number, then exit
|
||||
if ( options.version ) {
|
||||
println getVersion(fullVersion)
|
||||
return 0
|
||||
}
|
||||
|
||||
// -- print out the program help, then exit
|
||||
checkForHelp()
|
||||
|
||||
// launch the command
|
||||
command?.run()
|
||||
|
||||
if( log.isTraceEnabled())
|
||||
log.trace "Exit\n" + dumpThreads()
|
||||
return 0
|
||||
}
|
||||
|
||||
catch( AbortRunException e ) {
|
||||
return(1)
|
||||
}
|
||||
|
||||
catch ( AbortOperationException e ) {
|
||||
def message = e.getMessage()
|
||||
if( message )
|
||||
System.err.println(LoggerHelper.formatErrMessage(message,e))
|
||||
log.debug ("Operation aborted", e.cause ?: e)
|
||||
return(1)
|
||||
}
|
||||
|
||||
catch ( GitAPIException e ) {
|
||||
System.err.println e.getMessage() ?: e.toString()
|
||||
log.debug ("Operation aborted", e.cause ?: e)
|
||||
return(1)
|
||||
}
|
||||
|
||||
catch( ConfigParseException e ) {
|
||||
if( NF.isSyntaxParserV2() ) {
|
||||
log.error(e.message, e)
|
||||
}
|
||||
else {
|
||||
def message = e.message
|
||||
if( e.cause?.message ) {
|
||||
message += "\n\n${e.cause.message.toString().indent(' ')}"
|
||||
}
|
||||
log.error(message, e.cause ?: e)
|
||||
}
|
||||
return(1)
|
||||
}
|
||||
|
||||
catch( ScriptCompilationException e ) {
|
||||
log.error(e.message, e)
|
||||
return(1)
|
||||
}
|
||||
|
||||
catch ( ScriptRuntimeException | IllegalArgumentException e) {
|
||||
log.error(e.message, e)
|
||||
return(1)
|
||||
}
|
||||
|
||||
catch( IOException e ) {
|
||||
log.error(e.message, e)
|
||||
return(1)
|
||||
}
|
||||
|
||||
catch( Throwable fail ) {
|
||||
log.error("@unknown", fail)
|
||||
return(1)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* set up environment and system properties. It checks the following
|
||||
* environment variables:
|
||||
* <li>http_proxy</li>
|
||||
* <li>https_proxy</li>
|
||||
* <li>ftp_proxy</li>
|
||||
* <li>HTTP_PROXY</li>
|
||||
* <li>HTTPS_PROXY</li>
|
||||
* <li>FTP_PROXY</li>
|
||||
* <li>NO_PROXY</li>
|
||||
*/
|
||||
private void setupEnvironment() {
|
||||
|
||||
final env = System.getenv()
|
||||
setProxy('HTTP',env)
|
||||
setProxy('HTTPS',env)
|
||||
setProxy('FTP',env)
|
||||
|
||||
setProxy('http',env)
|
||||
setProxy('https',env)
|
||||
setProxy('ftp',env)
|
||||
|
||||
setNoProxy(env)
|
||||
|
||||
setHttpClientProperties(env)
|
||||
}
|
||||
|
||||
static void setHttpClientProperties(Map<String,String> env) {
|
||||
// Set the httpclient connection pool timeout to 10 seconds.
|
||||
// This required because the default is 20 minutes, which cause the error
|
||||
// "HTTP/1.1 header parser received no bytes" when in some circumstances
|
||||
// https://github.com/nextflow-io/nextflow/issues/3983#issuecomment-1702305137
|
||||
System.setProperty("jdk.httpclient.keepalive.timeout", env.getOrDefault("NXF_JDK_HTTPCLIENT_KEEPALIVE_TIMEOUT","10"))
|
||||
if( env.get("NXF_JDK_HTTPCLIENT_CONNECTIONPOOLSIZE") )
|
||||
System.setProperty("jdk.httpclient.connectionPoolSize", env.get("NXF_JDK_HTTPCLIENT_CONNECTIONPOOLSIZE"))
|
||||
}
|
||||
|
||||
/**
|
||||
* Set no proxy property if defined in the launching env
|
||||
*
|
||||
* See for details
|
||||
* https://docs.oracle.com/javase/8/docs/technotes/guides/net/proxies.html
|
||||
*
|
||||
* @param env
|
||||
*/
|
||||
@PackageScope
|
||||
static void setNoProxy(Map<String,String> env) {
|
||||
final noProxy = env.get('NO_PROXY') ?: env.get('no_proxy')
|
||||
if(noProxy) {
|
||||
System.setProperty('http.nonProxyHosts', noProxy.tokenize(',').join('|'))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Setup proxy system properties and optionally configure the network authenticator
|
||||
*
|
||||
* See:
|
||||
* http://docs.oracle.com/javase/6/docs/technotes/guides/net/proxies.html
|
||||
* https://github.com/nextflow-io/nextflow/issues/24
|
||||
*
|
||||
* @param qualifier Either {@code http/HTTP} or {@code https/HTTPS}.
|
||||
* @param env The environment variables system map
|
||||
*/
|
||||
@PackageScope
|
||||
static void setProxy(String qualifier, Map<String,String> env ) {
|
||||
assert qualifier in ['http','https','ftp','HTTP','HTTPS','FTP']
|
||||
def str = null
|
||||
def var = "${qualifier}_" + (qualifier.isLowerCase() ? 'proxy' : 'PROXY')
|
||||
|
||||
// -- setup HTTP proxy
|
||||
try {
|
||||
final proxy = ProxyConfig.parse(str = env.get(var.toString()))
|
||||
if( proxy ) {
|
||||
// set the expected protocol
|
||||
proxy.protocol = qualifier.toLowerCase()
|
||||
log.debug "Setting $qualifier proxy: $proxy"
|
||||
System.setProperty("${qualifier.toLowerCase()}.proxyHost", proxy.host)
|
||||
if( proxy.port )
|
||||
System.setProperty("${qualifier.toLowerCase()}.proxyPort", proxy.port)
|
||||
if( proxy.authenticator() ) {
|
||||
log.debug "Setting $qualifier proxy authenticator"
|
||||
Authenticator.setDefault(proxy.authenticator())
|
||||
}
|
||||
}
|
||||
}
|
||||
catch ( MalformedURLException e ) {
|
||||
log.warn "Not a valid $qualifier proxy: '$str' -- Check the value of variable `$var` in your environment"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Hey .. Nextflow starts here!
|
||||
*
|
||||
* @param args The program options as specified by the user on the CLI
|
||||
*/
|
||||
static void main(String... args) {
|
||||
LoggerHelper.bootstrapLogger()
|
||||
final status = new Launcher() .command(args) .run()
|
||||
if( status )
|
||||
System.exit(status)
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Print the application version number
|
||||
* @param full When {@code true} prints full version number including build timestamp
|
||||
* @return The version number string
|
||||
*/
|
||||
static String getVersion(boolean full = false) {
|
||||
|
||||
if ( full ) {
|
||||
SPLASH
|
||||
}
|
||||
else {
|
||||
"${APP_NAME} version ${BuildInfo.version}.${BuildInfo.buildNum}"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
* The application 'logo'
|
||||
*/
|
||||
/*
|
||||
* The application 'logo'
|
||||
*/
|
||||
static public final String SPLASH =
|
||||
|
||||
"""
|
||||
N E X T F L O W
|
||||
version ${BuildInfo.version} build ${BuildInfo.buildNum}
|
||||
created ${BuildInfo.timestampUTC} ${BuildInfo.timestampDelta}
|
||||
cite doi:10.1038/nbt.3820
|
||||
http://nextflow.io
|
||||
"""
|
||||
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
|
||||
import java.nio.file.Paths
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.Session
|
||||
import nextflow.config.ConfigBuilder
|
||||
import nextflow.exception.AbortOperationException
|
||||
import org.slf4j.Logger
|
||||
import org.slf4j.LoggerFactory
|
||||
import static PluginExecAware.CMD_SEP
|
||||
|
||||
/**
|
||||
* Abstract implementation for plugin commands
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
trait PluginAbstractExec implements PluginExecAware {
|
||||
|
||||
private static Logger log = LoggerFactory.getLogger(PluginAbstractExec)
|
||||
|
||||
private Session session
|
||||
private Launcher launcher
|
||||
|
||||
Session getSession() { session }
|
||||
|
||||
Launcher getLauncher() { launcher }
|
||||
|
||||
abstract List<String> getCommands()
|
||||
|
||||
@Override
|
||||
final int exec(Launcher launcher1, String pluginId, String cmd, List<String> args) {
|
||||
this.launcher = launcher1
|
||||
// create the config
|
||||
final config = new ConfigBuilder()
|
||||
.setOptions(launcher1.options)
|
||||
.setBaseDir(Paths.get('.'))
|
||||
.build()
|
||||
|
||||
if( !cmd || cmd !in getCommands() ) {
|
||||
def msg = cmd ? "Invalid command '$pluginId:$cmd' - usage: nextflow plugin ${pluginId}${CMD_SEP}<command>" : "Usage: nextflow plugin ${pluginId}${CMD_SEP}<command>"
|
||||
msg += "\nAvailable commands:"
|
||||
for( String it : getCommands() )
|
||||
msg += "\n $it"
|
||||
throw new AbortOperationException(msg)
|
||||
}
|
||||
|
||||
// create the session object
|
||||
this.session = new Session(config)
|
||||
// invoke the command
|
||||
try {
|
||||
exec(cmd, args)
|
||||
}
|
||||
catch (Throwable e) {
|
||||
log.error("Unexpected error on command: $cmd - cause: ${e.message}", e)
|
||||
}
|
||||
finally {
|
||||
session.destroy()
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
abstract int exec(String cmd, List<String> args)
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
|
||||
/**
|
||||
* Define the interface for plugin commands
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
interface PluginExecAware {
|
||||
|
||||
static final String CMD_SEP = ':'
|
||||
|
||||
int exec(Launcher launcher, String pluginId, String cmd, List<String> args)
|
||||
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli
|
||||
|
||||
/**
|
||||
* Command can implement this interface to provide a
|
||||
* custom usage description
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
interface UsageAware {
|
||||
|
||||
void usage()
|
||||
|
||||
void usage(List<String> args)
|
||||
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli.module
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.Parameters
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.cli.CmdBase
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.module.ModuleInfo
|
||||
|
||||
/**
|
||||
* Module create subcommand -- creates a new module skeleton
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
@Parameters(commandDescription = "Create a new module skeleton")
|
||||
class CmdModuleCreate extends CmdBase {
|
||||
|
||||
@Parameter(description = "[namespace/name]")
|
||||
List<String> args
|
||||
|
||||
@Override
|
||||
String getName() {
|
||||
return 'create'
|
||||
}
|
||||
|
||||
@Override
|
||||
void run() {
|
||||
String namespace
|
||||
String name
|
||||
|
||||
if( args && args.size() == 1 && args[0].contains('/') ) {
|
||||
// non-interactive: namespace/name passed as argument
|
||||
final slash = args[0].indexOf('/')
|
||||
namespace = args[0].substring(0, slash)
|
||||
name = args[0].substring(slash + 1)
|
||||
if( !namespace || !name )
|
||||
throw new AbortOperationException("Invalid module identifier -- expected format: namespace/name")
|
||||
}
|
||||
else if( args ) {
|
||||
throw new AbortOperationException("Invalid arguments -- usage: nextflow module create [namespace/name]")
|
||||
}
|
||||
else {
|
||||
// interactive mode
|
||||
print "Enter module namespace: "
|
||||
namespace = readLine()?.trim()
|
||||
if( !namespace )
|
||||
throw new AbortOperationException("Module namespace cannot be empty")
|
||||
|
||||
print "Enter module name: "
|
||||
name = readLine()?.trim()
|
||||
if( !name )
|
||||
throw new AbortOperationException("Module name cannot be empty")
|
||||
|
||||
println ""
|
||||
println " Module namespace : $namespace"
|
||||
println " Module name : $name"
|
||||
println " Directory : ./modules/$namespace/$name"
|
||||
println ""
|
||||
print "Are you OK to continue [y/N]? "
|
||||
final confirm = readLine()
|
||||
if( confirm?.toLowerCase() != 'y' ) {
|
||||
println "Module creation aborted."
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
validateSegment('namespace', namespace)
|
||||
validateSegments('name', name)
|
||||
createModule(namespace, name)
|
||||
}
|
||||
static private void validateSegment(String field, String value) {
|
||||
if( !value.matches('[a-zA-Z0-9][a-zA-Z0-9._\\-]*') )
|
||||
throw new AbortOperationException("Invalid module $field '${value}' -- only alphanumeric characters, hyphens, underscores and dots are allowed, and must start with an alphanumeric character")
|
||||
}
|
||||
|
||||
static private void validateSegments(String field, String value) {
|
||||
for( String segment : value.tokenize('/') ) {
|
||||
validateSegment(field, segment)
|
||||
}
|
||||
}
|
||||
|
||||
protected Path modulesBase() {
|
||||
return Path.of('modules')
|
||||
}
|
||||
|
||||
protected void createModule(String namespace, String name) {
|
||||
final moduleDir = modulesBase().resolve(namespace).resolve(name)
|
||||
if( Files.exists(moduleDir) )
|
||||
throw new AbortOperationException("Module directory already exists: $moduleDir")
|
||||
|
||||
// create directory structure
|
||||
Files.createDirectories(moduleDir)
|
||||
|
||||
// create main.nf
|
||||
moduleDir.resolve('main.nf').text = mainNf(namespace, name)
|
||||
|
||||
// create README.md
|
||||
moduleDir.resolve('README.md').text = readmeMd(namespace, name)
|
||||
|
||||
// create meta.yml
|
||||
moduleDir.resolve('meta.yml').text = metaYml(namespace, name)
|
||||
|
||||
// create .module-info so it's recognised as a Nextflow managed module
|
||||
Files.createFile(moduleDir.resolve(ModuleInfo.MODULE_INFO_FILE))
|
||||
|
||||
println "Module created successfully at path: $moduleDir"
|
||||
println ""
|
||||
println "To run the module:"
|
||||
println ""
|
||||
println " nextflow module run $namespace/$name --greeting 'Hello world!'"
|
||||
}
|
||||
|
||||
static private String readLine() {
|
||||
final console = System.console()
|
||||
return console != null
|
||||
? console.readLine()
|
||||
: new BufferedReader(new InputStreamReader(System.in)).readLine()
|
||||
}
|
||||
|
||||
static String mainNf(String namespace, String name) {
|
||||
"""\
|
||||
/*
|
||||
* Module: ${namespace}/${name}
|
||||
*/
|
||||
|
||||
process ${name.replaceAll('[^a-zA-Z0-9_]', '_').toUpperCase()} {
|
||||
input:
|
||||
val greeting
|
||||
|
||||
output:
|
||||
stdout
|
||||
|
||||
script:
|
||||
\"\"\"
|
||||
echo '\${greeting}'
|
||||
\"\"\"
|
||||
}
|
||||
""".stripIndent()
|
||||
}
|
||||
|
||||
static String metaYml(String namespace, String name) {
|
||||
"""\
|
||||
name: ${namespace}/${name}
|
||||
version: 1.0.0
|
||||
description: A brief description of the ${namespace}/${name} module
|
||||
license: Apache-2.0
|
||||
input:
|
||||
- name: greeting
|
||||
type: string
|
||||
description: A greeting string
|
||||
output:
|
||||
- name: stdout
|
||||
type: string
|
||||
description: The greeting message
|
||||
""".stripIndent()
|
||||
}
|
||||
|
||||
static String readmeMd(String namespace, String name) {
|
||||
"""\
|
||||
# ${namespace}/${name}
|
||||
|
||||
## Summary
|
||||
|
||||
A brief description of the `${namespace}/${name}` module.
|
||||
|
||||
## Get started
|
||||
|
||||
Include this module in your Nextflow pipeline:
|
||||
|
||||
```nextflow
|
||||
include { ${name.replaceAll('[^a-zA-Z0-9_]', '_').toUpperCase()} } from '${namespace}/${name}'
|
||||
```
|
||||
|
||||
## Dependencies
|
||||
|
||||
None.
|
||||
|
||||
## License
|
||||
|
||||
Apache-2.0
|
||||
""".stripIndent()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli.module
|
||||
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.Parameters
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import io.seqera.npr.client.RegistryClient
|
||||
import nextflow.cli.CmdBase
|
||||
import nextflow.config.ConfigBuilder
|
||||
import nextflow.config.RegistryConfig
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.module.ModuleReference
|
||||
import nextflow.module.RegistryClientFactory
|
||||
import nextflow.module.ModuleResolver
|
||||
import nextflow.module.ModuleSpecFactory
|
||||
|
||||
import nextflow.util.TestOnly
|
||||
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
|
||||
/**
|
||||
* Module install subcommand
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
@Slf4j
|
||||
@Parameters(commandDescription = "Install a module from the registry")
|
||||
@CompileStatic
|
||||
class CmdModuleInstall extends CmdBase {
|
||||
|
||||
@Parameter(names = ["-version"], description = "Module version")
|
||||
String version
|
||||
|
||||
@Parameter(names = ["-force"], description = "Force reinstall even if already installed", arity = 0)
|
||||
boolean force = false
|
||||
|
||||
@Parameter(description = "[scope/name]", required = true)
|
||||
List<String> args
|
||||
|
||||
@TestOnly
|
||||
protected Path root
|
||||
|
||||
@TestOnly
|
||||
protected RegistryClient client
|
||||
|
||||
@Override
|
||||
String getName() {
|
||||
return 'install'
|
||||
}
|
||||
|
||||
@Override
|
||||
void run() {
|
||||
if( !args || args.size() != 1 ) {
|
||||
throw new AbortOperationException("Incorrect number of arguments")
|
||||
}
|
||||
|
||||
def reference = ModuleReference.parse(args[0])
|
||||
|
||||
// Get config
|
||||
def baseDir = root ?: Paths.get('.').toAbsolutePath().normalize()
|
||||
def config = new ConfigBuilder()
|
||||
.setOptions(launcher.options)
|
||||
.setBaseDir(baseDir)
|
||||
.build()
|
||||
final registryConfig = config.navigate('registry') as RegistryConfig ?: new RegistryConfig()
|
||||
|
||||
// Create resolver and install
|
||||
def resolver = new ModuleResolver(baseDir, client ?: RegistryClientFactory.forConfig(registryConfig))
|
||||
|
||||
try {
|
||||
def installedMainFile = resolver.installModule(reference, version, force)
|
||||
// Read the installed version from meta.yml to avoid a redundant registry call
|
||||
def installedVersion = ModuleSpecFactory.fromYaml(installedMainFile.parent.resolve('meta.yml')).version
|
||||
|
||||
println "Module ${reference}@${installedVersion} installed and configured successfully"
|
||||
}
|
||||
catch( AbortOperationException e ) {
|
||||
throw e
|
||||
}
|
||||
catch( Exception e ) {
|
||||
throw new AbortOperationException("Installation failed: ${e.message}", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli.module
|
||||
|
||||
import com.beust.jcommander.IParameterValidator
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.ParameterException
|
||||
import com.beust.jcommander.Parameters
|
||||
import groovy.json.JsonOutput
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.cli.CmdBase
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.module.InstalledModule
|
||||
import nextflow.module.ModuleIntegrity
|
||||
import nextflow.module.ModuleStorage
|
||||
import nextflow.util.TestOnly
|
||||
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
|
||||
/**
|
||||
* Module list subcommand
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
@Parameters(commandDescription = "List all installed modules")
|
||||
class CmdModuleList extends CmdBase {
|
||||
|
||||
@Parameter(
|
||||
names = ['-o', '-output'],
|
||||
description = 'Output mode for reporting search results: table, json',
|
||||
validateWith = OutputModeValidator
|
||||
)
|
||||
String output = 'table'
|
||||
|
||||
static class OutputModeValidator implements IParameterValidator {
|
||||
|
||||
private static final List<String> MODES = List.of('table', 'json')
|
||||
|
||||
@Override
|
||||
void validate(String name, String value) {
|
||||
if( !MODES.contains(value) )
|
||||
throw new ParameterException("Output mode must be one of $MODES (found: $value)")
|
||||
}
|
||||
}
|
||||
|
||||
@TestOnly
|
||||
protected Path root
|
||||
|
||||
@Override
|
||||
String getName() {
|
||||
return 'list'
|
||||
}
|
||||
|
||||
@Override
|
||||
void run() {
|
||||
|
||||
// Get config
|
||||
def baseDir = root ?: Paths.get('.').toAbsolutePath().normalize()
|
||||
|
||||
|
||||
// Create resolver and list modules
|
||||
def storage = new ModuleStorage(baseDir)
|
||||
|
||||
try {
|
||||
def installed = storage.listInstalled()
|
||||
|
||||
if( installed.isEmpty() ) {
|
||||
println "No modules installed"
|
||||
return
|
||||
}
|
||||
|
||||
if( !output || output == 'table' ) {
|
||||
printFormattedList(installed)
|
||||
} else if( output == 'json' ) {
|
||||
printJsonList(installed)
|
||||
} else {
|
||||
throw new AbortOperationException("Not implemented output mode $output)")
|
||||
}
|
||||
|
||||
}
|
||||
catch( Exception e ) {
|
||||
log.error("Failed to list modules", e)
|
||||
throw new AbortOperationException("List failed: ${e.message}", e)
|
||||
}
|
||||
}
|
||||
|
||||
private void printFormattedList(List<InstalledModule> installed) {
|
||||
println ""
|
||||
println "Installed modules:"
|
||||
println ""
|
||||
println "Module".padRight(40) + "Version".padRight(15) + "Status"
|
||||
println("-" * 70)
|
||||
|
||||
installed.each { module ->
|
||||
def status = getStatusString(module.integrity)
|
||||
println "${module.reference.toString().padRight(40)}${(module.installedVersion ?: 'unknown').padRight(15)}${status}"
|
||||
}
|
||||
println ""
|
||||
}
|
||||
|
||||
private void printJsonList(List<InstalledModule> installed) {
|
||||
def modules = installed.collect { module ->
|
||||
[
|
||||
name : module.reference.toString(),
|
||||
version : module.installedVersion ?: 'unknown',
|
||||
integrity: module.integrity.toString(),
|
||||
directory: module.directory.toString(),
|
||||
registry : module.registryUrl ?: 'unknown'
|
||||
]
|
||||
}
|
||||
|
||||
// Simple JSON output (could use groovy.json.JsonOutput for better formatting)
|
||||
println JsonOutput.prettyPrint(JsonOutput.toJson(modules: modules))
|
||||
}
|
||||
|
||||
private String getStatusString(ModuleIntegrity integrity) {
|
||||
switch( integrity ) {
|
||||
case ModuleIntegrity.VALID:
|
||||
return 'OK'
|
||||
case ModuleIntegrity.MODIFIED:
|
||||
return 'MODIFIED'
|
||||
case ModuleIntegrity.NO_REMOTE_MODULE:
|
||||
return 'LOCAL'
|
||||
case ModuleIntegrity.CORRUPTED:
|
||||
return 'CORRUPTED'
|
||||
default:
|
||||
return 'UNKNOWN'
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli.module
|
||||
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.Parameters
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import io.seqera.npr.client.RegistryClient
|
||||
import nextflow.cli.CmdBase
|
||||
import nextflow.config.ConfigBuilder
|
||||
import nextflow.config.RegistryConfig
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.module.ModuleChecksum
|
||||
import nextflow.module.ModuleInfo
|
||||
import nextflow.module.ModuleSpec
|
||||
import nextflow.module.ModuleSpecFactory
|
||||
import nextflow.module.ModuleReference
|
||||
import nextflow.module.ModuleValidator
|
||||
import nextflow.module.RegistryClientFactory
|
||||
import nextflow.module.ModuleStorage
|
||||
import nextflow.util.TestOnly
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
|
||||
/**
|
||||
* Module publish subcommand
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
@Parameters(commandDescription = "Publish a module to the registry")
|
||||
class CmdModulePublish extends CmdBase {
|
||||
|
||||
@Parameter(names = ["-dry-run"], description = "Validate without uploading", arity=0)
|
||||
boolean dryRun = false
|
||||
|
||||
@Parameter(names = ["-registry"], description = "Target registry URL.")
|
||||
String registryUrl
|
||||
|
||||
@Parameter(description = "Module directory path or scope/name")
|
||||
List<String> args
|
||||
|
||||
@TestOnly
|
||||
protected Path root
|
||||
|
||||
@TestOnly
|
||||
protected RegistryClient client
|
||||
|
||||
//Flag if publish is invoked from a scope/name. In this case we should create/update the .module-info with the correct checksum
|
||||
private boolean useModuleReference = false
|
||||
|
||||
@Override
|
||||
String getName() {
|
||||
return 'publish'
|
||||
}
|
||||
|
||||
@Override
|
||||
void run() {
|
||||
if (!args || args.size() != 1) {
|
||||
throw new AbortOperationException("Incorrect number of arguments")
|
||||
}
|
||||
|
||||
Path moduleDir = determineModuleDir(args[0])
|
||||
|
||||
log.info "Publishing module from: ${moduleDir}"
|
||||
|
||||
// Step 1: Validate module structure and spec
|
||||
def validationErrors = ModuleValidator.validate(moduleDir)
|
||||
if (!validationErrors.isEmpty()) {
|
||||
throw new AbortOperationException(
|
||||
"Module validation failed:\n" + validationErrors.collect { " - ${it}" }.join('\n')
|
||||
)
|
||||
}
|
||||
|
||||
// Step 2: Load spec for publish metadata
|
||||
def manifestPath = moduleDir.resolve(ModuleStorage.MODULE_MANIFEST_FILE)
|
||||
def spec = ModuleSpecFactory.fromYaml(manifestPath)
|
||||
|
||||
log.info "Module validated: ${spec.name}@${spec.version}"
|
||||
|
||||
if (dryRun) {
|
||||
printDryRunInfo(spec)
|
||||
return
|
||||
}
|
||||
|
||||
// Step 3: Get authentication token
|
||||
def config = new ConfigBuilder()
|
||||
.setOptions(launcher.options)
|
||||
.setBaseDir(moduleDir)
|
||||
.build()
|
||||
|
||||
def registryConfig = config.navigate('registry') as RegistryConfig ?: new RegistryConfig()
|
||||
|
||||
publishModule(moduleDir, registryConfig, spec)
|
||||
|
||||
}
|
||||
|
||||
private void publishModule(Path moduleDir, RegistryConfig registryConfig, ModuleSpec spec){
|
||||
log.info "Creating module bundle..."
|
||||
def tempBundleFile = Files.createTempFile("nf-module-publish-", ".tar.gz")
|
||||
|
||||
try {
|
||||
def checksum = ModuleStorage.createBundle(moduleDir, tempBundleFile)
|
||||
log.info "Bundle checksum: ${checksum}"
|
||||
|
||||
// Read bundle content as bytes
|
||||
def bundleBytes = Files.readAllBytes(tempBundleFile)
|
||||
|
||||
// Create publish request as a map (npr-api will serialize it)
|
||||
def request = [
|
||||
version: spec.version,
|
||||
bundle: bundleBytes
|
||||
]
|
||||
|
||||
// Publish to registry
|
||||
final registry = registryUrl ?: registryConfig.url
|
||||
log.info "Publishing module to registry: ${registryUrl ?: registryConfig.url}"
|
||||
def registryClient = RegistryClientFactory.forConfig(registryConfig)
|
||||
def response = registryClient.publishModuleRelease(spec.name, request, registry)
|
||||
|
||||
if (useModuleReference) {
|
||||
// If publish is performed using the module reference we should create/update the .module-info with the correct checksum
|
||||
try {
|
||||
ModuleInfo.save(moduleDir, [checksum: ModuleChecksum.compute(moduleDir), registryUrl: registry] )
|
||||
}catch (Exception e){
|
||||
log.warn("Unable to save the checksum - ${e.message}")
|
||||
}
|
||||
}
|
||||
println "✓ Module published successfully!"
|
||||
println ""
|
||||
println "Module details:"
|
||||
println " Name: ${spec.name}"
|
||||
println " Version: ${spec.version}"
|
||||
println " DownloadUrl: ${response.downloadUrl}"
|
||||
|
||||
println ""
|
||||
println "Others can now install this module using:"
|
||||
println " nextflow module install ${spec.name}"
|
||||
|
||||
} finally {
|
||||
// Clean up temporary bundle file
|
||||
if (Files.exists(tempBundleFile)) {
|
||||
try {
|
||||
Files.delete(tempBundleFile)
|
||||
} catch (Exception e) {
|
||||
log.warn "Failed to clean up temporary bundle file: ${e.message}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void printDryRunInfo(ModuleSpec spec) {
|
||||
println "✓ Module structure is valid"
|
||||
println ""
|
||||
println "Module details:"
|
||||
println " Name: ${spec.name}"
|
||||
println " Version: ${spec.version}"
|
||||
println " Description: ${spec.description}"
|
||||
println " License: ${spec.license}"
|
||||
if( spec.authors ) {
|
||||
println " Authors: ${spec.authors.join(', ')}"
|
||||
}
|
||||
if( spec.keywords ) {
|
||||
println " Keywords: ${spec.keywords.join(', ')}"
|
||||
}
|
||||
if( spec.requires ) {
|
||||
println " Requires:"
|
||||
spec.requires.each { name, version ->
|
||||
println " - ${name}: ${version}"
|
||||
}
|
||||
}
|
||||
println ""
|
||||
println "Dry run complete. Module is ready to publish."
|
||||
println "Run without -dry-run to publish to the registry."
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine if the specified module is a local path or a reference
|
||||
* @param module
|
||||
* @return
|
||||
*/
|
||||
private Path determineModuleDir(String module) {
|
||||
//If local path exists return this path as module dir
|
||||
if (Paths.get(module).exists()){
|
||||
return Paths.get(module).toAbsolutePath().normalize()
|
||||
}
|
||||
|
||||
final ref = ModuleReference.parse(module)
|
||||
final localStorage = new ModuleStorage(root ?: Paths.get('.').toAbsolutePath().normalize())
|
||||
|
||||
if (!localStorage.isInstalled(ref)){
|
||||
throw new AbortOperationException("No module directory found for $module")
|
||||
}
|
||||
useModuleReference = true
|
||||
return localStorage.getModuleDir(ref)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli.module
|
||||
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.Parameters
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.cli.CmdBase
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.module.ModuleReference
|
||||
import nextflow.module.ModuleStorage
|
||||
|
||||
import nextflow.util.TestOnly
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
|
||||
/**
|
||||
* Module remove subcommand
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
@Parameters(commandDescription = "Remove an installed module")
|
||||
class CmdModuleRemove extends CmdBase {
|
||||
|
||||
@Parameter(description = "<module>", required = true)
|
||||
List<String> args
|
||||
|
||||
@Parameter(names = ["-keep-files"], description = "Remove only .module-info keeping the local files", arity = 0)
|
||||
boolean keepFiles = false
|
||||
|
||||
@Parameter(names = ["-force"], description = "Force remove", arity = 0)
|
||||
boolean force = false
|
||||
|
||||
@TestOnly
|
||||
protected Path root
|
||||
|
||||
@Override
|
||||
String getName() {
|
||||
return 'remove'
|
||||
}
|
||||
|
||||
@Override
|
||||
void run() {
|
||||
if( !args || args.size() != 1 ) {
|
||||
throw new AbortOperationException("Incorrect number of arguments")
|
||||
}
|
||||
if( keepFiles && force ) {
|
||||
throw new AbortOperationException("Cannot use both -keep-files and -force options")
|
||||
}
|
||||
|
||||
def reference = ModuleReference.parse(args[0])
|
||||
|
||||
// Get config
|
||||
def baseDir = root ?: Paths.get('.').toAbsolutePath().normalize()
|
||||
|
||||
// Create resolver and spec file manager
|
||||
def storage = new ModuleStorage(baseDir)
|
||||
|
||||
try {
|
||||
def filesRemoved = false
|
||||
|
||||
// Remove local files unless -keep-files is set
|
||||
if( !keepFiles ) {
|
||||
filesRemoved = storage.removeModule(reference, force)
|
||||
if( filesRemoved ) {
|
||||
println "Module ${reference} files removed successfully"
|
||||
} else {
|
||||
println "Module ${reference} not found locally"
|
||||
}
|
||||
} else {
|
||||
println "Keeping module files for ${reference} (-keep-files flag)"
|
||||
final moduleInfo = storage.getModuleInfo(reference)
|
||||
if( Files.exists(moduleInfo) ) {
|
||||
Files.delete(moduleInfo)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
catch( AbortOperationException e ) {
|
||||
throw e
|
||||
}
|
||||
catch( Exception e ) {
|
||||
throw new AbortOperationException("Failed to remove module $reference: ${e.message}", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli.module
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.Parameters
|
||||
import groovy.transform.CompileStatic
|
||||
import io.seqera.npr.client.RegistryClient
|
||||
import nextflow.Const
|
||||
import nextflow.cli.CmdRun
|
||||
import nextflow.config.ConfigBuilder
|
||||
import nextflow.config.RegistryConfig
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.module.ModuleReference
|
||||
import nextflow.module.ModuleResolver
|
||||
import nextflow.module.RegistryClientFactory
|
||||
import nextflow.util.TestOnly
|
||||
|
||||
/**
|
||||
* Module run subcommand
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
@CompileStatic
|
||||
@Parameters(commandDescription = "Run a module directly from the registry")
|
||||
class CmdModuleRun extends CmdRun {
|
||||
@Parameter(names = ["-version"], description = "Module version")
|
||||
String version
|
||||
|
||||
@TestOnly
|
||||
protected Path root
|
||||
|
||||
@TestOnly
|
||||
protected RegistryClient client
|
||||
|
||||
@Override
|
||||
String getName() {
|
||||
return 'run'
|
||||
}
|
||||
|
||||
@Override
|
||||
void run() {
|
||||
if( !args ) {
|
||||
throw new AbortOperationException("Module name/path not provided")
|
||||
}
|
||||
|
||||
final moduleFile = isLocalModule(args[0])
|
||||
? resolveLocalModule(args[0])
|
||||
: resolveRemoteModule(args[0], version)
|
||||
|
||||
if( moduleFile ) {
|
||||
args[0] = moduleFile.toAbsolutePath().toString()
|
||||
super.run()
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isLocalModule(String str) {
|
||||
return str.startsWith('/') || str.startsWith('./') || str.startsWith('../')
|
||||
}
|
||||
|
||||
private Path resolveLocalModule(String str) {
|
||||
final module = Path.of(str).toAbsolutePath().normalize()
|
||||
final path = module.isDirectory() ? module.resolve(Const.DEFAULT_MAIN_FILE_NAME) : module
|
||||
if( !path.exists() )
|
||||
throw new AbortOperationException("Invalid module path: ${str}")
|
||||
return path
|
||||
}
|
||||
|
||||
private Path resolveRemoteModule(String name, String version) {
|
||||
// Parse and validate module reference
|
||||
ModuleReference reference
|
||||
try {
|
||||
reference = ModuleReference.parse(name)
|
||||
} catch( Exception e ) {
|
||||
throw new AbortOperationException("Invalid module reference: ${name}", e)
|
||||
}
|
||||
|
||||
// Get config
|
||||
final baseDir = root ?: Path.of('.').toAbsolutePath().normalize()
|
||||
final config = new ConfigBuilder()
|
||||
.setOptions(launcher.options)
|
||||
.setBaseDir(baseDir)
|
||||
.build()
|
||||
|
||||
final registryConfig = new RegistryConfig(config.registry as Map ?: Collections.emptyMap())
|
||||
try {
|
||||
final resolver = new ModuleResolver(baseDir, client ?: RegistryClientFactory.forConfig(registryConfig))
|
||||
return resolver.installModule(reference, version)
|
||||
} catch( Exception e ) {
|
||||
throw new AbortOperationException("Unable to install module: ${name}", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
/*
|
||||
* Copyright 2013-2024, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli.module
|
||||
|
||||
import com.beust.jcommander.IParameterValidator
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.ParameterException
|
||||
import com.beust.jcommander.Parameters
|
||||
import groovy.json.JsonOutput
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import io.seqera.npr.api.schema.v1.ModuleSearchResult
|
||||
import io.seqera.npr.api.schema.v1.SearchModulesResponse
|
||||
import io.seqera.npr.client.RegistryClient
|
||||
import nextflow.cli.CmdBase
|
||||
import nextflow.config.ConfigBuilder
|
||||
import nextflow.config.RegistryConfig
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.module.RegistryClientFactory
|
||||
import nextflow.util.TestOnly
|
||||
|
||||
import java.nio.file.Paths
|
||||
|
||||
/**
|
||||
* Module search subcommand
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
@Parameters(commandDescription = "Search for modules in the registry")
|
||||
class CmdModuleSearch extends CmdBase {
|
||||
|
||||
@Parameter(names = ["-limit"], description = "Maximum number of results")
|
||||
int limit = 20
|
||||
|
||||
@Parameter(
|
||||
names = ['-o', '-output'],
|
||||
description = 'Output mode for reporting search results: simple, json',
|
||||
validateWith = OutputModeValidator
|
||||
)
|
||||
String output = 'simple'
|
||||
|
||||
static class OutputModeValidator implements IParameterValidator {
|
||||
|
||||
private static final List<String> MODES = List.of('simple', 'json')
|
||||
|
||||
@Override
|
||||
void validate(String name, String value) {
|
||||
if( !MODES.contains(value) )
|
||||
throw new ParameterException("Output mode must be one of $MODES (found: $value)")
|
||||
}
|
||||
}
|
||||
|
||||
@Parameter(description = "<query>", required = true)
|
||||
List<String> args
|
||||
|
||||
@TestOnly
|
||||
protected RegistryClient client
|
||||
|
||||
@Override
|
||||
String getName() {
|
||||
return 'search'
|
||||
}
|
||||
|
||||
@Override
|
||||
void run() {
|
||||
if( !args || args.size() != 1 ) {
|
||||
throw new AbortOperationException("Unexpected number of parameters")
|
||||
}
|
||||
String query = args[0]
|
||||
|
||||
// Get config
|
||||
def baseDir = Paths.get('.').toAbsolutePath().normalize()
|
||||
def config = new ConfigBuilder()
|
||||
.setOptions(launcher.options)
|
||||
.setBaseDir(baseDir)
|
||||
.build()
|
||||
|
||||
final registryConfig = config.navigate('registry') as RegistryConfig ?: new RegistryConfig()
|
||||
|
||||
// Create client to search
|
||||
final client = this.client ?: RegistryClientFactory.forConfig(registryConfig)
|
||||
|
||||
try {
|
||||
println "Searching for '${query}'..."
|
||||
final results = client.searchModules(query, limit)
|
||||
|
||||
if( !results || results.totalResults == 0 || !results.results || results.results.isEmpty() ) {
|
||||
println "No modules found"
|
||||
return
|
||||
}
|
||||
|
||||
if( !output || output == 'simple' ) {
|
||||
printFormattedResults(results)
|
||||
} else if( output == 'json' ) {
|
||||
printJsonResults(results)
|
||||
} else {
|
||||
throw new AbortOperationException("Not implemented output mode $output)")
|
||||
}
|
||||
}
|
||||
catch( AbortOperationException e ) {
|
||||
throw e
|
||||
}
|
||||
catch( Exception e ) {
|
||||
log.error("Failed to search modules", e)
|
||||
throw new AbortOperationException("Search failed: ${e.message}", e)
|
||||
}
|
||||
}
|
||||
|
||||
private void printFormattedResults(SearchModulesResponse response) {
|
||||
println ""
|
||||
println "Top ${response.totalResults} matching module(s):"
|
||||
println ""
|
||||
|
||||
response.results.each { ModuleSearchResult result ->
|
||||
println " ${result.name}"
|
||||
if( result.description ) {
|
||||
println " Description: ${result.description}"
|
||||
}
|
||||
println ""
|
||||
}
|
||||
}
|
||||
|
||||
private void printJsonResults(SearchModulesResponse response) {
|
||||
final modules = response.results.collect { ModuleSearchResult result ->
|
||||
[
|
||||
name : result.name,
|
||||
repositoryPath: result.repositoryPath,
|
||||
description : result.description,
|
||||
relevanceScore: result.relevanceScore,
|
||||
keywords : result.keywords,
|
||||
tools : result.tools,
|
||||
revoked : result.revoked
|
||||
]
|
||||
}
|
||||
|
||||
println JsonOutput.prettyPrint(JsonOutput.toJson(
|
||||
query: response.query,
|
||||
totalResults: response.totalResults,
|
||||
results: modules
|
||||
))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli.module
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.Parameters
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.cli.CmdBase
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.module.ModuleReference
|
||||
import nextflow.module.ModuleSpec
|
||||
import nextflow.module.ModuleSpecFactory
|
||||
import nextflow.module.ModuleStorage
|
||||
import nextflow.util.TestOnly
|
||||
|
||||
/**
|
||||
* Implements the {@code nextflow module spec} subcommand.
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
@Parameters(commandDescription = "Generate a meta.yml spec for a local module")
|
||||
class CmdModuleSpec extends CmdBase {
|
||||
|
||||
@Parameter(description = "Reference to local module (namespace/name) or module directory path")
|
||||
List<String> args
|
||||
|
||||
@Parameter(names = ["-dry-run"], description = "Print generated YAML to stdout without writing any file", arity = 0)
|
||||
boolean dryRun = false
|
||||
|
||||
@Parameter(names = ["-namespace"], description = "Module namespace")
|
||||
String moduleScope
|
||||
|
||||
@Parameter(names = ["-version"], description = "Module version (e.g. 1.0.0)")
|
||||
String moduleVersion
|
||||
|
||||
@Parameter(names = ["-description"], description = "Short description of what the module does")
|
||||
String description
|
||||
|
||||
@Parameter(names = ["-license"], description = "Module license identifier (e.g. MIT, Apache-2.0)")
|
||||
String license
|
||||
|
||||
@Parameter(names = ["-author"], description = "Module author; may be specified multiple times")
|
||||
List<String> authors
|
||||
|
||||
@TestOnly
|
||||
protected Path root
|
||||
|
||||
private String moduleName
|
||||
|
||||
@Override
|
||||
String getName() { 'spec' }
|
||||
|
||||
@Override
|
||||
void run() {
|
||||
if( !args ){
|
||||
throw new AbortOperationException("Reference to local module (namespace/name) or path is required")
|
||||
}
|
||||
final baseDir = root ?: Path.of('.').toAbsolutePath().normalize()
|
||||
def moduleDir = resolveAsModuleReference(baseDir, args[0])
|
||||
if( moduleDir ) {
|
||||
moduleName = args[0]
|
||||
if( !dryRun )
|
||||
log.info "Module name inferred as ${moduleName} ${moduleScope ? '(-namespace option is ignored)' : ''}"
|
||||
}
|
||||
else {
|
||||
moduleDir = resolveAsPath(baseDir, args[0])
|
||||
if( !moduleScope )
|
||||
throw new AbortOperationException("The -namespace option is required when referencing a module by path")
|
||||
}
|
||||
|
||||
final mainNfPath = moduleDir.resolve('main.nf')
|
||||
if( !Files.exists(mainNfPath) )
|
||||
throw new AbortOperationException("Missing module script (main.nf) in ${moduleDir}")
|
||||
|
||||
final metaYmlPath = moduleDir.resolve('meta.yml')
|
||||
if( Files.exists(metaYmlPath) && !dryRun )
|
||||
log.info "Using existing module spec: ${metaYmlPath}"
|
||||
|
||||
final yamlContent = generateSpec(mainNfPath, metaYmlPath).toYaml()
|
||||
|
||||
if( dryRun ) {
|
||||
println yamlContent
|
||||
return
|
||||
}
|
||||
|
||||
Files.writeString(metaYmlPath, yamlContent)
|
||||
println "Saved module spec to: ${metaYmlPath}"
|
||||
}
|
||||
|
||||
private Path resolveAsModuleReference(Path baseDir, String module) {
|
||||
try {
|
||||
final ref = ModuleReference.parse(module)
|
||||
final localStorage = new ModuleStorage(baseDir)
|
||||
if( localStorage.isInstalled(ref) ) {
|
||||
log.debug("Argument $module refers to a local module reference")
|
||||
return localStorage.getModuleDir(ref)
|
||||
}
|
||||
log.debug("Argument $module is not a local module reference")
|
||||
return null
|
||||
} catch( AbortOperationException e ) {
|
||||
log.debug("Argument $module is not a correct module reference")
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
private Path resolveAsPath(Path baseDir, String module) {
|
||||
final path = Path.of(module)
|
||||
final moduleDir = path.isAbsolute() ? path : baseDir.resolve(module)
|
||||
if( !Files.isDirectory(moduleDir) )
|
||||
throw new AbortOperationException("Not a directory: ${moduleDir}")
|
||||
return moduleDir
|
||||
}
|
||||
|
||||
private ModuleSpec generateSpec(Path mainNfPath, Path metaYmlPath) {
|
||||
final oldSpec = Files.exists(metaYmlPath)
|
||||
? ModuleSpecFactory.fromYaml(metaYmlPath)
|
||||
: new ModuleSpec()
|
||||
return ModuleSpecFactory.fromScript(
|
||||
mainNfPath,
|
||||
oldSpec,
|
||||
namespace: moduleScope,
|
||||
name: moduleName,
|
||||
version: moduleVersion,
|
||||
description: description,
|
||||
license: license,
|
||||
authors: authors
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli.module
|
||||
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.Parameters
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.cli.CmdBase
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.module.ModuleReference
|
||||
import nextflow.module.ModuleStorage
|
||||
import nextflow.module.ModuleValidator
|
||||
import nextflow.util.TestOnly
|
||||
|
||||
/**
|
||||
* Module validate subcommand -- validates module structure and meta.yml consistency
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
@Parameters(commandDescription = "Validate a module structure and metadata")
|
||||
class CmdModuleValidate extends CmdBase {
|
||||
|
||||
@Parameter(description = "[namespace/name or path]", required = true)
|
||||
List<String> args
|
||||
|
||||
@TestOnly
|
||||
protected Path root
|
||||
|
||||
@Override
|
||||
String getName() {
|
||||
return 'validate'
|
||||
}
|
||||
|
||||
@Override
|
||||
void run() {
|
||||
if( !args || args.size() != 1 )
|
||||
throw new AbortOperationException("Incorrect number of arguments -- usage: nextflow module validate <namespace/name>")
|
||||
|
||||
final moduleDir = determineModuleDir(args[0])
|
||||
final errors = ModuleValidator.validate(moduleDir)
|
||||
|
||||
if( errors ) {
|
||||
throw new AbortOperationException(
|
||||
"Module validation failed:\n" + errors.collect { " - ${it}" }.join('\n')
|
||||
)
|
||||
}
|
||||
|
||||
println "Module validation passed."
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve module path from argument (directory path or namespace/name reference)
|
||||
*/
|
||||
protected Path determineModuleDir(String module) {
|
||||
final path = Paths.get(module)
|
||||
if( path.exists() )
|
||||
return path.toAbsolutePath().normalize()
|
||||
|
||||
final ref = ModuleReference.parse(module)
|
||||
final storage = new ModuleStorage(root ?: Paths.get('.').toAbsolutePath().normalize())
|
||||
|
||||
if( !storage.isInstalled(ref) )
|
||||
throw new AbortOperationException("No module directory found for $module")
|
||||
|
||||
return storage.getModuleDir(ref)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,347 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cli.module
|
||||
|
||||
import com.beust.jcommander.IParameterValidator
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.beust.jcommander.ParameterException
|
||||
import com.beust.jcommander.Parameters
|
||||
import groovy.json.JsonOutput
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import io.seqera.npr.client.RegistryClient
|
||||
import io.seqera.npr.api.schema.v1.ModuleChannel
|
||||
import io.seqera.npr.api.schema.v1.ModuleChannelItem
|
||||
import io.seqera.npr.api.schema.v1.ModuleMetadata
|
||||
import io.seqera.npr.api.schema.v1.ModuleRelease
|
||||
import io.seqera.npr.api.schema.v1.ModuleTool
|
||||
import nextflow.cli.CmdBase
|
||||
import nextflow.config.ConfigBuilder
|
||||
import nextflow.config.RegistryConfig
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.module.ModuleReference
|
||||
import nextflow.module.RegistryClientFactory
|
||||
import nextflow.util.TestOnly
|
||||
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
|
||||
/**
|
||||
* Module view subcommand - displays module info and usage template
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
@Parameters(commandDescription = "Show module information and usage template")
|
||||
class CmdModuleView extends CmdBase {
|
||||
|
||||
@Parameter(names = ["-version"], description = "Module version")
|
||||
String version
|
||||
|
||||
@Parameter(
|
||||
names = ['-o', '-output'],
|
||||
description = 'Output mode for reporting search results: text, json',
|
||||
validateWith = OutputModeValidator
|
||||
)
|
||||
String output = 'text'
|
||||
|
||||
static class OutputModeValidator implements IParameterValidator {
|
||||
|
||||
private static final List<String> MODES = List.of('text', 'json')
|
||||
|
||||
@Override
|
||||
void validate(String name, String value) {
|
||||
if( !MODES.contains(value) )
|
||||
throw new ParameterException("Output mode must be one of $MODES (found: $value)")
|
||||
}
|
||||
}
|
||||
|
||||
@Parameter(description = "[scope/name]", required = true)
|
||||
List<String> args
|
||||
|
||||
@TestOnly
|
||||
protected Path root
|
||||
|
||||
@TestOnly
|
||||
protected RegistryClient client
|
||||
|
||||
@Override
|
||||
String getName() {
|
||||
return 'view'
|
||||
}
|
||||
|
||||
@Override
|
||||
List<String> getAliases() {
|
||||
return List.of('info')
|
||||
}
|
||||
|
||||
@Override
|
||||
void run() {
|
||||
if( !args || args.size() != 1 ) {
|
||||
throw new AbortOperationException("Incorrect number of arguments")
|
||||
}
|
||||
|
||||
def reference = ModuleReference.parse(args[0])
|
||||
|
||||
// Get config
|
||||
def baseDir = root ?: Paths.get('.').toAbsolutePath().normalize()
|
||||
def config = new ConfigBuilder()
|
||||
.setOptions(launcher.options)
|
||||
.setBaseDir(baseDir)
|
||||
.build()
|
||||
final registryConfig = config.navigate('registry') as RegistryConfig ?: new RegistryConfig()
|
||||
|
||||
|
||||
// Fetch full metadata from registry to get input/output parameters
|
||||
def registryClient = this.client ?: RegistryClientFactory.forConfig(registryConfig)
|
||||
ModuleRelease release = null
|
||||
|
||||
try {
|
||||
if( version ) {
|
||||
release = registryClient.getModuleRelease(reference.fullName, version)
|
||||
|
||||
} else {
|
||||
release = registryClient.getModule(reference.fullName).latest
|
||||
}
|
||||
} catch( Exception e ) {
|
||||
log.warn "Failed to fetch metadata from registry: ${e.message}"
|
||||
}
|
||||
if( !release ) {
|
||||
throw new AbortOperationException("No release information available for ${reference}")
|
||||
}
|
||||
if( !release.metadata ) {
|
||||
throw new AbortOperationException("No metadata found for ${reference}${release.version ? " (${release.version})" : ''}")
|
||||
}
|
||||
def moduleUrl = buildModuleUrl(registryConfig.url, reference, release.version)
|
||||
if( !output || output == 'text' ) {
|
||||
printFormattedInfo(reference, release, moduleUrl)
|
||||
} else if( output == 'json' ) {
|
||||
printJsonInfo(reference, release, moduleUrl)
|
||||
} else {
|
||||
throw new AbortOperationException("Not implemented output mode $output)")
|
||||
}
|
||||
}
|
||||
|
||||
private void printFormattedInfo(ModuleReference reference, ModuleRelease release, String moduleUrl) {
|
||||
ModuleMetadata metadata = release.metadata
|
||||
println ""
|
||||
println "Module: ${reference}"
|
||||
println "Version: ${release.version}"
|
||||
println "URL: ${moduleUrl}"
|
||||
println "Description: ${metadata.description ?: release.description ?: 'N/A'}"
|
||||
|
||||
if( metadata.authors ) {
|
||||
println "Authors: ${metadata.authors.join(', ')}"
|
||||
}
|
||||
|
||||
if( metadata.maintainers ) {
|
||||
println "Maintainers: ${metadata.maintainers.join(', ')}"
|
||||
}
|
||||
|
||||
if( metadata.keywords ) {
|
||||
println "Keywords: ${metadata.keywords.join(', ')}"
|
||||
}
|
||||
|
||||
printToolsInfo(metadata?.tools ?: [])
|
||||
|
||||
printInputsInfo(metadata.input ?: [])
|
||||
|
||||
printOutputsInfo(metadata.output ?: [:])
|
||||
|
||||
// Generate and display usage template
|
||||
println ""
|
||||
println "Usage Template:"
|
||||
println "---------------"
|
||||
println generateUsageTemplate(reference, metadata).join(" \\\n ")
|
||||
println ""
|
||||
}
|
||||
|
||||
private void printOutputsInfo(Map<String, ModuleChannel> outputs) {
|
||||
if( outputs ) {
|
||||
println ""
|
||||
println "Output:"
|
||||
outputs.each { name, output ->
|
||||
println "- ${name} ${output.tuple ? '(tuple)' : ''}"
|
||||
displayChannel("\t", output)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void printInputsInfo(List<ModuleChannel> inputs) {
|
||||
if( inputs ) {
|
||||
println ""
|
||||
println "Input:"
|
||||
inputs.each { input ->
|
||||
if( input.tuple ) {
|
||||
println "- (tuple)"
|
||||
displayChannel("\t", input)
|
||||
} else
|
||||
displayChannel("", input)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void printToolsInfo(List<ModuleTool> toolsList) {
|
||||
if( toolsList ) {
|
||||
println ""
|
||||
println "Tools:"
|
||||
toolsList.each { tool ->
|
||||
println " - ${tool.name}${tool.version ? ' v' + tool.version : ''}"
|
||||
if( tool.homepage ) {
|
||||
println " Homepage: ${tool.homepage}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void displayChannel(String prefix, ModuleChannel channel) {
|
||||
channel.items.each { ModuleChannelItem item ->
|
||||
println "${prefix}- ${item.name}${item.type ? ' (' + item.type + ')' : ''}"
|
||||
if( item.description ) {
|
||||
println "${prefix}\t${item.description.replaceAll(/\R/, ' ')}"
|
||||
}
|
||||
if( item.pattern ) {
|
||||
println "${prefix}\tPattern: ${item.pattern}"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private List<String> generateUsageTemplate(ModuleReference reference, ModuleMetadata metadata) {
|
||||
def template = new ArrayList<String>()
|
||||
template.add("nextflow module run ${reference}".toString())
|
||||
if( version )
|
||||
template.add(" -version $version".toString())
|
||||
|
||||
def inputs = metadata?.input ?: []
|
||||
inputs.each { input ->
|
||||
input.items.each { ModuleChannelItem item ->
|
||||
template.add(reference.scope == 'nf-core'
|
||||
? inferNfCoreParam(item.name, item.type)
|
||||
: inferNormalParam(item.name, item.type))
|
||||
|
||||
}
|
||||
}
|
||||
if( reference.scope == 'nf-core' ) {
|
||||
template.add('--outdir <OUTPUT_DIRECTORY>')
|
||||
}
|
||||
return template
|
||||
}
|
||||
|
||||
private static String inferNfCoreParam(String paramName, String type) {
|
||||
if( type?.equalsIgnoreCase("map") && paramName.equalsIgnoreCase("meta") ) {
|
||||
return "--${paramName}.id <ID>"
|
||||
}
|
||||
if( type?.equalsIgnoreCase("file") || type?.equalsIgnoreCase("path") ) {
|
||||
return "--${paramName} ${inferBioFilePlaceholder(paramName)}"
|
||||
}
|
||||
return inferNormalParam(paramName, type)
|
||||
}
|
||||
|
||||
private static String inferBioFilePlaceholder(String paramName) {
|
||||
final String lower = paramName.toLowerCase()
|
||||
if( lower.contains("fasta") ) return "<FASTA_FILE>"
|
||||
if( lower.contains("bam") ) return "<BAM_FILE>"
|
||||
if( lower.contains("fastq") || lower.equals("reads") ) return "<FASTQ_FILE>"
|
||||
if( lower.contains("vcf") ) return "<VCF_FILE>"
|
||||
if( lower.contains("ref") ) return "<REFERENCE_FILE>"
|
||||
if( lower.contains("bed") ) return "<BED_FILE>"
|
||||
if( lower.contains("gff") || lower.contains("gtf") ) return "<ANNOTATION_FILE>"
|
||||
|
||||
return "<${paramName.toUpperCase().replaceAll(/[^A-Z0-9]/, '_')}_PATH>"
|
||||
}
|
||||
|
||||
private static String inferNormalParam(String paramName, String type) {
|
||||
final paramPlaceholder = paramName.toUpperCase().replaceAll(/[^A-Z0-9]/, '_')
|
||||
if( type?.equalsIgnoreCase("map") ) {
|
||||
return "--${paramName}.<KEY> <${paramPlaceholder}_KEY_VALUE>"
|
||||
}
|
||||
if( type?.equalsIgnoreCase("file") || type?.equalsIgnoreCase("path") ) {
|
||||
return "--${paramName} <${paramPlaceholder}_PATH>"
|
||||
}
|
||||
return "--${paramName} <${paramPlaceholder}>"
|
||||
}
|
||||
|
||||
private static String buildModuleUrl(String registryUrl, ModuleReference reference, String version) {
|
||||
// Strip /api suffix to get the base UI URL
|
||||
def baseUrl = registryUrl.endsWith('/api') ? registryUrl[0..-5] : registryUrl
|
||||
return "${baseUrl}/modules/${reference.fullName}@${version}"
|
||||
}
|
||||
|
||||
private void printJsonInfo(ModuleReference reference, ModuleRelease release, String moduleUrl) {
|
||||
def metadata = release?.metadata
|
||||
def info = [
|
||||
name : reference.toString(),
|
||||
fullName : reference.fullName,
|
||||
version : release.version,
|
||||
url : moduleUrl,
|
||||
description: metadata.description ?: release.description,
|
||||
authors : metadata.authors,
|
||||
keywords : metadata.keywords,
|
||||
]
|
||||
|
||||
def toolsList = metadata.tools ?: []
|
||||
if( toolsList ) {
|
||||
info.tools = toolsList.collect { tool ->
|
||||
return [
|
||||
name : tool.name,
|
||||
version : tool.version,
|
||||
homepage : tool.homepage,
|
||||
documentation: tool.documentation
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
def inputs = metadata.input ?: []
|
||||
if( inputs ) {
|
||||
info.input = inputs.collect { input ->
|
||||
return [
|
||||
tuple: input.tuple,
|
||||
items: input.items?.collect { item ->
|
||||
[
|
||||
name : item.name,
|
||||
type : item.type,
|
||||
description: item.description,
|
||||
pattern : item.pattern
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
def outputs = metadata.output ?: [:]
|
||||
if( outputs ) {
|
||||
info.output = outputs.collectEntries { name, output ->
|
||||
return [name, [
|
||||
tuple: output.tuple,
|
||||
items: output.items?.collect { item ->
|
||||
[
|
||||
name : item.name,
|
||||
type : item.type,
|
||||
description: item.description,
|
||||
pattern : item.pattern
|
||||
]
|
||||
}
|
||||
]]
|
||||
}
|
||||
}
|
||||
|
||||
info.usageTemplate = generateUsageTemplate(reference, metadata).join(" ")
|
||||
|
||||
println JsonOutput.prettyPrint(JsonOutput.toJson(info))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud
|
||||
import groovy.transform.CompileStatic
|
||||
/**
|
||||
* Exception thrown when a spot instance is retired by the
|
||||
* provider
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Deprecated
|
||||
@CompileStatic
|
||||
class CloudSpotTerminationException extends RuntimeException {
|
||||
|
||||
String termination
|
||||
|
||||
CloudSpotTerminationException(String termination) {
|
||||
super("Cloud spot termination notice detected: $termination")
|
||||
this.termination = termination
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud
|
||||
|
||||
import nextflow.util.Duration
|
||||
|
||||
/**
|
||||
* Define common cloud data transfer options
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
interface CloudTransferOptions {
|
||||
|
||||
static final public int MAX_TRANSFER = 4
|
||||
|
||||
static final public int MAX_TRANSFER_ATTEMPTS = 1
|
||||
|
||||
static final public Duration DEFAULT_DELAY_BETWEEN_ATTEMPTS = Duration.of('10sec')
|
||||
|
||||
int getMaxParallelTransfers()
|
||||
int getMaxTransferAttempts()
|
||||
Duration getDelayBetweenAttempts()
|
||||
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud.types
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.Immutable
|
||||
/**
|
||||
* Model a cloud instance
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Immutable
|
||||
@CompileStatic
|
||||
class CloudInstance implements Serializable, Cloneable {
|
||||
|
||||
/**
|
||||
* The instance unique identifier as assigned by the provider
|
||||
*/
|
||||
String id
|
||||
|
||||
/**
|
||||
* The instance current state
|
||||
*/
|
||||
String state
|
||||
|
||||
/**
|
||||
* The instance public DNS name
|
||||
*/
|
||||
String publicDnsName
|
||||
|
||||
/**
|
||||
* The instance private DNS name
|
||||
*/
|
||||
String privateDnsName
|
||||
|
||||
/**
|
||||
* The instance private DNS name
|
||||
*/
|
||||
String publicIpAddress
|
||||
|
||||
/**
|
||||
* The instance private IP address
|
||||
*/
|
||||
String privateIpAddress
|
||||
|
||||
/**
|
||||
* The instance role in the cluster, either {@link nextflow.Const#ROLE_MASTER} or {@link nextflow.Const#ROLE_WORKER}
|
||||
*/
|
||||
String role
|
||||
|
||||
/**
|
||||
* The name of the cluster to which this instance belong to
|
||||
*/
|
||||
String clusterName
|
||||
|
||||
/**
|
||||
* @return The instance public address, falling back on the private when the public is not available
|
||||
*/
|
||||
String getAddress() {
|
||||
publicDnsName ?: publicIpAddress ?: privateDnsName ?: privateIpAddress
|
||||
}
|
||||
|
||||
boolean hasPublicAddress() {
|
||||
publicDnsName || publicIpAddress
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud.types
|
||||
|
||||
/**
|
||||
* Model a cloud instance status
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
enum CloudInstanceStatus implements Serializable, Cloneable {
|
||||
|
||||
/**
|
||||
* The instance is started, bootstrap/initialisation may be on-going
|
||||
*/
|
||||
STARTED,
|
||||
|
||||
/**
|
||||
* The instance is running and available for use
|
||||
*/
|
||||
READY,
|
||||
|
||||
/**
|
||||
* The instance ws terminated
|
||||
*/
|
||||
TERMINATED
|
||||
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud.types
|
||||
|
||||
import groovy.transform.Immutable
|
||||
import nextflow.util.MemoryUnit
|
||||
|
||||
/**
|
||||
* Models a cloud instance type
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Immutable
|
||||
class CloudInstanceType implements Serializable, Cloneable {
|
||||
|
||||
/**
|
||||
* Instance type ID as assigned by the cloud provider
|
||||
*/
|
||||
String id
|
||||
|
||||
/**
|
||||
* Number of CPUs
|
||||
*/
|
||||
int cpus
|
||||
|
||||
/**
|
||||
* Amount of memory (RAM)
|
||||
*/
|
||||
MemoryUnit memory
|
||||
|
||||
/**
|
||||
* Amount of local storage
|
||||
*/
|
||||
MemoryUnit disk
|
||||
|
||||
/**
|
||||
* Number of disks
|
||||
*/
|
||||
int numOfDisks
|
||||
|
||||
String toString() {
|
||||
"id=$id; cpus=$cpus; mem=$memory; disk=$disk x $numOfDisks"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud.types
|
||||
|
||||
import groovy.transform.Canonical
|
||||
import groovy.transform.CompileStatic
|
||||
/**
|
||||
* Model cloud job metadata
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
@Canonical
|
||||
class CloudMachineInfo {
|
||||
String type
|
||||
String zone
|
||||
PriceModel priceModel
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud.types
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.Immutable
|
||||
|
||||
/**
|
||||
* Models an instance spot price record
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
|
||||
@Immutable
|
||||
@CompileStatic
|
||||
class CloudSpotPrice implements Serializable, Cloneable {
|
||||
|
||||
/**
|
||||
* The instance type identifier e.g. {@code m4.xlarge}
|
||||
*/
|
||||
String type
|
||||
|
||||
/**
|
||||
* The spot price in USD
|
||||
*/
|
||||
String price
|
||||
|
||||
/**
|
||||
* The instance availability zone
|
||||
*/
|
||||
String zone
|
||||
|
||||
/**
|
||||
* The instance description
|
||||
*/
|
||||
String description
|
||||
|
||||
Date timestamp
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud.types
|
||||
|
||||
/**
|
||||
* Model different cloud price models
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
enum PriceModel {
|
||||
standard, spot;
|
||||
}
|
||||
@@ -0,0 +1,434 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.conda
|
||||
|
||||
import java.nio.file.FileSystems
|
||||
import java.nio.file.NoSuchFileException
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.Memoized
|
||||
import groovy.transform.PackageScope
|
||||
import groovy.util.logging.Slf4j
|
||||
import groovyx.gpars.dataflow.DataflowVariable
|
||||
import groovyx.gpars.dataflow.LazyDataflowVariable
|
||||
import nextflow.Global
|
||||
import nextflow.SysEnv
|
||||
import nextflow.file.FileMutex
|
||||
import nextflow.util.CacheHelper
|
||||
import nextflow.util.Duration
|
||||
import nextflow.util.Escape
|
||||
import nextflow.util.TestOnly
|
||||
/**
|
||||
* Handle Conda environment creation and caching
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class CondaCache {
|
||||
static final private Object condaLock = new Object()
|
||||
|
||||
/**
|
||||
* Cache the prefix path for each Conda environment
|
||||
*/
|
||||
static final private Map<String,DataflowVariable<Path>> condaPrefixPaths = new ConcurrentHashMap<>()
|
||||
|
||||
/**
|
||||
* The Conda settings defined in the nextflow config file
|
||||
*/
|
||||
private CondaConfig config
|
||||
|
||||
/**
|
||||
* Timeout after which the environment creation is aborted
|
||||
*/
|
||||
private Duration createTimeout
|
||||
|
||||
private String createOptions
|
||||
|
||||
private boolean useMamba
|
||||
|
||||
private boolean useMicromamba
|
||||
|
||||
private Path configCacheDir0
|
||||
|
||||
private List<String> channels = Collections.emptyList()
|
||||
|
||||
@PackageScope String getCreateOptions() { createOptions }
|
||||
|
||||
@PackageScope Duration getCreateTimeout() { createTimeout }
|
||||
|
||||
@PackageScope Map<String,String> getEnv() { SysEnv.get() }
|
||||
|
||||
@PackageScope Path getConfigCacheDir0() { configCacheDir0 }
|
||||
|
||||
@PackageScope List<String> getChannels() { channels }
|
||||
|
||||
@PackageScope String getBinaryName() {
|
||||
if (useMamba)
|
||||
return "mamba"
|
||||
if (useMicromamba)
|
||||
return "micromamba"
|
||||
return "conda"
|
||||
}
|
||||
|
||||
@TestOnly
|
||||
protected CondaCache() {}
|
||||
|
||||
/**
|
||||
* Create a Conda env cache object
|
||||
*
|
||||
* @param config A {@link Map} object
|
||||
*/
|
||||
CondaCache(CondaConfig config) {
|
||||
this.config = config
|
||||
|
||||
if( config.createTimeout() )
|
||||
createTimeout = config.createTimeout()
|
||||
|
||||
if( config.createOptions() )
|
||||
createOptions = config.createOptions()
|
||||
|
||||
if( config.cacheDir() )
|
||||
configCacheDir0 = config.cacheDir().toAbsolutePath()
|
||||
|
||||
if( config.useMamba() && config.useMicromamba() )
|
||||
throw new IllegalArgumentException("Both conda.useMamba and conda.useMicromamba were enabled -- Please choose only one")
|
||||
|
||||
if( config.useMamba() ) {
|
||||
useMamba = config.useMamba()
|
||||
}
|
||||
|
||||
if( config.useMicromamba() )
|
||||
useMicromamba = config.useMicromamba()
|
||||
|
||||
if( config.getChannels() )
|
||||
channels = config.getChannels()
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve the directory where store the conda environment.
|
||||
*
|
||||
* If tries these setting in the following order:
|
||||
* 1) {@code conda.cacheDir} setting in the nextflow config file;
|
||||
* 2) the {@code $workDir/conda} path
|
||||
*
|
||||
* @return
|
||||
* the {@code Path} where store the conda envs
|
||||
*/
|
||||
@PackageScope
|
||||
Path getCacheDir() {
|
||||
|
||||
def cacheDir = configCacheDir0
|
||||
|
||||
if( !cacheDir && getEnv().NXF_CONDA_CACHEDIR )
|
||||
cacheDir = getEnv().NXF_CONDA_CACHEDIR as Path
|
||||
|
||||
if( !cacheDir )
|
||||
cacheDir = getSessionWorkDir().resolve('conda')
|
||||
|
||||
if( cacheDir.fileSystem != FileSystems.default ) {
|
||||
throw new IOException("Cannot store Conda environments to a remote work directory -- Use a POSIX compatible work directory or specify an alternative path with the `conda.cacheDir` config setting")
|
||||
}
|
||||
|
||||
if( !cacheDir.exists() && !cacheDir.mkdirs() ) {
|
||||
throw new IOException("Failed to create Conda cache directory: $cacheDir -- Make sure a file with the same name does not exist and you have write permission")
|
||||
}
|
||||
|
||||
return cacheDir
|
||||
}
|
||||
|
||||
@PackageScope Path getSessionWorkDir() {
|
||||
Global.session.workDir
|
||||
}
|
||||
|
||||
@PackageScope
|
||||
boolean isYamlFilePath(String str) {
|
||||
(str.endsWith('.yml') || str.endsWith('.yaml')) && !str.contains('\n')
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the given string is a path to a conda explicit file
|
||||
* by verifying it contains the @EXPLICIT marker in the first 20 lines
|
||||
*
|
||||
* @param str The conda environment string
|
||||
* @return {@code true} if it's a path to an explicit file, {@code false} otherwise
|
||||
*/
|
||||
@PackageScope
|
||||
@Memoized // <-- annotate as "Memoized" to avoid parsing multiple time the same file
|
||||
boolean isExplicitFile(String str) {
|
||||
if( str.contains('\n') )
|
||||
return false
|
||||
try {
|
||||
final path = str as Path
|
||||
if( !path.exists() )
|
||||
return false
|
||||
return containsExplicitMarker(path)
|
||||
}
|
||||
catch( Exception e ) {
|
||||
log.debug "Failed to check explicit file: $str - ${e.message}"
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a file contains the @EXPLICIT marker in the first 20 lines
|
||||
*
|
||||
* @param path The file path to check
|
||||
* @return {@code true} if the marker is found, {@code false} otherwise
|
||||
*/
|
||||
private boolean containsExplicitMarker(Path path) {
|
||||
try {
|
||||
try(final reader = path.newReader()) {
|
||||
for( int i = 0; i < 20; i++ ) {
|
||||
final line = reader.readLine()
|
||||
if( line == null )
|
||||
break
|
||||
if( line.trim() == '@EXPLICIT' )
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
catch( Exception e ) {
|
||||
log.debug "Failed to check for @EXPLICIT marker in file: $path - ${e.message}"
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the path on the file system where store a Conda environment
|
||||
*
|
||||
* @param condaEnv The conda environment
|
||||
* @return the conda unique prefix {@link Path} where the env is created
|
||||
*/
|
||||
@PackageScope
|
||||
Path condaPrefixPath(String condaEnv) {
|
||||
assert condaEnv
|
||||
|
||||
String content
|
||||
String name = 'env'
|
||||
// check if it's a remote uri
|
||||
if( isYamlUriPath(condaEnv) ) {
|
||||
content = condaEnv
|
||||
}
|
||||
// check if it's a YAML file
|
||||
else if( isYamlFilePath(condaEnv) ) {
|
||||
try {
|
||||
final path = condaEnv as Path
|
||||
content = path.text
|
||||
|
||||
}
|
||||
catch( NoSuchFileException e ) {
|
||||
throw new IllegalArgumentException("Conda environment file does not exist: $condaEnv")
|
||||
}
|
||||
catch( Exception e ) {
|
||||
throw new IllegalArgumentException("Error parsing Conda environment YAML file: $condaEnv -- Check the log file for details", e)
|
||||
}
|
||||
}
|
||||
// check if it's a conda explicit file (contains @EXPLICIT marker)
|
||||
else if( isExplicitFile(condaEnv) ) {
|
||||
try {
|
||||
final path = condaEnv as Path
|
||||
content = path.text
|
||||
}
|
||||
catch( Exception e ) {
|
||||
throw new IllegalArgumentException("Error reading Conda explicit file: $condaEnv -- Check the log file for details", e)
|
||||
}
|
||||
}
|
||||
// it's interpreted as user provided prefix directory
|
||||
else if( condaEnv.contains('/') ) {
|
||||
final prefix = condaEnv as Path
|
||||
if( !prefix.isDirectory() )
|
||||
throw new IllegalArgumentException("Conda prefix path does not exist or is not a directory: $prefix")
|
||||
if( prefix.fileSystem != FileSystems.default )
|
||||
throw new IllegalArgumentException("Conda prefix path must be a POSIX file path: $prefix")
|
||||
|
||||
return prefix
|
||||
}
|
||||
else if( condaEnv.contains('\n') ) {
|
||||
throw new IllegalArgumentException("Invalid Conda environment definition: $condaEnv")
|
||||
}
|
||||
else {
|
||||
content = condaEnv
|
||||
}
|
||||
|
||||
final hash = CacheHelper.hasher(content).hash().toString()
|
||||
getCacheDir().resolve("$name-$hash")
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the conda tool to create an environment in the file system.
|
||||
*
|
||||
* @param condaEnv The conda environment definition
|
||||
* @return the conda environment prefix {@link Path}
|
||||
*/
|
||||
@PackageScope
|
||||
Path createLocalCondaEnv(String condaEnv, Path prefixPath) {
|
||||
|
||||
if( prefixPath.isDirectory() ) {
|
||||
log.debug "${binaryName} found local env for environment=$condaEnv; path=$prefixPath"
|
||||
return prefixPath
|
||||
}
|
||||
|
||||
final file = new File("${prefixPath.parent}/.${prefixPath.name}.lock")
|
||||
final wait = "Another Nextflow instance is creating the conda environment $condaEnv -- please wait till it completes"
|
||||
final err = "Unable to acquire exclusive lock after $createTimeout on file: $file"
|
||||
|
||||
final mutex = new FileMutex(target: file, timeout: createTimeout, waitMessage: wait, errorMessage: err)
|
||||
try {
|
||||
mutex .lock { createLocalCondaEnv0(condaEnv, prefixPath) }
|
||||
}
|
||||
finally {
|
||||
file.delete()
|
||||
}
|
||||
|
||||
return prefixPath
|
||||
}
|
||||
|
||||
@PackageScope
|
||||
Path makeAbsolute( String envFile ) {
|
||||
Paths.get(envFile).toAbsolutePath()
|
||||
}
|
||||
|
||||
@PackageScope boolean isYamlUriPath(String env) {
|
||||
env.startsWith('http://') || env.startsWith('https://')
|
||||
}
|
||||
|
||||
@PackageScope
|
||||
Path createLocalCondaEnv0(String condaEnv, Path prefixPath) {
|
||||
if( prefixPath.isDirectory() ) {
|
||||
log.debug "${binaryName} found local env for environment=$condaEnv; path=$prefixPath"
|
||||
return prefixPath
|
||||
}
|
||||
|
||||
log.info "Creating env using ${binaryName}: $condaEnv [cache $prefixPath]"
|
||||
|
||||
String opts = createOptions ? "$createOptions " : ''
|
||||
|
||||
def cmd
|
||||
if( isYamlFilePath(condaEnv) ) {
|
||||
final target = isYamlUriPath(condaEnv) ? condaEnv : Escape.path(makeAbsolute(condaEnv))
|
||||
final yesOpt = binaryName=="mamba" || binaryName == "micromamba" ? '--yes ' : ''
|
||||
cmd = "${binaryName} env create ${yesOpt}--prefix ${Escape.path(prefixPath)} --file ${target}"
|
||||
}
|
||||
else if( isExplicitFile(condaEnv) ) {
|
||||
cmd = "${binaryName} create ${opts}--yes --quiet --prefix ${Escape.path(prefixPath)} --file ${Escape.path(makeAbsolute(condaEnv))}"
|
||||
}
|
||||
|
||||
else {
|
||||
final channelsOpt = channels.collect(it -> "-c $it ").join('')
|
||||
cmd = "${binaryName} create ${opts}--yes --quiet --prefix ${Escape.path(prefixPath)} ${channelsOpt}$condaEnv"
|
||||
}
|
||||
|
||||
try {
|
||||
// Parallel execution of conda causes data and package corruption.
|
||||
// https://github.com/nextflow-io/nextflow/issues/4233
|
||||
// https://github.com/conda/conda/issues/13037
|
||||
// Should be removed as soon as the upstream bug is fixed and released.
|
||||
synchronized(condaLock) {
|
||||
runCommand( cmd )
|
||||
}
|
||||
log.debug "'${binaryName}' create complete env=$condaEnv path=$prefixPath"
|
||||
}
|
||||
catch( Exception e ){
|
||||
// clean-up to avoid to keep eventually corrupted image file
|
||||
prefixPath.delete()
|
||||
throw e
|
||||
}
|
||||
return prefixPath
|
||||
}
|
||||
|
||||
@PackageScope
|
||||
int runCommand( String cmd ) {
|
||||
log.trace """${binaryName} create
|
||||
command: $cmd
|
||||
timeout: $createTimeout""".stripIndent(true)
|
||||
|
||||
final max = createTimeout.toMillis()
|
||||
final builder = new ProcessBuilder(['bash','-c',cmd])
|
||||
final proc = builder.redirectErrorStream(true).start()
|
||||
final err = new StringBuilder()
|
||||
final consumer = proc.consumeProcessOutputStream(err)
|
||||
proc.waitForOrKill(max)
|
||||
def status = proc.exitValue()
|
||||
if( status != 0 ) {
|
||||
consumer.join()
|
||||
def msg = "Failed to create Conda environment\n command: $cmd\n status : $status\n message:\n"
|
||||
msg += err.toString().trim().indent(' ')
|
||||
throw new IllegalStateException(msg)
|
||||
}
|
||||
return status
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a remote image URL returns a {@link DataflowVariable} which holds
|
||||
* the local image path.
|
||||
*
|
||||
* This method synchronise multiple concurrent requests so that only one
|
||||
* image download is actually executed.
|
||||
*
|
||||
* @param condaEnv
|
||||
* Conda environment string
|
||||
* @return
|
||||
* The {@link DataflowVariable} which hold (and pull) the local image file
|
||||
*/
|
||||
@PackageScope
|
||||
DataflowVariable<Path> getLazyImagePath(String condaEnv) {
|
||||
final prefixPath = condaPrefixPath(condaEnv)
|
||||
final condaEnvPath = prefixPath.toString()
|
||||
if( condaEnvPath in condaPrefixPaths ) {
|
||||
log.trace "${binaryName} found local environment `$condaEnv`"
|
||||
return condaPrefixPaths[condaEnvPath]
|
||||
}
|
||||
|
||||
synchronized (condaPrefixPaths) {
|
||||
def result = condaPrefixPaths[condaEnvPath]
|
||||
if( result == null ) {
|
||||
result = new LazyDataflowVariable<Path>({ createLocalCondaEnv(condaEnv, prefixPath) })
|
||||
condaPrefixPaths[condaEnvPath] = result
|
||||
}
|
||||
else {
|
||||
log.trace "${binaryName} found local cache for environment `$condaEnv` (2)"
|
||||
}
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a conda environment caching it in the file system.
|
||||
*
|
||||
* This method synchronise multiple concurrent requests so that only one
|
||||
* environment is actually created.
|
||||
*
|
||||
* @param condaEnv The conda environment string
|
||||
* @return the local environment path prefix {@link Path}
|
||||
*/
|
||||
Path getCachePathFor(String condaEnv) {
|
||||
def promise = getLazyImagePath(condaEnv)
|
||||
def result = promise.getVal()
|
||||
if( promise.isError() )
|
||||
throw new IllegalStateException(promise.getError())
|
||||
if( !result )
|
||||
throw new IllegalStateException("Cannot create Conda environment `$condaEnv`")
|
||||
log.trace "Conda cache for env `$condaEnv` path=$result"
|
||||
return result
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.conda
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.config.spec.ConfigOption
|
||||
import nextflow.config.spec.ConfigScope
|
||||
import nextflow.config.spec.ScopeName
|
||||
import nextflow.script.dsl.Description
|
||||
import nextflow.util.Duration
|
||||
|
||||
/**
|
||||
* Model Conda configuration
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@ScopeName("conda")
|
||||
@Description("""
|
||||
The `conda` scope controls the creation of Conda environments by the Conda package manager.
|
||||
""")
|
||||
@CompileStatic
|
||||
class CondaConfig implements ConfigScope {
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Execute tasks with Conda environments (default: `false`).
|
||||
""")
|
||||
final boolean enabled
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The path where Conda environments are stored. It should be accessible from all compute nodes when using a shared file system.
|
||||
""")
|
||||
final String cacheDir
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The list of Conda channels that can be used to resolve Conda packages. Channel priority decreases from left to right.
|
||||
""")
|
||||
final List<String> channels
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Extra command line options for the `conda create` command. See the [Conda documentation](https://docs.conda.io/projects/conda/en/latest/commands/create.html) for more information.
|
||||
""")
|
||||
final String createOptions
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The amount of time to wait for the Conda environment to be created before failing (default: `20 min`).
|
||||
""")
|
||||
final Duration createTimeout
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Use [Mamba](https://github.com/mamba-org/mamba) instead of `conda` to create Conda environments (default: `false`).
|
||||
""")
|
||||
final boolean useMamba
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Use [Micromamba](https://mamba.readthedocs.io/en/latest/user_guide/micromamba.html) instead of `conda` to create Conda environments (default: `false`).
|
||||
""")
|
||||
final boolean useMicromamba
|
||||
|
||||
/* required by extension point -- do not remove */
|
||||
CondaConfig() {}
|
||||
|
||||
CondaConfig(Map opts, Map<String, String> env) {
|
||||
enabled = opts.enabled != null
|
||||
? opts.enabled as boolean
|
||||
: (env.NXF_CONDA_ENABLED?.toString() == 'true')
|
||||
cacheDir = opts.cacheDir
|
||||
channels = parseChannels(opts.channels)
|
||||
createOptions = opts.createOptions
|
||||
createTimeout = opts.createTimeout as Duration ?: Duration.of('20min')
|
||||
useMamba = opts.useMamba as boolean
|
||||
useMicromamba = opts.useMicromamba as boolean
|
||||
|
||||
if( useMamba && useMicromamba )
|
||||
throw new IllegalArgumentException("Both conda.useMamba and conda.useMicromamba were enabled -- Please choose only one")
|
||||
}
|
||||
|
||||
private List<String> parseChannels(Object value) {
|
||||
if( !value )
|
||||
return ['conda-forge','bioconda']
|
||||
if( value instanceof List )
|
||||
return value
|
||||
if( value instanceof CharSequence )
|
||||
return value.tokenize(',').collect(it -> it.trim())
|
||||
throw new IllegalArgumentException("Unexpected conda.channels value: $value")
|
||||
}
|
||||
|
||||
Duration createTimeout() {
|
||||
createTimeout
|
||||
}
|
||||
|
||||
String createOptions() {
|
||||
createOptions
|
||||
}
|
||||
|
||||
Path cacheDir() {
|
||||
cacheDir as Path
|
||||
}
|
||||
|
||||
boolean useMamba() {
|
||||
useMamba
|
||||
}
|
||||
|
||||
boolean useMicromamba() {
|
||||
useMicromamba
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.config
|
||||
import java.lang.reflect.Method
|
||||
|
||||
import groovy.transform.CompileDynamic
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.Memoized
|
||||
/**
|
||||
* Configuration object which fallback to a parent {@link CascadingConfig} object
|
||||
* when an attribute value is missing.
|
||||
*
|
||||
* @see ConfigField
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
abstract class CascadingConfig<K,V> {
|
||||
|
||||
protected Map<K,V> config = [:]
|
||||
|
||||
protected CascadingConfig<K,V> parent
|
||||
|
||||
CascadingConfig() {}
|
||||
|
||||
CascadingConfig(Map<K,V> config, CascadingConfig<K,V> parent) {
|
||||
this.config = config
|
||||
this.parent = parent
|
||||
config.keySet().each { K it -> checkField(it) }
|
||||
}
|
||||
|
||||
@CompileDynamic
|
||||
private Set<String> discoverFields(Closure<Boolean> accept) {
|
||||
def result = new HashSet<String>()
|
||||
def clazz = this.getClass()
|
||||
while( clazz != null ) {
|
||||
def methods = clazz.getMethods()
|
||||
def names = (methods as List<Method>)?.findResults { field ->
|
||||
def annotation = field.getAnnotation(ConfigField)
|
||||
if( annotation && accept(annotation) )
|
||||
return annotation.value() ?: field.getName().replaceFirst('^get','').uncapitalize()
|
||||
return null
|
||||
}
|
||||
result.addAll(names)
|
||||
clazz = clazz.getSuperclass()
|
||||
}
|
||||
|
||||
return result ?: Collections.<String>emptySet()
|
||||
}
|
||||
|
||||
@Memoized
|
||||
protected Set<String> validFields() {
|
||||
return discoverFields({ true })
|
||||
}
|
||||
|
||||
protected Set<String> privateFields() {
|
||||
return discoverFields({ ConfigField field -> field._private() })
|
||||
}
|
||||
|
||||
final protected void checkField(K name) throws IllegalArgumentException {
|
||||
def fields = validFields()
|
||||
|
||||
if( !fields.contains(name.toString()) )
|
||||
throw new IllegalArgumentException("Not a valid config attribute: `$name`")
|
||||
}
|
||||
|
||||
protected Map<String,Object> copyPublicAttributes() {
|
||||
def skip = privateFields()
|
||||
def copy = new LinkedHashMap()
|
||||
this.config.each { k,v -> if(!skip.contains(k)) copy[k]=v }
|
||||
return copy
|
||||
}
|
||||
|
||||
boolean isEmpty() { config.isEmpty() }
|
||||
|
||||
boolean containsAttribute( K key ) {
|
||||
config.containsKey(key) ?: ( parent ? parent.containsAttribute(key) : false )
|
||||
}
|
||||
|
||||
V getAttribute(K key) {
|
||||
getAttribute(key, null)
|
||||
}
|
||||
|
||||
V getAttribute(K key, V defValue) {
|
||||
checkField(key)
|
||||
return config.containsKey(key) ? config.get(key) : ( parent && parent.containsAttribute(key) ? parent.getAttribute(key) : defValue )
|
||||
}
|
||||
|
||||
V getOrCreateAttribute(K key, Closure<V> missing) {
|
||||
def result = getAttribute(key)
|
||||
if( result == null ) {
|
||||
result = missing.call()
|
||||
config.put(key, result)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
void setAttribute( K key, V value ) {
|
||||
checkField(key)
|
||||
config.put(key,value)
|
||||
}
|
||||
|
||||
Set<K> getAttributeNames() {
|
||||
new HashSet<K>(config.keySet())
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Convert this object to an equivalent {@link ConfigObject}
|
||||
*
|
||||
* @return A {@link ConfigObject} holding the same data
|
||||
*/
|
||||
ConfigObject toConfigObject() {
|
||||
toConfigObject0(this.config)
|
||||
}
|
||||
|
||||
@CompileDynamic
|
||||
protected ConfigObject toConfigObject0( Map map ) {
|
||||
|
||||
def result = new ConfigObject()
|
||||
map.each { key, value ->
|
||||
if( value instanceof Map ) {
|
||||
result.put( key, toConfigObject0((Map)value) )
|
||||
}
|
||||
else if( value instanceof CascadingConfig ) {
|
||||
result.put( key, value.toConfigObject() )
|
||||
}
|
||||
else {
|
||||
result.put( key, value )
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,916 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.config
|
||||
|
||||
import static nextflow.util.ConfigHelper.*
|
||||
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
|
||||
import groovy.transform.Memoized
|
||||
import groovy.transform.PackageScope
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.Const
|
||||
import nextflow.NF
|
||||
import nextflow.SysEnv
|
||||
import nextflow.cli.CliOptions
|
||||
import nextflow.cli.CmdConfig
|
||||
import nextflow.cli.CmdNode
|
||||
import nextflow.cli.CmdRun
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.exception.ConfigParseException
|
||||
import nextflow.secret.SecretsLoader
|
||||
import nextflow.secret.SecretsProvider
|
||||
import nextflow.util.HistoryFile
|
||||
import nextflow.util.SecretHelper
|
||||
/**
|
||||
* Builds up the Nextflow configuration object
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
class ConfigBuilder {
|
||||
|
||||
static final public String DEFAULT_PROFILE = 'standard'
|
||||
|
||||
CliOptions options
|
||||
|
||||
CmdRun cmdRun
|
||||
|
||||
CmdNode cmdNode
|
||||
|
||||
Path baseDir
|
||||
|
||||
Path homeDir
|
||||
|
||||
Path currentDir
|
||||
|
||||
Map<String,?> cliParams
|
||||
|
||||
boolean showAllProfiles
|
||||
|
||||
String profile = DEFAULT_PROFILE
|
||||
|
||||
boolean validateProfile
|
||||
|
||||
List<Path> userConfigFiles = []
|
||||
|
||||
List<Path> parsedConfigFiles = []
|
||||
|
||||
boolean showClosures
|
||||
|
||||
boolean stripSecrets
|
||||
|
||||
boolean showMissingVariables
|
||||
|
||||
SecretsProvider secretsProvider
|
||||
|
||||
Map<ConfigObject, String> emptyVariables = new LinkedHashMap<>(10)
|
||||
|
||||
Map<String,String> env = new HashMap<>(SysEnv.get())
|
||||
|
||||
List<String> warnings = new ArrayList<>(10)
|
||||
|
||||
Map<String,Object> declaredParams
|
||||
|
||||
ConfigBuilder() {
|
||||
setHomeDir(Const.APP_HOME_DIR)
|
||||
setCurrentDir(Paths.get('.'))
|
||||
}
|
||||
|
||||
ConfigBuilder setShowClosures(boolean value) {
|
||||
this.showClosures = value
|
||||
return this
|
||||
}
|
||||
|
||||
ConfigBuilder setStripSecrets(boolean value) {
|
||||
this.stripSecrets = value
|
||||
return this
|
||||
}
|
||||
|
||||
ConfigBuilder showMissingVariables(boolean value) {
|
||||
this.showMissingVariables = value
|
||||
return this
|
||||
}
|
||||
|
||||
ConfigBuilder setSecretsProvider(SecretsProvider value) {
|
||||
this.secretsProvider = value
|
||||
return this
|
||||
}
|
||||
|
||||
ConfigBuilder setOptions( CliOptions options ) {
|
||||
this.options = options
|
||||
return this
|
||||
}
|
||||
|
||||
ConfigBuilder setCmdRun( CmdRun cmdRun ) {
|
||||
this.cmdRun = cmdRun
|
||||
setProfile(cmdRun.profile)
|
||||
return this
|
||||
}
|
||||
|
||||
ConfigBuilder setCliParams( Map<String,?> cliParams ) {
|
||||
this.cliParams = cliParams
|
||||
return this
|
||||
}
|
||||
|
||||
ConfigBuilder setBaseDir( Path path ) {
|
||||
this.baseDir = path.complete()
|
||||
return this
|
||||
}
|
||||
|
||||
ConfigBuilder setCurrentDir( Path path ) {
|
||||
this.currentDir = path.complete()
|
||||
return this
|
||||
}
|
||||
|
||||
ConfigBuilder setHomeDir( Path path ) {
|
||||
this.homeDir = path.complete()
|
||||
return this
|
||||
}
|
||||
|
||||
ConfigBuilder setCmdNode( CmdNode node ) {
|
||||
this.cmdNode = node
|
||||
return this
|
||||
}
|
||||
|
||||
ConfigBuilder setCmdConfig( CmdConfig cmdConfig ) {
|
||||
showAllProfiles = cmdConfig.showAllProfiles
|
||||
setProfile(cmdConfig.profile)
|
||||
return this
|
||||
}
|
||||
|
||||
ConfigBuilder setProfile( String value ) {
|
||||
profile = value ?: DEFAULT_PROFILE
|
||||
validateProfile = value as boolean
|
||||
return this
|
||||
}
|
||||
|
||||
ConfigBuilder setShowAllProfiles(boolean value) {
|
||||
this.showAllProfiles = value
|
||||
return this
|
||||
}
|
||||
|
||||
ConfigBuilder setUserConfigFiles( Path... files ) {
|
||||
setUserConfigFiles(files as List)
|
||||
return this
|
||||
}
|
||||
|
||||
ConfigBuilder setUserConfigFiles( List<Path> files ) {
|
||||
if( files )
|
||||
userConfigFiles.addAll(files)
|
||||
return this
|
||||
}
|
||||
|
||||
Map<String,Object> getConfigParams() {
|
||||
return declaredParams
|
||||
}
|
||||
|
||||
static private wrapValue( value ) {
|
||||
if( !value )
|
||||
return ''
|
||||
|
||||
value = value.toString().trim()
|
||||
if( value == 'true' || value == 'false')
|
||||
return value
|
||||
|
||||
if( value.isNumber() )
|
||||
return value
|
||||
|
||||
return "'$value'"
|
||||
}
|
||||
|
||||
/**
|
||||
* Transform the specified list of string to a list of files, verifying their existence.
|
||||
* <p>
|
||||
* If a file in the list does not exist an exception of type {@code CliArgumentException} is thrown.
|
||||
* <p>
|
||||
* If the specified list is empty it tries to return of default configuration files located at:
|
||||
* <li>$HOME/.nextflow/taskConfig
|
||||
* <li>$PWD/nextflow.taskConfig
|
||||
*
|
||||
* @param files
|
||||
* @return
|
||||
*/
|
||||
@PackageScope
|
||||
List<Path> validateConfigFiles( List<String> files ) {
|
||||
|
||||
def result = []
|
||||
if ( files ) {
|
||||
for( String fileName : files ) {
|
||||
def thisFile = currentDir.resolve(fileName)
|
||||
if(!thisFile.exists()) {
|
||||
throw new AbortOperationException("The specified configuration file does not exist: $thisFile -- check the name or choose another file")
|
||||
}
|
||||
result << thisFile
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
/*
|
||||
* config file in the nextflow home
|
||||
*/
|
||||
def home = homeDir.resolve('config')
|
||||
if( home.exists() ) {
|
||||
log.debug "Found config home: $home"
|
||||
result << home
|
||||
}
|
||||
|
||||
/**
|
||||
* Config file in the pipeline base dir
|
||||
* This config file name should be predictable, therefore cannot be overridden
|
||||
*/
|
||||
def base = null
|
||||
if( baseDir && baseDir != currentDir ) {
|
||||
base = baseDir.resolve('nextflow.config')
|
||||
if( base.exists() ) {
|
||||
log.debug "Found config base: $base"
|
||||
result << base
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Local or user provided file
|
||||
* Default config file name can be overridden with `NXF_CONFIG_FILE` env variable
|
||||
*/
|
||||
def configFileName = env.get('NXF_CONFIG_FILE') ?: 'nextflow.config'
|
||||
def local = currentDir.resolve(configFileName)
|
||||
if( local.exists() && local != base ) {
|
||||
log.debug "Found config local: $local"
|
||||
result << local
|
||||
}
|
||||
|
||||
def customConfigs = []
|
||||
if( userConfigFiles ) customConfigs.addAll(userConfigFiles)
|
||||
if( options?.userConfig ) customConfigs.addAll(options.userConfig)
|
||||
if( cmdRun?.runConfig ) customConfigs.addAll(cmdRun.runConfig)
|
||||
if( customConfigs ) {
|
||||
for( def item : customConfigs ) {
|
||||
def configFile = item instanceof Path ? item : currentDir.resolve(item.toString())
|
||||
if(!configFile.exists()) {
|
||||
throw new AbortOperationException("The specified configuration file does not exist: $configFile -- check the name or choose another file")
|
||||
}
|
||||
|
||||
log.debug "User config file: $configFile"
|
||||
result << configFile
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Create the nextflow configuration {@link ConfigObject} given a one or more
|
||||
* config files
|
||||
*
|
||||
* @param files A list of config files {@link Path}
|
||||
* @return The resulting {@link ConfigObject} instance
|
||||
*/
|
||||
@PackageScope
|
||||
ConfigObject buildGivenFiles(List<Path> files) {
|
||||
|
||||
final Map<String,String> vars = cmdRun?.env
|
||||
final boolean exportSysEnv = cmdRun?.exportSysEnv
|
||||
|
||||
def items = []
|
||||
if( files ) for( Path file : files ) {
|
||||
log.debug "Parsing config file: ${file.complete()}"
|
||||
if (!file.exists()) {
|
||||
log.warn "The specified configuration file cannot be found: $file"
|
||||
}
|
||||
else {
|
||||
items << file
|
||||
}
|
||||
}
|
||||
|
||||
Map env = [:]
|
||||
if( exportSysEnv ) {
|
||||
log.debug "Adding current system environment to session environment"
|
||||
env.putAll(SysEnv.get())
|
||||
}
|
||||
if( vars ) {
|
||||
log.debug "Adding the following variables to session environment: $vars"
|
||||
env.putAll(vars)
|
||||
}
|
||||
|
||||
// set the cluster options for the node command
|
||||
if( cmdNode?.clusterOptions ) {
|
||||
def str = new StringBuilder()
|
||||
cmdNode.clusterOptions.each { k, v ->
|
||||
str << "cluster." << k << '=' << wrapValue(v) << '\n'
|
||||
}
|
||||
items << str
|
||||
}
|
||||
|
||||
// -- add the executor obj from the command line args
|
||||
if( cmdRun?.clusterOptions ) {
|
||||
def str = new StringBuilder()
|
||||
cmdRun.clusterOptions.each { k, v ->
|
||||
str << "cluster." << k << '=' << wrapValue(v) << '\n'
|
||||
}
|
||||
items << str
|
||||
}
|
||||
|
||||
if( cmdRun?.executorOptions ) {
|
||||
def str = new StringBuilder()
|
||||
cmdRun.executorOptions.each { k, v ->
|
||||
str << "executor." << k << '=' << wrapValue(v) << '\n'
|
||||
}
|
||||
items << str
|
||||
}
|
||||
|
||||
buildConfig0( env, items )
|
||||
}
|
||||
|
||||
@PackageScope
|
||||
ConfigObject buildGivenFiles(Path... files) {
|
||||
buildGivenFiles(files as List<Path>)
|
||||
}
|
||||
|
||||
protected Map configVars() {
|
||||
// this is needed to make sure to reuse the same
|
||||
// instance of the config vars across different instances of the ConfigBuilder
|
||||
// and prevent multiple parsing of the same params file (which can even be remote resource)
|
||||
final secretContext = secretsProvider
|
||||
? SecretsLoader.secretContext(secretsProvider)
|
||||
: SecretsLoader.secretContext()
|
||||
return getConfigVars(baseDir, secretContext)
|
||||
}
|
||||
|
||||
@Memoized
|
||||
static Map getConfigVars(Path base, Object secretContext) {
|
||||
final binding = new HashMap(10)
|
||||
binding.put('baseDir', base)
|
||||
binding.put('projectDir', base)
|
||||
binding.put('launchDir', Paths.get('.').toRealPath())
|
||||
binding.put('outputDir', Paths.get('results').complete())
|
||||
binding.put('secrets', secretContext)
|
||||
return binding
|
||||
}
|
||||
|
||||
protected ConfigObject buildConfig0( Map env, List configEntries ) {
|
||||
assert env != null
|
||||
|
||||
final ignoreIncludes = options ? options.ignoreConfigIncludes : false
|
||||
final ansiLog = options ? options.ansiLog : false
|
||||
final parser = ConfigParserFactory.create()
|
||||
.setRenderClosureAsString(showClosures)
|
||||
.setStripSecrets(stripSecrets)
|
||||
.setIgnoreIncludes(ignoreIncludes)
|
||||
.setAnsiLog(ansiLog)
|
||||
ConfigObject result = new ConfigObject()
|
||||
|
||||
if( cliParams )
|
||||
parser.setParams(cliParams)
|
||||
|
||||
// add the user specified environment to the session env
|
||||
env.sort().each { name, value -> result.env.put(name,value) }
|
||||
|
||||
if( configEntries ) {
|
||||
// the configuration object binds always the current environment
|
||||
// so that in the configuration file may be referenced any variable
|
||||
// in the current environment
|
||||
final binding = new HashMap(SysEnv.get())
|
||||
binding.putAll(env)
|
||||
binding.putAll(configVars())
|
||||
|
||||
parser.setBinding(binding)
|
||||
|
||||
// merge of the provided configuration files
|
||||
for( def entry : configEntries ) {
|
||||
|
||||
try {
|
||||
merge0(result, parser, entry)
|
||||
}
|
||||
catch( ConfigParseException e ) {
|
||||
throw e
|
||||
}
|
||||
catch( Exception e ) {
|
||||
def message = (entry instanceof Path ? "Unable to parse config file: '$entry'" : "Unable to parse configuration ")
|
||||
throw new ConfigParseException(message,e)
|
||||
}
|
||||
}
|
||||
|
||||
if( validateProfile ) {
|
||||
checkValidProfile(parser.getDeclaredProfiles())
|
||||
}
|
||||
|
||||
this.declaredParams = parser.getDeclaredParams()
|
||||
}
|
||||
|
||||
// guarantee top scopes
|
||||
for( String name : ['env','session','params','process','executor']) {
|
||||
if( !result.isSet(name) ) result.put(name, new ConfigObject())
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge the main config with a separate config file
|
||||
*
|
||||
* @param result The main {@link ConfigObject}
|
||||
* @param parser The {@ConfigParser} instance
|
||||
* @param entry The next config snippet/file to be parsed
|
||||
* @return
|
||||
*/
|
||||
protected void merge0(ConfigObject result, ConfigParser parser, entry) {
|
||||
if( !entry )
|
||||
return
|
||||
|
||||
// select the profile
|
||||
if( !showAllProfiles ) {
|
||||
log.debug "Applying config profile: `${profile}`"
|
||||
parser.setProfiles(profile.tokenize(','))
|
||||
}
|
||||
|
||||
final config = parse0(parser, entry)
|
||||
if( NF.getSyntaxParserVersion() == 'v1' )
|
||||
validate(config, entry)
|
||||
result.merge(config)
|
||||
}
|
||||
|
||||
protected ConfigObject parse0(ConfigParser parser, entry) {
|
||||
if( entry instanceof File ) {
|
||||
final path = entry.toPath()
|
||||
parsedConfigFiles << path
|
||||
return parser.parse(path)
|
||||
}
|
||||
|
||||
if( entry instanceof Path ) {
|
||||
parsedConfigFiles << entry
|
||||
return parser.parse(entry)
|
||||
}
|
||||
|
||||
if( entry instanceof CharSequence ) {
|
||||
return parser.parse(entry.toString())
|
||||
}
|
||||
|
||||
throw new IllegalStateException("Unexpected config entry: ${entry}")
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate a config object verifying is does not contains unresolved attributes
|
||||
*
|
||||
* @param config The {@link ConfigObject} to verify
|
||||
* @param file The source config file/snippet
|
||||
* @return
|
||||
*/
|
||||
protected void validate(ConfigObject config, file, String parent=null, List stack = new ArrayList()) {
|
||||
for( String key : new ArrayList<>(config.keySet()) ) {
|
||||
final value = config.get(key)
|
||||
if( value instanceof ConfigObject ) {
|
||||
final fqKey = parent ? "${parent}.${key}": key as String
|
||||
if( value.isEmpty() ) {
|
||||
final msg = "Unknown config attribute `$fqKey` -- check config file: $file".toString()
|
||||
if( showMissingVariables ) {
|
||||
emptyVariables.put(value, key)
|
||||
warnings.add(msg)
|
||||
}
|
||||
else {
|
||||
log.debug("In the following config snippet the attribute `$fqKey` is empty:\n${->config.prettyPrint().indent(' ')}")
|
||||
throw new ConfigParseException(msg)
|
||||
}
|
||||
}
|
||||
else {
|
||||
stack.push(config)
|
||||
try {
|
||||
if( !stack.contains(value)) {
|
||||
validate(value, file, fqKey, stack)
|
||||
}
|
||||
else {
|
||||
log.debug("Found a recursive config property: `$fqKey`")
|
||||
}
|
||||
}
|
||||
finally {
|
||||
stack.pop()
|
||||
}
|
||||
}
|
||||
}
|
||||
else if( value instanceof GString && showMissingVariables ) {
|
||||
final str = (GString) value
|
||||
for( int i=0; i<str.values.length; i++ ) {
|
||||
// try replace empty interpolated strings with variable handle
|
||||
final arg = str.values[i]
|
||||
final name = emptyVariables.get(arg)
|
||||
if( name )
|
||||
str.values[i] = '$' + name
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected void checkValidProfile(Collection<String> validNames) {
|
||||
if( !profile || profile == DEFAULT_PROFILE ) {
|
||||
return
|
||||
}
|
||||
|
||||
log.debug "Available config profiles: $validNames"
|
||||
for( String name : profile.tokenize(',') ) {
|
||||
if( name in validNames )
|
||||
continue
|
||||
|
||||
def message = "Unknown configuration profile: '${name}'"
|
||||
def choices = validNames.closest(name)
|
||||
if( choices ) {
|
||||
message += "\n\nDid you mean one of these?\n"
|
||||
choices.each { message += " ${it}\n" }
|
||||
message += '\n'
|
||||
}
|
||||
|
||||
throw new AbortOperationException(message)
|
||||
}
|
||||
}
|
||||
|
||||
private String normalizeResumeId( String uniqueId ) {
|
||||
if( !uniqueId )
|
||||
return null
|
||||
if( uniqueId == 'last' || uniqueId == 'true' ) {
|
||||
if( HistoryFile.disabled() )
|
||||
throw new AbortOperationException("The resume session id should be specified via `-resume` option when history file tracking is disabled")
|
||||
uniqueId = HistoryFile.DEFAULT.getLast()?.sessionId
|
||||
|
||||
if( !uniqueId ) {
|
||||
log.warn "It appears you have never run this project before -- Option `-resume` is ignored"
|
||||
}
|
||||
}
|
||||
|
||||
return uniqueId
|
||||
}
|
||||
|
||||
@PackageScope
|
||||
void configRunOptions(ConfigObject config, Map env, CmdRun cmdRun) {
|
||||
|
||||
// -- set config options
|
||||
if( cmdRun.cacheable != null )
|
||||
config.cacheable = cmdRun.cacheable
|
||||
|
||||
// -- set the run name
|
||||
if( cmdRun.runName )
|
||||
config.runName = cmdRun.runName
|
||||
|
||||
if( cmdRun.stubRun )
|
||||
config.stubRun = cmdRun.stubRun
|
||||
|
||||
// -- set the output directory
|
||||
if( cmdRun.outputDir )
|
||||
config.outputDir = cmdRun.outputDir
|
||||
|
||||
if( cmdRun.outputFormat )
|
||||
config.outputFormat = cmdRun.outputFormat
|
||||
|
||||
if( cmdRun.preview )
|
||||
config.preview = cmdRun.preview
|
||||
|
||||
if( cmdRun.plugins )
|
||||
config.plugins = cmdRun.plugins.tokenize(',')
|
||||
|
||||
// -- sets the working directory
|
||||
if( cmdRun.workDir )
|
||||
config.workDir = cmdRun.workDir
|
||||
|
||||
else if( !config.workDir )
|
||||
config.workDir = env.get('NXF_WORK') ?: 'work'
|
||||
|
||||
if( cmdRun.bucketDir )
|
||||
config.bucketDir = cmdRun.bucketDir
|
||||
|
||||
// -- sets the library path
|
||||
if( cmdRun.libPath )
|
||||
config.libDir = cmdRun.libPath
|
||||
|
||||
else if ( !config.isSet('libDir') && env.get('NXF_LIB') )
|
||||
config.libDir = env.get('NXF_LIB')
|
||||
|
||||
// -- override 'process' parameters defined on the cmd line
|
||||
cmdRun.process.each { name, value ->
|
||||
config.process[name] = parseValue(value)
|
||||
}
|
||||
|
||||
if( cmdRun.withoutConda && config.conda instanceof Map ) {
|
||||
// disable conda execution
|
||||
log.debug "Disabling execution with Conda as requested by command-line option `-without-conda`"
|
||||
config.conda.enabled = false
|
||||
}
|
||||
|
||||
// -- apply the conda environment
|
||||
if( cmdRun.withConda ) {
|
||||
if( cmdRun.withConda != '-' )
|
||||
config.process.conda = cmdRun.withConda
|
||||
config.conda.enabled = true
|
||||
}
|
||||
|
||||
if( cmdRun.withoutSpack && config.spack instanceof Map ) {
|
||||
// disable spack execution
|
||||
log.debug "Disabling execution with Spack as requested by command-line option `-without-spack`"
|
||||
config.spack.enabled = false
|
||||
}
|
||||
|
||||
// -- apply the spack environment
|
||||
if( cmdRun.withSpack ) {
|
||||
if( cmdRun.withSpack != '-' )
|
||||
config.process.spack = cmdRun.withSpack
|
||||
config.spack.enabled = true
|
||||
}
|
||||
|
||||
// -- sets the resume option
|
||||
if( cmdRun.resume )
|
||||
config.resume = cmdRun.resume
|
||||
|
||||
if( config.isSet('resume') )
|
||||
config.resume = normalizeResumeId(config.resume as String)
|
||||
|
||||
// -- sets `dumpHashes` option
|
||||
if( cmdRun.dumpHashes ) {
|
||||
config.dumpHashes = cmdRun.dumpHashes != '-' ? cmdRun.dumpHashes : 'default'
|
||||
}
|
||||
|
||||
if( cmdRun.dumpChannels )
|
||||
config.dumpChannels = cmdRun.dumpChannels.tokenize(',')
|
||||
|
||||
// -- other configuration parameters
|
||||
if( cmdRun.poolSize ) {
|
||||
config.poolSize = cmdRun.poolSize
|
||||
}
|
||||
if( cmdRun.queueSize ) {
|
||||
config.executor.queueSize = cmdRun.queueSize
|
||||
}
|
||||
if( cmdRun.pollInterval ) {
|
||||
config.executor.pollInterval = cmdRun.pollInterval
|
||||
}
|
||||
|
||||
// -- sets trace file options
|
||||
if( cmdRun.withTrace ) {
|
||||
if( !(config.trace instanceof Map) )
|
||||
config.trace = [:]
|
||||
config.trace.enabled = true
|
||||
if( cmdRun.withTrace != '-' )
|
||||
config.trace.file = cmdRun.withTrace
|
||||
}
|
||||
|
||||
// -- sets report report options
|
||||
if( cmdRun.withReport ) {
|
||||
if( !(config.report instanceof Map) )
|
||||
config.report = [:]
|
||||
config.report.enabled = true
|
||||
if( cmdRun.withReport != '-' )
|
||||
config.report.file = cmdRun.withReport
|
||||
}
|
||||
|
||||
// -- sets timeline report options
|
||||
if( cmdRun.withTimeline ) {
|
||||
if( !(config.timeline instanceof Map) )
|
||||
config.timeline = [:]
|
||||
config.timeline.enabled = true
|
||||
if( cmdRun.withTimeline != '-' )
|
||||
config.timeline.file = cmdRun.withTimeline
|
||||
}
|
||||
|
||||
// -- sets DAG report options
|
||||
if( cmdRun.withDag ) {
|
||||
if( !(config.dag instanceof Map) )
|
||||
config.dag = [:]
|
||||
config.dag.enabled = true
|
||||
if( cmdRun.withDag != '-' )
|
||||
config.dag.file = cmdRun.withDag
|
||||
}
|
||||
|
||||
if( cmdRun.withNotification ) {
|
||||
if( !(config.notification instanceof Map) )
|
||||
config.notification = [:]
|
||||
if( cmdRun.withNotification in ['true','false']) {
|
||||
config.notification.enabled = cmdRun.withNotification == 'true'
|
||||
}
|
||||
else {
|
||||
config.notification.enabled = true
|
||||
config.notification.to = cmdRun.withNotification
|
||||
}
|
||||
}
|
||||
|
||||
// -- sets the messages options
|
||||
if( cmdRun.withWebLog ) {
|
||||
log.warn "The command line option '-with-weblog' is deprecated - consider enabling this feature by setting 'weblog.enabled=true' in your configuration file"
|
||||
if( !(config.weblog instanceof Map) )
|
||||
config.weblog = [:]
|
||||
config.weblog.enabled = true
|
||||
if( cmdRun.withWebLog != '-' )
|
||||
config.weblog.url = cmdRun.withWebLog
|
||||
else if( !config.weblog.url )
|
||||
config.weblog.url = 'http://localhost'
|
||||
}
|
||||
|
||||
// -- sets tower options
|
||||
if( cmdRun.withTower ) {
|
||||
if( !(config.tower instanceof Map) )
|
||||
config.tower = [:]
|
||||
config.tower.enabled = true
|
||||
if( cmdRun.withTower != '-' )
|
||||
config.tower.endpoint = cmdRun.withTower
|
||||
}
|
||||
|
||||
// -- set wave options
|
||||
if( cmdRun.withWave ) {
|
||||
if( !(config.wave instanceof Map) )
|
||||
config.wave = [:]
|
||||
config.wave.enabled = true
|
||||
if( cmdRun.withWave != '-' )
|
||||
config.wave.endpoint = cmdRun.withWave
|
||||
}
|
||||
|
||||
// -- set fusion options
|
||||
if( cmdRun.withFusion ) {
|
||||
if( !(config.fusion instanceof Map) )
|
||||
config.fusion = [:]
|
||||
config.fusion.enabled = cmdRun.withFusion == 'true'
|
||||
}
|
||||
|
||||
// -- set cloudcache options
|
||||
final envCloudPath = env.get('NXF_CLOUDCACHE_PATH')
|
||||
if( cmdRun.cloudCachePath || envCloudPath ) {
|
||||
if( !(config.cloudcache instanceof Map) )
|
||||
config.cloudcache = [:]
|
||||
if( !config.cloudcache.isSet('enabled') )
|
||||
config.cloudcache.enabled = true
|
||||
if( cmdRun.cloudCachePath && cmdRun.cloudCachePath != '-' )
|
||||
config.cloudcache.path = cmdRun.cloudCachePath
|
||||
else if( !config.cloudcache.isSet('path') && envCloudPath )
|
||||
config.cloudcache.path = envCloudPath
|
||||
}
|
||||
|
||||
// -- add the command line parameters to the 'taskConfig' object
|
||||
if( cliParams )
|
||||
config.params = mergeMaps( (Map)config.params, cliParams, NF.strictMode )
|
||||
|
||||
if( cmdRun.withoutDocker && config.docker instanceof Map ) {
|
||||
// disable docker execution
|
||||
log.debug "Disabling execution in Docker container as requested by command-line option `-without-docker`"
|
||||
config.docker.enabled = false
|
||||
}
|
||||
|
||||
if( cmdRun.withDocker ) {
|
||||
configContainer(config, 'docker', cmdRun.withDocker)
|
||||
}
|
||||
|
||||
if( cmdRun.withPodman ) {
|
||||
configContainer(config, 'podman', cmdRun.withPodman)
|
||||
}
|
||||
|
||||
if( cmdRun.withSingularity ) {
|
||||
configContainer(config, 'singularity', cmdRun.withSingularity)
|
||||
}
|
||||
|
||||
if( cmdRun.withApptainer ) {
|
||||
configContainer(config, 'apptainer', cmdRun.withApptainer)
|
||||
}
|
||||
|
||||
if( cmdRun.withCharliecloud ) {
|
||||
configContainer(config, 'charliecloud', cmdRun.withCharliecloud)
|
||||
}
|
||||
}
|
||||
|
||||
private void configContainer(ConfigObject config, String engine, def cli) {
|
||||
log.debug "Enabling execution in ${engine.capitalize()} container as requested by command-line option `-with-$engine ${cmdRun.withDocker}`"
|
||||
|
||||
if( !config.containsKey(engine) )
|
||||
config.put(engine, [:])
|
||||
|
||||
if( !(config.get(engine) instanceof Map) )
|
||||
throw new AbortOperationException("Invalid `$engine` definition in the config file")
|
||||
|
||||
def containerConfig = (Map)config.get(engine)
|
||||
containerConfig.enabled = true
|
||||
if( cli != '-' ) {
|
||||
// this is supposed to be a docker image name
|
||||
config.process.container = cli
|
||||
}
|
||||
else if( containerConfig.image ) {
|
||||
config.process.container = containerConfig.image
|
||||
}
|
||||
}
|
||||
|
||||
ConfigObject buildConfigObject() {
|
||||
// -- configuration file(s)
|
||||
def configFiles = validateConfigFiles(options?.config)
|
||||
def config = buildGivenFiles(configFiles)
|
||||
|
||||
if( cmdRun )
|
||||
configRunOptions(config, SysEnv.get(), cmdRun)
|
||||
|
||||
return config
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @return A the application options hold in a {@code ConfigObject} instance
|
||||
*/
|
||||
ConfigMap build() {
|
||||
toConfigMap(buildConfigObject())
|
||||
}
|
||||
|
||||
protected static ConfigMap toConfigMap(ConfigObject config) {
|
||||
assert config != null
|
||||
(ConfigMap)normalize0((Map)config)
|
||||
}
|
||||
|
||||
static private normalize0( config ) {
|
||||
|
||||
if( config instanceof Map ) {
|
||||
ConfigMap result = new ConfigMap(config.size())
|
||||
for( String name : config.keySet() ) {
|
||||
def value = (config as Map).get(name)
|
||||
result.put(name, normalize0(value))
|
||||
}
|
||||
return result
|
||||
}
|
||||
else if( config instanceof Collection ) {
|
||||
List result = new ArrayList(config.size())
|
||||
for( entry in config ) {
|
||||
result << normalize0(entry)
|
||||
}
|
||||
return result
|
||||
}
|
||||
else {
|
||||
return config
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge two maps recursively avoiding keys to be overwritten
|
||||
*
|
||||
* @param config
|
||||
* @param params
|
||||
* @return a map resulting of merging result and right maps
|
||||
*/
|
||||
protected Map mergeMaps(Map config, Map params, boolean strict, List keys=[]) {
|
||||
if( config==null )
|
||||
config = new LinkedHashMap()
|
||||
|
||||
for( Map.Entry entry : params ) {
|
||||
final key = entry.key.toString()
|
||||
final value = entry.value
|
||||
final previous = getConfigVal0(config, key)
|
||||
keys << entry.key
|
||||
|
||||
if( previous==null ) {
|
||||
config[key] = value
|
||||
}
|
||||
else if( previous instanceof Map && value instanceof Map ) {
|
||||
mergeMaps(previous, value, strict, keys)
|
||||
}
|
||||
else {
|
||||
if( previous instanceof Map || value instanceof Map ) {
|
||||
final msg = "Configuration setting type with key '${keys.join('.')}' does not match the parameter with the same key - Config value=$previous; parameter value=$value"
|
||||
if(strict)
|
||||
throw new AbortOperationException(msg)
|
||||
log.warn(msg)
|
||||
}
|
||||
config[key] = value
|
||||
}
|
||||
}
|
||||
|
||||
return config
|
||||
}
|
||||
|
||||
private Object getConfigVal0(Map config, String key) {
|
||||
if( config instanceof ConfigObject ) {
|
||||
return config.isSet(key) ? config.get(key) : null
|
||||
}
|
||||
else {
|
||||
return config.get(key)
|
||||
}
|
||||
}
|
||||
|
||||
static String resolveConfig(Path baseDir, CmdRun cmdRun, Map cliParams) {
|
||||
|
||||
final config = new ConfigBuilder()
|
||||
.setShowClosures(true)
|
||||
.setStripSecrets(true)
|
||||
.setOptions(cmdRun.launcher.options)
|
||||
.setCmdRun(cmdRun)
|
||||
.setCliParams(cliParams)
|
||||
.setBaseDir(baseDir)
|
||||
.buildConfigObject()
|
||||
|
||||
// strip secrets
|
||||
SecretHelper.hideSecrets(config)
|
||||
// compute config
|
||||
final result = toCanonicalString(config, false)
|
||||
// dump config for debugging
|
||||
log.trace "Resolved config:\n${result.indent('\t')}"
|
||||
return result
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.config
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
/**
|
||||
* Placeholder class that replacing closure definitions in the nextflow configuration
|
||||
* file in order to print the closure content itself
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@EqualsAndHashCode
|
||||
@CompileStatic
|
||||
class ConfigClosurePlaceholder {
|
||||
|
||||
private String str
|
||||
|
||||
ConfigClosurePlaceholder(String str) {
|
||||
this.str = str
|
||||
}
|
||||
|
||||
@Override String toString() { str }
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.config
|
||||
|
||||
import java.lang.annotation.ElementType
|
||||
import java.lang.annotation.Retention
|
||||
import java.lang.annotation.RetentionPolicy
|
||||
import java.lang.annotation.Target
|
||||
|
||||
/**
|
||||
* Annotation used to marked fields of {@link CascadingConfig} subclasses
|
||||
*
|
||||
* @see CascadingConfig
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
@Target(ElementType.METHOD)
|
||||
@interface ConfigField {
|
||||
/**
|
||||
* @return The field name (default: empty string)
|
||||
*/
|
||||
String value() default ''
|
||||
|
||||
/**
|
||||
* Mark the configuration field as private
|
||||
*
|
||||
* @return {@code true} when the field is private (default: false)
|
||||
*/
|
||||
boolean _private() default false
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.config
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.config.spec.ConfigOption
|
||||
import nextflow.config.spec.ConfigScope
|
||||
import nextflow.config.spec.ScopeName
|
||||
import nextflow.script.dsl.Description
|
||||
/**
|
||||
* Represent Nextflow config as Map
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@ScopeName('')
|
||||
@CompileStatic
|
||||
class ConfigMap extends LinkedHashMap implements ConfigScope {
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The remote work directory used by hybrid workflows. Equivalent to the `-bucket-dir` option of the `run` command.
|
||||
""")
|
||||
final String bucketDir
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Delete all files associated with a run in the work directory when the run completes successfully (default: `false`).
|
||||
""")
|
||||
final boolean cleanup
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The pipeline output directory. Equivalent to the `-output-dir` option of the `run` command.
|
||||
""")
|
||||
final String outputDir
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Enable the use of previously cached task executions. Equivalent to the `-resume` option of the `run` command.
|
||||
""")
|
||||
final boolean resume
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The pipeline work directory. Equivalent to the `-work-dir` option of the `run` command.
|
||||
""")
|
||||
final String workDir
|
||||
|
||||
ConfigMap() {
|
||||
}
|
||||
|
||||
ConfigMap(int initialCapacity) {
|
||||
super(initialCapacity)
|
||||
}
|
||||
|
||||
ConfigMap(Map opts) {
|
||||
super(opts)
|
||||
bucketDir = opts.bucketDir
|
||||
cleanup = opts.cleanup as boolean
|
||||
outputDir = opts.outputDir
|
||||
resume = opts.resume as boolean
|
||||
workDir = opts.workDir
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.config
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
/**
|
||||
* Interface for Nextflow config parsers.
|
||||
*
|
||||
* @author Ben Sherman <bentshermann@gmail.com>
|
||||
*/
|
||||
interface ConfigParser {
|
||||
|
||||
/**
|
||||
* Toggle whether config include statements should be ignored.
|
||||
*
|
||||
* @param value
|
||||
*/
|
||||
ConfigParser setIgnoreIncludes(boolean value)
|
||||
|
||||
/**
|
||||
* Toggle whether to strip secrets when rendering the config.
|
||||
*
|
||||
* @param value
|
||||
*/
|
||||
ConfigParser setStripSecrets(boolean value)
|
||||
|
||||
/**
|
||||
* Toggle whether to render the source code of closures.
|
||||
*
|
||||
* @param value
|
||||
*/
|
||||
ConfigParser setRenderClosureAsString(boolean value)
|
||||
|
||||
/**
|
||||
* Toggle whether to raise an error if a missing property is accessed.
|
||||
*
|
||||
* @param value
|
||||
*/
|
||||
ConfigParser setStrict(boolean value)
|
||||
|
||||
/**
|
||||
* Define variables which will be made available to the config script.
|
||||
*
|
||||
* @param vars
|
||||
*/
|
||||
ConfigParser setBinding(Map vars)
|
||||
|
||||
/**
|
||||
* Define pipeline parameters which will be made available to the config script.
|
||||
*
|
||||
* @param vars
|
||||
*/
|
||||
ConfigParser setParams(Map vars)
|
||||
|
||||
/**
|
||||
* Set the profiles that should be applied.
|
||||
*/
|
||||
ConfigParser setProfiles(List<String> profiles)
|
||||
|
||||
/**
|
||||
* Toggle whether to render compilation errors with ANSI colors.
|
||||
*
|
||||
* @param value
|
||||
*/
|
||||
default ConfigParser setAnsiLog(boolean value) {
|
||||
return this
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a config object from the given source.
|
||||
*/
|
||||
ConfigObject parse(String text)
|
||||
ConfigObject parse(File file)
|
||||
ConfigObject parse(Path path)
|
||||
|
||||
/**
|
||||
* Get the set of declared profiles.
|
||||
*/
|
||||
Set<String> getDeclaredProfiles()
|
||||
|
||||
/**
|
||||
* Get the map of declared params.
|
||||
*/
|
||||
Map<String,Object> getDeclaredParams()
|
||||
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.config
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.NF
|
||||
import nextflow.config.parser.v1.ConfigParserV1
|
||||
import nextflow.config.parser.v2.ConfigParserV2
|
||||
|
||||
/**
|
||||
* Factory for creating an instance of {@link ConfigParser}.
|
||||
*
|
||||
* @author Ben Sherman <bentshermann@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class ConfigParserFactory {
|
||||
|
||||
static ConfigParser create() {
|
||||
final parser = NF.getSyntaxParserVersion()
|
||||
if( parser == 'v1' ) {
|
||||
return new ConfigParserV1()
|
||||
}
|
||||
if( parser == 'v2' ) {
|
||||
log.debug "Using config parser v2"
|
||||
return new ConfigParserV2()
|
||||
}
|
||||
throw new IllegalStateException("Invalid NXF_SYNTAX_PARSER setting -- should be either 'v1' or 'v2'")
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.config
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.config.spec.ConfigScope
|
||||
import nextflow.config.spec.SpecNode
|
||||
import nextflow.config.spec.ScopeName
|
||||
import nextflow.plugin.Plugins
|
||||
import nextflow.script.dsl.Description
|
||||
/**
|
||||
* Validate the Nextflow configuration
|
||||
*
|
||||
* @author Ben Sherman <bentshermann@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class ConfigValidator {
|
||||
|
||||
/**
|
||||
* Hidden options added by ConfigBuilder
|
||||
*/
|
||||
private static final List<String> HIDDEN_OPTIONS = List.of(
|
||||
'cacheable',
|
||||
'dumpChannels',
|
||||
'dumpHashes',
|
||||
'libDir',
|
||||
'outputFormat',
|
||||
'poolSize',
|
||||
'preview',
|
||||
'runName',
|
||||
'stubRun',
|
||||
);
|
||||
|
||||
/**
|
||||
* Core plugin scopes which can only be validated when
|
||||
* the plugin is loaded.
|
||||
*/
|
||||
private static final List<String> CORE_PLUGIN_SCOPES = List.of(
|
||||
'aws',
|
||||
'azure',
|
||||
'cloudcache',
|
||||
'google',
|
||||
'k8s',
|
||||
'tower',
|
||||
'wave'
|
||||
)
|
||||
|
||||
/**
|
||||
* Additional config scopes added by third-party plugins
|
||||
*/
|
||||
private SpecNode.Scope pluginScopes
|
||||
|
||||
ConfigValidator() {
|
||||
loadPluginScopes()
|
||||
}
|
||||
|
||||
private void loadPluginScopes() {
|
||||
final children = new HashMap<String, SpecNode>()
|
||||
for( final scope : Plugins.getExtensions(ConfigScope) ) {
|
||||
final clazz = scope.getClass()
|
||||
final name = clazz.getAnnotation(ScopeName)?.value()
|
||||
final description = clazz.getAnnotation(Description)?.value()
|
||||
if( name == '' ) {
|
||||
children.putAll(SpecNode.Scope.of(clazz, '').children())
|
||||
continue
|
||||
}
|
||||
if( !name )
|
||||
continue
|
||||
if( name in children ) {
|
||||
log.warn "Plugin config scope `${clazz.name}` conflicts with existing scope: `${name}`"
|
||||
continue
|
||||
}
|
||||
children.put(name, SpecNode.Scope.of(clazz, description))
|
||||
}
|
||||
pluginScopes = new SpecNode.Scope('', children)
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate a config block within the given scope.
|
||||
*
|
||||
* @param config
|
||||
* @param scopes
|
||||
*/
|
||||
void validate(Map<String,?> config, List<String> scopes=[]) {
|
||||
for( final entry : config.entrySet() ) {
|
||||
final key = entry.key
|
||||
final value = entry.value
|
||||
|
||||
final names = scopes + [key]
|
||||
|
||||
if( names.size() == 2 && names.first() == 'profiles' )
|
||||
names.clear()
|
||||
|
||||
if( value instanceof Map ) {
|
||||
if( isSelector(key) )
|
||||
names.removeLast()
|
||||
if( isMapOption(names) )
|
||||
continue
|
||||
validate(value, names)
|
||||
}
|
||||
else {
|
||||
validateOption(names)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine whether a scope name is a process selector.
|
||||
*
|
||||
* @param name
|
||||
*/
|
||||
private boolean isSelector(String name) {
|
||||
return name.startsWith('withLabel:') || name.startsWith('withName:')
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate a config option given by the list of names.
|
||||
*
|
||||
* For example, the option 'process.resourceLimits' is represented
|
||||
* as ['process', 'resourceLimits'].
|
||||
*
|
||||
* @param names
|
||||
*/
|
||||
void validateOption(List<String> names) {
|
||||
final scope = names.first()
|
||||
if( scope == 'env' ) {
|
||||
checkEnv(names.last())
|
||||
return
|
||||
}
|
||||
if( scope == 'params' )
|
||||
return
|
||||
if( isMissingCorePluginScope(scope) )
|
||||
return
|
||||
if( isValid(names) )
|
||||
return
|
||||
log.warn1 "Unrecognized config option '${names.join('.')}'"
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine whether a config option is defined in the spec.
|
||||
*
|
||||
* @param names
|
||||
*/
|
||||
boolean isValid(List<String> names) {
|
||||
if( names.size() == 1 && names.first() in HIDDEN_OPTIONS )
|
||||
return true
|
||||
final child = SpecNode.ROOT.getChild(names)
|
||||
if( child instanceof SpecNode.Option || child instanceof SpecNode.DslOption )
|
||||
return true
|
||||
if( pluginScopes.getOption(names) )
|
||||
return true
|
||||
return false
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine whether a config scope is from a core plugin
|
||||
* which is not currently loaded.
|
||||
*
|
||||
* @param name
|
||||
*/
|
||||
private boolean isMissingCorePluginScope(String name) {
|
||||
return name in CORE_PLUGIN_SCOPES
|
||||
&& !pluginScopes.children().containsKey(name)
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine whether a config option is a map option.
|
||||
*
|
||||
* This method is needed to distinguish between config scopes
|
||||
* and config options that happen to be maps, since this distinction
|
||||
* is lost when the config is resolved.
|
||||
*
|
||||
* @param names
|
||||
*/
|
||||
private boolean isMapOption(List<String> names) {
|
||||
return isMapOption0(SpecNode.ROOT, names)
|
||||
|| isMapOption0(pluginScopes, names)
|
||||
}
|
||||
|
||||
private static boolean isMapOption0(SpecNode.Scope scope, List<String> names) {
|
||||
final node = scope.getOption(names)
|
||||
return node != null && node.types().contains(Map.class)
|
||||
}
|
||||
|
||||
/**
|
||||
* Warn about setting `NXF_*` environment variables in the config.
|
||||
*
|
||||
* @param name
|
||||
*/
|
||||
private void checkEnv(String name) {
|
||||
if( name.startsWith('NXF_') && name!='NXF_DEBUG' )
|
||||
log.warn "Nextflow environment variables must be defined in the launch environment -- the following environment variable in the config will be ignored: '$name'"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,251 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.config
|
||||
|
||||
import java.util.stream.Collectors
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import nextflow.config.spec.ConfigOption
|
||||
import nextflow.config.spec.ConfigScope
|
||||
import nextflow.config.spec.ScopeName
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.script.dsl.Description
|
||||
|
||||
import static nextflow.Const.DEFAULT_MAIN_FILE_NAME
|
||||
/**
|
||||
* Models the nextflow config manifest settings
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@ScopeName("manifest")
|
||||
@Description("""
|
||||
The `manifest` scope allows you to define some metadata that is useful when publishing or running your pipeline.
|
||||
""")
|
||||
@CompileStatic
|
||||
class Manifest implements ConfigScope {
|
||||
|
||||
@Deprecated
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Project author name (use a comma to separate multiple names).
|
||||
""")
|
||||
final String author
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
List of project contributors. Should be a list of maps.
|
||||
""")
|
||||
final List<Contributor> contributors
|
||||
|
||||
@Deprecated
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Git repository default branch (default: `master`).
|
||||
""")
|
||||
final String defaultBranch
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Free text describing the workflow project.
|
||||
""")
|
||||
final String description
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Project documentation URL.
|
||||
""")
|
||||
final String docsUrl
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Project related publication DOI identifier.
|
||||
""")
|
||||
final String doi
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Controls whether git sub-modules should be cloned with the main repository.
|
||||
|
||||
Can be either a boolean value, a list of submodule names, or a comma-separated string of submodule names.
|
||||
""")
|
||||
final Object gitmodules
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Project home page URL.
|
||||
""")
|
||||
final String homePage
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Project related icon location (Relative path or URL).
|
||||
""")
|
||||
final String icon
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Project license.
|
||||
""")
|
||||
final String license
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Project main script (default: `main.nf`).
|
||||
""")
|
||||
final String mainScript
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Project short name.
|
||||
""")
|
||||
final String name
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Minimum required Nextflow version.
|
||||
""")
|
||||
final String nextflowVersion
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Project organization.
|
||||
""")
|
||||
final String organization
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Pull submodules recursively from the Git repository.
|
||||
""")
|
||||
final boolean recurseSubmodules
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Project version number.
|
||||
""")
|
||||
final String version
|
||||
|
||||
/* required by extension point -- do not remove */
|
||||
Manifest() {}
|
||||
|
||||
Manifest(Map opts) {
|
||||
author = opts.author as String
|
||||
contributors = parseContributors(opts.contributors)
|
||||
defaultBranch = opts.defaultBranch as String
|
||||
description = opts.description as String
|
||||
docsUrl = opts.docsUrl as String
|
||||
doi = opts.doi as String
|
||||
gitmodules = opts.gitmodules
|
||||
homePage = opts.homePage as String
|
||||
icon = opts.icon as String
|
||||
license = opts.license as String
|
||||
mainScript = opts.mainScript as String ?: DEFAULT_MAIN_FILE_NAME
|
||||
name = opts.name as String
|
||||
nextflowVersion = opts.nextflowVersion as String
|
||||
organization = opts.organization as String
|
||||
recurseSubmodules = opts.recurseSubmodules as boolean
|
||||
version = opts.version as String
|
||||
}
|
||||
|
||||
private List<Contributor> parseContributors(Object value) {
|
||||
if( !value )
|
||||
return Collections.emptyList()
|
||||
|
||||
try {
|
||||
final contributors = value as List<Map>
|
||||
return contributors.stream()
|
||||
.map(opts -> new Contributor(opts))
|
||||
.toList()
|
||||
}
|
||||
catch( IllegalArgumentException e ){
|
||||
throw new AbortOperationException(e.message)
|
||||
}
|
||||
catch( ClassCastException e ){
|
||||
throw new AbortOperationException("Invalid setting for `manifest.contributors` config option -- should be a list of maps")
|
||||
}
|
||||
}
|
||||
|
||||
Map toMap() {
|
||||
return [
|
||||
author: author,
|
||||
contributors: contributors.stream().map(c -> c.toMap()).toList(),
|
||||
defaultBranch: defaultBranch,
|
||||
description: description,
|
||||
homePage: homePage,
|
||||
gitmodules: gitmodules,
|
||||
mainScript: mainScript,
|
||||
version: version,
|
||||
nextflowVersion: nextflowVersion,
|
||||
doi: doi,
|
||||
docsUrl: docsUrl,
|
||||
icon: icon,
|
||||
organization: organization,
|
||||
license: license,
|
||||
]
|
||||
}
|
||||
|
||||
@EqualsAndHashCode
|
||||
static class Contributor {
|
||||
final String name
|
||||
final String affiliation
|
||||
final String email
|
||||
final String github
|
||||
final List<ContributionType> contribution
|
||||
final String orcid
|
||||
|
||||
Contributor(Map opts) {
|
||||
name = opts.name as String
|
||||
affiliation = opts.affiliation as String
|
||||
email = opts.email as String
|
||||
github = opts.github as String
|
||||
contribution = parseContributionTypes(opts.contribution as List<String>)
|
||||
orcid = opts.orcid as String
|
||||
}
|
||||
|
||||
private List<ContributionType> parseContributionTypes(List<String> values) {
|
||||
if( values == null )
|
||||
return []
|
||||
final result = new LinkedList<ContributionType>()
|
||||
for( final value : values ) {
|
||||
try {
|
||||
result.add(ContributionType.valueOf(value.toUpperCase()))
|
||||
}
|
||||
catch( IllegalArgumentException e ) {
|
||||
throw new IllegalArgumentException("Invalid contribution type '$value' in `manifest.contributors` config option")
|
||||
}
|
||||
}
|
||||
return result.toSorted()
|
||||
}
|
||||
|
||||
Map toMap() {
|
||||
return [
|
||||
name: name,
|
||||
affiliation: affiliation,
|
||||
email: email,
|
||||
github: github,
|
||||
contribution: contribution.stream().map(c -> c.toString().toLowerCase()).toList(),
|
||||
orcid: orcid,
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
static enum ContributionType {
|
||||
AUTHOR,
|
||||
MAINTAINER,
|
||||
CONTRIBUTOR
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.config
|
||||
|
||||
import java.lang.annotation.ElementType
|
||||
import java.lang.annotation.Retention
|
||||
import java.lang.annotation.RetentionPolicy
|
||||
import java.lang.annotation.Target
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.config.control.StripSecretsVisitor
|
||||
import org.codehaus.groovy.ast.ASTNode
|
||||
import org.codehaus.groovy.ast.ClassNode
|
||||
import org.codehaus.groovy.control.CompilePhase
|
||||
import org.codehaus.groovy.control.SourceUnit
|
||||
import org.codehaus.groovy.transform.ASTTransformation
|
||||
import org.codehaus.groovy.transform.GroovyASTTransformation
|
||||
import org.codehaus.groovy.transform.GroovyASTTransformationClass
|
||||
|
||||
/**
|
||||
* AST transformation that replaces properties prefixed with `secrets.`
|
||||
* with a static string
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Retention(RetentionPolicy.SOURCE)
|
||||
@Target(ElementType.METHOD)
|
||||
@GroovyASTTransformationClass(classes = [StripSecretsXformImpl])
|
||||
@interface StripSecretsXform {
|
||||
|
||||
@CompileStatic
|
||||
@GroovyASTTransformation(phase = CompilePhase.CONVERSION)
|
||||
class StripSecretsXformImpl implements ASTTransformation {
|
||||
|
||||
@Override
|
||||
void visit(ASTNode[] nodes, SourceUnit source) {
|
||||
new StripSecretsVisitor(source).visitClass((ClassNode)nodes[1])
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package nextflow.config
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.config.spec.ConfigOption
|
||||
import nextflow.config.spec.ConfigScope
|
||||
import nextflow.config.spec.ScopeName
|
||||
import nextflow.script.dsl.Description
|
||||
|
||||
@ScopeName("workflow")
|
||||
@Description("""
|
||||
The `workflow` scope provides workflow execution options.
|
||||
""")
|
||||
@CompileStatic
|
||||
class WorkflowConfig implements ConfigScope {
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
When `true`, the pipeline will exit with a non-zero exit code if any failed tasks are ignored using the `ignore` error strategy (default: `false`).
|
||||
""")
|
||||
final boolean failOnIgnore
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Specify a closure that will be invoked at the end of a workflow run (including failed runs).
|
||||
""")
|
||||
final Closure onComplete
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Specify a closure that will be invoked if a workflow run is terminated.
|
||||
""")
|
||||
final Closure onError
|
||||
|
||||
@Description("""
|
||||
The `workflow.output` scope provides options for publishing workflow outputs.
|
||||
|
||||
[Read more](https://nextflow.io/docs/latest/reference/config.html#workflow)
|
||||
""")
|
||||
final WorkflowOutputConfig output
|
||||
|
||||
/* required by extension point -- do not remove */
|
||||
WorkflowConfig() {}
|
||||
|
||||
WorkflowConfig(Map opts) {
|
||||
failOnIgnore = opts.failOnIgnore as boolean
|
||||
onComplete = opts.onComplete as Closure
|
||||
onError = opts.onError as Closure
|
||||
output = new WorkflowOutputConfig(opts.output as Map ?: Collections.emptyMap())
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@CompileStatic
|
||||
class WorkflowOutputConfig implements ConfigScope {
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
*Currently only supported for S3.*
|
||||
|
||||
Specify the media type, also known as [MIME type](https://developer.mozilla.org/en-US/docs/Web/HTTP/MIME_types), of published files (default: `false`). Can be a string (e.g. `'text/html'`), or `true` to infer the content type from the file extension.
|
||||
""")
|
||||
final Object contentType
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
*Currently only supported for local and shared filesystems.*
|
||||
|
||||
Copy file attributes (such as the last modified timestamp) to the published file (default: `false`).
|
||||
""")
|
||||
final boolean copyAttributes
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Enable or disable publishing (default: `true`).
|
||||
""")
|
||||
final boolean enabled
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
When `true`, the workflow will not fail if a file can't be published for some reason (default: `false`).
|
||||
""")
|
||||
final boolean ignoreErrors
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The file publishing method (default: `'symlink'`).
|
||||
""")
|
||||
final String mode
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
When `true` any existing file in the specified folder will be overwritten (default: `'standard'`).
|
||||
""")
|
||||
final Object overwrite
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
*Currently only supported for S3.*
|
||||
|
||||
Specify the storage class for published files.
|
||||
""")
|
||||
final String storageClass
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
*Currently only supported for S3.*
|
||||
|
||||
Specify arbitrary tags for published files.
|
||||
""")
|
||||
final Map tags
|
||||
|
||||
/* required by extension point -- do not remove */
|
||||
WorkflowOutputConfig() {}
|
||||
|
||||
WorkflowOutputConfig(Map opts) {
|
||||
contentType = opts.contentType
|
||||
copyAttributes = opts.copyAttributes as boolean
|
||||
enabled = opts.enabled as boolean
|
||||
ignoreErrors = opts.ignoreErrors as boolean
|
||||
mode = opts.mode
|
||||
overwrite = opts.overwrite
|
||||
storageClass = opts.storageClass
|
||||
tags = opts.tags as Map
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.config.parser.v1
|
||||
|
||||
import java.nio.file.NoSuchFileException
|
||||
import java.nio.file.Path
|
||||
|
||||
import ch.artecat.grengine.Grengine
|
||||
import groovy.transform.Memoized
|
||||
import nextflow.SysEnv
|
||||
import nextflow.config.StripSecretsXform
|
||||
import nextflow.exception.IllegalConfigException
|
||||
import nextflow.file.FileHelper
|
||||
import nextflow.util.Duration
|
||||
import nextflow.util.MemoryUnit
|
||||
import org.codehaus.groovy.control.CompilerConfiguration
|
||||
import org.codehaus.groovy.control.customizers.ASTTransformationCustomizer
|
||||
import org.codehaus.groovy.control.customizers.ImportCustomizer
|
||||
import org.slf4j.Logger
|
||||
import org.slf4j.LoggerFactory
|
||||
/**
|
||||
* Extends a {@link Script} class adding a method that allows a configuration
|
||||
* file to include other configuration files.
|
||||
* <p>
|
||||
* This class is used to base class when parsing the groovy config object
|
||||
* <p>
|
||||
*
|
||||
* Based on
|
||||
* http://naleid.com/blog/2009/07/30/modularizing-groovy-config-files-with-a-dash-of-meta-programming
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
abstract class ConfigBase extends Script {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(ConfigBase)
|
||||
|
||||
private Stack<Path> configStack
|
||||
|
||||
private boolean ignoreIncludes
|
||||
|
||||
private boolean renderClosureAsString
|
||||
|
||||
private boolean stripSecrets
|
||||
|
||||
protected void setStripSecrets( boolean value ) {
|
||||
this.stripSecrets = value
|
||||
}
|
||||
|
||||
protected void setIgnoreIncludes( boolean value ) {
|
||||
this.ignoreIncludes = value
|
||||
}
|
||||
|
||||
protected void setRenderClosureAsString( boolean value ) {
|
||||
this.renderClosureAsString = value
|
||||
}
|
||||
|
||||
protected void setConfigPath(Path path) {
|
||||
if( configStack == null )
|
||||
configStack = new Stack<>()
|
||||
configStack.push(path)
|
||||
}
|
||||
|
||||
protected void setConfigStack(Stack<Path> stack) {
|
||||
this.configStack = stack
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the value of an environment variable from the launch environment.
|
||||
*
|
||||
* @param name
|
||||
* The environment variable name to be referenced
|
||||
* @return
|
||||
* The value associate with the specified variable name or {@code null} if the variable does not exist.
|
||||
*/
|
||||
String env(String name) {
|
||||
return SysEnv.get(name)
|
||||
}
|
||||
|
||||
/**
|
||||
* Implements the config file include
|
||||
*/
|
||||
def includeConfig( includeFile ) {
|
||||
if( !includeFile )
|
||||
throw new IllegalConfigException("includeConfig argument cannot be empty")
|
||||
|
||||
if( ignoreIncludes )
|
||||
return
|
||||
|
||||
if( configStack == null )
|
||||
configStack = new Stack<>()
|
||||
|
||||
Path owner = configStack ? this.configStack.peek() : null
|
||||
Path includePath = FileHelper.asPath(includeFile.toString())
|
||||
log.trace "Include config file: $includeFile [parent: $owner]"
|
||||
|
||||
if( !includePath.isAbsolute() && owner ) {
|
||||
includePath = owner.resolveSibling(includeFile.toString())
|
||||
}
|
||||
|
||||
def configText = readConfigFile(includePath)
|
||||
|
||||
// -- set the required base script
|
||||
def config = new CompilerConfiguration()
|
||||
config.scriptBaseClass = ConfigBase.class.name
|
||||
if( stripSecrets )
|
||||
config.addCompilationCustomizers(new ASTTransformationCustomizer(StripSecretsXform))
|
||||
def params = [:]
|
||||
if( renderClosureAsString )
|
||||
params.put('renderClosureAsString', true)
|
||||
config.addCompilationCustomizers(new ASTTransformationCustomizer(params, ConfigTransform))
|
||||
// -- add implicit types
|
||||
def importCustomizer = new ImportCustomizer()
|
||||
importCustomizer.addImports( Duration.name )
|
||||
importCustomizer.addImports( MemoryUnit.name )
|
||||
config.addCompilationCustomizers(importCustomizer)
|
||||
|
||||
// -- setup the grengine instance
|
||||
def engine = new Grengine(this.class.classLoader,config)
|
||||
def clazz = engine.load(configText)
|
||||
|
||||
// -- push this file on the stack
|
||||
this.configStack.push(includePath)
|
||||
|
||||
/*
|
||||
* here it is the magic code
|
||||
*/
|
||||
def script = (ConfigBase)clazz.newInstance()
|
||||
script.setConfigStack(this.configStack)
|
||||
script.setBinding(this.getBinding())
|
||||
script.metaClass.getProperty = { String name -> this.metaClass.getProperty(this, name) }
|
||||
script.metaClass.invokeMethod = { String name, args -> this.metaClass.invokeMethod(this, name, args)}
|
||||
script.&run.call()
|
||||
|
||||
// remove the path from the stack
|
||||
this.configStack.pop()
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads the content of a config file. The result is cached to
|
||||
* avoid multiple reads.
|
||||
*
|
||||
* @param includePath The config file path
|
||||
* @return The file content
|
||||
*/
|
||||
@Memoized
|
||||
protected static String readConfigFile(Path includePath) {
|
||||
def configText
|
||||
try {
|
||||
configText = includePath.getText()
|
||||
}
|
||||
catch (NoSuchFileException | FileNotFoundException ignored) {
|
||||
throw new NoSuchFileException("Config file does not exist: ${includePath.toUriString()}")
|
||||
}
|
||||
catch (IOException e) {
|
||||
throw new IOException("Cannot read config file include: ${includePath.toUriString()}", e)
|
||||
}
|
||||
return configText
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,491 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.config.parser.v1
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
import ch.artecat.grengine.Grengine
|
||||
import com.google.common.hash.Hashing
|
||||
import groovy.transform.PackageScope
|
||||
import nextflow.ast.NextflowXform
|
||||
import nextflow.config.ConfigParser
|
||||
import nextflow.config.StripSecretsXform
|
||||
import nextflow.exception.ConfigParseException
|
||||
import nextflow.extension.Bolts
|
||||
import nextflow.file.FileHelper
|
||||
import nextflow.util.Duration
|
||||
import nextflow.util.MemoryUnit
|
||||
import org.codehaus.groovy.control.CompilerConfiguration
|
||||
import org.codehaus.groovy.control.customizers.ASTTransformationCustomizer
|
||||
import org.codehaus.groovy.control.customizers.ImportCustomizer
|
||||
import org.codehaus.groovy.runtime.InvokerHelper
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* A ConfigSlurper that allows to include a config file into another. For example:
|
||||
*
|
||||
* <pre>
|
||||
* process {
|
||||
* foo = 1
|
||||
* that = 2
|
||||
*
|
||||
* includeConfig( 'path/to/another/config/file' )
|
||||
*
|
||||
* }
|
||||
*
|
||||
* </pre>
|
||||
*
|
||||
* See http://naleid.com/blog/2009/07/30/modularizing-groovy-config-files-with-a-dash-of-meta-programming
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*
|
||||
*/
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* ConfigSlurper is a utility class for reading configuration files defined in the form of Groovy
|
||||
* scripts. Configuration settings can be defined using dot notation or scoped using closures
|
||||
*
|
||||
* <pre><code>
|
||||
* grails.webflow.stateless = true
|
||||
* smtp {
|
||||
* mail.host = 'smtp.myisp.com'
|
||||
* mail.auth.user = 'server'
|
||||
* }
|
||||
* resources.URL = "http://localhost:80/resources"
|
||||
* </pre></code>
|
||||
*
|
||||
* <p>Settings can either be bound into nested maps or onto a specified JavaBean instance. In the case
|
||||
* of the latter an error will be thrown if a property cannot be bound.
|
||||
*
|
||||
* @author Graeme Rocher
|
||||
* @author Andres Almiray
|
||||
* @since 1.5
|
||||
*/
|
||||
class ConfigParserV1 implements ConfigParser {
|
||||
private Map bindingVars = [:]
|
||||
private Map paramVars = [:]
|
||||
|
||||
private final Map<String, List<String>> conditionValues = [:]
|
||||
private final Stack<Map<String, ConfigObject>> conditionalBlocks = new Stack<Map<String,ConfigObject>>()
|
||||
private final Set<String> conditionalNames = new HashSet<>()
|
||||
private final Set<String> profileNames = new HashSet<>()
|
||||
|
||||
private boolean ignoreIncludes
|
||||
|
||||
private boolean renderClosureAsString
|
||||
|
||||
private boolean stripSecrets
|
||||
|
||||
private Grengine grengine
|
||||
|
||||
@Override
|
||||
ConfigParser setProfiles(List<String> profiles) {
|
||||
final blockName = 'profiles'
|
||||
if (!profiles) {
|
||||
conditionValues.remove(blockName)
|
||||
}
|
||||
else {
|
||||
conditionValues[blockName] = profiles
|
||||
}
|
||||
return this
|
||||
}
|
||||
|
||||
@Override
|
||||
Set<String> getDeclaredProfiles() {
|
||||
Collections.unmodifiableSet(conditionalNames)
|
||||
}
|
||||
|
||||
@Override
|
||||
Map<String,Object> getDeclaredParams() {
|
||||
[:]
|
||||
}
|
||||
|
||||
private Grengine getGrengine() {
|
||||
if( grengine ) {
|
||||
return grengine
|
||||
}
|
||||
|
||||
// set the required base script
|
||||
def config = new CompilerConfiguration()
|
||||
config.scriptBaseClass = ConfigBase.class.name
|
||||
if( stripSecrets )
|
||||
config.addCompilationCustomizers(new ASTTransformationCustomizer(StripSecretsXform))
|
||||
def params = [:]
|
||||
if( renderClosureAsString )
|
||||
params.put('renderClosureAsString', true)
|
||||
config.addCompilationCustomizers(new ASTTransformationCustomizer(params, ConfigTransform))
|
||||
config.addCompilationCustomizers(new ASTTransformationCustomizer(NextflowXform))
|
||||
// add implicit types
|
||||
def importCustomizer = new ImportCustomizer()
|
||||
importCustomizer.addImports( Duration.name )
|
||||
importCustomizer.addImports( MemoryUnit.name )
|
||||
config.addCompilationCustomizers(importCustomizer)
|
||||
grengine = new Grengine(config)
|
||||
}
|
||||
|
||||
@Override
|
||||
ConfigParser setRenderClosureAsString(boolean value) {
|
||||
this.renderClosureAsString = value
|
||||
return this
|
||||
}
|
||||
|
||||
@Override
|
||||
ConfigParser setStrict(boolean value) {
|
||||
// not supported
|
||||
return this
|
||||
}
|
||||
|
||||
@Override
|
||||
ConfigParser setStripSecrets(boolean value) {
|
||||
this.stripSecrets = value
|
||||
return this
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets any additional variables that should be placed into the binding when evaluating Config scripts
|
||||
*/
|
||||
@Override
|
||||
ConfigParser setBinding(Map vars) {
|
||||
this.bindingVars = vars
|
||||
return this
|
||||
}
|
||||
|
||||
@Override
|
||||
ConfigParser setParams(Map vars) {
|
||||
// deep clone the map to prevent side-effect
|
||||
// see https://github.com/nextflow-io/nextflow/issues/1923
|
||||
this.paramVars = Bolts.deepClone(vars)
|
||||
return this
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Creates a unique name for the config class in order to avoid collision
|
||||
* with top level configuration scopes
|
||||
*
|
||||
* @param text
|
||||
* @return
|
||||
*/
|
||||
private String createUniqueName(String text) {
|
||||
def hash = Hashing
|
||||
.murmur3_32()
|
||||
.newHasher()
|
||||
.putUnencodedChars(text)
|
||||
.hash()
|
||||
return "_nf_config_$hash"
|
||||
}
|
||||
|
||||
private Script loadScript(String text) {
|
||||
(Script)getGrengine().load(text, createUniqueName(text)).newInstance()
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a ConfigObject instances from an instance of java.util.Properties
|
||||
* @param The java.util.Properties instance
|
||||
*/
|
||||
ConfigObject parse(Properties properties) {
|
||||
ConfigObject config = new ConfigObject()
|
||||
for (key in properties.keySet()) {
|
||||
def tokens = key.split(/\./)
|
||||
|
||||
def current = config
|
||||
def last
|
||||
def lastToken
|
||||
def foundBase = false
|
||||
for (token in tokens) {
|
||||
if (foundBase) {
|
||||
// handle not properly nested tokens by ignoring
|
||||
// hierarchy below this point
|
||||
lastToken += "." + token
|
||||
current = last
|
||||
} else {
|
||||
last = current
|
||||
lastToken = token
|
||||
current = current."${token}"
|
||||
if (!(current instanceof ConfigObject)) foundBase = true
|
||||
}
|
||||
}
|
||||
|
||||
if (current instanceof ConfigObject) {
|
||||
if (last[lastToken]) {
|
||||
def flattened = last.flatten()
|
||||
last.clear()
|
||||
flattened.each { k2, v2 -> last[k2] = v2 }
|
||||
last[lastToken] = properties.get(key)
|
||||
}
|
||||
else {
|
||||
last[lastToken] = properties.get(key)
|
||||
}
|
||||
}
|
||||
current = config
|
||||
}
|
||||
return config
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the given script as a string and return the configuration object
|
||||
*
|
||||
* @see ConfigParser#parse(groovy.lang.Script)
|
||||
*/
|
||||
@Override
|
||||
ConfigObject parse(String text) {
|
||||
return parse(loadScript(text))
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the given script into a configuration object (a Map)
|
||||
* (This method creates a new class to parse the script each time it is called.)
|
||||
* @param script The script to parse
|
||||
* @return A Map of maps that can be navigating with dot de-referencing syntax to obtain configuration entries
|
||||
*/
|
||||
@Deprecated
|
||||
ConfigObject parse(Script script) {
|
||||
return parse(script, null)
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a Script represented by the given URL into a ConfigObject
|
||||
*
|
||||
* @param location The location of the script to parse
|
||||
* @return The ConfigObject instance
|
||||
*/
|
||||
@Deprecated
|
||||
ConfigObject parse(URL location) {
|
||||
return parse(loadScript(location.text), FileHelper.asPath(location.toURI()))
|
||||
}
|
||||
|
||||
@Override
|
||||
ConfigObject parse(File file) {
|
||||
return parse(file.toPath())
|
||||
}
|
||||
|
||||
@Override
|
||||
ConfigObject parse(Path path) {
|
||||
return parse(loadScript(path.text), path)
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the passed groovy.lang.Script instance using the second argument to allow the ConfigObject
|
||||
* to retain an reference to the original location other Groovy script
|
||||
*
|
||||
* @param script The groovy.lang.Script instance
|
||||
* @param location The original location of the Script as a URL
|
||||
* @return The ConfigObject instance
|
||||
*/
|
||||
ConfigObject parse(Script _script, Path location) {
|
||||
final script = (ConfigBase)_script
|
||||
Stack<String> currentConditionalBlock = new Stack<String>()
|
||||
def config = location ? new ConfigObject(location.toUri().toURL()) : new ConfigObject()
|
||||
GroovySystem.metaClassRegistry.removeMetaClass(script.class)
|
||||
def mc = script.class.metaClass
|
||||
def prefix = ""
|
||||
LinkedList stack = new LinkedList()
|
||||
LinkedList profileStack = new LinkedList()
|
||||
stack << [config: config, scope: [:]]
|
||||
boolean withinProfile = false
|
||||
|
||||
def pushStack = { co ->
|
||||
stack << [config: co, scope: stack.last.scope.clone()]
|
||||
}
|
||||
def assignName = { name, co ->
|
||||
def current = stack.last
|
||||
current.config[name] = co
|
||||
current.scope[name] = co
|
||||
}
|
||||
mc.getProperty = { String name ->
|
||||
def current = stack.last
|
||||
def result
|
||||
if (current.config.get(name)) {
|
||||
result = current.config.get(name)
|
||||
} else if (current.scope.get(name)) {
|
||||
result = current.scope[name]
|
||||
} else {
|
||||
try {
|
||||
result = InvokerHelper.getProperty(this, name)
|
||||
} catch (GroovyRuntimeException e) {
|
||||
result = new ConfigObject()
|
||||
assignName.call(name, result)
|
||||
}
|
||||
}
|
||||
if( name=='params' && result instanceof Map && paramVars ) {
|
||||
result.putAll(Bolts.deepMerge(result, paramVars))
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
ConfigObject overrides = new ConfigObject()
|
||||
mc.invokeMethod = { String name, args ->
|
||||
def result
|
||||
if (args.length == 1 && args[0] instanceof Closure) {
|
||||
if( profileStack && profileStack.last == 'profiles' )
|
||||
profileNames.add(name)
|
||||
|
||||
if (name in conditionValues.keySet()) {
|
||||
try {
|
||||
if( name == 'profiles' ){
|
||||
withinProfile=true
|
||||
}
|
||||
currentConditionalBlock.push(name)
|
||||
conditionalBlocks.push([:])
|
||||
args[0].call()
|
||||
} finally {
|
||||
currentConditionalBlock.pop()
|
||||
for (entry in conditionalBlocks.pop().entrySet()) {
|
||||
def c = stack.last.config
|
||||
(c != config? c : overrides).merge(entry.value)
|
||||
}
|
||||
if( name == 'profiles' ){
|
||||
withinProfile=false
|
||||
}
|
||||
}
|
||||
} else if (currentConditionalBlock.size() > 0) {
|
||||
String conditionalBlockKey = currentConditionalBlock.peek()
|
||||
conditionalNames.add(name)
|
||||
if (name in conditionValues[conditionalBlockKey]) {
|
||||
def co = conditionalBlocks.peek()[conditionalBlockKey]
|
||||
if( co == null ) {
|
||||
co = new ConfigObject()
|
||||
conditionalBlocks.peek()[conditionalBlockKey] = co
|
||||
}
|
||||
|
||||
pushStack.call(co)
|
||||
try {
|
||||
currentConditionalBlock.pop()
|
||||
args[0].call()
|
||||
} finally {
|
||||
currentConditionalBlock.push(conditionalBlockKey)
|
||||
}
|
||||
stack.removeLast()
|
||||
}
|
||||
}
|
||||
else if( name == 'plugins' ) {
|
||||
if( stack.size()>1 )
|
||||
throw new ConfigParseException("Plugins definition is only allowed in config top-most scope")
|
||||
// Implements `plugins` mini-dsl for plugins definition
|
||||
def dsl = new PluginsDsl()
|
||||
def clo = args[0] as Closure
|
||||
clo.delegate = dsl
|
||||
clo.resolveStrategy = Closure.DELEGATE_ONLY
|
||||
clo.call()
|
||||
assignName.call(name, dsl.plugins)
|
||||
}
|
||||
else {
|
||||
def current = name=='profiles' || withinProfile ? stack.first : stack.last
|
||||
def co
|
||||
if (current.config.containsKey(name) && current.config.get(name) instanceof ConfigObject) {
|
||||
co = current.config.get(name)
|
||||
}
|
||||
else if (current.scope.containsKey(name) && current.scope.get(name) instanceof ConfigObject) {
|
||||
co = current.scope.get(name).clone()
|
||||
}
|
||||
else {
|
||||
co = new ConfigObject()
|
||||
}
|
||||
|
||||
profileStack.add(name)
|
||||
assignName.call(name, co)
|
||||
pushStack.call(co)
|
||||
args[0].call()
|
||||
stack.removeLast()
|
||||
profileStack.removeLast()
|
||||
|
||||
if (current.scope.containsKey(name) && current.scope.get(name) instanceof ConfigObject) {
|
||||
if( current.scope.get(name) != co) {
|
||||
current.scope.get(name).merge(co)
|
||||
}
|
||||
} else {
|
||||
current.scope.put(name,co)
|
||||
}
|
||||
}
|
||||
} else if (args.length == 2 && args[1] instanceof Closure) {
|
||||
try {
|
||||
prefix = name + '.'
|
||||
assignName.call(name, args[0])
|
||||
args[1].call()
|
||||
} finally { prefix = "" }
|
||||
} else {
|
||||
MetaMethod mm = mc.getMetaMethod(name, args)
|
||||
if (mm) {
|
||||
result = mm.invoke(delegate, args)
|
||||
} else {
|
||||
throw new MissingMethodException(name, getClass(), args)
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
script.metaClass = mc
|
||||
|
||||
def setProperty = { String name, value ->
|
||||
assignName.call(prefix + name, value)
|
||||
}
|
||||
def binding = new ConfigBinding(setProperty)
|
||||
if (this.bindingVars) {
|
||||
binding.getVariables().putAll(this.bindingVars)
|
||||
}
|
||||
|
||||
// add the script file location into the binding
|
||||
if( location ) {
|
||||
script.setConfigPath(location)
|
||||
}
|
||||
|
||||
// disable include parsing when required
|
||||
script.setIgnoreIncludes(ignoreIncludes)
|
||||
script.setRenderClosureAsString(renderClosureAsString)
|
||||
script.setStripSecrets(stripSecrets)
|
||||
|
||||
// -- set the binding and run
|
||||
script.binding = binding
|
||||
script.run()
|
||||
config.merge(overrides)
|
||||
|
||||
return config
|
||||
}
|
||||
|
||||
/**
|
||||
* Disable parsing of {@code includeConfig} directive
|
||||
*
|
||||
* @param value A boolean value, when {@code true} includes are disabled
|
||||
* @return The {@link ConfigParser} object itself
|
||||
*/
|
||||
@Override
|
||||
ConfigParser setIgnoreIncludes(boolean value) {
|
||||
this.ignoreIncludes = value
|
||||
return this
|
||||
}
|
||||
|
||||
/**
|
||||
* Since Groovy Script doesn't support overriding setProperty, we have to using a trick with the Binding to provide this
|
||||
* functionality
|
||||
*/
|
||||
@PackageScope
|
||||
static class ConfigBinding extends Binding {
|
||||
Closure callable
|
||||
|
||||
ConfigBinding(Closure c) {
|
||||
this.callable = c
|
||||
}
|
||||
|
||||
void setVariable(String name, Object value) {
|
||||
callable(name, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.config.parser.v1
|
||||
|
||||
import java.lang.annotation.ElementType
|
||||
import java.lang.annotation.Retention
|
||||
import java.lang.annotation.RetentionPolicy
|
||||
import java.lang.annotation.Target
|
||||
|
||||
import org.codehaus.groovy.transform.GroovyASTTransformationClass
|
||||
/**
|
||||
* Nextflow configuration file AST xform marker interface
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Retention(RetentionPolicy.SOURCE)
|
||||
@Target(ElementType.METHOD)
|
||||
@GroovyASTTransformationClass(classes = [ConfigTransformImpl])
|
||||
@interface ConfigTransform {
|
||||
/**
|
||||
* hack to pass a parameter in the {@link ConfigTransformImpl} class -- do not remove
|
||||
*
|
||||
* See {@link ConfigTransformImpl#renderClosureAsString}
|
||||
*/
|
||||
boolean renderClosureAsString()
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.config.parser.v1
|
||||
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.config.ConfigClosurePlaceholder
|
||||
import org.codehaus.groovy.ast.ASTNode
|
||||
import org.codehaus.groovy.ast.AnnotationNode
|
||||
import org.codehaus.groovy.ast.ClassCodeVisitorSupport
|
||||
import org.codehaus.groovy.ast.ClassNode
|
||||
import org.codehaus.groovy.ast.expr.ArgumentListExpression
|
||||
import org.codehaus.groovy.ast.expr.BinaryExpression
|
||||
import org.codehaus.groovy.ast.expr.ClosureExpression
|
||||
import org.codehaus.groovy.ast.expr.ConstantExpression
|
||||
import org.codehaus.groovy.ast.expr.ConstructorCallExpression
|
||||
import org.codehaus.groovy.ast.expr.Expression
|
||||
import org.codehaus.groovy.ast.expr.MapEntryExpression
|
||||
import org.codehaus.groovy.ast.expr.MapExpression
|
||||
import org.codehaus.groovy.ast.expr.MethodCallExpression
|
||||
import org.codehaus.groovy.ast.stmt.ExpressionStatement
|
||||
import org.codehaus.groovy.control.CompilePhase
|
||||
import org.codehaus.groovy.control.SourceUnit
|
||||
import org.codehaus.groovy.transform.ASTTransformation
|
||||
import org.codehaus.groovy.transform.GroovyASTTransformation
|
||||
/**
|
||||
* Implements Nextflow configuration file AST xform
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
@GroovyASTTransformation(phase = CompilePhase.CONVERSION)
|
||||
class ConfigTransformImpl implements ASTTransformation {
|
||||
|
||||
private boolean renderClosureAsString
|
||||
|
||||
@Override
|
||||
void visit(ASTNode[] astNodes, SourceUnit unit) {
|
||||
final annot = (AnnotationNode)astNodes[0]
|
||||
final clazz = (ClassNode)astNodes[1]
|
||||
// the following line is mostly an hack to pass a parameter to this xform instance
|
||||
this.renderClosureAsString = annot.getMember('renderClosureAsString') != null
|
||||
createVisitor(unit).visitClass(clazz)
|
||||
}
|
||||
|
||||
protected ClassCodeVisitorSupport createVisitor(SourceUnit unit) {
|
||||
return (renderClosureAsString
|
||||
? new RetainClosureSourceCodeVisitorSupport(unit: unit)
|
||||
: new DefaultConfigCodeVisitor(unit: unit) )
|
||||
}
|
||||
|
||||
/**
|
||||
* Nextflow config file visitor support class. Apply default transformations to
|
||||
* the config object to implements config syntax sugars
|
||||
*/
|
||||
@CompileStatic
|
||||
static class DefaultConfigCodeVisitor extends ClassCodeVisitorSupport {
|
||||
|
||||
protected SourceUnit unit
|
||||
|
||||
@Override
|
||||
protected SourceUnit getSourceUnit() { unit }
|
||||
|
||||
@Override
|
||||
void visitExpressionStatement(ExpressionStatement stm) {
|
||||
if( stm.expression instanceof MethodCallExpression && stm.getStatementLabel() == 'withLabel' ) {
|
||||
replaceMethodName( stm.expression as MethodCallExpression, 'withLabel' )
|
||||
}
|
||||
else if( stm.expression instanceof MethodCallExpression && stm.getStatementLabel() == 'withName' ) {
|
||||
replaceMethodName( stm.expression as MethodCallExpression, 'withName' )
|
||||
}
|
||||
super.visitExpressionStatement(stm)
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace the name of the invoked method pre-pending with the specified string
|
||||
*
|
||||
* @param call A object representing a method call expression
|
||||
* @param prefix A string to prepend to the name of the invoked method
|
||||
*/
|
||||
protected void replaceMethodName(MethodCallExpression call, String prefix) {
|
||||
call.setMethod( new ConstantExpression(prefix + ":" + call.method.text) )
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* This visitor is only used to render the closure source code
|
||||
* when is required to visualise the nextflow config content
|
||||
*/
|
||||
@CompileStatic
|
||||
static class RetainClosureSourceCodeVisitorSupport extends DefaultConfigCodeVisitor {
|
||||
|
||||
/**
|
||||
* Visit expression statements replacing a binary assignment such as:
|
||||
*
|
||||
* foo = { closure }
|
||||
*
|
||||
* with an equivalent assignment replacing the closure with placeholder
|
||||
* class holding the closure source code
|
||||
*
|
||||
* @param stm
|
||||
*
|
||||
* @see ConfigClosurePlaceholder
|
||||
*/
|
||||
@Override
|
||||
void visitExpressionStatement(ExpressionStatement stm) {
|
||||
if( stm.expression instanceof BinaryExpression ) {
|
||||
replaceClosureAssignment(stm, stm.expression as BinaryExpression)
|
||||
}
|
||||
super.visitExpressionStatement(stm)
|
||||
}
|
||||
|
||||
protected void replaceClosureAssignment(ExpressionStatement stm, BinaryExpression expr ) {
|
||||
if( expr.operation.text == '=' && expr.rightExpression instanceof ClosureExpression ) {
|
||||
final value = closureToString(expr.rightExpression)
|
||||
final replace = new BinaryExpression(expr.leftExpression, expr.getOperation(), value)
|
||||
stm.setExpression(replace)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Visit Map expressions replacing closure values such as :
|
||||
*
|
||||
* [key: { closure }]
|
||||
*
|
||||
* with an equivalent Map in which the closure is replaced with a placeholder
|
||||
* class holding the closure source code
|
||||
*
|
||||
* @param expr
|
||||
*
|
||||
* @see ConfigClosurePlaceholder
|
||||
*/
|
||||
@Override
|
||||
void visitMapExpression(MapExpression expr) {
|
||||
for( int i=0; i<expr.mapEntryExpressions.size(); i++ ) {
|
||||
def entry = expr.mapEntryExpressions[i]
|
||||
if( entry.valueExpression instanceof ClosureExpression ) {
|
||||
expr.mapEntryExpressions[i] = replaceMapEntryAssignment(entry)
|
||||
}
|
||||
}
|
||||
super.visitMapExpression(expr)
|
||||
}
|
||||
|
||||
protected MapEntryExpression replaceMapEntryAssignment(MapEntryExpression expr) {
|
||||
if( expr.valueExpression instanceof ClosureExpression ) {
|
||||
final value = closureToString(expr.valueExpression)
|
||||
new MapEntryExpression(expr.keyExpression, value)
|
||||
}
|
||||
else
|
||||
return expr
|
||||
}
|
||||
|
||||
protected Expression closureToString( Expression closure ) {
|
||||
def buffer = new StringBuilder()
|
||||
readSource(closure, buffer)
|
||||
def str = new ConstantExpression(buffer.toString())
|
||||
|
||||
def type = new ClassNode(ConfigClosurePlaceholder)
|
||||
def args = new ArgumentListExpression( [str] as List<Expression> )
|
||||
return new ConstructorCallExpression(type,args)
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the user provided script source string
|
||||
*
|
||||
* @param expr
|
||||
* @param buffer
|
||||
* @param unit
|
||||
*/
|
||||
protected void readSource(Expression expr, StringBuilder buffer) {
|
||||
final colBegin = Math.max(expr.getColumnNumber()-1, 0)
|
||||
final colEnd = Math.max(expr.getLastColumnNumber()-1, 0)
|
||||
final lineFirst = expr.getLineNumber()
|
||||
final lineLast = expr.getLastLineNumber()
|
||||
|
||||
for( int i=lineFirst; i<=lineLast; i++ ) {
|
||||
def line = unit.source.getLine(i, null)
|
||||
if( i==lineFirst ) {
|
||||
def str = i==lineLast ? line.substring(colBegin,colEnd) : line.substring(colBegin)
|
||||
buffer.append(str)
|
||||
}
|
||||
else {
|
||||
def str = i==lineLast ? line.substring(0, colEnd) : line
|
||||
buffer.append('\n')
|
||||
buffer.append(str)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package nextflow.config.parser.v1
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
|
||||
/**
|
||||
* Model a mini-dsl for plugins configuration
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class PluginsDsl {
|
||||
|
||||
private Set<String> plugins = []
|
||||
|
||||
Set<String> getPlugins() { plugins }
|
||||
|
||||
void id( String plg ) {
|
||||
if( !plg )
|
||||
throw new IllegalArgumentException("Plugin id cannot be empty or null")
|
||||
plugins << plg
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.config.parser.v2;
|
||||
|
||||
import nextflow.config.ConfigClosurePlaceholder;
|
||||
import org.codehaus.groovy.ast.ClassCodeVisitorSupport;
|
||||
import org.codehaus.groovy.ast.ClassNode;
|
||||
import org.codehaus.groovy.ast.expr.ArgumentListExpression;
|
||||
import org.codehaus.groovy.ast.expr.ClosureExpression;
|
||||
import org.codehaus.groovy.ast.expr.Expression;
|
||||
import org.codehaus.groovy.ast.expr.MapExpression;
|
||||
import org.codehaus.groovy.ast.expr.MethodCallExpression;
|
||||
import org.codehaus.groovy.control.SourceUnit;
|
||||
|
||||
import static org.codehaus.groovy.ast.tools.GeneralUtils.*;
|
||||
/**
|
||||
* AST transformation to render closure source text
|
||||
*
|
||||
* @author Ben Sherman <bentshermann@gmail.com>
|
||||
*/
|
||||
class ClosureToStringVisitor extends ClassCodeVisitorSupport {
|
||||
|
||||
protected SourceUnit sourceUnit;
|
||||
|
||||
public ClosureToStringVisitor(SourceUnit sourceUnit) {
|
||||
this.sourceUnit = sourceUnit;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected SourceUnit getSourceUnit() {
|
||||
return sourceUnit;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void visitMethodCallExpression(MethodCallExpression node) {
|
||||
var name = node.getMethodAsString();
|
||||
if( !"assign".equals(name) ) {
|
||||
super.visitMethodCallExpression(node);
|
||||
return;
|
||||
}
|
||||
|
||||
var arguments = (ArgumentListExpression) node.getArguments();
|
||||
if( arguments.getExpressions().size() != 2 )
|
||||
return;
|
||||
|
||||
var secondArg = arguments.getExpression(1);
|
||||
if( secondArg instanceof MapExpression me ) {
|
||||
for( var entry : me.getMapEntryExpressions() ) {
|
||||
if( entry.getValueExpression() instanceof ClosureExpression ce )
|
||||
entry.setValueExpression(wrapExprAsPlaceholder(ce));
|
||||
}
|
||||
}
|
||||
else if( secondArg instanceof ClosureExpression ce ) {
|
||||
node.setArguments(args(arguments.getExpression(0), wrapExprAsPlaceholder(ce)));
|
||||
}
|
||||
}
|
||||
|
||||
protected Expression wrapExprAsPlaceholder(Expression node) {
|
||||
var type = new ClassNode(ConfigClosurePlaceholder.class);
|
||||
var text = constX(getSourceText(node));
|
||||
return ctorX(type, args(text));
|
||||
}
|
||||
|
||||
protected String getSourceText(Expression node) {
|
||||
var builder = new StringBuilder();
|
||||
var colBegin = Math.max(node.getColumnNumber()-1, 0);
|
||||
var colEnd = Math.max(node.getLastColumnNumber()-1, 0);
|
||||
var lineFirst = node.getLineNumber();
|
||||
var lineLast = node.getLastLineNumber();
|
||||
|
||||
for( int i = lineFirst; i <= lineLast; i++ ) {
|
||||
var line = sourceUnit.getSource().getLine(i, null);
|
||||
if( i == lineFirst ) {
|
||||
var str = i == lineLast ? line.substring(colBegin,colEnd) : line.substring(colBegin);
|
||||
builder.append(str);
|
||||
}
|
||||
else {
|
||||
var str = i == lineLast ? line.substring(0, colEnd) : line;
|
||||
builder.append('\n');
|
||||
builder.append(str);
|
||||
}
|
||||
}
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.config.parser.v2;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
|
||||
import com.google.common.hash.Hashing;
|
||||
import groovy.lang.Binding;
|
||||
import groovy.lang.GroovyClassLoader;
|
||||
import groovy.lang.Script;
|
||||
import nextflow.config.control.ConfigResolveVisitor;
|
||||
import nextflow.config.control.ConfigToGroovyVisitor;
|
||||
import nextflow.config.control.ResolveIncludeVisitor;
|
||||
import nextflow.config.control.StringReaderSourceWithURI;
|
||||
import nextflow.config.control.StripSecretsVisitor;
|
||||
import nextflow.config.parser.ConfigParserPluginFactory;
|
||||
import nextflow.script.control.Compiler;
|
||||
import nextflow.script.control.GStringToStringVisitor;
|
||||
import nextflow.script.control.PathCompareVisitor;
|
||||
import org.codehaus.groovy.ast.ASTNode;
|
||||
import org.codehaus.groovy.ast.ClassHelper;
|
||||
import org.codehaus.groovy.ast.ClassNode;
|
||||
import org.codehaus.groovy.control.CompilationUnit;
|
||||
import org.codehaus.groovy.control.CompilerConfiguration;
|
||||
import org.codehaus.groovy.control.Phases;
|
||||
import org.codehaus.groovy.control.SourceUnit;
|
||||
import org.codehaus.groovy.control.customizers.ImportCustomizer;
|
||||
import org.codehaus.groovy.control.io.StringReaderSource;
|
||||
import org.codehaus.groovy.control.messages.SyntaxErrorMessage;
|
||||
import org.codehaus.groovy.runtime.InvokerHelper;
|
||||
|
||||
/**
|
||||
* Compile a Nextflow config file into a Groovy class.
|
||||
*
|
||||
* @see groovy.lang.GroovyShell::parse()
|
||||
* @see groovy.lang.GroovyClassLoader::doParseClass()
|
||||
*
|
||||
* @author Ben Sherman <bentshermann@gmail.com>
|
||||
*/
|
||||
public class ConfigCompiler {
|
||||
|
||||
private static final List<String> DEFAULT_IMPORT_NAMES = List.of(
|
||||
"nextflow.util.Duration",
|
||||
"nextflow.util.MemoryUnit"
|
||||
);
|
||||
private static final String BASE_CLASS_NAME = "nextflow.config.parser.v2.ConfigDsl";
|
||||
|
||||
private final CompilerConfiguration config;
|
||||
|
||||
private final GroovyClassLoader loader;
|
||||
|
||||
private boolean renderClosureAsString;
|
||||
|
||||
private boolean stripSecrets;
|
||||
|
||||
private Compiler compiler;
|
||||
|
||||
private SourceUnit sourceUnit;
|
||||
|
||||
public ConfigCompiler(boolean renderClosureAsString, boolean stripSecrets) {
|
||||
this.config = getConfig();
|
||||
this.loader = new GroovyClassLoader(new GroovyClassLoader(), config);
|
||||
this.renderClosureAsString = renderClosureAsString;
|
||||
this.stripSecrets = stripSecrets;
|
||||
}
|
||||
|
||||
private static CompilerConfiguration getConfig() {
|
||||
var importCustomizer = new ImportCustomizer();
|
||||
for ( var name : DEFAULT_IMPORT_NAMES )
|
||||
importCustomizer.addImports(name);
|
||||
|
||||
var config = new CompilerConfiguration();
|
||||
config.addCompilationCustomizers(importCustomizer);
|
||||
config.setScriptBaseClass(BASE_CLASS_NAME);
|
||||
config.setPluginFactory(new ConfigParserPluginFactory());
|
||||
|
||||
return config;
|
||||
}
|
||||
|
||||
public Script compile(String text, Path path) throws IOException {
|
||||
return compile0(text, path);
|
||||
}
|
||||
|
||||
private Script compile0(String text, Path path) throws IOException {
|
||||
// prepare compiler and source unit
|
||||
var unit = new ConfigCompilationUnit(config, loader);
|
||||
sourceUnit = unit.createSourceUnit(text, path);
|
||||
var collector = new ConfigClassLoader(loader).createCollector(unit, sourceUnit);
|
||||
|
||||
compiler = new Compiler(unit);
|
||||
compiler.addSource(sourceUnit);
|
||||
|
||||
// compile config file
|
||||
unit.addSource(sourceUnit);
|
||||
unit.setClassgenCallback(collector);
|
||||
unit.compile(Phases.CLASS_GENERATION);
|
||||
|
||||
// return compiled script class
|
||||
var clazz = (Class) collector.getLoadedClasses().stream().findFirst().orElse(null);
|
||||
return InvokerHelper.createScript(clazz, new Binding());
|
||||
}
|
||||
|
||||
public SourceUnit getSource() {
|
||||
return sourceUnit;
|
||||
}
|
||||
|
||||
public List<SyntaxErrorMessage> getErrors() {
|
||||
if( sourceUnit == null )
|
||||
return null;
|
||||
return sourceUnit
|
||||
.getErrorCollector()
|
||||
.getErrors()
|
||||
.stream()
|
||||
.map(e -> e instanceof SyntaxErrorMessage sem ? sem : null)
|
||||
.filter(sem -> sem != null)
|
||||
.toList();
|
||||
}
|
||||
|
||||
private static class ConfigClassLoader extends GroovyClassLoader {
|
||||
ConfigClassLoader(GroovyClassLoader parent) {
|
||||
super(parent);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected ClassCollector createCollector(CompilationUnit unit, SourceUnit su) {
|
||||
return super.createCollector(unit, su);
|
||||
}
|
||||
}
|
||||
|
||||
private class ConfigCompilationUnit extends CompilationUnit {
|
||||
|
||||
private static final List<ClassNode> DEFAULT_IMPORTS = defaultImports();
|
||||
|
||||
private static List<ClassNode> defaultImports() {
|
||||
return DEFAULT_IMPORT_NAMES.stream()
|
||||
.map(name -> ClassHelper.makeWithoutCaching(name))
|
||||
.toList();
|
||||
}
|
||||
|
||||
ConfigCompilationUnit(CompilerConfiguration configuration, GroovyClassLoader loader) {
|
||||
super(configuration, null, loader);
|
||||
super.addPhaseOperation(source -> analyze(source), Phases.CONVERSION);
|
||||
}
|
||||
|
||||
private void analyze(SourceUnit source) {
|
||||
// initialize script class
|
||||
var cn = source.getAST().getClasses().get(0);
|
||||
|
||||
// perform strict syntax checking
|
||||
if( source.getSource() instanceof StringReaderSourceWithURI ) {
|
||||
var includeResolver = new ResolveIncludeVisitor(source);
|
||||
includeResolver.visit();
|
||||
for( var error : includeResolver.getErrors() )
|
||||
source.getErrorCollector().addErrorAndContinue(error);
|
||||
}
|
||||
new ConfigResolveVisitor(source, this, DEFAULT_IMPORTS).visit();
|
||||
if( source.getErrorCollector().hasErrors() )
|
||||
return;
|
||||
|
||||
// convert to Groovy
|
||||
new ConfigToGroovyVisitor(source).visit();
|
||||
new PathCompareVisitor(source).visitClass(cn);
|
||||
if( stripSecrets )
|
||||
new StripSecretsVisitor(source).visitClass(cn);
|
||||
if( renderClosureAsString )
|
||||
new ClosureToStringVisitor(source).visitClass(cn);
|
||||
new GStringToStringVisitor(source).visitClass(cn);
|
||||
}
|
||||
|
||||
SourceUnit createSourceUnit(String source, Path path) {
|
||||
var readerSource = path != null
|
||||
? new StringReaderSourceWithURI(source, path.toUri(), getConfiguration())
|
||||
: new StringReaderSource(source, getConfiguration());
|
||||
return new SourceUnit(
|
||||
uniqueClassName(source),
|
||||
readerSource,
|
||||
getConfiguration(),
|
||||
getClassLoader(),
|
||||
getErrorCollector());
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a unique name for the config class in order to avoid collision
|
||||
* with config DSL
|
||||
*
|
||||
* @param text
|
||||
*/
|
||||
private String uniqueClassName(String text) {
|
||||
var hash = Hashing
|
||||
.sipHash24()
|
||||
.newHasher()
|
||||
.putUnencodedChars(text)
|
||||
.hash();
|
||||
return "_nf_config_" + hash;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,396 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.config.parser.v2
|
||||
|
||||
import java.nio.file.NoSuchFileException
|
||||
import java.nio.file.Path
|
||||
|
||||
import groovy.transform.CompileDynamic
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.Memoized
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.SysEnv
|
||||
import nextflow.exception.ConfigParseException
|
||||
import nextflow.extension.Bolts
|
||||
import nextflow.file.FileHelper
|
||||
/**
|
||||
* Builder DSL for Nextflow config files.
|
||||
*
|
||||
* @author Ben Sherman <bentshermann@gmail.com>
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class ConfigDsl extends Script {
|
||||
|
||||
private boolean ignoreIncludes
|
||||
|
||||
private boolean renderClosureAsString
|
||||
|
||||
private boolean strict
|
||||
|
||||
private boolean stripSecrets
|
||||
|
||||
private Path configPath
|
||||
|
||||
private Map cliParams
|
||||
|
||||
private List<String> profiles
|
||||
|
||||
private Map target = [params: [:]]
|
||||
|
||||
private Set<String> declaredProfiles = []
|
||||
|
||||
private Map<String,Object> declaredParams = [:]
|
||||
|
||||
void setIgnoreIncludes(boolean value) {
|
||||
this.ignoreIncludes = value
|
||||
}
|
||||
|
||||
void setRenderClosureAsString(boolean value) {
|
||||
this.renderClosureAsString = value
|
||||
}
|
||||
|
||||
void setStrict(boolean value) {
|
||||
this.strict = value
|
||||
}
|
||||
|
||||
void setStripSecrets(boolean value) {
|
||||
this.stripSecrets = value
|
||||
}
|
||||
|
||||
void setConfigPath(Path path) {
|
||||
this.configPath = path
|
||||
}
|
||||
|
||||
void setParams(Map params) {
|
||||
this.cliParams = params
|
||||
(target.params as Map).putAll(params)
|
||||
}
|
||||
|
||||
void setConfigParams(Map params) {
|
||||
(target.params as Map).putAll(params)
|
||||
}
|
||||
|
||||
void setProfiles(List<String> profiles) {
|
||||
this.profiles = profiles
|
||||
}
|
||||
|
||||
void declareProfile(String profile) {
|
||||
declaredProfiles.add(profile)
|
||||
}
|
||||
|
||||
Set<String> getDeclaredProfiles() {
|
||||
return declaredProfiles
|
||||
}
|
||||
|
||||
void declareParam(String name, Object value) {
|
||||
declaredParams.put(name, value)
|
||||
}
|
||||
|
||||
Map<String,Object> getDeclaredParams() {
|
||||
return declaredParams
|
||||
}
|
||||
|
||||
Map getTarget() {
|
||||
return target
|
||||
}
|
||||
|
||||
Object run() {}
|
||||
|
||||
@Override
|
||||
def getProperty(String name) {
|
||||
if( name == 'params' )
|
||||
return target.params
|
||||
|
||||
try {
|
||||
return super.getProperty(name)
|
||||
}
|
||||
catch( MissingPropertyException e ) {
|
||||
if( strict )
|
||||
throw e
|
||||
else
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Assign a value to a config option.
|
||||
*
|
||||
* When assigning a param, if the param was specified
|
||||
* on the command line, then the command line value takes
|
||||
* precedence. CLI params are applied here in order to ensure
|
||||
* that if the param is referenced later in the config file,
|
||||
* the command line value is used.
|
||||
*
|
||||
* @param names
|
||||
* @param value
|
||||
*/
|
||||
void assign(List<String> names, Object value) {
|
||||
if( names.size() == 2 && names.first() == 'params' ) {
|
||||
final name = names.last()
|
||||
declareParam(name, value)
|
||||
if( cliParams.containsKey(name) )
|
||||
value = asDeclaredType(cliParams[name], value)
|
||||
}
|
||||
navigate(names.init()).put(names.last(), value)
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a CLI param override to an appropriate type based
|
||||
* on the default param value in the config file.
|
||||
*
|
||||
* Note: this applies only to numbers and booleans.
|
||||
*
|
||||
* @param value
|
||||
* @param declValue
|
||||
*/
|
||||
private Object asDeclaredType(Object value, Object declValue) {
|
||||
if( value == null )
|
||||
return null
|
||||
|
||||
if( value !instanceof CharSequence )
|
||||
return value
|
||||
|
||||
final str = value.toString()
|
||||
|
||||
if( declValue instanceof Boolean ) {
|
||||
if( str.toLowerCase() == 'true' ) return Boolean.TRUE
|
||||
if( str.toLowerCase() == 'false' ) return Boolean.FALSE
|
||||
}
|
||||
|
||||
if( declValue instanceof Number ) {
|
||||
if( str.isInteger() ) return str.toInteger()
|
||||
if( str.isLong() ) return str.toLong()
|
||||
if( str.isBigInteger() ) return str.toBigInteger()
|
||||
if( str.isFloat() ) return str.toFloat()
|
||||
if( str.isDouble() ) return str.toDouble()
|
||||
if( str.isBigDecimal() ) return str.toBigDecimal()
|
||||
}
|
||||
|
||||
return value
|
||||
}
|
||||
|
||||
private Map navigate(List<String> names) {
|
||||
Map ctx = target
|
||||
for( final name : names ) {
|
||||
if( name !in ctx ) ctx[name] = [:]
|
||||
ctx = ctx[name] as Map
|
||||
}
|
||||
return ctx
|
||||
}
|
||||
|
||||
void block(String name, Closure closure) {
|
||||
block([name], closure)
|
||||
}
|
||||
|
||||
void block(List<String> names, Closure closure) {
|
||||
final dsl = blockDsl(names)
|
||||
final cl = (Closure)closure.clone()
|
||||
cl.setResolveStrategy(Closure.DELEGATE_FIRST)
|
||||
cl.setDelegate(dsl)
|
||||
cl.call()
|
||||
dsl.apply()
|
||||
}
|
||||
|
||||
private ConfigBlockDsl blockDsl(List<String> names) {
|
||||
if( names.size() == 1 && names.first() == 'plugins' )
|
||||
return new PluginsDsl(this)
|
||||
|
||||
final relativeNames = names.size() == 3 && names.first() == 'profiles'
|
||||
? List.of(names.last())
|
||||
: names
|
||||
|
||||
if( relativeNames.size() == 1 && relativeNames.last() == 'process' )
|
||||
return new ProcessDsl(this, names)
|
||||
|
||||
if( names.size() == 1 && names.first() == 'profiles' )
|
||||
return new ProfilesDsl(this, profiles)
|
||||
|
||||
return new ConfigBlockDsl(this, names)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the value of an environment variable from the launch environment.
|
||||
*
|
||||
* @param name
|
||||
*/
|
||||
String env(String name) {
|
||||
return SysEnv.get(name)
|
||||
}
|
||||
|
||||
void includeConfig(String includeFile) {
|
||||
includeConfig([], includeFile)
|
||||
}
|
||||
|
||||
void includeConfig(List<String> names, String includeFile) {
|
||||
assert includeFile
|
||||
|
||||
if( ignoreIncludes )
|
||||
return
|
||||
|
||||
Path includePath = FileHelper.asPath(includeFile)
|
||||
log.trace "Include config file: $includeFile [parent: $configPath]"
|
||||
|
||||
if( !includePath.isAbsolute() && configPath )
|
||||
includePath = configPath.resolveSibling(includeFile)
|
||||
|
||||
final configText = readConfigFile(includePath)
|
||||
final parser = new ConfigParserV2()
|
||||
.setIgnoreIncludes(ignoreIncludes)
|
||||
.setRenderClosureAsString(renderClosureAsString)
|
||||
.setStrict(strict)
|
||||
.setStripSecrets(stripSecrets)
|
||||
.setBinding(binding.getVariables())
|
||||
.setParams(cliParams)
|
||||
.setConfigParams(target.params as Map)
|
||||
.setProfiles(profiles)
|
||||
final config = parser.parse(configText, includePath)
|
||||
declaredProfiles.addAll(parser.getDeclaredProfiles())
|
||||
declaredParams.putAll(parser.getDeclaredParams())
|
||||
|
||||
final ctx = navigate(names)
|
||||
ctx.putAll(Bolts.deepMerge(ctx, config))
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the content of a config file. The result is cached to
|
||||
* avoid multiple reads.
|
||||
*
|
||||
* @param includePath
|
||||
*/
|
||||
@Memoized
|
||||
@CompileDynamic // required to support ProviderPath::getText() over NioExtensions::getText()
|
||||
protected static String readConfigFile(Path includePath) {
|
||||
try {
|
||||
return includePath.getText()
|
||||
}
|
||||
catch (NoSuchFileException | FileNotFoundException ignored) {
|
||||
throw new NoSuchFileException("Config file does not exist: ${includePath.toUriString()}")
|
||||
}
|
||||
catch (IOException e) {
|
||||
throw new IOException("Cannot read config file include: ${includePath.toUriString()}", e)
|
||||
}
|
||||
}
|
||||
|
||||
private static class ConfigBlockDsl {
|
||||
protected ConfigDsl dsl
|
||||
protected List<String> scope
|
||||
|
||||
ConfigBlockDsl(ConfigDsl dsl, List<String> scope) {
|
||||
this.dsl = dsl
|
||||
this.scope = scope
|
||||
}
|
||||
|
||||
void assign(List<String> names, Object value) {
|
||||
dsl.assign(scope + names, value)
|
||||
}
|
||||
|
||||
void block(String name, Closure closure) {
|
||||
dsl.block(scope + [name], closure)
|
||||
}
|
||||
|
||||
void withLabel(String label, Closure closure) {
|
||||
throw new ConfigParseException("Process selectors are only allowed in the `process` scope (offending scope: `${scope.join('.')}`)")
|
||||
}
|
||||
|
||||
void withName(String selector, Closure closure) {
|
||||
throw new ConfigParseException("Process selectors are only allowed in the `process` scope (offending scope: `${scope.join('.')}`)")
|
||||
}
|
||||
|
||||
void includeConfig(String includeFile) {
|
||||
dsl.includeConfig(scope, includeFile)
|
||||
}
|
||||
|
||||
void apply() {
|
||||
}
|
||||
}
|
||||
|
||||
private static class PluginsDsl extends ConfigBlockDsl {
|
||||
PluginsDsl(ConfigDsl dsl) {
|
||||
super(dsl, Collections.<String>emptyList())
|
||||
}
|
||||
|
||||
void id(String value) {
|
||||
final target = dsl.getTarget()
|
||||
final plugins = (Set) target.computeIfAbsent('plugins', (k) -> new HashSet<>())
|
||||
plugins.add(value)
|
||||
}
|
||||
}
|
||||
|
||||
private static class ProcessDsl extends ConfigBlockDsl {
|
||||
ProcessDsl(ConfigDsl dsl, List<String> scope) {
|
||||
super(dsl, scope)
|
||||
}
|
||||
|
||||
@Override
|
||||
void withLabel(String label, Closure closure) {
|
||||
dsl.block(scope + ["withLabel:${label}".toString()], closure)
|
||||
}
|
||||
|
||||
@Override
|
||||
void withName(String selector, Closure closure) {
|
||||
dsl.block(scope + ["withName:${selector}".toString()], closure)
|
||||
}
|
||||
}
|
||||
|
||||
private static class ProfilesDsl extends ConfigBlockDsl {
|
||||
private List<String> profiles
|
||||
private Map<String,Closure> blocks = [:]
|
||||
|
||||
ProfilesDsl(ConfigDsl dsl, List<String> profiles) {
|
||||
super(dsl, Collections.<String>emptyList())
|
||||
this.profiles = profiles
|
||||
}
|
||||
|
||||
@Override
|
||||
void assign(List<String> names, Object value) {
|
||||
throw new ConfigParseException("Only profile blocks are allowed in the `profiles` scope")
|
||||
}
|
||||
|
||||
@Override
|
||||
void block(String name, Closure closure) {
|
||||
blocks[name] = closure
|
||||
dsl.declareProfile(name)
|
||||
}
|
||||
|
||||
@Override
|
||||
void includeConfig(String includeFile) {
|
||||
throw new ConfigParseException("Only profile blocks are allowed in the `profiles` scope")
|
||||
}
|
||||
|
||||
@Override
|
||||
void apply() {
|
||||
if( profiles != null ) {
|
||||
// apply profiles in the order they were specified
|
||||
for( final name : profiles ) {
|
||||
final closure = blocks[name]
|
||||
if( closure )
|
||||
dsl.block(scope, closure)
|
||||
}
|
||||
}
|
||||
else {
|
||||
// append all profiles to the config map
|
||||
for( final name : blocks.keySet() ) {
|
||||
final closure = blocks[name]
|
||||
dsl.block(['profiles', name], closure)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.config.parser.v2
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
import groovy.transform.CompileDynamic
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.config.ConfigParser
|
||||
import nextflow.exception.ConfigParseException
|
||||
import nextflow.extension.Bolts
|
||||
import nextflow.script.parser.v2.StandardErrorListener
|
||||
import org.codehaus.groovy.control.CompilationFailedException
|
||||
import org.codehaus.groovy.control.SourceUnit
|
||||
|
||||
/**
|
||||
* The parser for Nextflow config files.
|
||||
*
|
||||
* @author Ben Sherman <bentshermann@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class ConfigParserV2 implements ConfigParser {
|
||||
|
||||
private Map bindingVars = [:]
|
||||
|
||||
private Map cliParams = [:]
|
||||
|
||||
private Map configParams = [:]
|
||||
|
||||
private boolean ignoreIncludes = false
|
||||
|
||||
private boolean renderClosureAsString = false
|
||||
|
||||
private boolean strict = true
|
||||
|
||||
private boolean stripSecrets
|
||||
|
||||
private boolean ansiLog
|
||||
|
||||
private List<String> appliedProfiles
|
||||
|
||||
private Set<String> declaredProfiles = []
|
||||
|
||||
private Map<String,Object> declaredParams = [:]
|
||||
|
||||
private GroovyShell groovyShell
|
||||
|
||||
@Override
|
||||
ConfigParserV2 setProfiles(List<String> profiles) {
|
||||
this.appliedProfiles = profiles
|
||||
return this
|
||||
}
|
||||
|
||||
@Override
|
||||
ConfigParserV2 setIgnoreIncludes(boolean value) {
|
||||
this.ignoreIncludes = value
|
||||
return this
|
||||
}
|
||||
|
||||
@Override
|
||||
ConfigParserV2 setRenderClosureAsString(boolean value) {
|
||||
this.renderClosureAsString = value
|
||||
return this
|
||||
}
|
||||
|
||||
@Override
|
||||
ConfigParserV2 setStrict(boolean value) {
|
||||
this.strict = value
|
||||
return this
|
||||
}
|
||||
|
||||
@Override
|
||||
ConfigParserV2 setStripSecrets(boolean value) {
|
||||
this.stripSecrets = value
|
||||
return this
|
||||
}
|
||||
|
||||
@Override
|
||||
ConfigParserV2 setAnsiLog(boolean value) {
|
||||
this.ansiLog = value
|
||||
return this
|
||||
}
|
||||
|
||||
@Override
|
||||
ConfigParserV2 setBinding(Map vars) {
|
||||
this.bindingVars = vars
|
||||
return this
|
||||
}
|
||||
|
||||
@Override
|
||||
ConfigParserV2 setParams(Map params) {
|
||||
// deep clone the map to prevent side effects with nested params
|
||||
// see https://github.com/nextflow-io/nextflow/issues/1923
|
||||
this.cliParams = Bolts.deepClone(params)
|
||||
return this
|
||||
}
|
||||
|
||||
ConfigParserV2 setConfigParams(Map params) {
|
||||
this.configParams = params
|
||||
return this
|
||||
}
|
||||
|
||||
@Override
|
||||
Set<String> getDeclaredProfiles() {
|
||||
return declaredProfiles
|
||||
}
|
||||
|
||||
@Override
|
||||
Map<String,Object> getDeclaredParams() {
|
||||
return declaredParams
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the given script as a string and return the configuration object
|
||||
*
|
||||
* @param text
|
||||
* @param path
|
||||
*/
|
||||
@Override
|
||||
ConfigObject parse(String text) {
|
||||
parse(text, null)
|
||||
}
|
||||
|
||||
ConfigObject parse(String text, Path path) {
|
||||
final compiler = getCompiler()
|
||||
try {
|
||||
final script = (ConfigDsl) compiler.compile(text, path)
|
||||
script.setBinding(new Binding(bindingVars))
|
||||
if( path )
|
||||
script.setConfigPath(path)
|
||||
script.setIgnoreIncludes(ignoreIncludes)
|
||||
script.setRenderClosureAsString(renderClosureAsString)
|
||||
script.setStrict(strict)
|
||||
script.setStripSecrets(stripSecrets)
|
||||
script.setParams(cliParams)
|
||||
script.setConfigParams(configParams)
|
||||
script.setProfiles(appliedProfiles)
|
||||
script.run()
|
||||
|
||||
final target = script.getTarget()
|
||||
declaredProfiles.addAll(script.getDeclaredProfiles())
|
||||
declaredParams.putAll(script.getDeclaredParams())
|
||||
return Bolts.toConfigObject(target)
|
||||
}
|
||||
catch( CompilationFailedException e ) {
|
||||
if( path )
|
||||
printErrors(path)
|
||||
throw new ConfigParseException("Config parsing failed", e)
|
||||
}
|
||||
}
|
||||
|
||||
private void printErrors(Path path) {
|
||||
final source = compiler.getSource()
|
||||
final errorListener = new StandardErrorListener('full', ansiLog)
|
||||
println()
|
||||
errorListener.beforeErrors()
|
||||
for( final message : compiler.getErrors() ) {
|
||||
final cause = message.getCause()
|
||||
final filename = getRelativePath(source, path)
|
||||
errorListener.onError(cause, filename, source)
|
||||
}
|
||||
errorListener.afterErrors()
|
||||
}
|
||||
|
||||
private String getRelativePath(SourceUnit source, Path path) {
|
||||
final uri = source.getSource().getURI()
|
||||
return path.getParent().relativize(Path.of(uri)).toString()
|
||||
}
|
||||
|
||||
@Override
|
||||
ConfigObject parse(File file) {
|
||||
return parse(file.toPath())
|
||||
}
|
||||
|
||||
@Override
|
||||
@CompileDynamic // required to support ProviderPath::getText() over NioExtensions::getText()
|
||||
ConfigObject parse(Path path) {
|
||||
return parse(path.getText(), path)
|
||||
}
|
||||
|
||||
private ConfigCompiler compiler
|
||||
|
||||
private ConfigCompiler getCompiler() {
|
||||
if( !compiler )
|
||||
compiler = new ConfigCompiler(renderClosureAsString, stripSecrets)
|
||||
return compiler
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package nextflow.config.spec
|
||||
|
||||
import groovy.transform.TypeChecked
|
||||
import nextflow.plugin.Plugins
|
||||
import nextflow.script.dsl.Description
|
||||
|
||||
@TypeChecked
|
||||
class MarkdownRenderer {
|
||||
|
||||
String render() {
|
||||
final entries = getSpec().entrySet().sort { entry -> entry.key }
|
||||
final result = new StringBuilder()
|
||||
entries.each { entry ->
|
||||
final scopeName = entry.key
|
||||
|
||||
final anchor = scopeName == ''
|
||||
? '(config-unscoped)='
|
||||
: "(config-$scopeName)="
|
||||
result.append("\n$anchor\n")
|
||||
|
||||
final title = scopeName == ''
|
||||
? 'Unscoped options'
|
||||
: "`$scopeName`"
|
||||
result.append("\n## $title\n")
|
||||
|
||||
final scope = entry.value
|
||||
final description = scope.description()
|
||||
if( description )
|
||||
result.append("\n${fromDescription(description)}\n")
|
||||
result.append("\nThe following settings are available:\n")
|
||||
|
||||
final options = scope.children().findAll { name, node -> node instanceof SpecNode.Option }
|
||||
renderOptions(options, scopeName, result)
|
||||
|
||||
final scopes = scope.children().findAll { name, node -> node instanceof SpecNode.Scope }
|
||||
renderOptions(scopes, scopeName, result)
|
||||
}
|
||||
return result.toString()
|
||||
}
|
||||
|
||||
private static Map<String,SpecNode.Scope> getSpec() {
|
||||
final result = new HashMap<String,SpecNode.Scope>()
|
||||
for( final scope : Plugins.getExtensions(ConfigScope) ) {
|
||||
final clazz = scope.getClass()
|
||||
final scopeName = clazz.getAnnotation(ScopeName)?.value()
|
||||
final description = clazz.getAnnotation(Description)?.value()
|
||||
if( scopeName == null )
|
||||
continue
|
||||
final node = SpecNode.Scope.of(clazz, description)
|
||||
result.put(scopeName, node)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
private static String fromDescription(String description) {
|
||||
return description.stripIndent(true).trim()
|
||||
}
|
||||
|
||||
private static void renderOptions(Map<String,SpecNode> nodes, String scopeName, StringBuilder result) {
|
||||
final prefix = scopeName ? scopeName + '.' : ''
|
||||
final entries = nodes.entrySet().sort { entry -> entry.key }
|
||||
entries.each { entry ->
|
||||
final name = entry.key
|
||||
final node = entry.value
|
||||
if( node instanceof SpecNode.Option )
|
||||
renderOption("${prefix}${name}", node, result)
|
||||
else if( node instanceof SpecNode.Placeholder )
|
||||
renderOptions(node.scope().children(), "${prefix}${name}.${node.placeholderName()}", result)
|
||||
else if( node instanceof SpecNode.Scope )
|
||||
renderOptions(node.children(), "${prefix}${name}", result)
|
||||
else
|
||||
throw new IllegalStateException()
|
||||
}
|
||||
}
|
||||
|
||||
private static void renderOption(String name, SpecNode.Option node, StringBuilder result) {
|
||||
final description = fromDescription(node.description())
|
||||
if( !description )
|
||||
return
|
||||
result.append("\n`${name}`")
|
||||
description.eachLine { line ->
|
||||
if( line )
|
||||
result.append("\n: ${line}")
|
||||
}
|
||||
result.append('\n')
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.container
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
|
||||
/**
|
||||
* Wrap a task execution inside an Apple container (apple/container) runtime.
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class AppleContainerBuilder extends ContainerBuilder<AppleContainerBuilder> {
|
||||
|
||||
private boolean remove
|
||||
|
||||
private boolean tty
|
||||
|
||||
private String name
|
||||
|
||||
private String capAdd
|
||||
|
||||
private String removeCommand
|
||||
|
||||
private String killCommand
|
||||
|
||||
private kill = true
|
||||
|
||||
AppleContainerBuilder(String name, AppleContainerConfig config) {
|
||||
this.image = name
|
||||
|
||||
if( config.engineOptions )
|
||||
addEngineOptions(config.engineOptions)
|
||||
|
||||
if( config.runOptions )
|
||||
addRunOptions(config.runOptions)
|
||||
|
||||
if( config.temp )
|
||||
this.temp = config.temp
|
||||
|
||||
this.remove = config.remove
|
||||
this.tty = config.tty
|
||||
|
||||
if( !config.writableInputMounts )
|
||||
this.readOnlyInputs = true
|
||||
}
|
||||
|
||||
AppleContainerBuilder(String name) {
|
||||
this(name, new AppleContainerConfig([:]))
|
||||
}
|
||||
|
||||
@Override
|
||||
AppleContainerBuilder params( Map params ) {
|
||||
if( !params ) return this
|
||||
|
||||
if( params.containsKey('entry') )
|
||||
this.entryPoint = params.entry
|
||||
|
||||
if( params.containsKey('kill') )
|
||||
this.kill = params.kill
|
||||
|
||||
if( params.containsKey('capAdd') )
|
||||
this.capAdd = params.capAdd
|
||||
|
||||
return this
|
||||
}
|
||||
|
||||
@Override
|
||||
AppleContainerBuilder setName( String name ) {
|
||||
this.name = name
|
||||
return this
|
||||
}
|
||||
|
||||
@Override
|
||||
AppleContainerBuilder build(StringBuilder result) {
|
||||
assert image
|
||||
|
||||
result << 'container '
|
||||
|
||||
if( engineOptions )
|
||||
result << engineOptions.join(' ') << ' '
|
||||
|
||||
result << 'run -i '
|
||||
|
||||
if( tty )
|
||||
result << '-t '
|
||||
|
||||
if( cpus )
|
||||
result << "--cpus ${cpus} "
|
||||
|
||||
if( memory )
|
||||
result << "-m ${memory} "
|
||||
|
||||
if( platform )
|
||||
result << "--platform ${platform} "
|
||||
|
||||
// environment variables
|
||||
appendEnv(result)
|
||||
|
||||
if( temp )
|
||||
result << "-v $temp:/tmp "
|
||||
|
||||
// volume mounts
|
||||
result << makeVolumes(mounts)
|
||||
result << '-w "$NXF_TASK_WORKDIR" '
|
||||
|
||||
if( entryPoint )
|
||||
result << '--entrypoint ' << entryPoint << ' '
|
||||
|
||||
if( runOptions )
|
||||
result << runOptions.join(' ') << ' '
|
||||
|
||||
if( capAdd )
|
||||
result << '--cap-add ' << capAdd << ' '
|
||||
|
||||
if( name )
|
||||
result << '--name ' << name << ' '
|
||||
|
||||
// image is the final positional argument
|
||||
result << image
|
||||
|
||||
runCommand = result.toString()
|
||||
|
||||
if( remove && name ) {
|
||||
removeCommand = 'container rm ' + name
|
||||
}
|
||||
|
||||
if( kill ) {
|
||||
killCommand = 'container stop '
|
||||
if( kill instanceof String ) killCommand = "container kill -s $kill "
|
||||
killCommand += name
|
||||
}
|
||||
|
||||
return this
|
||||
}
|
||||
|
||||
@Override
|
||||
String getRemoveCommand() { removeCommand }
|
||||
|
||||
@Override
|
||||
String getKillCommand() { killCommand }
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package nextflow.container
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.config.spec.ConfigOption
|
||||
import nextflow.config.spec.ConfigScope
|
||||
import nextflow.config.spec.ScopeName
|
||||
import nextflow.script.dsl.Description
|
||||
|
||||
@ScopeName("appleContainer")
|
||||
@Description("""
|
||||
The `appleContainer` scope controls how [Apple container](https://github.com/apple/container) runtime is used by Nextflow.
|
||||
""")
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
@EqualsAndHashCode
|
||||
class AppleContainerConfig implements ConfigScope, ContainerConfig {
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Enable Apple container execution (default: `false`).
|
||||
""")
|
||||
boolean enabled
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Specify additional options supported by the `container` CLI i.e. `container [OPTIONS] run`.
|
||||
""")
|
||||
final String engineOptions
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Comma separated list of environment variable names to be included in the container environment.
|
||||
""")
|
||||
final List<String> envWhitelist
|
||||
|
||||
@ConfigOption(types=[String,Boolean])
|
||||
@Description("""
|
||||
""")
|
||||
final Object kill
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The registry from where container images are pulled. It should NOT include the protocol prefix i.e. `http://`.
|
||||
""")
|
||||
final String registry
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Clean up the container after the execution (default: `true`).
|
||||
""")
|
||||
final boolean remove
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Specify extra command line options supported by the `container run` command.
|
||||
""")
|
||||
final String runOptions
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Mounts a path of your choice as the `/tmp` directory in the container.
|
||||
""")
|
||||
final String temp
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Allocates a pseudo-tty (default: `false`).
|
||||
""")
|
||||
final boolean tty
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
When `false`, mount input directories as read-only (default: `true`).
|
||||
""")
|
||||
final boolean writableInputMounts
|
||||
|
||||
/* required by extension point -- do not remove */
|
||||
AppleContainerConfig() {}
|
||||
|
||||
AppleContainerConfig(Map opts) {
|
||||
enabled = opts.enabled as boolean
|
||||
engineOptions = opts.engineOptions
|
||||
envWhitelist = ContainerHelper.parseEnvWhitelist(opts.envWhitelist)
|
||||
kill = opts.kill != null ? opts.kill : true
|
||||
registry = opts.registry
|
||||
remove = opts.remove != null ? opts.remove as boolean : true
|
||||
runOptions = opts.runOptions
|
||||
temp = opts.temp
|
||||
tty = opts.tty as boolean
|
||||
writableInputMounts = opts.writableInputMounts != null ? opts.writableInputMounts as boolean : true
|
||||
}
|
||||
|
||||
@Override
|
||||
String getEngine() {
|
||||
return 'apple-container'
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean canRunOciImage() {
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.container
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
|
||||
/**
|
||||
* Implements a builder for Apptainer containerisation
|
||||
*
|
||||
* see https://apptainer.org
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class ApptainerBuilder extends SingularityBuilder {
|
||||
|
||||
ApptainerBuilder(String name, ApptainerConfig config) {
|
||||
super(name)
|
||||
applyConfig(config)
|
||||
}
|
||||
|
||||
ApptainerBuilder(String name) {
|
||||
this(name, new ApptainerConfig([:]))
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String getBinaryName() { 'apptainer' }
|
||||
|
||||
protected void applyConfig(ApptainerConfig config) {
|
||||
|
||||
if( config.autoMounts != null )
|
||||
this.autoMounts = config.autoMounts
|
||||
|
||||
if( config.engineOptions )
|
||||
this.addEngineOptions(config.engineOptions)
|
||||
|
||||
if( config.runOptions )
|
||||
this.addRunOptions(config.runOptions)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.container
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
|
||||
/**
|
||||
* Handle caching of remote Apptainer images
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class ApptainerCache extends SingularityCache {
|
||||
|
||||
ApptainerCache(ApptainerConfig config, Map<String,String> env=null) {
|
||||
super(config.cacheDir, config.libraryDir, config.noHttps, config.pullTimeout, env)
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String getBinaryName() { 'apptainer' }
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package nextflow.container
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.config.spec.ConfigOption
|
||||
import nextflow.config.spec.ConfigScope
|
||||
import nextflow.config.spec.ScopeName
|
||||
import nextflow.script.dsl.Description
|
||||
import nextflow.util.Duration
|
||||
|
||||
@ScopeName("apptainer")
|
||||
@Description("""
|
||||
The `apptainer` scope controls how [Apptainer](https://apptainer.org) containers are executed by Nextflow.
|
||||
""")
|
||||
@CompileStatic
|
||||
class ApptainerConfig implements ConfigScope, ContainerConfig {
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Automatically mount host paths in the executed container (default: `true`). It requires the `user bind control` feature to be enabled in your Apptainer installation.
|
||||
""")
|
||||
final Boolean autoMounts
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The directory where remote Apptainer images are stored. When using a computing cluster it must be a shared folder accessible to all compute nodes.
|
||||
""")
|
||||
final String cacheDir
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Execute tasks with Apptainer containers (default: `false`).
|
||||
""")
|
||||
final boolean enabled
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Specify additional options supported by the Apptainer engine i.e. `apptainer [OPTIONS]`.
|
||||
""")
|
||||
final String engineOptions
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Comma separated list of environment variable names to be included in the container environment.
|
||||
""")
|
||||
final List<String> envWhitelist
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Directory where remote Apptainer images are retrieved. When using a computing cluster it must be a shared folder accessible to all compute nodes.
|
||||
""")
|
||||
final String libraryDir
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Pull the Apptainer image with http protocol (default: `false`).
|
||||
""")
|
||||
final boolean noHttps
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
When enabled, OCI (and Docker) container images are pulled and converted to the SIF format by the Apptainer run command, instead of Nextflow (default: `false`).
|
||||
""")
|
||||
final boolean ociAutoPull
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The amount of time the Apptainer pull can last, exceeding which the process is terminated (default: `20 min`).
|
||||
""")
|
||||
final Duration pullTimeout
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The registry from where Docker images are pulled. It should be only used to specify a private registry server. It should NOT include the protocol prefix i.e. `http://`.
|
||||
""")
|
||||
final String registry
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Specify extra command line options supported by `apptainer exec`.
|
||||
""")
|
||||
final String runOptions
|
||||
|
||||
/* required by extension point -- do not remove */
|
||||
ApptainerConfig() {}
|
||||
|
||||
ApptainerConfig(Map opts) {
|
||||
autoMounts = opts.autoMounts as Boolean
|
||||
cacheDir = opts.cacheDir
|
||||
enabled = opts.enabled as boolean
|
||||
engineOptions = opts.engineOptions
|
||||
envWhitelist = ContainerHelper.parseEnvWhitelist(opts.envWhitelist)
|
||||
libraryDir = opts.libraryDir
|
||||
noHttps = opts.noHttps as boolean
|
||||
ociAutoPull = opts.ociAutoPull as boolean
|
||||
pullTimeout = opts.pullTimeout as Duration ?: Duration.of('20min')
|
||||
registry = opts.registry
|
||||
runOptions = opts.runOptions
|
||||
}
|
||||
|
||||
@Override
|
||||
String getEngine() {
|
||||
return 'apptainer'
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean canRunOciImage() {
|
||||
return ociAutoPull
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.container
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.Global
|
||||
/**
|
||||
* Implements a builder for Charliecloud containerisation
|
||||
*
|
||||
* see https://hpc.github.io/charliecloud/
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
* @author Patrick Hüther <patrick.huether@gmail.com>
|
||||
* @author Laurent Modolo <laurent.modolo@ens-lyon.fr>
|
||||
* @author Niklas Schandry <niklas@bio.lmu.de>
|
||||
*/
|
||||
@CompileStatic
|
||||
@Slf4j
|
||||
class CharliecloudBuilder extends ContainerBuilder<CharliecloudBuilder> {
|
||||
|
||||
private boolean writeFake
|
||||
|
||||
CharliecloudBuilder(String name, CharliecloudConfig config) {
|
||||
this.image = name
|
||||
|
||||
if( config.runOptions )
|
||||
addRunOptions(config.runOptions)
|
||||
|
||||
if( config.temp )
|
||||
this.temp = config.temp
|
||||
|
||||
if( !config.writableInputMounts )
|
||||
this.readOnlyInputs = true
|
||||
|
||||
this.writeFake = config.writeFake
|
||||
}
|
||||
|
||||
CharliecloudBuilder(String name) {
|
||||
this(name, new CharliecloudConfig([:]))
|
||||
}
|
||||
|
||||
@Override
|
||||
CharliecloudBuilder params(Map params) {
|
||||
|
||||
if( params.containsKey('entry') )
|
||||
this.entryPoint = params.entry
|
||||
|
||||
return this
|
||||
}
|
||||
|
||||
CharliecloudBuilder addRunOptions(String str) {
|
||||
runOptions.add(str)
|
||||
return this
|
||||
}
|
||||
|
||||
@Override
|
||||
CharliecloudBuilder build(StringBuilder result) {
|
||||
|
||||
assert image
|
||||
def imageStorage = Paths.get(image).parent.parent
|
||||
def imageName = image.split('/')[-1]
|
||||
|
||||
result << 'ch-run --unset-env="*" -c "$NXF_TASK_WORKDIR" --set-env '
|
||||
|
||||
if ( writeFake )
|
||||
result << '--write-fake '
|
||||
|
||||
if ( !writeFake && !readOnlyInputs ) {
|
||||
// -w and CH_IMAGE_STORAGE are incompatible.
|
||||
if(System.getenv('CH_IMAGE_STORAGE') == imageStorage)
|
||||
throw new Exception('It is not possible to run writeable images from `$CH_IMAGE_STORAGE`')
|
||||
result << '-w '
|
||||
}
|
||||
|
||||
appendEnv(result)
|
||||
|
||||
if( temp )
|
||||
result << "-b $temp:/tmp "
|
||||
|
||||
makeVolumes(mounts, result)
|
||||
|
||||
if( runOptions )
|
||||
result << runOptions.join(' ') << ' '
|
||||
|
||||
if( writeFake && System.getenv('CH_IMAGE_STORAGE') ) {
|
||||
// Run by name if writeFake is true and CH_IMAGE_STORAGE is set
|
||||
result << imageName
|
||||
} else {
|
||||
// Otherwise run by path
|
||||
result << image
|
||||
}
|
||||
|
||||
result << ' --'
|
||||
|
||||
runCommand = result.toString()
|
||||
|
||||
return this
|
||||
}
|
||||
|
||||
protected String getRoot(String path) {
|
||||
def rootPath = path.tokenize("/")
|
||||
|
||||
if (rootPath.size() >= 1 && path[0] == '/')
|
||||
rootPath = "/${rootPath[0]}"
|
||||
else
|
||||
throw new IllegalArgumentException("Not a valid working directory value (absolute path?): ${path}")
|
||||
|
||||
return rootPath
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String composeVolumePath(String path, boolean readOnlyInputs = false) {
|
||||
def mountCmd = "-b ${escape(path)}"
|
||||
if (readOnlyInputs)
|
||||
mountCmd = "-b ${getRoot(escape(path))}"
|
||||
return mountCmd
|
||||
}
|
||||
|
||||
@Override
|
||||
protected StringBuilder makeEnv( env, StringBuilder result = new StringBuilder() ) {
|
||||
|
||||
if( env instanceof Map ) {
|
||||
short index = 0
|
||||
for( Map.Entry entry : env.entrySet() ) {
|
||||
if( index++ ) result << ' '
|
||||
result << ("--set-env=${entry.key}=${entry.value}")
|
||||
}
|
||||
}
|
||||
else if( env instanceof String && env.contains('=') ) {
|
||||
result << "--set-env=" << env
|
||||
}
|
||||
else if( env instanceof String ) {
|
||||
result << "\${$env:+--set-env=$env=\$$env}"
|
||||
}
|
||||
else if( env ) {
|
||||
throw new IllegalArgumentException("Not a valid environment value: $env [${env.class.name}]")
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user