add nextflow d30e48d

This commit is contained in:
2026-04-29 23:01:54 +02:00
parent d0b12d668d
commit 97cc9058d3
2840 changed files with 730250 additions and 0 deletions

View File

@@ -0,0 +1,119 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
plugins {
id "com.gradleup.shadow" version "9.3.1"
}
apply plugin: 'groovy'
apply plugin: 'application'
sourceSets {
main.java.srcDirs = []
main.groovy.srcDirs = ['src/main/java', 'src/main/groovy']
main.resources.srcDirs = ['src/main/resources']
test.java.srcDirs = []
test.groovy.srcDirs = ['src/test/groovy']
test.resources.srcDirs = ['src/test/resources']
}
compileGroovy {
options.compilerArgs = ['-XDignore.symbol.file']
}
configurations {
lineageImplementation
}
dependencies {
api(project(':nf-commons'))
api(project(':nf-httpfs'))
api(project(':nf-lang'))
api "org.apache.groovy:groovy:4.0.31"
api "org.apache.groovy:groovy-nio:4.0.31"
api "org.apache.groovy:groovy-xml:4.0.31"
api "org.apache.groovy:groovy-json:4.0.31"
api "org.apache.groovy:groovy-templates:4.0.31"
api "org.apache.groovy:groovy-yaml:4.0.31"
api "org.slf4j:jcl-over-slf4j:2.0.17"
api "org.slf4j:jul-to-slf4j:2.0.17"
api "org.slf4j:log4j-over-slf4j:2.0.17"
api "ch.qos.logback:logback-classic:1.5.26"
api "ch.qos.logback:logback-core:1.5.26"
api "org.codehaus.gpars:gpars:1.2.1"
api("ch.artecat.grengine:grengine:3.0.2") { exclude group: 'org.codehaus.groovy' }
api "org.apache.commons:commons-lang3:3.18.0"
api "commons-codec:commons-codec:1.15"
api "commons-io:commons-io:2.15.1"
api "com.beust:jcommander:1.35"
api("com.esotericsoftware.kryo:kryo:2.24.0") { exclude group: 'com.esotericsoftware.minlog', module: 'minlog' }
api('org.iq80.leveldb:leveldb:0.12')
api('org.eclipse.jgit:org.eclipse.jgit:7.1.1.202505221757-r')
api ('javax.activation:activation:1.1.1')
api ('javax.mail:mail:1.4.7')
api ('org.yaml:snakeyaml:2.2')
api ('org.jsoup:jsoup:1.15.4')
api 'jline:jline:2.9'
api 'org.pf4j:pf4j:3.14.1'
api 'dev.failsafe:failsafe:3.1.0'
api 'io.seqera:lib-trace:0.1.0'
api 'com.fasterxml.woodstox:woodstox-core:7.1.1'
api 'org.apache.commons:commons-compress:1.27.1' // For tar.gz extraction
api 'io.seqera:npr-api:0.22.0'
api 'io.seqera:npr-client:0.22.0'
testImplementation 'org.subethamail:subethasmtp:3.1.7'
testImplementation (project(':nf-lineage'))
testImplementation 'org.wiremock:wiremock:3.13.1'
// test configuration
testFixturesApi ("org.apache.groovy:groovy-test:4.0.31") { exclude group: 'org.apache.groovy' }
testFixturesApi ("org.objenesis:objenesis:3.4")
testFixturesApi ("net.bytebuddy:byte-buddy:1.14.17")
testFixturesApi ("org.spockframework:spock-core:2.4-groovy-4.0") { exclude group: 'org.apache.groovy' }
testFixturesApi ('org.spockframework:spock-junit4:2.4-groovy-4.0') { exclude group: 'org.apache.groovy' }
testFixturesApi 'com.google.jimfs:jimfs:1.2'
// note: declare as separate dependency to avoid a circular dependency
lineageImplementation (project(':nf-lineage'))
}
test {
minHeapSize = "512m"
maxHeapSize = "4096m"
}
application {
mainClass = 'nextflow.cli.Launcher'
}
run{
args( (project.hasProperty("runCmd") ? project.findProperty("runCmd") : "set a cmd to run").split(' ') )
}
shadowJar {
// add 'lineage' because it cannot be added to this project
// explicitly otherwise it will result into a circular dependency
configurations = [project.configurations.runtimeClasspath, project.configurations.lineageImplementation]
archiveClassifier='one'
manifest {
attributes 'Main-Class': application.mainClass.get()
}
mergeServiceFiles()
mergeGroovyExtensionModules()
transform(com.github.jengelman.gradle.plugins.shadow.transformers.AppendingTransformer) {
resource = 'META-INF/extensions.idx'
}
}

View File

@@ -0,0 +1,653 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow
import static nextflow.util.CheckHelper.*
import java.nio.file.FileSystem
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
import java.util.concurrent.CompletableFuture
import java.util.regex.Pattern
import groovy.transform.CompileStatic
import groovy.transform.PackageScope
import groovy.util.logging.Slf4j
import groovyx.gpars.dataflow.DataflowQueue
import groovyx.gpars.dataflow.DataflowReadChannel
import groovyx.gpars.dataflow.DataflowVariable
import groovyx.gpars.dataflow.DataflowWriteChannel
import groovyx.gpars.dataflow.operator.ControlMessage
import nextflow.dag.NodeMarker
import nextflow.datasource.SraExplorer
import nextflow.exception.AbortOperationException
import nextflow.extension.CH
import nextflow.extension.GroupTupleOp
import nextflow.extension.LinExtension
import nextflow.extension.MapOp
import nextflow.file.DirListener
import nextflow.file.DirWatcher
import nextflow.file.DirWatcherV2
import nextflow.file.FileHelper
import nextflow.file.FilePatternSplitter
import nextflow.file.PathVisitor
import nextflow.plugin.Plugins
import nextflow.plugin.extension.PluginExtensionProvider
import nextflow.util.Duration
import nextflow.util.TestOnly
import org.codehaus.groovy.runtime.InvokerHelper
import org.codehaus.groovy.runtime.NullObject
/**
* Channel factory object
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
class Channel {
static public ControlMessage STOP = CH.stop()
static public NullObject VOID = NullObject.getNullObject()
@TestOnly
private static CompletableFuture fromPath0Future
static private Session getSession() { Global.session as Session }
/**
* Allow the dynamic loading of plugin provided channel extension methods
*
* @param name The name of the method
* @param args The method arguments
* @return The method return value
*/
static def $static_methodMissing(String name, Object args) {
PluginExtensionProvider.INSTANCE().invokeFactoryExtensionMethod(name, InvokerHelper.asArray(args))
}
static Object[] array0(Object value) {
if( value instanceof Object[] )
return (Object[])value
if( value instanceof Collection )
return value.toArray()
else
return new Object[] {value}
}
static private checkNoChannels(String name, Object value) {
if( value==null )
return
Object[] items = array0(value)
if( items.size()==1 && CH.isChannel(items[0]))
throw new IllegalArgumentException("Argument of '$name' method cannot be a channel object — Likely you can replace the use of '$name' with the channel object itself")
for( int i=0; i<items.size(); i++ ) {
if( CH.isChannel(items[i]) )
throw new IllegalArgumentException("Argument ${nth(i+1)} of method '$name' cannot be a channel object")
}
}
static private String nth(int i) {
if( i==1 ) return '1-st'
if( i==2 ) return '2-nd'
if( i==3 ) return '3-rd'
return "$i-th"
}
static DataflowWriteChannel topic(String name) {
return CH.topic(name)
}
/**
* Create a empty channel i.e. only emits a STOP signal
*
* @return The channel instance
*/
static DataflowWriteChannel empty() {
final result = CH.emit(CH.queue(), STOP)
NodeMarker.addSourceNode('channel.empty', result)
return result
}
static DataflowWriteChannel of(Object ... items) {
checkNoChannels('channel.of', items)
final result = CH.create()
final values = new ArrayList()
if( items == null ) {
values.add(null)
}
else {
for( int i=0; i<items.size(); i++ )
addToList0(values, items[i])
}
values.add(STOP)
CH.emitValues(result, values)
NodeMarker.addSourceNode('channel.of', result)
return result
}
static private void addToList0(List list, obj) {
if( obj instanceof Range ) {
for( def x : obj ) {
list.add(x)
}
}
else {
list.add(obj)
}
}
/**
* Creates a channel sending the items in the collection over it
*
* @param items
* @return
*/
@Deprecated
static DataflowWriteChannel from( Collection items ) {
final result = from0(items)
NodeMarker.addSourceNode('channel.from', result)
return result
}
static private DataflowWriteChannel from0( Collection items ) {
final result = CH.create()
if( items != null )
CH.emitAndClose(result, items)
return result
}
static DataflowWriteChannel fromList( Collection items ) {
final result = CH.create()
CH.emitAndClose(result, items as List)
NodeMarker.addSourceNode('channel.fromList', result)
return result
}
/**
* Creates a channel sending the items in the collection over it
*
* @param items
* @return
*/
@Deprecated
static DataflowWriteChannel from( Object... items ) {
checkNoChannels('channel.from', items)
for( Object it : items ) if(CH.isChannel(it))
throw new IllegalArgumentException("channel.from argument is already a channel object")
final result = from0(items as List)
NodeMarker.addSourceNode('channel.from', result)
return result
}
static DataflowVariable value( obj = null ) {
checkNoChannels('channel.value', obj)
obj != null ? CH.value(obj) : CH.value()
}
/**
* Creates a channel emitting a sequence of integers spaced by a given time interval
*
* @param duration
* @return
*/
static DataflowWriteChannel interval(String duration) {
final result = interval0( duration, { index -> index })
NodeMarker.addSourceNode('channel.interval', result)
return result
}
/**
* Creates a channel emitting a sequence of value given by the closure and spaced by a given time interval.
*
* To stop the interval return the special value {@code #STOP}
*
* @param duration
* @return
*/
static DataflowWriteChannel interval(String duration, Closure closure ) {
final result = interval0(duration, closure)
NodeMarker.addSourceNode('channel.interval', result)
return result
}
static private DataflowWriteChannel interval0(String duration, Closure closure) {
def millis = Duration.of(duration).toMillis()
def timer = new Timer()
def result = CH.create()
long index = 0
def task = {
def value = closure.call(index++)
result << value
if( value == STOP ) {
timer.cancel()
}
}
session.addIgniter {
timer.schedule( task as TimerTask, 0, millis )
}
return result
}
/*
* valid parameters for fromPath operator
*/
static private Map VALID_FROM_PATH_PARAMS = [
type:['file','dir','any'],
followLinks: Boolean,
hidden: Boolean,
maxDepth: Integer,
checkIfExists: Boolean,
glob: Boolean,
relative: Boolean
]
/**
* Implements {@code Channel.fromPath} factory method
*
* @param opts
* A map object holding the optional method parameters
* @param pattern
* A file path or a glob pattern matching the required files.
* Multiple paths or patterns can be using a list object
* @return
* A channel emitting the matching files
*/
static DataflowWriteChannel<Path> fromPath( Map opts = null, pattern ) {
if( !pattern ) throw new AbortOperationException("Missing `fromPath` parameter")
checkNoChannels('channel.fromPath', pattern)
// verify that the 'type' parameter has a valid value
checkParams( 'path', opts, VALID_FROM_PATH_PARAMS )
final result = fromPath0(opts, pattern instanceof List ? pattern : [pattern])
NodeMarker.addSourceNode('channel.fromPath', result)
return result
}
private static DataflowWriteChannel<Path> fromPath0( Map opts, List allPatterns ) {
final result = CH.create()
session.addIgniter {
pumpFiles0(result, opts, allPatterns)
}
return result
}
private static void pumpFiles0(DataflowWriteChannel result, Map opts, List allPatterns) {
def future = CompletableFuture.completedFuture(null)
for( int index=0; index<allPatterns.size(); index++ ) {
def factory = new PathVisitor(target: result, opts: opts)
factory.closeChannelOnComplete = index==allPatterns.size()-1
future = factory.applyAsync(future, allPatterns[index])
}
// abort the execution when an exception is raised
fromPath0Future = future.exceptionally(Channel.&handleException)
}
static private void handleException(Throwable e) {
final error = e.cause ?: e
log.error(error.message, error)
session?.abort(error)
}
static private DataflowWriteChannel watchImpl( String syntax, String folder, String pattern, boolean skipHidden, String events, FileSystem fs ) {
final result = CH.create()
final legacy = System.getenv('NXF_DIRWATCHER_LEGACY')
final DirListener watcher = legacy=='true'
? new DirWatcher(syntax,folder,pattern,skipHidden,events,fs)
: new DirWatcherV2(syntax,folder,pattern,skipHidden,events,fs)
watcher.onComplete { result.bind(STOP) }
Global.onCleanup(it -> watcher.terminate())
session.addIgniter {
watcher.apply { Path file -> result.bind(file.toAbsolutePath()) }
}
return result
}
/**
* Watch the a folder for the specified events emitting the files that matches
* the specified regular expression.
*
*
* @param filePattern
* The file pattern to match e.g. /*.fasta/
*
* @param events
* The events to watch, a comma separated string of the following values:
* {@code create}, {@code modify}, {@code delete}
*
* @return A dataflow channel that will emit the matching files
*
*/
static DataflowWriteChannel watchPath( Pattern filePattern, String events = 'create' ) {
assert filePattern
// split the folder and the pattern
final splitter = FilePatternSplitter.regex().parse(filePattern.toString())
def fs = FileHelper.fileSystemForScheme(splitter.scheme)
def result = watchImpl( 'regex', splitter.parent, splitter.fileName, false, events, fs )
NodeMarker.addSourceNode('channel.watchPath', result)
return result
}
/**
* Watch the a folder for the specified events emitting the files that matches
* the specified {@code glob} pattern.
*
* @link http://docs.oracle.com/javase/tutorial/essential/io/fileOps.html#glob
*
* @param filePattern
* The file pattern to match e.g. /some/path/*.fasta
*
* @param events
* The events to watch, a comma separated string of the following values:
* {@code create}, {@code modify}, {@code delete}
*
* @return A dataflow channel that will emit the matching files
*
*/
static DataflowWriteChannel watchPath( String filePattern, String events = 'create' ) {
if( filePattern.endsWith('/') )
filePattern += '*'
else if(Files.isDirectory(Paths.get(filePattern)))
filePattern += '/*'
final splitter = FilePatternSplitter.glob().parse(filePattern)
final fs = FileHelper.fileSystemForScheme(splitter.scheme)
final folder = splitter.parent
final pattern = splitter.fileName
def result = watchImpl('glob', folder, pattern, pattern.startsWith('*'), events, fs)
NodeMarker.addSourceNode('channel.watchPath', result)
return result
}
static DataflowWriteChannel watchPath( Path path, String events = 'create' ) {
final fs = path.getFileSystem()
final splitter = FilePatternSplitter.glob().parse(path.toString())
final folder = splitter.parent
final pattern = splitter.fileName
final result = watchImpl('glob', folder, pattern, pattern.startsWith('*'), events, fs)
NodeMarker.addSourceNode('channel.watchPath', result)
return result
}
/**
* Implements the `fromFilePairs` channel factory method
*
* @param options
* A {@link Map} holding the optional parameters
* - type: either `file`, `dir` or `any`
* - followLinks: Boolean
* - hidden: Boolean
* - maxDepth: Integer
* - glob: Boolean
* - relative: Boolean
* @param pattern
* One or more path patterns eg. `/some/path/*_{1,2}.fastq`
* @return
* A channel emitting the file pairs matching the specified pattern(s)
*/
static DataflowWriteChannel fromFilePairs(Map options = null, pattern) {
checkNoChannels('channel.fromFilePairs', pattern)
final allPatterns = pattern instanceof List ? pattern : [pattern]
final allGrouping = new ArrayList(allPatterns.size())
for( int i=0; i<allPatterns.size(); i++ ) {
final template = allPatterns[i]
allGrouping[i] = { Path path -> readPrefix(path,template) }
}
final result = fromFilePairs0(options, allPatterns, allGrouping)
NodeMarker.addSourceNode('channel.fromFilePairs', result)
return result
}
/**
* Implements the `fromFilePairs` channel factory method
*
* @param options
* A {@link Map} holding the optional parameters
* - type: either `file`, `dir` or `any`
* - followLinks: Boolean
* - hidden: Boolean
* - maxDepth: Integer
* - glob: Boolean
* - relative: Boolean
* @param pattern
* One or more path patterns eg. `/some/path/*_{1,2}.fastq`
* @param grouping
* A closure implementing a pair grouping rule for the specified
* file patterns
* @return
* A channel emitting the file pairs matching the specified pattern(s)
*/
static DataflowWriteChannel fromFilePairs(Map options = null, pattern, Closure grouping) {
checkNoChannels('channel.fromFilePairs', pattern)
final allPatterns = pattern instanceof List ? pattern : [pattern]
final allGrouping = new ArrayList(allPatterns.size())
for( int i=0; i<allPatterns.size(); i++ ) {
allGrouping[i] = grouping
}
final result = fromFilePairs0(options, allPatterns, allGrouping)
NodeMarker.addSourceNode('channel.fromFilePairs', result)
return result
}
private static DataflowWriteChannel fromFilePairs0(Map options, List allPatterns, List<Closure> grouping) {
assert allPatterns.size() == grouping.size()
if( !allPatterns ) throw new AbortOperationException("Missing `fromFilePairs` parameter")
if( !grouping ) throw new AbortOperationException("Missing `fromFilePairs` grouping parameter")
// -- a channel from the path
final fromOpts = fetchParams0(VALID_FROM_PATH_PARAMS, options)
final files = new DataflowQueue()
session.addIgniter {
pumpFilePairs0(files, fromOpts, allPatterns)
}
// -- map the files to a tuple like ( ID, filePath )
final mapper = { path, int index ->
def prefix = grouping[index].call(path)
return [ prefix, path ]
}
final mapChannel = (DataflowReadChannel)new MapOp(files, mapper)
.setTarget(new DataflowQueue())
.apply()
boolean anyPattern=false
for( int index=0; index<allPatterns.size(); index++ ) {
anyPattern |= FilePatternSplitter.isMatchingPattern(allPatterns.get(index))
}
// -- result the files having the same ID
def DEF_SIZE = anyPattern ? 2 : 1
def size = (options?.size ?: DEF_SIZE)
def isFlat = options?.flat == true
def groupOpts = [sort: true, size: size]
def groupChannel = isFlat ? new DataflowQueue<>() : CH.create()
new GroupTupleOp(groupOpts, mapChannel)
.setTarget(groupChannel)
.apply()
// -- flat the group resulting tuples
DataflowWriteChannel result
if( isFlat ) {
def makeFlat = { id, List items ->
def tuple = new ArrayList(items.size()+1);
tuple.add(id)
tuple.addAll(items)
return tuple
}
result = new MapOp((DataflowReadChannel)groupChannel,makeFlat).apply()
}
else {
result = groupChannel
}
return result
}
static private void pumpFilePairs0(DataflowWriteChannel files, Map fromOpts, List allPatterns) {
def future = CompletableFuture.completedFuture(null)
for( int index=0; index<allPatterns.size(); index++ ) {
def factory = new PathVisitor(opts: fromOpts, target: files)
factory.bindPayload = index
factory.closeChannelOnComplete = index == allPatterns.size()-1
future = factory.applyAsync( future, allPatterns.get(index) )
}
// abort the execution when an exception is raised
fromPath0Future = future.exceptionally(Channel.&handleException)
}
static private Map fetchParams0(Map valid, Map actual ) {
if( actual==null ) return null
def result = [:]
def keys = valid.keySet()
keys.each {
if( actual.containsKey(it) ) result.put(it, actual.get(it))
}
return result
}
/*
* Helper function, given a file Path
* returns the file name region matching a specified glob pattern
* starting from the beginning of the name up to last matching group.
*
* For example:
* readPrefix('/some/data/file_alpha_1.fa', 'file*_1.fa' )
*
* Returns:
* 'file_alpha'
*/
@PackageScope
static String readPrefix( Path actual, template ) {
final fileName = actual.getFileName().toString()
def filePattern = template.toString()
int p = filePattern.lastIndexOf('/')
if( p != -1 )
filePattern = filePattern.substring(p+1)
final indexOfWildcards = filePattern.findIndexOf { it=='*' || it=='?' }
final indexOfBrackets = filePattern.findIndexOf { it=='{' || it=='[' }
if( indexOfWildcards==-1 && indexOfBrackets==-1 ) {
// when the pattern does not contain any glob characters
// then it can only have been used only to filter files with exactly
// that name, therefore if the name is matching return it (without suffix)
if( fileName == filePattern )
return actual.getSimpleName()
throw new IllegalArgumentException("Not a valid file pair globbing pattern: pattern=$filePattern file=$fileName")
}
// count the `*` and `?` wildcard before any {} and [] glob pattern
int groupCount = 0
for( int i=0; i<filePattern.size(); i++ ) {
def ch = filePattern[i]
if( ch=='?' || ch=='*' )
groupCount++
else if( ch=='{' || ch=='[' )
break
}
def regex = filePattern
.replace('.','\\.')
.replace('*','(.*)')
.replace('?','(.?)')
.replace('{','(?:')
.replace('}',')')
.replace(',','|')
def matcher = (fileName =~ /$regex/)
if( matcher.matches() ) {
def c=Math.min(groupCount, matcher.groupCount())
def end = c ? matcher.end(c) : ( indexOfBrackets != -1 ? indexOfBrackets : fileName.size() )
def prefix = fileName.substring(0,end)
while(prefix.endsWith('-') || prefix.endsWith('_') || prefix.endsWith('.') )
prefix=prefix[0..-2]
return prefix
}
return null
}
static DataflowWriteChannel fromSRA(query) {
checkNoChannels('channel.fromSRA', query)
fromSRA( Collections.emptyMap(), query )
}
static DataflowWriteChannel fromSRA(Map opts, query) {
checkParams('fromSRA', opts, SraExplorer.PARAMS)
checkNoChannels('channel.fromSRA', query)
def target = new DataflowQueue()
def explorer = new SraExplorer(target, opts).setQuery(query)
session.addIgniter {
fetchSraFiles0(explorer)
}
NodeMarker.addSourceNode('channel.fromSRA', target)
return target
}
static private void fetchSraFiles0(SraExplorer explorer) {
def future = CompletableFuture.runAsync(() -> {
explorer.apply()
})
fromPath0Future = future.exceptionally(Channel.&handleException)
}
static DataflowWriteChannel fromLineage(Map<String,?> params) {
checkParams('fromLineage', params, LinExtension.PARAMS)
final result = CH.create()
session.addIgniter {
fromLineage0(result, params)
}
return result
}
private static void fromLineage0(DataflowWriteChannel channel, Map<String,?> params) {
final linExt = Plugins.getExtension(LinExtension)
if( !linExt )
throw new IllegalStateException("Unable to load lineage extensions.")
final session = Global.session as Session
final future = CompletableFuture.runAsync(() -> {
linExt.fromLineage(session, channel, params)
})
future.exceptionally(Channel.&handleException)
}
}

View File

@@ -0,0 +1,68 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow
import groovy.runtime.metaclass.NextflowDelegatingMetaClass
import nextflow.extension.CH
import nextflow.plugin.extension.PluginExtensionProvider
import nextflow.script.ExecutionStack
import nextflow.script.WorkflowBinding
/**
* Helper class to shortcut common api
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
class NF {
static String getSyntaxParserVersion() {
return SysEnv.get('NXF_SYNTAX_PARSER', 'v2')
}
static boolean isSyntaxParserV2() {
return getSyntaxParserVersion() == 'v2'
}
static void init() {
NextflowDelegatingMetaClass.provider = PluginExtensionProvider.INSTANCE()
CH.init()
WorkflowBinding.init()
}
static boolean hasOperator(String name) {
NextflowDelegatingMetaClass.provider.operatorNames().contains(name)
}
static Binding getBinding() {
ExecutionStack.binding()
}
static String lookupVariable(value) {
return WorkflowBinding.lookup(value)
}
static boolean isStrictMode() {
NextflowMeta.instance.isStrictModeEnabled()
}
static boolean isModuleBinariesEnabled() {
NextflowMeta.instance.isModuleBinariesEnabled()
}
static boolean isRecurseEnabled() {
NextflowMeta.instance.preview.recursion
}
}

View File

@@ -0,0 +1,433 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow
import static nextflow.file.FileHelper.*
import java.nio.file.FileSystem
import java.nio.file.Files
import java.nio.file.NoSuchFileException
import java.nio.file.Path
import groovyx.gpars.dataflow.DataflowReadChannel
import nextflow.exception.StopSplitIterationException
import nextflow.exception.WorkflowScriptErrorException
import nextflow.extension.GroupKey
import nextflow.extension.OperatorImpl
import nextflow.file.FileHelper
import nextflow.file.FilePatternSplitter
import nextflow.mail.Mailer
import nextflow.script.TokenBranchDef
import nextflow.script.TokenMultiMapDef
import nextflow.splitter.FastaSplitter
import nextflow.splitter.FastqSplitter
import nextflow.util.ArrayTuple
import nextflow.util.CacheHelper
import nextflow.util.RecordMap
import org.slf4j.Logger
import org.slf4j.LoggerFactory
/**
* Defines the main methods imported by default in the script scope
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
class Nextflow {
static private Session getSession() { Global.session as Session }
// note: groovy `Slf4j` annotation causes a bizarre issue
// https://issues.apache.org/jira/browse/GROOVY-7371
// declare public so that can be accessed from the user script
public static final Logger log = LoggerFactory.getLogger(Nextflow)
private static final Random random = new Random()
/**
* Get the value of an environment variable from the launch environment.
*
* @param name
* The environment variable name to be referenced
* @return
* The value associate with the specified variable name or {@code null} if the variable does not exist.
*/
static String env(String name) {
return SysEnv.get(name)
}
private static List<Path> fileNamePattern( FilePatternSplitter splitter, Map opts ) {
final scheme = splitter.scheme
final target = scheme ? "$scheme://$splitter.parent" : splitter.parent
final folder = toCanonicalPath(target)
final pattern = splitter.fileName
if( opts == null ) opts = [:]
if( !opts.type ) opts.type = 'file'
def result = new LinkedList<Path>()
try {
FileHelper.visitFiles(opts, folder, pattern) { Path it -> result.add(it) }
}
catch (NoSuchFileException e) {
log.debug "No such file or directory: $folder -- Skipping visit"
}
return result
}
static private String str0(value) {
if( value==null )
return null
if( value instanceof CharSequence )
return value.toString()
if( value instanceof File )
return value.toString()
if( value instanceof Path )
return value.toUriString()
throw new IllegalArgumentException("Invalid file path type - offending value: $value [${value.getClass().getName()}]")
}
/**
* Get one or more file object given the specified path or glob pattern.
*
* @param options
* A map holding one or more of the following options:
* - type: Type of paths returned, either `file`, `dir` or `any` (default: `file`)
* - hidden: When `true` includes hidden files in the resulting paths (default: `false`)
* - maxDepth: Maximum number of directory levels to visit (default: no limit)
* - followLinks: When `true` it follows symbolic links during directories tree traversal, otherwise they are managed as files (default: `true`)
*
* @param path A file path eventually including a glob pattern e.g. /some/path/file*.txt
* @return An instance of {@link Path} when a single file is matched or a list of {@link Path}s
*/
static file(Map options = null, def filePattern) {
if( !filePattern )
throw new IllegalArgumentException("Argument of `file()` function cannot be ${filePattern==null?'null':'empty'}")
final result = file0(options, filePattern)
if( result instanceof Collection && result.size() != 1 && NF.isSyntaxParserV2() )
log.warn "The `file()` function was called with a glob pattern that matched a collection of files -- use `files()` instead."
return result
}
private static file0( Map options = null, def filePattern ) {
final path = filePattern as Path
final glob = options?.containsKey('glob') ? options.glob as boolean : isGlobAllowed(path)
if( !glob ) {
return checkIfExists(path, options)
}
// if it isn't a glob pattern simply return it a normalized absolute Path object
final strPattern = str0(filePattern)
final splitter = FilePatternSplitter.glob().parse(strPattern)
if( !splitter.isPattern() ) {
final normalised = splitter.strip(strPattern)
return checkIfExists(asPath(normalised), options)
}
// revolve the glob pattern returning all matches
final result = fileNamePattern(splitter, options)
if( glob && options?.checkIfExists && result.isEmpty() )
throw new NoSuchFileException(path.toString())
return result
}
static Collection<Path> files(Map options=null, def filePattern) {
if( !filePattern )
throw new IllegalArgumentException("Argument of `files()` function cannot be ${filePattern==null?'null':'empty'}")
final result = file0(options, filePattern)
return result instanceof Collection ? result : [result]
}
/**
* Creates a {@link RecordMap} from the given named arguments.
*
* @param props
*/
static RecordMap record(Map<String,?> props) {
return new RecordMap(props)
}
/**
* Creates a {@link ArrayTuple} object with the given open array items
*
* @param args The items used to created the tuple object
* @return An instance of {@link ArrayTuple} populated with the given argument(s)
*/
static ArrayTuple tuple( def value ) {
if( !value )
return new ArrayTuple()
new ArrayTuple( value instanceof Collection ? (Collection)value : [value] )
}
/**
* Creates a {@link ArrayTuple} object with the given open array items
*
* @param args The items used to created the tuple object
* @return An instance of {@link ArrayTuple} populated with the given argument(s)
*/
static ArrayTuple tuple( Object ... args ) {
tuple( args as List )
}
/**
* Creates a FASTQ splitter handler for the given object
*
* @param obj The object to be managed as a FASTQ
* @return An instance of {@link FastqSplitter
*/
@Deprecated
static FastqSplitter fastq( obj ) {
(FastqSplitter)new FastqSplitter('fastq').target(obj)
}
/**
* Creates a FASTA splitter handler for the given object
*
* @param obj The object to be managed as a FASTA
* @return An instance of {@link FastqSplitter
*/
@Deprecated
static FastaSplitter fasta( obj ) {
(FastaSplitter)new FastaSplitter('fasta').target(obj)
}
/**
* Interrupts the iteration when using a split operators
*/
static void stop() {
throw new StopSplitIterationException()
}
/**
* Stop the current execution returning an error code and message
*
* @param exitCode The exit code to be returned
* @param message The message that will be reported in the log file (optional)
*/
@Deprecated
static void exit(int exitCode, String message = null) {
if( session.aborted ) {
log.debug "Ignoring exit because execution is already aborted -- message=$message"
return
}
if ( exitCode && message ) {
log.error message
}
else if ( message ) {
log.info message
}
System.exit(exitCode)
}
/**
* Stop the current execution returning a 0 error code and the specified message
*
* @param message The message that will be reported in the log file
*/
@Deprecated
static void exit( String message ) {
exit(0, message)
}
/**
* Throws a script runtime error
* @param message An optional error message
*/
static void error( String message = null ) {
throw message ? new WorkflowScriptErrorException(message) : new WorkflowScriptErrorException()
}
/**
* Create a folder for the given key. It guarantees to return the same folder name
* the same provided object key.
*
* @param key An object to be used as cache-key creating the folder, it can be any object
* or an array or objects to use multi-objects key
*
* @return The {@code Path} to the cached directory or a newly created folder for the specified key
*/
@Deprecated
static Path cacheableDir( Object key ) {
assert key, "Please specify the 'key' argument on 'cacheableDir' method"
final session = (Session)Global.session
if( !session )
throw new IllegalStateException("Invalid access to `cacheableDir` method -- Session object not yet defined")
def hash = CacheHelper.hasher([ session.uniqueId, key, session.cacheable ? 0 : random.nextInt() ]).hash()
def file = FileHelper.getWorkFolder(session.workDir, hash)
if( !file.exists() && !file.mkdirs() ) {
throw new IOException("Unable to create directory: $file -- Check file system permissions" )
}
return file
}
/**
* Create a file for the given key. It guarantees to return the same file name
* the same provided object key.
*
* @param key
* @param name
* @return
*/
@Deprecated
static Path cacheableFile( Object key, String name = null ) {
// the cacheability is guaranteed by the folder
def folder = cacheableDir(key)
if( !name ) {
if( key instanceof File ) {
name = key.getName()
}
else if( key instanceof Path ) {
name = key.getName()
}
else {
name = key.toString()
}
}
return folder.resolve(name)
}
/**
* This method is exposed as a public API to script, it should be removed
*
* @return Create a temporary directory
*/
@Deprecated
static Path tempDir( String name = null, boolean create = true ) {
final session = (ISession)Global.session
if( !session )
throw new IllegalStateException("Invalid access to `tempDir` method -- Session object not yet defined")
def path = FileHelper.createTempFolder(session.workDir)
if( name )
path = path.resolve(name)
if( !path.exists() && create && !path.mkdirs() )
throw new IOException("Unable to create directory: $path -- Check file system permissions" )
return path
}
/**
* This method is exposed as a public API to script, it should be removed
*
* @return Create a temporary file
*/
@Deprecated
static Path tempFile( String name = null, boolean create = false ) {
if( !name )
name = 'file.tmp'
def folder = tempDir()
def result = folder.resolve(name)
if( create )
Files.createFile(result)
return result
}
/**
* Implements built-in send mail functionality
*
* @param params
* A map object holding the mail parameters
* - to: String or a List of strings representing the mail recipients
* - cc: String or a List of strings representing the mail recipients in carbon copy
* - from: String representing the mail sender address
* - subject: The mail subject
* - content: The mail content
* - attach: One or more list attachment
*/
static void sendMail( Map params ) {
final opts = Global.session.config.mail as Map ?: Collections.emptyMap()
new Mailer(opts).send(params)
}
/**
* Implements built-in send mail functionality
*
* @param params
* A closure representing the mail message to send eg
* <code>
* sendMail {
* to 'me@dot.com'
* from 'your@name.com'
* attach '/some/file/path'
* subject 'Hello'
* content '''
* Hi there,
* Hope this email find you well
* '''
* }
* <code>
*/
static void sendMail( Closure params ) {
final opts = Global.session.config.mail as Map ?: Collections.emptyMap()
new Mailer(opts).send(params)
}
/**
* Creates a groupTuple dynamic key
*
* @param key
* @param size
* @return
*/
static GroupKey groupKey(key, int size) {
new GroupKey(key,size)
}
/**
* Marker method to create a closure to be passed to {@link OperatorImpl#branch(DataflowReadChannel, groovy.lang.Closure)}
* operator.
*
* Despite apparently is doing nothing, this method is needed as marker to apply the {@link OpXform} AST
* transformation required to interpret the closure content as required for the branch evaluation.
*
* @see OperatorImpl#branch(DataflowReadChannel, Closure)
* @see OpXformImpl
*
* @param closure
* @return
*/
static Closure<TokenBranchDef> branchCriteria(Closure<TokenBranchDef> closure) { closure }
/**
* Marker method to create a closure to be passed to {@link OperatorImpl#fork(DataflowReadChannel, Closure)}
* operator.
*
* Despite apparently is doing nothing, this method is needed as marker to apply the {@link OpXform} AST
* transformation required to interpret the closure content as required for the branch evaluation.
*
* @see OperatorImpl#multiMap(groovyx.gpars.dataflow.DataflowReadChannel, groovy.lang.Closure) (DataflowReadChannel, Closure)
* @see OpXformImpl
*
* @param closure
* @return
*/
static Closure<TokenMultiMapDef> multiMapCriteria(Closure<TokenBranchDef> closure) { closure }
}

View File

@@ -0,0 +1,130 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow
import static nextflow.extension.Bolts.*
import java.text.SimpleDateFormat
import groovy.transform.CompileStatic
import groovy.transform.EqualsAndHashCode
import groovy.transform.ToString
import groovy.util.logging.Slf4j
import nextflow.exception.AbortOperationException
import nextflow.util.VersionNumber
/**
* Models nextflow script properties and metadata
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
@Singleton(strict = false)
@ToString(includeNames = true)
@EqualsAndHashCode
class NextflowMeta {
static trait Flags {
abstract float dsl
abstract boolean strict
abstract boolean moduleBinaries
}
@Slf4j
static class Preview implements Flags {
@Deprecated volatile float dsl
@Deprecated boolean strict
boolean recursion
boolean moduleBinaries
void setRecursion(Boolean recursion) {
if( recursion )
log.warn "NEXTFLOW RECURSION IS A PREVIEW FEATURE - SYNTAX AND FUNCTIONALITY CAN CHANGE IN FUTURE RELEASES"
this.recursion = recursion
}
}
static class Features implements Flags {
volatile float dsl
boolean strict
boolean moduleBinaries
}
final VersionNumber version
final int build
/*
* Timestamp as dd-MM-yyyy HH:mm UTC formatted string
*/
final String timestamp
final Preview preview = new Preview()
final Features enable = new Features()
private NextflowMeta() {
version = new VersionNumber(BuildInfo.version)
build = BuildInfo.buildNum as int
timestamp = BuildInfo.timestampUTC
}
protected NextflowMeta(String ver, int build, String timestamp ) {
this.version = new VersionNumber(ver)
this.build = build
this.timestamp = timestamp
}
Map featuresMap() {
final result = new LinkedHashMap()
if( isStrictModeEnabled() )
result.strict = true
if( isModuleBinariesEnabled() )
result.moduleBinaries = true
return result
}
Map toJsonMap() {
final result = new LinkedHashMap<>(5)
result.version = version.toString()
result.build = build
result.timestamp = parseDateStr(timestamp)
result.enable = featuresMap()
return result
}
private Date parseDateStr(String str) {
def fmt = new SimpleDateFormat(DATETIME_FORMAT + ' Z')
fmt.parse(str)
}
boolean isStrictModeEnabled() {
return enable.strict
}
void strictMode(boolean mode) {
enable.strict = mode
}
boolean isModuleBinariesEnabled() {
return enable.moduleBinaries
}
void moduleBinaries(boolean mode) {
enable.moduleBinaries = mode
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,152 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.ast
import groovy.transform.CompileStatic
import org.codehaus.groovy.ast.ASTNode
import org.codehaus.groovy.ast.ClassNode
import org.codehaus.groovy.ast.expr.ArgumentListExpression
import org.codehaus.groovy.ast.expr.BinaryExpression
import org.codehaus.groovy.ast.expr.CastExpression
import org.codehaus.groovy.ast.expr.ClosureExpression
import org.codehaus.groovy.ast.expr.ConstantExpression
import org.codehaus.groovy.ast.expr.ConstructorCallExpression
import org.codehaus.groovy.ast.expr.DeclarationExpression
import org.codehaus.groovy.ast.expr.Expression
import org.codehaus.groovy.ast.expr.MapExpression
import org.codehaus.groovy.ast.expr.MethodCallExpression
import org.codehaus.groovy.ast.expr.TupleExpression
import org.codehaus.groovy.ast.expr.VariableExpression
import org.codehaus.groovy.ast.stmt.BlockStatement
import org.codehaus.groovy.ast.stmt.ExpressionStatement
import org.codehaus.groovy.ast.stmt.ReturnStatement
import org.codehaus.groovy.ast.stmt.Statement
import org.codehaus.groovy.ast.tools.GeneralUtils
import org.codehaus.groovy.control.SourceUnit
import org.codehaus.groovy.syntax.SyntaxException
import org.codehaus.groovy.syntax.Types
/**
* Helper methods to handle AST xforms
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class ASTHelpers {
static void syntaxError(String message, ASTNode node, SourceUnit unit) {
int line = node.lineNumber
int coln = node.columnNumber
final ex = new SyntaxException(message,line,coln)
if( unit )
unit.addError(ex)
else
throw ex
}
/**
* Creates a new {@code ConstructorCallExpression} for the specified class and arguments
*
* @param clazz The {@code Class} for which the create a constructor call expression
* @param args The arguments to be passed to the constructor
* @return The instance for the constructor call
*/
static Expression createX(Class clazz, TupleExpression args ) {
def type = new ClassNode(clazz)
return new ConstructorCallExpression(type,args)
}
/**
* Creates a new {@code ConstructorCallExpression} for the specified class and arguments
* specified using an open array.
*
* @param clazz The {@code Class} for which the create a constructor call expression
* @param args The arguments to be passed to the constructor, they will be wrapped by in a {@code ArgumentListExpression}
* @return The instance for the constructor call
*/
static Expression createX(Class clazz, Expression... params ) {
def type = new ClassNode(clazz)
def args = new ArgumentListExpression(params as List<Expression>)
return new ConstructorCallExpression(type,args)
}
static Expression createX(Class clazz, List<Expression> params ) {
def type = new ClassNode(clazz)
def args = new ArgumentListExpression(params as List<Expression>)
return new ConstructorCallExpression(type,args)
}
static Expression declX(Expression left, Expression right) {
new DeclarationExpression(left, GeneralUtils.ASSIGN, right)
}
static MethodCallExpression isMethodCallX(Expression expr) {
return expr instanceof MethodCallExpression ? expr : null
}
static VariableExpression isVariableX(Expression expr) {
return expr instanceof VariableExpression ? expr : null
}
static CastExpression isCastX(Expression expr) {
return expr instanceof CastExpression ? expr : null
}
static VariableExpression isThisX(Expression expr) {
isVariableX(expr)?.name == 'this' ? (VariableExpression)expr : null
}
static BinaryExpression isBinaryX(Expression expr) {
expr instanceof BinaryExpression ? expr : null
}
static ArgumentListExpression isArgsX(Expression expr) {
expr instanceof ArgumentListExpression ? expr : null
}
static TupleExpression isTupleX(Expression expr) {
expr instanceof TupleExpression ? expr : null
}
static MapExpression isMapX(Expression exp) {
exp instanceof MapExpression ? exp : null
}
static ConstantExpression isConstX(Expression exp) {
exp instanceof ConstantExpression ? exp : null
}
static BinaryExpression isAssignX(Expression expr) {
isBinaryX(expr)?.operation?.type == Types.ASSIGN ? (BinaryExpression)expr : null
}
static ClosureExpression isClosureX(Expression exp) {
exp instanceof ClosureExpression ? exp : null
}
static BlockStatement isBlockStmt(Statement stm) {
stm instanceof BlockStatement ? stm : null
}
static ExpressionStatement isStmtX(Statement stm) {
stm instanceof ExpressionStatement ? stm : null
}
static ReturnStatement isReturnS(Statement stmt) {
stmt instanceof ReturnStatement ? stmt : null
}
}

View File

@@ -0,0 +1,32 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.ast
import java.lang.annotation.ElementType
import java.lang.annotation.Retention
import java.lang.annotation.RetentionPolicy
import java.lang.annotation.Target
import org.codehaus.groovy.transform.GroovyASTTransformationClass
/**
* Marker interface which to apply AST transformation to {@code process} declaration
*/
@Retention(RetentionPolicy.SOURCE)
@Target(ElementType.METHOD)
@GroovyASTTransformationClass(classes = [NextflowDSLImpl])
@interface NextflowDSL {}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,53 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.ast
import java.lang.annotation.ElementType
import java.lang.annotation.Retention
import java.lang.annotation.RetentionPolicy
import java.lang.annotation.Target
import groovy.transform.CompileStatic
import nextflow.script.control.PathCompareVisitor
import org.codehaus.groovy.ast.ASTNode
import org.codehaus.groovy.ast.ClassNode
import org.codehaus.groovy.control.CompilePhase
import org.codehaus.groovy.control.SourceUnit
import org.codehaus.groovy.transform.ASTTransformation
import org.codehaus.groovy.transform.GroovyASTTransformation
import org.codehaus.groovy.transform.GroovyASTTransformationClass
/**
* Replace path comparisons with explicit method calls.
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Retention(RetentionPolicy.SOURCE)
@Target(ElementType.METHOD)
@GroovyASTTransformationClass(classes = [NextflowXformImpl])
@interface NextflowXform {
@CompileStatic
@GroovyASTTransformation(phase = CompilePhase.CONVERSION)
class NextflowXformImpl implements ASTTransformation {
@Override
void visit(ASTNode[] nodes, SourceUnit source) {
new PathCompareVisitor(source).visitClass((ClassNode)nodes[1])
}
}
}

View File

@@ -0,0 +1,55 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.ast
import java.lang.annotation.ElementType
import java.lang.annotation.Retention
import java.lang.annotation.RetentionPolicy
import java.lang.annotation.Target
import nextflow.script.control.OpCriteriaVisitor
import groovy.transform.CompileStatic
import org.codehaus.groovy.ast.ASTNode
import org.codehaus.groovy.ast.ClassNode
import org.codehaus.groovy.control.CompilePhase
import org.codehaus.groovy.control.SourceUnit
import org.codehaus.groovy.transform.ASTTransformation
import org.codehaus.groovy.transform.GroovyASTTransformation
import org.codehaus.groovy.transform.GroovyASTTransformationClass
/**
* Transform closure arguments for branch and multiMap
* into the appropriate criteria objects.
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Retention(RetentionPolicy.SOURCE)
@Target(ElementType.METHOD)
@GroovyASTTransformationClass(classes = [OpXformImpl])
@interface OpXform {
@CompileStatic
@GroovyASTTransformation(phase = CompilePhase.CONVERSION)
class OpXformImpl implements ASTTransformation {
@Override
void visit(ASTNode[] nodes, SourceUnit source) {
new OpCriteriaVisitor(source).visitClass((ClassNode)nodes[1])
}
}
}

View File

@@ -0,0 +1,57 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.ast
import java.lang.annotation.ElementType
import java.lang.annotation.Retention
import java.lang.annotation.RetentionPolicy
import java.lang.annotation.Target
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.script.control.TaskCmdXformVisitor
import org.codehaus.groovy.ast.ASTNode
import org.codehaus.groovy.ast.ClassNode
import org.codehaus.groovy.control.CompilePhase
import org.codehaus.groovy.control.SourceUnit
import org.codehaus.groovy.transform.ASTTransformation
import org.codehaus.groovy.transform.GroovyASTTransformation
import org.codehaus.groovy.transform.GroovyASTTransformationClass
/**
* Declares an AST xform to escape and manipulate task command
* special value
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Retention(RetentionPolicy.SOURCE)
@Target(ElementType.METHOD)
@GroovyASTTransformationClass(classes = [TaskCmdXformImpl])
@interface TaskCmdXform {
//--- === implementation === ---
@Slf4j
@CompileStatic
@GroovyASTTransformation(phase = CompilePhase.CONVERSION)
class TaskCmdXformImpl implements ASTTransformation {
@Override
void visit(ASTNode[] nodes, SourceUnit source) {
new TaskCmdXformVisitor(source).visitClass((ClassNode)nodes[1])
}
}
}

View File

@@ -0,0 +1,52 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.ast
import java.lang.annotation.ElementType
import java.lang.annotation.Retention
import java.lang.annotation.RetentionPolicy
import java.lang.annotation.Target
import groovy.transform.CompileStatic
import org.codehaus.groovy.ast.ASTNode
import org.codehaus.groovy.ast.ClassNode
import org.codehaus.groovy.control.CompilePhase
import org.codehaus.groovy.control.SourceUnit
import org.codehaus.groovy.transform.ASTTransformation
import org.codehaus.groovy.transform.GroovyASTTransformation
import org.codehaus.groovy.transform.GroovyASTTransformationClass
/**
* Capture variables that are declared in a task template.
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Retention(RetentionPolicy.SOURCE)
@Target(ElementType.METHOD)
@GroovyASTTransformationClass(classes = [TaskTemplateVarsXformImpl])
@interface TaskTemplateVarsXform {
@CompileStatic
@GroovyASTTransformation(phase = CompilePhase.CONVERSION)
class TaskTemplateVarsXformImpl implements ASTTransformation {
@Override
void visit(ASTNode[] nodes, SourceUnit source) {
new TaskTemplateVisitor(source).visitClass((ClassNode)nodes[1])
}
}
}

View File

@@ -0,0 +1,108 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.ast
import groovy.transform.CompileStatic
import org.codehaus.groovy.ast.ClassNode
import org.codehaus.groovy.ast.ConstructorNode
import org.codehaus.groovy.ast.expr.ArgumentListExpression
import org.codehaus.groovy.ast.expr.ConstantExpression
import org.codehaus.groovy.ast.expr.GStringExpression
import org.codehaus.groovy.ast.expr.ListExpression
import org.codehaus.groovy.ast.expr.MethodCallExpression
import org.codehaus.groovy.ast.expr.VariableExpression
import org.codehaus.groovy.ast.stmt.BlockStatement
import org.codehaus.groovy.ast.stmt.ExpressionStatement
import org.codehaus.groovy.ast.stmt.Statement
import org.codehaus.groovy.control.SourceUnit
/**
* Inject string var names in the script binding object
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class TaskTemplateVisitor extends VariableVisitor {
int withinGString
TaskTemplateVisitor(SourceUnit unit) {
super(unit)
}
@Override
protected boolean scopeEnabled() {
return withinGString>0
}
@Override
void visitGStringExpression(GStringExpression expression) {
try {
withinGString++
super.visitGStringExpression(expression)
}
finally {
withinGString--
}
}
@Override
protected void visitObjectInitializerStatements(ClassNode node) {
injectMetadata(node)
super.visitObjectInitializerStatements(node)
}
protected void injectMetadata(ClassNode node) {
// inject the invocation of a custom method
// as last statement of the constructor invocation
for( ConstructorNode constructor : node.getDeclaredConstructors() ) {
def code = constructor.getCode()
if( code instanceof BlockStatement ) {
code.addStatement(makeSetVariableNamesStm())
}
else if( code instanceof ExpressionStatement ) {
def expr = code
def block = new BlockStatement()
block.addStatement(expr)
block.addStatement(makeSetVariableNamesStm())
constructor.setCode(block)
}
else
throw new IllegalStateException("Invalid constructor expression: $code")
}
}
protected Statement makeSetVariableNamesStm() {
final names = new ListExpression()
for( String it: getAllVariableNames() ) {
names.addExpression(new ConstantExpression(it))
}
final varArgs = new ArgumentListExpression()
varArgs.addExpression( new ConstantExpression('__$$_template_vars') )
varArgs.addExpression( names )
// some magic code
// this generates the invocation of the method:
// this.getBinding().setVariable('__$$_template_vars', [variable names])
final thiz = new VariableExpression('this')
final bind = new MethodCallExpression( thiz, 'getBinding', ArgumentListExpression.EMPTY_ARGUMENTS )
final call = new MethodCallExpression( bind, 'setVariable', varArgs )
final stm = new ExpressionStatement(call)
return stm
}
}

View File

@@ -0,0 +1,144 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.ast
import groovy.transform.CompileStatic
import nextflow.script.TokenValRef
import org.codehaus.groovy.ast.ClassCodeVisitorSupport
import org.codehaus.groovy.ast.expr.ClosureExpression
import org.codehaus.groovy.ast.expr.ConstantExpression
import org.codehaus.groovy.ast.expr.DeclarationExpression
import org.codehaus.groovy.ast.expr.PropertyExpression
import org.codehaus.groovy.ast.expr.VariableExpression
import org.codehaus.groovy.control.SourceUnit
import org.codehaus.groovy.syntax.SyntaxException
/**
* Visit a closure and collect all referenced variable names
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class VariableVisitor extends ClassCodeVisitorSupport {
final Map<String, TokenValRef> fAllVariables = [:]
final Set<String> localDef = []
final SourceUnit sourceUnit
private boolean declaration
private int deep
VariableVisitor( SourceUnit unit ) {
this.sourceUnit = unit
}
protected boolean scopeEnabled() { true }
protected boolean isNormalized(PropertyExpression expr) {
if( !(expr.getProperty() instanceof ConstantExpression) )
return false
def target = expr.getObjectExpression()
while( target instanceof PropertyExpression) {
target = (target as PropertyExpression).getObjectExpression()
}
return target instanceof VariableExpression
}
@Override
void visitClosureExpression(ClosureExpression expression) {
if( deep++ == 0 )
super.visitClosureExpression(expression)
}
@Override
void visitDeclarationExpression(DeclarationExpression expr) {
declaration = true
try {
visit(expr.getLeftExpression())
}
finally {
declaration = false
}
visit(expr.getRightExpression())
}
@Override
void visitPropertyExpression(PropertyExpression expr) {
if( isNormalized(expr)) {
final name = expr.text.replace('?','')
final line = expr.lineNumber
final coln = expr.columnNumber
if( !name.startsWith('this.') && !fAllVariables.containsKey(name) && scopeEnabled() ) {
fAllVariables[name] = new TokenValRef(name,line,coln)
}
}
else
super.visitPropertyExpression(expr)
}
@Override
void visitVariableExpression(VariableExpression var) {
final name = var.name
final line = var.lineNumber
final coln = var.columnNumber
if( name == 'this' )
return
if( declaration ) {
if( fAllVariables.containsKey(name) )
sourceUnit.addError( new SyntaxException("Variable `$name` already declared in the process scope", line, coln))
else
localDef.add(name)
}
// Note: variable declared in the process scope are not added
// to the set of referenced variables. Only global ones are tracked
else if( !localDef.contains(name) && !fAllVariables.containsKey(name) && scopeEnabled() ) {
fAllVariables[name] = new TokenValRef(name,line,coln)
}
}
@Override
protected SourceUnit getSourceUnit() {
return sourceUnit
}
/**
* @return The set of all variables referenced in the script.
* NOTE: it includes properties in the form {@code object.propertyName}
*/
Set<TokenValRef> getAllVariables() {
new HashSet<TokenValRef>(fAllVariables.values())
}
Set<String> getAllVariableNames() {
def all = getAllVariables()
def result = new HashSet(all.size())
for( TokenValRef ref : all )
result.add(ref.name)
return result
}
}

View File

@@ -0,0 +1,268 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cache
import com.google.common.hash.HashCode
import groovy.transform.CompileStatic
import groovy.transform.PackageScope
import groovy.transform.stc.ClosureParams
import groovy.transform.stc.SimpleType
import groovy.util.logging.Slf4j
import groovyx.gpars.agent.Agent
import nextflow.processor.TaskContext
import nextflow.processor.TaskEntry
import nextflow.processor.TaskHandler
import nextflow.processor.TaskProcessor
import nextflow.trace.TraceRecord
import nextflow.util.KryoHelper
/**
* Manages nextflow cache DB
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
class CacheDB implements Closeable {
/** An agent used to apply asynchronously DB write operations */
private Agent writer
private CacheStore store
CacheDB(CacheStore store) {
this.store = store
this.writer = new Agent()
}
/**
* Initialise the database structure on the underlying file system
*
* @return The {@link CacheDB} instance itself
*/
CacheDB open() {
store.open()
return this
}
/**
* Open the database in read mode
*
* @return The {@link CacheDB} instance itself
*/
CacheDB openForRead() {
store.openForRead()
return this
}
/**
* Retrieve a task runtime information from the cache DB
*
* @param taskHash The {@link HashCode} of the task to retrieve
* @param processor The {@link TaskProcessor} instance to be assigned to the retrieved task
* @return A {link TaskEntry} instance or {@code null} if a task for the given hash does not exist
*/
TaskEntry getTaskEntry(HashCode taskHash, TaskProcessor processor) {
final payload = store.getEntry(taskHash)
if( !payload )
return null
final record = (List)KryoHelper.deserialize(payload)
TraceRecord trace = TraceRecord.deserialize( (byte[])record[0] )
TaskContext ctx = record[1]!=null && processor!=null ? TaskContext.deserialize(processor, (byte[])record[1]) : null
return new TaskEntry(trace,ctx)
}
void incTaskEntry( HashCode hash ) {
final payload = store.getEntry(hash)
if( !payload ) {
log.debug "Can't increment reference for cached task with key: $hash"
return
}
final record = (List)KryoHelper.deserialize(payload)
// third record contains the reference count for this record
record[2] = ((Integer)record[2]) +1
// save it again
store.putEntry(hash, KryoHelper.serialize(record))
}
boolean removeTaskEntry( HashCode hash ) {
final payload = store.getEntry(hash)
if( !payload ) {
log.debug "Can't increment reference for cached task with key: $hash"
return false
}
final record = (List)KryoHelper.deserialize(payload)
// third record contains the reference count for this record
def count = record[2] = ((Integer)record[2]) -1
// save or delete
if( count > 0 ) {
store.putEntry(hash, KryoHelper.serialize(record))
return false
}
else {
store.deleteEntry(hash)
return true
}
}
/**
* Save task runtime information to th cache DB
*
* @param handler A {@link TaskHandler} instance
*/
@PackageScope
void writeTaskEntry0( TaskHandler handler, TraceRecord trace ) {
final task = handler.task
final proc = task.processor
final key = task.hash
// save the context map for caching purpose
// only the 'cache' is active and
TaskContext ctx = proc.isCacheable() && task.hasCacheableValues() ? task.context : null
def record = new ArrayList(3)
record[0] = trace.serialize()
record[1] = ctx != null ? ctx.serialize() : null
record[2] = 1
// -- save in the db
store.putEntry( key, KryoHelper.serialize(record) )
}
void putTaskAsync( TaskHandler handler, TraceRecord trace ) {
writer.send { writeTaskEntry0(handler, trace) }
}
void cacheTaskAsync( TaskHandler handler ) {
writer.send {
writeTaskIndex0(handler,true)
incTaskEntry(handler.task.hash)
}
}
void putIndexAsync(TaskHandler handler ) {
writer.send { writeTaskIndex0(handler) }
}
@PackageScope
void writeTaskIndex0( TaskHandler handler, boolean cached = false ) {
store.writeIndex(handler.task.hash, cached)
}
void deleteIndex() {
store.deleteIndex()
}
void drop() {
store.drop()
}
/**
* Iterate the tasks cache using the index file
* @param closure The operation to applied
* @return The {@link CacheDB} instance itself
*/
CacheDB eachRecord( Closure closure ) {
assert closure
final itr = store.iterateIndex()
while( itr.hasNext() ) {
final index = itr.next()
final payload = store.getEntry(index.key)
if( !payload ) {
log.trace "Unable to retrieve cache record for key: ${-> index.key}"
continue
}
final record = (List<byte[]>)KryoHelper.deserialize(payload)
TraceRecord trace = TraceRecord.deserialize(record[0])
trace.setCached(index.cached)
final refCount = record[2] as Integer
final len=closure.maximumNumberOfParameters
if( len==1 )
closure.call(trace)
else if( len==2 )
closure.call(index.key, trace)
else if( len==3 )
closure.call(index.key, trace, refCount)
else
throw new IllegalArgumentException("Invalid closure signature -- Too many parameters")
}
return this
}
TraceRecord getTraceRecord( HashCode hashCode ) {
final result = getTaskEntry(hashCode, null)
return result ? result.trace : null
}
TraceRecord findTraceRecord( @ClosureParams(value = SimpleType.class, options = "nextflow.trace.TraceRecord") Closure<Boolean> criteria ) {
final itr = store.iterateIndex()
while( itr.hasNext() ) {
final index = itr.next()
final payload = store.getEntry(index.key)
if( !payload ) {
log.trace "Unable to retrieve cache record for key: ${-> index.key}"
continue
}
final record = (List<byte[]>)KryoHelper.deserialize(payload)
TraceRecord trace = TraceRecord.deserialize(record[0])
trace.setCached(index.cached)
final len=criteria.maximumNumberOfParameters
if( len!=1 ) {
throw new IllegalArgumentException("Invalid criteria signature -- Too many parameters")
}
if( criteria.call(trace) )
return trace
}
// no matches
return null
}
/**
* Close the underlying database and index file
*/
@Override
void close() {
log.trace "Closing CacheDB.."
writer.await()
log.trace "Closing CacheDB index"
store.close()
log.debug "Closing CacheDB done"
}
}

View File

@@ -0,0 +1,46 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cache
import java.nio.file.Path
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.plugin.Plugins
import org.pf4j.ExtensionPoint
/**
* Factory class that create an instance of the {@link CacheDB}
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
abstract class CacheFactory implements ExtensionPoint {
protected abstract CacheDB newInstance(UUID uniqueId, String runName, Path home=null)
static CacheDB create(UUID uniqueId, String runName, Path home=null) {
final all = Plugins.getPriorityExtensions(CacheFactory)
if( !all )
throw new IllegalStateException("Unable to find Nextflow cache factory")
final factory = all.first()
log.debug "Using Nextflow cache factory: ${factory.getClass().getName()}"
return factory.newInstance(uniqueId, runName, home)
}
}

View File

@@ -0,0 +1,48 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cache
import com.google.common.hash.HashCode
import groovy.transform.TupleConstructor
/**
* Defines the contract for a pluggable cache storage
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
interface CacheStore {
@TupleConstructor
static class Index {
final HashCode key
final boolean cached
}
CacheStore open()
CacheStore openForRead()
void close()
void drop()
byte[] getEntry(HashCode key)
void putEntry(HashCode key, byte[] value)
void deleteEntry(HashCode key)
void writeIndex(HashCode key, boolean cached)
Iterator<Index> iterateIndex()
void deleteIndex()
}

View File

@@ -0,0 +1,44 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cache
import java.nio.file.Path
import groovy.transform.CompileStatic
import nextflow.exception.AbortOperationException
import nextflow.plugin.Priority
/**
* Implements the default cache factory
*
* @see DefaultCacheStore
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
@Priority(0)
class DefaultCacheFactory extends CacheFactory {
@Override
protected CacheDB newInstance(UUID uniqueId, String runName, Path home) {
if( !uniqueId ) throw new AbortOperationException("Missing cache `uuid`")
if( !runName ) throw new AbortOperationException("Missing cache `runName`")
final store = new DefaultCacheStore(uniqueId, runName, home)
return new CacheDB(store)
}
}

View File

@@ -0,0 +1,188 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cache
import java.nio.file.Path
import com.google.common.hash.HashCode
import groovy.transform.CompileStatic
import nextflow.Const
import nextflow.exception.AbortOperationException
import nextflow.util.CacheHelper
import org.iq80.leveldb.DB
import org.iq80.leveldb.Options
import org.iq80.leveldb.impl.Iq80DBFactory
/**
* Implement the default nextflow cache store that save the cache data
* into the local directory using an embedded LevelDVB instance
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class DefaultCacheStore implements CacheStore {
/** The underlying Level DB instance */
private DB db
/** The session UUID */
private UUID uniqueId
/** The unique run name associated to this cache instance */
private String runName
/** The base folder against which the cache is located. Default: current working directory */
private Path baseDir
/** The actual path where DB and index file are located */
private Path dataDir
private final int KEY_SIZE
/** The path to the index file */
private Path indexFile
/** Index file read/write handle */
private RandomAccessFile indexHandle
DefaultCacheStore(UUID uniqueId, String runName, Path home=null) {
this.KEY_SIZE = CacheHelper.hasher('x').hash().asBytes().size()
this.uniqueId = uniqueId
this.runName = runName
this.baseDir = home ?: Const.appCacheDir.toAbsolutePath()
this.dataDir = baseDir.resolve("cache/$uniqueId")
this.indexFile = dataDir.resolve("index.$runName")
}
private void openDb() {
// make sure the db path exists
dataDir.mkdirs()
// open a LevelDB instance
final file=dataDir.resolve('db').toFile()
try {
db = Iq80DBFactory.@factory.open(file, new Options().createIfMissing(true))
}
catch( Exception e ) {
String msg
if( e.message?.startsWith('Unable to acquire lock') ) {
msg = "Unable to acquire lock on session with ID $uniqueId"
msg += "\n\n"
msg += "Common reasons for this error are:"
msg += "\n - You are trying to resume the execution of an already running pipeline"
msg += "\n - A previous execution was abruptly interrupted, leaving the session open"
msg += '\n'
msg += '\nYou can see which process is holding the lock file by using the following command:'
msg += "\n - lsof $file/LOCK"
throw new IOException(msg)
}
else {
msg = "Can't open cache DB: $file"
msg += '\n\n'
msg += "Nextflow needs to be executed in a shared file system that supports file locks.\n"
msg += "Alternatively, you can run it in a local directory and specify the shared work\n"
msg += "directory by using the `-w` command line option."
throw new IOException(msg, e)
}
}
}
@Override
DefaultCacheStore open() {
openDb()
//
indexFile.delete()
indexHandle = new RandomAccessFile(indexFile.toFile(), 'rw')
return this
}
@Override
DefaultCacheStore openForRead() {
openDb()
if( !indexFile.exists() )
throw new AbortOperationException("Missing cache index file: $indexFile")
indexHandle = new RandomAccessFile(indexFile.toFile(), 'r')
return this
}
@Override
void drop() {
dataDir.deleteDir()
}
@Override
void close() {
indexHandle.closeQuietly()
db.closeQuietly()
}
@Override
void writeIndex(HashCode key, boolean cached) {
indexHandle.write(key.asBytes())
indexHandle.writeBoolean(cached)
}
@Override
void deleteIndex() {
indexFile.delete()
}
@Override
Iterator<Index> iterateIndex() {
return new Iterator<Index>() {
private Index next
{
next = fetch()
}
@Override
boolean hasNext() {
return next != null
}
@Override
Index next() {
final result = next
next = fetch()
return result
}
private Index fetch() {
byte[] key = new byte[KEY_SIZE]
if( indexHandle.read(key) == -1 )
return null
final cached = indexHandle.readBoolean()
return new Index (HashCode.fromBytes(key), cached)
}
}
}
@Override
byte[] getEntry(HashCode key) {
return db.get(key.asBytes())
}
@Override
void putEntry(HashCode key, byte[] value) {
db.put(key.asBytes(), value)
}
@Override
void deleteEntry(HashCode key) {
db.delete(key.asBytes())
}
}

View File

@@ -0,0 +1,99 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
import java.nio.file.Path
import groovy.transform.CompileStatic
import groovy.transform.PackageScope
import nextflow.cache.CacheDB
import nextflow.cache.CacheFactory
import nextflow.exception.AbortOperationException
import nextflow.util.HistoryFile
/**
* Common cache operations shared by {@link CmdLog} and {@link CmdClean}
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
trait CacheBase {
@PackageScope
Path basePath
@PackageScope
HistoryFile history
abstract String getBut()
abstract String getBefore()
abstract String getAfter()
abstract List<String> getArgs()
void init() {
if( !history ) {
history = !basePath ? HistoryFile.DEFAULT : new HistoryFile(basePath.resolve(HistoryFile.defaultFileName()))
}
if( !history.exists() || history.empty() )
throw new AbortOperationException("It looks like no pipeline was executed in this folder (or execution history is empty)")
if( after && before )
throw new AbortOperationException("Options `after` and `before` cannot be used in the same command")
if( after && but )
throw new AbortOperationException("Options `after` and `but` cannot be used in the same command")
if( before && but )
throw new AbortOperationException("Options `before` and `but` cannot be used in the same command")
}
CacheDB cacheFor(HistoryFile.Record entry) {
CacheFactory.create(entry.sessionId, entry.runName, basePath)
}
List<HistoryFile.Record> listIds() {
if( but ) {
return history.findBut(but)
}
if( before ) {
return history.findBefore(before)
}
else if( after ) {
return history.findAfter(after)
}
// -- get the session ID from the command line if specified or retrieve from
if( !args )
return history.findByIdOrName('last')
List<HistoryFile.Record> result = []
for( String name : args ) {
result.addAll(history.findByIdOrName(name))
}
return result
}
}

View File

@@ -0,0 +1,134 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
import com.beust.jcommander.DynamicParameter
import com.beust.jcommander.Parameter
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.SysEnv
import nextflow.exception.AbortOperationException
import org.fusesource.jansi.Ansi
/**
* Main application command line options
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
class CliOptions {
/**
* The packages to debug
*/
@Parameter(hidden = true, names='-debug')
List<String> debug
@Parameter(names=['-log'], description = 'Set nextflow log file path')
String logFile
@Parameter(names=['-c','-config'], description = 'Add the specified file to configuration set')
List<String> userConfig
@Parameter(names=['-config-ignore-includes'], description = 'Disable the parsing of config includes')
boolean ignoreConfigIncludes
@Parameter(names=['-C'], description = 'Use the specified configuration file(s) overriding any defaults')
List<String> config
/**
* the packages to trace
*/
@Parameter(names='-trace', description = 'Enable trace level logging for the specified package name - multiple packages can be provided separating them with a comma e.g. \'-trace nextflow,io.seqera\'')
List<String> trace
/**
* Enable syslog appender
*/
@Parameter(names = ['-syslog'], description = 'Send logs to syslog server (eg. localhost:514)' )
String syslog
/**
* Print out the version number and exit
*/
@Parameter(names = ['-v','-version'], description = 'Print the program version')
boolean version
/**
* Print out the 'help' and exit
*/
@Parameter(names = ['-h'], description = 'Print this help', help = true)
boolean help
@Parameter(names = ['-q','-quiet'], description = 'Do not print information messages' )
boolean quiet
@Parameter(names = ['-bg'], description = 'Execute nextflow in background', arity = 0)
boolean background
@DynamicParameter(names = ['-D'], description = 'Set JVM properties' )
Map<String,String> jvmOpts = [:]
@Parameter(names = ['-self-update'], description = 'Update nextflow to the latest version', arity = 0, hidden = true)
boolean selfUpdate
@Parameter(names=['-remote-debug'], description = "Enable JVM interactive remote debugging (experimental)")
boolean remoteDebug
Boolean ansiLog
boolean getAnsiLog() {
if( ansiLog && quiet )
throw new AbortOperationException("Command line options `quiet` and `ansi-log` cannot be used together")
if( ansiLog != null )
return ansiLog
if( background )
return ansiLog = false
if( quiet )
return ansiLog = false
final env = SysEnv.get('NXF_ANSI_LOG')
if( env ) try {
return Boolean.parseBoolean(env)
}
catch (Exception e) {
log.warn "Invalid boolean value for variable NXF_ANSI_LOG: $env -- it must be 'true' or 'false'"
}
// Check NO_COLOR environment variable (https://no-color.org/)
final noColor = SysEnv.get('NO_COLOR')
if( noColor ) {
return ansiLog = false
}
// Disable ANSI log in agent mode for plain, parseable output
if( SysEnv.isAgentMode() ) {
return ansiLog = false
}
return Ansi.isEnabled()
}
boolean hasAnsiLogFlag() {
ansiLog==true || SysEnv.get('NXF_ANSI_LOG')=='true'
}
}

View File

@@ -0,0 +1,498 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
import com.beust.jcommander.Parameter
import com.beust.jcommander.Parameters
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.Const
import nextflow.SysEnv
import nextflow.config.ConfigBuilder
import nextflow.exception.AbortOperationException
import nextflow.platform.PlatformHelper
import nextflow.plugin.Plugins
import org.pf4j.ExtensionPoint
/**
* Command-line interface for managing Seqera Platform authentication and configuration.
*
* <p>This class provides the {@code nextflow auth} command with four sub-commands for managing
* authentication credentials and platform settings:
*
* <ul>
* <li>{@code login} - Authenticate with Seqera Platform using OAuth2 (Cloud) or PAT (Enterprise)
* <li>{@code logout} - Revoke access token and remove authentication from local config
* <li>{@code config} - Configure platform settings (workspace, monitoring, compute environment)
* <li>{@code status} - Display current authentication status and configuration sources
* </ul>
*
* <h2>Authentication Flow</h2>
* <p>The authentication mechanism varies based on the platform type:
* <ul>
* <li><b>Seqera Platform Cloud</b>: Uses OAuth2 device authorization flow with Auth0
* <li><b>Seqera Platform Enterprise</b>: Prompts for Personal Access Token (PAT)
* </ul>
*
* <h2>Configuration Management</h2>
* <p>Authentication credentials and settings are stored in the user's home directory:
* <pre>
* ~/.nextflow/config (includes seqera-auth.config)
* ~/.nextflow/seqera-auth.config (contains tower.* settings)
* </pre>
*
* <p>Configuration values can be sourced from:
* <ol>
* <li>Nextflow config files (highest priority)
* <li>Environment variables (TOWER_ACCESS_TOKEN, TOWER_API_ENDPOINT, etc.)
* <li>Default values (lowest priority)
* </ol>
*
* <h2>Plugin Architecture</h2>
* <p>The actual authentication implementation is provided by the {@code nf-tower} plugin through
* the {@link AuthCommand} extension point. This allows the authentication logic to be maintained
* separately from the CLI interface.
*
* @author Phil Ewels <phil.ewels@seqera.io>
* @see AuthCommand
* @see nextflow.platform.PlatformHelper
*/
@Slf4j
@CompileStatic
@Parameters(commandDescription = "Manage Seqera Platform authentication")
class CmdAuth extends CmdBase implements UsageAware {
/**
* Interface for auth sub-commands that defines the contract for command execution and help text.
*
* <p>Each sub-command (login, logout, config, status) implements this interface to provide:
* <ul>
* <li>Command name identification
* <li>Argument processing and execution logic
* <li>Usage/help text generation
* </ul>
*/
interface SubCmd {
/**
* @return the name of this sub-command (e.g., "login", "logout")
*/
String getName()
/**
* Executes the sub-command with the provided arguments.
*
* @param result the command-line arguments passed to this sub-command
*/
void apply(List<String> result)
/**
* Generates usage/help text for this sub-command.
*
* @param result the list to which usage text lines should be added
*/
void usage(List<String> result)
}
/**
* Extension point interface for authentication command implementations.
*
* <p>This interface is implemented by the {@code nf-tower} plugin to provide the actual
* authentication logic while keeping the CLI interface in the core Nextflow codebase.
* The plugin system uses PF4J to discover and load implementations at runtime.
*
* <p>The implementation handles:
* <ul>
* <li>OAuth2 device flow for Seqera Platform Cloud
* <li>PAT (Personal Access Token) authentication for Enterprise
* <li>Token storage and management in config files
* <li>Workspace and compute environment configuration
* </ul>
*
* @see io.seqera.tower.plugin.auth.AuthCommandImpl
*/
interface AuthCommand extends ExtensionPoint {
/**
* Authenticates with Seqera Platform and saves credentials to config.
*
* @param url the Seqera Platform API endpoint URL (null for default)
*/
void login(String url)
/**
* Revokes access token and removes authentication from local config.
*/
void logout()
/**
* Configures Seqera Platform settings (workspace, monitoring, compute environment).
*/
void config()
/**
* Displays current authentication status and configuration sources.
*/
void status()
}
static public final String NAME = 'auth'
private List<SubCmd> commands = []
private AuthCommand operation
String getName() {
return NAME
}
@Parameter(hidden = true)
List<String> args
@Parameter(names = ['-u', '-url'], description = 'Seqera Platform API endpoint')
String apiUrl
CmdAuth() {
commands.add(new LoginCmd())
commands.add(new LogoutCmd())
commands.add(new ConfigCmd())
commands.add(new StatusCmd())
}
void usage() {
usage(args)
}
void usage(List<String> args) {
List<String> result = []
if (!args) {
result << this.getClass().getAnnotation(Parameters).commandDescription()
result << 'Usage: nextflow auth <sub-command> [options]'
result << ''
result << 'Commands:'
result << ' login Authenticate with Seqera Platform'
result << ' logout Remove authentication and revoke access token'
result << ' status Show current authentication status and configuration'
result << ' config Configure Seqera Platform settings'
result << ''
} else {
def sub = commands.find { it.name == args[0] }
if (sub)
sub.usage(result)
else {
throw new AbortOperationException("Unknown auth sub-command: ${args[0]}")
}
}
println result.join('\n').toString()
}
@Override
void run() {
if (!args) {
usage()
return
}
// load the Auth command implementation
this.operation = loadOperation()
if( !operation )
throw new IllegalStateException("Unable to load auth extensions.")
// consume the first argument
getCmd(args).apply(args.drop(1))
}
protected AuthCommand loadOperation(){
// setup the plugins system and load the secrets provider
Plugins.init()
// load the config
Plugins.start('nf-tower')
// get Auth command operations implementation from plugins
return Plugins.getExtension(AuthCommand)
}
protected SubCmd getCmd(List<String> args) {
def cmd = commands.find { it.name == args[0] }
if (cmd) {
return cmd
}
def matches = commands.collect { it.name }.closest(args[0])
def msg = "Unknown auth sub-command: ${args[0]}"
if (matches)
msg += " -- Did you mean one of these?\n" + matches.collect { " $it" }.join('\n')
throw new AbortOperationException(msg)
}
/**
* Implements the {@code nextflow auth login} sub-command for authenticating with Seqera Platform.
*
* <p>This command initiates the authentication process, which varies based on the platform type:
* <ul>
* <li><b>Seqera Platform Cloud</b>: Launches OAuth2 device authorization flow
* <ol>
* <li>Requests device code from Auth0
* <li>Displays verification URL and code to user
* <li>Opens browser for authentication
* <li>Polls for token completion
* <li>Generates Personal Access Token from OAuth token
* </ol>
* </li>
* <li><b>Seqera Platform Enterprise</b>: Prompts for manual PAT entry
* <ol>
* <li>Displays URL for token creation
* <li>Prompts user to paste PAT
* <li>Validates token with API
* </ol>
* </li>
* </ul>
*
* <p>Upon successful authentication, the command:
* <ul>
* <li>Saves {@code tower.accessToken} to {@code ~/.nextflow/seqera-auth.config}
* <li>Configures {@code tower.endpoint} with the API URL
* <li>Enables {@code tower.enabled} for automatic monitoring
* <li>Adds {@code includeConfig 'seqera-auth.config'} to main config
* </ul>
*
* <p><b>Usage:</b> {@code nextflow auth login [-u <endpoint>]}
*/
class LoginCmd implements SubCmd {
@Override
String getName() { 'login' }
@Override
void apply(List<String> args) {
if (args.size() > 0) {
throw new AbortOperationException("Too many arguments for ${name} command")
}
operation.login(apiUrl)
}
@Override
void usage(List<String> result) {
// Read config to get the actual resolved endpoint value
final builder = new ConfigBuilder().setHomeDir(Const.APP_HOME_DIR).setCurrentDir(Const.APP_HOME_DIR)
final config = builder.buildConfigObject().flatten()
final towerConfig = config.findAll { it.key.toString().startsWith('tower.') }
.collectEntries { k, v -> [(k.toString().substring(6)): v] }
def defaultEndpoint = PlatformHelper.getEndpoint(towerConfig, SysEnv.get())
result << 'Authenticate with Seqera Platform'
result << "Usage: nextflow auth $name [-u <endpoint>]".toString()
result << ''
result << 'Options:'
result << " -u, -url <endpoint> Seqera Platform API endpoint (default: ${defaultEndpoint})".toString()
result << ''
result << 'This command will:'
result << ' 1. Display a URL and device code for OAuth2 authentication (Cloud) or prompt for PAT (Enterprise)'
result << ' 2. Wait for user to complete authentication in web browser'
result << ' 3. Generate and save access token to home-directory Nextflow config'
result << ' 4. Configure tower.accessToken, tower.endpoint, and tower.enabled settings'
result << ''
}
}
/**
* Implements the {@code nextflow auth logout} sub-command for removing authentication credentials.
*
* <p>This command performs a safe logout by:
* <ol>
* <li>Checking if {@code tower.accessToken} is configured in {@code ~/.nextflow/seqera-auth.config}
* <li>Validating the token with the Seqera Platform API
* <li>Displaying current configuration details to the user
* <li>Prompting for confirmation before proceeding
* <li>Deleting the Personal Access Token from Platform (Cloud only)
* <li>Removing {@code seqera-auth.config} file
* <li>Removing {@code includeConfig} line from main config
* </ol>
*
* <p><b>Important behavior differences:</b>
* <ul>
* <li><b>Seqera Platform Cloud</b>: Deletes the PAT from the platform via API before removing local config
* <li><b>Seqera Platform Enterprise</b>: Only removes local config; does not delete PAT from platform
* </ul>
*
* <p><b>Note:</b> This command only removes credentials from Nextflow config files. If
* {@code TOWER_ACCESS_TOKEN} environment variable is set, it will remain unchanged and
* the user will be warned about this.
*
* <p><b>Usage:</b> {@code nextflow auth logout}
*/
class LogoutCmd implements SubCmd {
@Override
String getName() { 'logout' }
@Override
void apply(List<String> args) {
if (args.size() > 0) {
throw new AbortOperationException("Too many arguments for ${name} command")
}
operation.logout()
}
@Override
void usage(List<String> result) {
result << 'Log out and remove Seqera Platform authentication'
result << "Usage: nextflow auth $name".toString()
result << ''
result << 'This command will:'
result << ' 1. Check if tower.accessToken is configured'
result << ' 2. Validate the token with Seqera Platform'
result << ' 3. Delete the PAT from Platform (only if Seqera Platform Cloud)'
result << ' 4. Remove the authentication from Nextflow config'
result << ''
}
}
/**
* Implements the {@code nextflow auth config} sub-command for configuring Seqera Platform settings.
*
* <p>This interactive command guides users through configuring their Seqera Platform integration.
* It requires an existing authentication (see {@link LoginCmd}) and allows configuration of:
*
* <h3>Workflow Monitoring</h3>
* <ul>
* <li>Configures {@code tower.enabled} setting
* <li>When enabled: all workflow runs are automatically monitored
* <li>When disabled: monitoring requires {@code -with-tower} flag per run
* </ul>
*
* <h3>Default Workspace</h3>
* <ul>
* <li>Configures {@code tower.workspaceId} setting
* <li>Lists all accessible workspaces organized by organization
* <li>For many workspaces: uses two-stage selection (org → workspace)
* <li>For few workspaces: displays all at once
* <li>Option to use Personal workspace (no workspace ID)
* </ul>
*
* <h3>Primary Compute Environment</h3>
* <ul>
* <li>Displays compute environments available in selected workspace
* <li>Allows setting a primary compute environment
* <li>Primary compute environment is used by default for pipeline execution
* </ul>
*
* <p>All configuration changes are saved to {@code ~/.nextflow/seqera-auth.config}.
*
* <p><b>Usage:</b> {@code nextflow auth config}
*/
class ConfigCmd implements SubCmd {
@Override
String getName() { 'config' }
@Override
void apply(List<String> args) {
if (args.size() > 0) {
throw new AbortOperationException("Too many arguments for ${name} command")
}
operation.config()
}
@Override
void usage(List<String> result) {
result << 'Configure Seqera Platform settings'
result << "Usage: nextflow auth $name".toString()
result << ''
result << 'This command will:'
result << ' 1. Check authentication status'
result << ' 2. Configure tower.enabled setting for workflow monitoring'
result << ' 3. Configure default workspace (tower.workspaceId)'
result << ''
}
}
/**
* Implements the {@code nextflow auth status} sub-command for displaying authentication status.
*
* <p>This command provides a comprehensive overview of the current Seqera Platform configuration,
* displaying all settings in a formatted table with their values and sources.
*
* <h3>Information Displayed</h3>
* <table border="1">
* <tr><th>Setting</th><th>Description</th><th>Possible Sources</th></tr>
* <tr>
* <td>API endpoint</td>
* <td>Seqera Platform API URL</td>
* <td>nextflow config, env var $TOWER_API_ENDPOINT, default</td>
* </tr>
* <tr>
* <td>API connection</td>
* <td>Whether the API is reachable</td>
* <td>Health check result (✔ OK or ERROR)</td>
* </tr>
* <tr>
* <td>Authentication</td>
* <td>Token status and username</td>
* <td>nextflow config, env var $TOWER_ACCESS_TOKEN, not set</td>
* </tr>
* <tr>
* <td>Workflow monitoring</td>
* <td>Whether monitoring is enabled</td>
* <td>nextflow config, default (No)</td>
* </tr>
* <tr>
* <td>Default workspace</td>
* <td>Configured workspace ID and name</td>
* <td>nextflow config, env var $TOWER_WORKSPACE_ID, default (Personal)</td>
* </tr>
* <tr>
* <td>Primary compute env</td>
* <td>Primary compute environment name</td>
* <td>Workspace setting</td>
* </tr>
* <tr>
* <td>Default work dir</td>
* <td>Working directory from compute env</td>
* <td>Compute environment setting</td>
* </tr>
* </table>
*
* <p>The source column shows where each setting's value comes from, making it easy to
* understand configuration precedence and troubleshoot issues.
*
* <p><b>Usage:</b> {@code nextflow auth status}
*/
class StatusCmd implements SubCmd {
@Override
String getName() { 'status' }
@Override
void apply(List<String> args) {
if (args.size() > 0) {
throw new AbortOperationException("Too many arguments for ${name} command")
}
operation.status()
}
@Override
void usage(List<String> result) {
result << 'Show authentication status and configuration'
result << "Usage: nextflow auth $name".toString()
result << ''
result << 'This command shows:'
result << ' - Authentication status (yes/no) and source'
result << ' - API endpoint and source'
result << ' - Monitoring enabled status and source'
result << ' - Default workspace and source'
result << ' - System health status (API connection and authentication)'
result << ''
}
}
}

View File

@@ -0,0 +1,45 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
import com.beust.jcommander.Parameter
/**
* Implement command shared methods
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
abstract class CmdBase implements Runnable {
private Launcher launcher
private List<String> unknownOptions
abstract String getName()
List<String> getAliases() { Collections.emptyList() }
protected List<String> getUnknownOptions(){ return this.unknownOptions }
void setUnknownOptions(List<String> options){ this.unknownOptions = options }
Launcher getLauncher() { launcher }
void setLauncher( Launcher value ) { this.launcher = value }
@Parameter(names=['-h','-help'], description = 'Print the command usage', arity = 0, help = true)
boolean help
}

View File

@@ -0,0 +1,281 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
import java.nio.file.FileVisitResult
import java.nio.file.FileVisitor
import java.nio.file.Files
import java.nio.file.NoSuchFileException
import java.nio.file.Path
import java.nio.file.Paths
import java.nio.file.attribute.BasicFileAttributes
import com.beust.jcommander.Parameter
import com.beust.jcommander.Parameters
import com.google.common.hash.HashCode
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.cache.CacheDB
import nextflow.Global
import nextflow.ISession
import nextflow.Session
import nextflow.config.ConfigBuilder
import nextflow.exception.AbortOperationException
import nextflow.file.FileHelper
import nextflow.plugin.Plugins
import nextflow.trace.TraceRecord
import nextflow.util.HistoryFile.Record
/**
* Implements cache clean up command
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
* @author Lorenz Gerber <lorenzottogerber@gmail.com>
*/
@Slf4j
@CompileStatic
@Parameters(commandDescription = "Clean up project cache and work directories")
class CmdClean extends CmdBase implements CacheBase {
static final public NAME = 'clean'
@Parameter(names=['-q', '-quiet'], description = 'Do not print names of files removed', arity = 0)
boolean quiet
@Parameter(names=['-f', '-force'], description = 'Force clean command', arity = 0)
boolean force
@Parameter(names=['-n', '-dry-run'], description = 'Print names of file to be removed without deleting them' , arity = 0)
boolean dryRun
@Parameter(names='-after', description = 'Clean up runs executed after the specified one')
String after
@Parameter(names='-before', description = 'Clean up runs executed before the specified one')
String before
@Parameter(names='-but', description = 'Clean up all runs except the specified one')
String but
@Parameter(names=['-k', '-keep-logs'], description = 'Removes only temporary files but retains execution log entries and metadata')
boolean keepLogs
@Parameter
List<String> args
private CacheDB currentCacheDb
private Map<HashCode, Short> dryHash = new HashMap<>()
/**
* @return The name of this command {@code clean}
*/
@Override
String getName() {
return NAME
}
/**
* Command entry method
*/
@Override
void run() {
init()
validateOptions()
createSession()
Plugins.init()
listIds().each { entry -> cleanup(entry) }
}
/**
* Create the NF session which can be required to access cloud file store
*/
private void createSession() {
Global.setLazySession {
final builder = new ConfigBuilder()
.setShowClosures(true)
.showMissingVariables(true)
.setOptions(launcher.options)
.setBaseDir(Paths.get('.'))
final config = builder.buildConfigObject()
return (ISession) new Session(config)
}
}
/**
* Extra CLI option validation
*/
private void validateOptions() {
if( !dryRun && !force )
throw new AbortOperationException("Neither -f or -n specified -- refused to clean")
}
/**
* Given a history entry clean up execution cache, deleting
* task work directories and cache DB records
*
* @param entry
* A {@link Record} object representing a row in the history log file
*/
private void cleanup(Record entry) {
currentCacheDb = cacheFor(entry).openForRead()
// -- remove each entry and work dir
currentCacheDb.eachRecord(this.&removeRecord)
// -- close the cache
currentCacheDb.close()
// -- STOP HERE !
if( dryRun || keepLogs ) return
// -- remove the index file
currentCacheDb.deleteIndex()
// -- remove the session from the history file
history.deleteEntry(entry)
// -- check if exists another history entry for the same session
if( !history.checkExistsById(entry.sessionId)) {
currentCacheDb.drop()
}
}
/**
* Check if a tasks can be removed during a dry-run simulation.
*
* @param hash
* The task unique hash code
* @param refCount
* The number of times the task cache is references by other run instances
* @return
* {@code true} when task will be removed by the clean command, {@code false} otherwise i.e.
* entry cannot be deleted because is referenced by other run instances
*/
private boolean wouldRemove(HashCode hash, Integer refCount) {
if( dryHash.containsKey(hash) ) {
refCount = dryHash.get(hash)-1
}
if( refCount == 1 ) {
dryHash.remove(hash)
return true
}
else {
dryHash.put(hash, (short)refCount)
return false
}
}
/**
* Delete task cache entry
*
* @param hash The task unique hash code
* @param record The task {@link TraceRecord}
* @param refCount The number of times the task cache is references by other run instances
*/
private void removeRecord(HashCode hash, TraceRecord record, int refCount) {
if( dryRun ) {
if( wouldRemove(hash,refCount) )
printMessage(record.workDir,true)
return
}
// decrement the ref count in the db
def proceed = keepLogs || currentCacheDb.removeTaskEntry(hash)
if( proceed ) {
// delete folder
if( deleteFolder(FileHelper.asPath(record.workDir), keepLogs)) {
if(!quiet) printMessage(record.workDir,false)
}
}
}
private printMessage(String path, boolean dryRun) {
if( dryRun ) {
println keepLogs ? "Would remove temp files from ${path}" : "Would remove ${path}"
}
else {
println keepLogs ? "Removed temp files from ${path}" : "Removed ${path}"
}
}
/**
* Traverse a directory structure and delete all the content
*
* @param folder
* The directory to delete
* @return
* {@code true} in the directory was removed, {@code false} otherwise
*/
private boolean deleteFolder( Path folder, boolean keepLogs ) {
def result = true
Files.walkFileTree(folder, new FileVisitor<Path>() {
@Override
FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
result ? FileVisitResult.CONTINUE : FileVisitResult.TERMINATE
}
@Override
FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
final canDelete = !keepLogs || ( keepLogs && !(file.name.startsWith('.command.') || file.name == '.exitcode'))
if( canDelete && !delete0(file,false) ) {
result = false
if(!quiet) System.err.println "Failed to remove ${file.toUriString()}"
}
result ? FileVisitResult.CONTINUE : FileVisitResult.TERMINATE
}
@Override
FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
FileVisitResult.CONTINUE
}
@Override
FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
if( !keepLogs && result && !delete0(dir,true) ) {
result = false
if(!quiet) System.err.println "Failed to remove ${dir.toUriString()}"
}
result ? FileVisitResult.CONTINUE : FileVisitResult.TERMINATE
}
})
return result
}
private static delete0(Path path, boolean dir) {
try {
log.trace "Deleting path [dir=$dir]: ${path.toUriString()}"
Files.delete(path)
return true
}
catch( IOException e ) {
// kind of hack: directory deletion
if( dir && path.scheme=='gs' && e instanceof NoSuchFileException )
return true
log.debug("Failed to remove path: ${path.toUriString()}", e)
return false
}
}
}

View File

@@ -0,0 +1,76 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
import com.beust.jcommander.Parameter
import com.beust.jcommander.Parameters
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.exception.AbortOperationException
import nextflow.plugin.Plugins
import nextflow.scm.AssetManager
/**
* CLI sub-command clone
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
@Parameters(commandDescription = "Clone a project into a folder")
class CmdClone extends CmdBase implements HubOptions {
static final public NAME = 'clone'
@Parameter(required=true, description = 'name of the project to clone')
List<String> args
@Parameter(names='-r', description = 'Revision of the project to clone (either a git branch, tag or commit SHA number)')
String revision
@Parameter(names=['-d','-deep'], description = 'Create a shallow clone of the specified depth')
Integer deep
@Override
final String getName() { NAME }
@Override
void run() {
// init plugin system
Plugins.init()
// the pipeline name
String pipeline = args[0]
try (final manager = new AssetManager(pipeline, this)) {
// the target directory is the second parameter
// otherwise default the current pipeline name
def target = new File(args.size()> 1 ? args[1] : manager.getBaseName())
if( target.exists() ) {
if( target.isFile() )
throw new AbortOperationException("A file with the same name already exists: $target")
if( !target.empty() )
throw new AbortOperationException("Clone target directory must be empty: $target")
}
else if( !target.mkdirs() ) {
throw new AbortOperationException("Cannot create clone target directory: $target")
}
manager.checkValidRemoteRepo()
print "Cloning ${manager.getProjectWithRevision()} ..."
manager.clone(target, revision, deep)
print "\r"
println "${manager.getProjectWithRevision()} cloned to: $target"
}
}
}

View File

@@ -0,0 +1,245 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
import java.nio.file.Path
import java.nio.file.Paths
import com.beust.jcommander.Parameter
import com.beust.jcommander.Parameters
import groovy.transform.CompileStatic
import groovy.transform.PackageScope
import groovy.util.logging.Slf4j
import nextflow.NF
import nextflow.config.ConfigBuilder
import nextflow.config.ConfigValidator
import nextflow.exception.AbortOperationException
import nextflow.plugin.Plugins
import nextflow.scm.AssetManager
import nextflow.util.ConfigHelper
/**
* Prints the pipeline configuration
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
@Parameters(commandDescription = "Print a project configuration")
class CmdConfig extends CmdBase {
static final public NAME = 'config'
static final List<String> FORMATS = ['flat','properties','canonical','json','yaml']
@Parameter(description = 'project name')
List<String> args = []
@Parameter(names=['-r','-revision'], description = 'Revision of the project (either a git branch, tag or commit SHA number)')
String revision
@Parameter(names=['-a','-show-profiles'], description = 'Show all configuration profiles')
boolean showAllProfiles
@Parameter(names=['-profile'], description = 'Choose a configuration profile')
String profile
@Deprecated
@Parameter(names = '-properties', description = 'Prints config using Java properties notation (deprecated: use `-o properties` instead)')
boolean printProperties
@Deprecated
@Parameter(names = '-flat', description = 'Print config using flat notation (deprecated: use `-o flat` instead)')
boolean printFlatten
@Parameter(names = '-sort', description = 'Sort config attributes')
boolean sort
@Parameter(names = '-value', description = 'Print the value of a config option, or fail if the option is not defined')
String printValue
@Parameter(names = ['-o','-output'], description = 'Print the config using the specified format: canonical,properties,flat,json,yaml')
String outputFormat
@Override
String getName() { NAME }
private OutputStream stdout = System.out
@Override
void run() {
Plugins.init()
Path base = null
if( args ) base = getBaseDir(args[0])
if( !base ) base = Paths.get('.')
// -- validate command line options
if( profile && showAllProfiles ) {
throw new AbortOperationException("Option `-profile` conflicts with option `-show-profiles`")
}
if( printProperties && printFlatten )
throw new AbortOperationException("Option `-flat` and `-properties` conflicts each other")
if ( printValue && printFlatten )
throw new AbortOperationException("Option `-value` and `-flat` conflicts each other")
if ( printValue && printProperties )
throw new AbortOperationException("Option `-value` and `-properties` conflicts each other")
if( printValue && outputFormat )
throw new AbortOperationException("Option `-value` and `-output` conflicts each other")
if( printFlatten )
outputFormat = 'flat'
if( printProperties )
outputFormat = 'properties'
// -- build the config
final builder = new ConfigBuilder()
.setShowClosures(true)
.setStripSecrets(true)
.showMissingVariables(true)
.setOptions(launcher.options)
.setBaseDir(base)
.setCmdConfig(this)
final config = builder.buildConfigObject()
// -- validate config options
if( NF.isSyntaxParserV2() ) {
Plugins.load(config)
new ConfigValidator().validate(config)
}
// -- print config options
if( printValue ) {
printValue0(config, printValue, stdout)
}
else if( outputFormat=='properties' ) {
printProperties0(config, stdout)
}
else if( outputFormat=='flat' ) {
printFlatten0(config, stdout)
}
else if( outputFormat=='yaml' ) {
printYaml0(config, stdout)
}
else if( outputFormat=='json') {
printJson0(config, stdout)
}
else if( !outputFormat || outputFormat=='canonical' ) {
printCanonical0(config, stdout)
}
else {
def msg = "Unknown output format: $outputFormat"
def suggest = FORMATS.closest(outputFormat)
if( suggest )
msg += " - did you mean '${suggest.first()}' instead?"
throw new AbortOperationException(msg)
}
for( String msg : builder.warnings )
log.warn(msg)
}
/**
* Prints a {@link ConfigObject} using Java {@link Properties} in canonical format
* ie. any nested config object is printed within curly brackets
*
* @param config The {@link ConfigObject} representing the parsed workflow configuration
* @param output The stream where output the formatted configuration notation
*/
@PackageScope void printCanonical0(ConfigObject config, OutputStream output) {
output << ConfigHelper.toCanonicalString(config, sort)
}
/**
* Prints a {@link ConfigObject} using Java {@link Properties} format
*
* @param config The {@link ConfigObject} representing the parsed workflow configuration
* @param output The stream where output the formatted configuration notation
*/
@PackageScope void printProperties0(ConfigObject config, OutputStream output) {
output << ConfigHelper.toPropertiesString(config, sort)
}
/**
* Prints a property of a {@link ConfigObject}.
*
* @param config The {@link ConfigObject} representing the parsed workflow configuration
* @param name The {@link String} representing the property name using dot notation
* @param output The stream where output the formatted configuration notation
*/
@PackageScope void printValue0(ConfigObject config, String name, OutputStream output) {
final map = config.flatten()
if( !map.containsKey(name) )
throw new AbortOperationException("Configuration option '$name' not found")
output << map.get(name).toString() << '\n'
}
/**
* Prints a {@link ConfigObject} using properties dot notation.
* String values are enclosed in single quote characters.
*
* @param config The {@link ConfigObject} representing the parsed workflow configuration
* @param output The stream where output the formatted configuration notation
*/
@PackageScope void printFlatten0(ConfigObject config, OutputStream output) {
output << ConfigHelper.toFlattenString(config, sort)
}
/**
* Prints the {@link ConfigObject} configuration object using the default notation
*
* @param config The {@link ConfigObject} representing the parsed workflow configuration
* @param output The stream where output the formatted configuration notation
*/
@PackageScope void printDefault0(ConfigObject config, OutputStream output) {
def writer = new PrintWriter(output,true)
config.writeTo( writer )
}
@PackageScope void printJson0(ConfigObject config, OutputStream output) {
output << ConfigHelper.toJsonString(config, sort) << '\n'
}
@PackageScope void printYaml0(ConfigObject config, OutputStream output) {
output << ConfigHelper.toYamlString(config, sort)
}
Path getBaseDir(String path) {
def file = Paths.get(path)
if( file.isDirectory() )
return file
if( file.exists() ) {
return file.parent ?: Paths.get('/')
}
try (final manager = new AssetManager(path, revision)) {
if( revision && manager.isUsingLegacyStrategy() ){
log.warn("The local asset for ${path} does not support multi-revision - 'revision' option is ignored\n" +
"Consider updating the project using 'nextflow pull ${path} -r $revision -migrate'")
}
return manager.isLocal() ? manager.localPath.toPath() : manager.configFile?.parent
}
}
}

View File

@@ -0,0 +1,52 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
import com.beust.jcommander.Parameter
import com.beust.jcommander.Parameters
import groovy.transform.CompileStatic
import nextflow.plugin.Plugins
import nextflow.ui.console.ConsoleExtension
/**
* Launch the Nextflow Console plugin
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
@Parameters(commandDescription = "Launch Nextflow interactive console")
class CmdConsole extends CmdBase {
@Parameter(description = 'Nextflow console arguments')
List<String> args
String getName() { 'console' }
void run() {
Plugins.init()
Plugins.start('nf-console')
final console = Plugins.getExtension(ConsoleExtension)
if( !console )
throw new IllegalStateException("Failed to find Nextflow Console extension")
// normalise the console args prepending the `console` command itself
if( args == null )
args = []
args.add(0, 'console')
// go !
console.run(args as String[])
}
}

View File

@@ -0,0 +1,57 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
import com.beust.jcommander.Parameter
import com.beust.jcommander.Parameters
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.plugin.Plugins
import nextflow.scm.AssetManager
/**
* CLI sub-command DROP
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
@Parameters(commandDescription = "Delete the local copy of a project")
class CmdDrop extends CmdBase {
static final public NAME = 'drop'
@Parameter(required=true, description = 'name of the project to drop')
List<String> args
@Parameter(names=['-r','-revision'], description = 'Revision of the project to drop (either a git branch, tag or commit SHA number)')
String revision
@Parameter(names='-f', description = 'Delete the repository without taking care of local changes')
boolean force
@Override
final String getName() { NAME }
@Override
void run() {
Plugins.init()
try (final manager = new AssetManager(args[0])) {
manager.drop(revision, force)
}
}
}

View File

@@ -0,0 +1,324 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
import static nextflow.file.FileHelper.toCanonicalPath
import java.nio.charset.Charset
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
import java.nio.file.attribute.BasicFileAttributes
import com.beust.jcommander.Parameter
import com.beust.jcommander.Parameters
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.Global
import nextflow.Session
import nextflow.config.ConfigBuilder
import nextflow.exception.AbortOperationException
import nextflow.extension.FilesEx
import nextflow.file.FileHelper
import nextflow.file.FilePatternSplitter
import nextflow.plugin.Plugins
/**
* Implements `fs` command
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
@Slf4j
@Parameters(commandDescription = "Perform filesystem operations")
class CmdFs extends CmdBase implements UsageAware {
static final public NAME = 'fs'
static final List<SubCmd> commands = new ArrayList<>()
static {
commands << new CmdCopy()
commands << new CmdMove()
commands << new CmdList()
commands << new CmdCat()
commands << new CmdRemove()
commands << new CmdStat()
}
trait SubCmd {
abstract int getArity()
abstract String getName()
abstract String getDescription()
abstract void apply(Path source, Path target)
String usage() {
"Usage: nextflow fs ${name} " + (arity==1 ? "<path>" : "source_file target_file")
}
}
static class CmdCopy implements SubCmd {
@Override
int getArity() { 2 }
@Override
String getName() { 'cp' }
String getDescription() { 'Copy a file' }
@Override
void apply(Path source, Path target) {
FilesEx.copyTo(source, target)
}
}
static class CmdMove implements SubCmd {
@Override
int getArity() { 2 }
@Override
String getName() { 'mv' }
String getDescription() { 'Move a file' }
@Override
void apply(Path source, Path target) {
FilesEx.moveTo(source, target)
}
}
static class CmdList implements SubCmd {
@Override
int getArity() { 1 }
String getDescription() { 'List the content of a folder' }
@Override
String getName() { 'ls' }
@Override
void apply(Path source, Path target) {
println source.name
}
}
static class CmdCat implements SubCmd {
@Override
int getArity() { 1 }
@Override
String getName() { 'cat' }
@Override
String getDescription() { 'Print a file to the stdout' }
@Override
void apply(Path source, Path target) {
String line
def reader = Files.newBufferedReader(source, Charset.defaultCharset())
while( line = reader.readLine() )
println line
}
}
static class CmdStat implements SubCmd {
@Override
int getArity() { 1 }
@Override
String getName() { 'stat' }
@Override
String getDescription() { 'Print file to meta info' }
@Override
void apply(Path source, Path target) {
try {
final attr = Files.readAttributes(source, BasicFileAttributes)
print """\
name : ${source.name}
size : ${attr.size()}
is directory : ${attr.isDirectory()}
last modified : ${attr.lastModifiedTime() ?: '-'}
creation time : ${attr.creationTime() ?: '-'}
""".stripIndent()
}
catch (IOException e) {
log.warn "Unable to read attributes for file: ${source.toUriString()} - cause: $e.message", e
}
}
}
static class CmdRemove implements SubCmd {
@Override
int getArity() { 1 }
@Override
String getName() { 'rm' }
@Override
String getDescription() { 'Remove a file' }
@Override
void apply(Path source, Path target) {
Files.isDirectory(source) ? FilesEx.deleteDir(source) : FilesEx.delete(source)
}
}
@Parameter
List<String> args
@Override
String getName() {
return NAME
}
private Session createSession() {
// create the config
final config = new ConfigBuilder()
.setOptions(getLauncher().getOptions())
.setBaseDir(Paths.get('.'))
.build()
return new Session(config)
}
@Override
void run() {
if( !args ) {
usage()
return
}
Plugins.init()
final session = createSession()
try {
run0()
}
finally {
try {
session.destroy()
Global.cleanUp()
Plugins.stop()
} catch (Throwable t) {
log.warn "Unexpected error while destroying the session object - cause: ${t.message}"
}
}
}
private void run0() {
final cmd = findCmd(args[0])
if( !cmd ) {
throw new AbortOperationException("Unknown fs sub-command: `$cmd`")
}
Path target
String source
if( cmd.arity==1 ) {
if( args.size() < 2 )
throw new AbortOperationException(cmd.usage())
source = args[1]
target = null
}
else {
if( args.size() < 3 )
throw new AbortOperationException(cmd.usage())
source = args[1]
target = args[2] as Path
}
traverse(source) { Path path -> cmd.apply(path, target) }
}
private SubCmd findCmd( String name ) {
commands.find { it.name == name }
}
private void traverse( String source, Closure op ) {
// if it isn't a glob pattern simply return it a normalized absolute Path object
def splitter = FilePatternSplitter.glob().parse(source)
if( splitter.isPattern() ) {
final scheme = splitter.scheme
final target = scheme ? "$scheme://$splitter.parent" : splitter.parent
final folder = toCanonicalPath(target)
final pattern = splitter.fileName
def opts = [:]
opts.type = 'any'
FileHelper.visitFiles(opts, folder, pattern, op)
}
else {
def normalised = splitter.strip(source)
op.call(FileHelper.asPath(normalised))
}
}
/**
* Print the command usage help
*/
@Override
void usage() {
usage(args)
}
/**
* Print the command usage help
*
* @param args The arguments as entered by the user
*/
@Override
void usage(List<String> args) {
def result = []
if( !args ) {
result << 'Usage: nextflow fs <command> [arg]'
result << ''
result << 'Commands:'
commands.each {
result << " ${it.name}\t${it.description}"
}
result << ''
println result.join('\n').toString()
}
else {
def sub = findCmd(args[0])
if( sub )
println sub.usage()
else {
throw new AbortOperationException("Unknown fs sub-command: ${args[0]}")
}
}
}
}

View File

@@ -0,0 +1,53 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
import com.beust.jcommander.Parameter
import com.beust.jcommander.Parameters
import groovy.transform.CompileStatic
/**
* CLI sub-command HELP
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
@Parameters(commandDescription = "Print the usage help for a command")
class CmdHelp extends CmdBase {
static final public NAME = 'help'
@Override
final String getName() { NAME }
@Parameter(description = 'command name', arity = 1)
List<String> args
private UsageAware getUsage( List<String> args ) {
def result = args ? launcher.findCommand(args[0]) : null
result instanceof UsageAware ? result as UsageAware: null
}
@Override
void run() {
def cmd = getUsage(args)
if( cmd ) {
cmd.usage(args.size()>1 ? args[1..-1] : Collections.<String>emptyList())
}
else {
launcher.usage(args ? args[0] : null)
}
}
}

View File

@@ -0,0 +1,51 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
/**
* Cmd CLI helper methods
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
class CmdHelper {
static private String operators = '<>!='
/**
* Expand single `=` to `==` in filter string
*
* @param filter A filter string e.g. `cpus = 4`
* @return A filter in which `=` char is expanded to `==` operator
*/
static String fixEqualsOp( String filter ) {
if( !filter ) return filter
def result = new StringBuilder()
int i=0
int len=filter.length()
while( i<len ) {
def ch = filter[i++]
result << ch
if( i<len-1 && filter[i]=='=' && filter[i+1]!='=' && !operators.contains(ch)) {
result.append('=')
}
}
return result.toString()
}
}

View File

@@ -0,0 +1,356 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
import java.lang.management.ManagementFactory
import java.nio.file.spi.FileSystemProvider
import com.beust.jcommander.Parameter
import com.beust.jcommander.Parameters
import com.sun.management.OperatingSystemMXBean
import groovy.json.JsonOutput
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.BuildInfo
import nextflow.exception.AbortOperationException
import nextflow.plugin.DefaultPlugins
import nextflow.plugin.Plugins
import nextflow.scm.AssetManager
import nextflow.util.MemoryUnit
import nextflow.util.Threads
import org.yaml.snakeyaml.Yaml
/**
* CLI sub-command INFO
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
@Parameters(commandDescription = "Print project and system runtime information")
class CmdInfo extends CmdBase {
static final public NAME = 'info'
private PrintStream out = System.out
@Parameter(description = 'project name')
List<String> args
@Parameter(names='-d',description = 'Show detailed information', arity = 0)
boolean detailed
@Parameter(names='-dd', hidden = true, arity = 0)
boolean moreDetailed
@Parameter(names='-o', description = 'Output format, either: text (default), json, yaml')
String format
@Parameter(names=['-u','-check-updates'], description = 'Check for remote updates')
boolean checkForUpdates
@Override
final String getName() { NAME }
@Override
void run() {
int level = moreDetailed ? 2 : ( detailed ? 1 : 0 )
if( !args ) {
println getInfo(level)
return
}
Plugins.init()
try (final manager = new AssetManager(args[0])) {
if( manager.isNotInitialized() ) {
throw new AbortOperationException("Unknown project `${args[0]}`")
}
if( !format || format == 'text' ) {
printText(manager,level)
return
}
def map = createMap(manager)
if( format == 'json' ) {
printJson(map)
}
else if( format == 'yaml' ) {
printYaml(map)
}
else
throw new AbortOperationException("Unknown output format: $format");
}
}
protected printText(AssetManager manager, int level) {
final manifest = manager.getManifest()
out.println " project name: ${manager.project}"
out.println " repository : ${manager.repositoryUrl}"
out.println " local path : ${manager.projectPath}"
out.println " main script : ${manager.mainScriptName}"
if( manager.homePage && manager.homePage != manager.repositoryUrl )
out.println " home page : ${manager.homePage}"
if( manifest.description )
out.println " description : ${manifest.description}"
if( manifest.author )
out.println " author : ${manifest.author}"
def revs = manager.getRevisions(level)
if( revs.size() == 1 )
out.println " revision : ${revs[0]}"
else {
out.println " revisions : "
revs.each { out.println " $it" }
}
if( !checkForUpdates )
return
def updates = manager.getUpdates(level)
if( updates ) {
if( updates.size() == 1 && revs.size() == 1 )
out.println " updates : ${updates[0]}"
else {
out.println " updates : "
updates.each { out.println " $it" }
}
}
out.flush()
}
protected Map createMap(AssetManager manager) {
def result = [:]
result.projectName = manager.project
result.repository = manager.repositoryUrl
result.localPath = manager.projectPath?.toString()
result.manifest = manager.manifest.toMap()
result.revisions = manager.getBranchesAndTags(checkForUpdates)
return result
}
protected printJson(Map map) {
out.println JsonOutput.prettyPrint(JsonOutput.toJson(map))
out.flush()
}
protected printYaml(Map map) {
out.println new Yaml().dump(map).toString()
out.flush()
}
final static private BLANK = ' '
final static private NEWLINE = '\n'
static String status(boolean detailed = false) {
getInfo( detailed ? 1 : 0, true )
}
static private String totMem(OperatingSystemMXBean os) {
try {
new MemoryUnit(os.totalPhysicalMemorySize).toString()
}
catch (Throwable t) {
log.debug "Unable to fetch totalPhysicalMemorySize - ${t.message ?: t}"
return '-'
}
}
static private String freeMem(OperatingSystemMXBean os) {
try {
return new MemoryUnit(os.freePhysicalMemorySize).toString()
}
catch (Throwable t) {
log.debug "Unable to fetch freePhysicalMemorySize - ${t.message ?: t}"
return '-'
}
}
static private String totSwap(OperatingSystemMXBean os) {
try {
new MemoryUnit(os.totalSwapSpaceSize).toString()
}
catch (Throwable t) {
log.debug "Unable to fetch totalSwapSpaceSize - ${t.message ?: t}"
return '-'
}
}
static private String freeSwap(OperatingSystemMXBean os) {
try {
return new MemoryUnit(os.freeSwapSpaceSize).toString()
}
catch (Throwable t) {
log.debug "Unable to fetch freeSwapSpaceSize - ${t.message ?: t}"
return '-'
}
}
/**
* @return A string containing some system runtime information
*/
static String getInfo(int level, boolean printProc=false) {
def props = System.getProperties()
def result = new StringBuilder()
result << BLANK << "Version: ${BuildInfo.version} build ${BuildInfo.buildNum}" << NEWLINE
result << BLANK << "Created: ${BuildInfo.timestampUTC} ${BuildInfo.timestampDelta}" << NEWLINE
result << BLANK << "System: ${props['os.name']} ${props['os.version']}" << NEWLINE
result << BLANK << "Runtime: Groovy ${GroovySystem.getVersion()} on ${System.getProperty('java.vm.name')} ${props['java.runtime.version']}" << NEWLINE
result << BLANK << "Encoding: ${System.getProperty('file.encoding')} (${System.getProperty('sun.jnu.encoding')})" << NEWLINE
if( printProc ) {
final os = (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean()
result << BLANK << "Process: ${ManagementFactory.getRuntimeMXBean().getName()} " << getLocalAddress() << NEWLINE
result << BLANK << "CPUs: ${os.availableProcessors} - Mem: ${totMem(os)} (${freeMem(os)}) - Swap: ${totSwap(os)} (${freeSwap(os)})"
if( Threads.useVirtual() )
result << " - Virtual threads ON"
}
if( level == 0 )
return result.toString()
List<String> capsule = []
List<String> args = []
ManagementFactory
.getRuntimeMXBean()
.getInputArguments()
.each { String it ->
if( it.startsWith('-Dcapsule.'))
capsule << it.substring(2)
else
args << it
}
// file system
result << BLANK << "File systems: "
result << FileSystemProvider.installedProviders().collect { it.scheme }.join(', ')
result << NEWLINE
// plugins
result << BLANK << "Core plugins: "
result << DefaultPlugins.INSTANCE.toSortedString(', ')
result << NEWLINE
// JVM options
result << BLANK << "JVM opts:" << NEWLINE
for( String entry : args ) {
int p = entry.indexOf('=')
if( p!=-1 ) {
def key = entry.substring(0,p)
def value = entry.substring(p+1)
dump(key,value,2, result)
}
else {
dump(entry,null,2, result)
}
}
// Capsule options
dump("Capsule", capsule, 1, result)
// Env
result << BLANK << "Environment:" << NEWLINE
def entries = System.getenv().keySet().sort()
for( def key : entries ) {
if( key.startsWith('NXF_') || level>1 ) {
dump( key, System.getenv().get(key), 2, result )
}
}
// java properties
if( level>1 ) {
result << BLANK << "Properties:" << NEWLINE
entries = System.getProperties().keySet().sort()
for( String key : entries ) {
if( key == 'java.class.path' ) continue
dump( key, System.getProperties().get(key), 2, result )
}
}
// Class path
dump("Class-path" , System.getProperty('java.class.path'), 1, result)
// final string
return result.toString()
}
static private void dump(String key, def value, int indent, StringBuilder result) {
if( value instanceof String && value.count(':') && (key=~/.*PATH$/ || key=='PERL5LIB' || key.contains('.path') || key.contains('-path') || key.contains('.dir')) ) {
value = value.split(":") as Collection
}
else if( value?.class?.isArray() ) {
value = value as Collection
}
if ( value instanceof Collection ) {
blanks(indent, result)
result << key << (indent==1 ? ':' : '=')
for( String item : value ) {
result << NEWLINE
blanks(indent+1, result)
result << item
}
}
else if( value ) {
blanks(indent, result)
result << key << (indent==1 ? ':' : '=')
if( value == '\n' ) {
result << '\\n'
}
else if( value == '\r' ) {
result << '\\r'
}
else if( value == '\n\r' ) {
result << '\\n\\r'
}
else if( value == '\r\n' ) {
result << '\\r\\n'
}
else {
result << value
}
}
else if( key ) {
blanks(indent, result)
result << key
}
result << NEWLINE
}
static private blanks( int n, StringBuilder result ) {
for( int i=0; i<n; i++ ) {
result << BLANK
}
}
/**
* @return A string holding the local host name and address used for logging
*/
static private getLocalAddress() {
try {
return "[${InetAddress.getLocalHost().getHostAddress()}]"
}
catch(Exception e) {
return null
}
}
}

View File

@@ -0,0 +1,114 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
import com.beust.jcommander.DynamicParameter
import com.beust.jcommander.Parameter
import com.beust.jcommander.Parameters
import groovy.transform.CompileDynamic
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.Session
import nextflow.container.inspect.ContainerInspectMode
import nextflow.container.inspect.ContainersInspector
import nextflow.util.LoggerHelper
/**
* Implement `inspect` command
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
@Parameters(commandDescription = "Inspect process settings in a pipeline project")
class CmdInspect extends CmdBase {
@Override
String getName() {
return 'inspect'
}
@Parameter(names=['-concretize'], description = "Build the container images resolved by the inspect command")
boolean concretize
@Parameter(names=['-c','-config'], hidden = true)
List<String> runConfig
@Parameter(names=['-format'], description = "Inspect output format. Can be 'json' or 'config'")
String format = 'json'
@Parameter(names=['-i','-ignore-errors'], description = 'Ignore errors while inspecting the pipeline')
boolean ignoreErrors
@DynamicParameter(names = '--', hidden = true)
Map<String,String> params = new LinkedHashMap<>()
@Parameter(names='-params-file', description = 'Load script parameters from a JSON/YAML file')
String paramsFile
@Parameter(names=['-profile'], description = 'Use the given configuration profile(s)')
String profile
@Parameter(names=['-r','-revision'], description = 'Revision of the project to inspect (either a git branch, tag or commit SHA number)')
String revision
@Parameter(description = 'Project name or repository url')
List<String> args
@Override
void run() {
ContainerInspectMode.activate(!concretize)
// configure quiet mode
LoggerHelper.setQuiet(true)
// setup the target run command
final target = new CmdRun()
target.launcher = this.launcher
target.args = args
target.profile = this.profile
target.revision = this.revision
target.runConfig = this.runConfig
target.params = this.params
target.paramsFile = this.paramsFile
target.preview = true
target.previewAction = this.&applyInspect
target.ansiLog = false
target.skipHistoryFile = true
// run it
target.run()
}
protected void applyInspect(Session session) {
// slow down max rate when concretize is specified
if( concretize ) {
configureMaxRate(session.config)
}
// run the inspector
new ContainersInspector(concretize)
.withFormat(format)
.withIgnoreErrors(ignoreErrors)
.printContainers()
}
@CompileDynamic
protected void configureMaxRate(Map config) {
if( config.wave == null )
config.wave = new HashMap()
if( config.wave.httpClient == null )
config.wave.httpClient = new HashMap()
config.wave.httpClient.maxRate = '5/30sec'
}
}

View File

@@ -0,0 +1,109 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
import com.beust.jcommander.Parameter
import com.beust.jcommander.Parameters
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.exception.AbortOperationException
import nextflow.plugin.Plugins
import org.pf4j.ExtensionPoint
/**
* Extends `run` command to support Kubernetes deployment
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
@Parameters(commandDescription = "Execute a workflow in a Kubernetes cluster (experimental)")
class CmdKubeRun extends CmdRun {
interface KubeCommand extends ExtensionPoint {
int run(CmdKubeRun cmd, String pipeline, List<String> args)
}
static private String POD_NAME = /[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/
/**
* One or more volume claims mounts
*/
@Parameter(names = ['-v','-volume-mount'], description = 'Volume claim mounts eg. my-pvc:/mnt/path')
List<String> volMounts
@Parameter(names = ['-n','-namespace'], description = 'Specify the K8s namespace to use')
String namespace
@Parameter(names = '-head-image', description = 'Specify the container image for the Nextflow driver pod')
String headImage
@Parameter(names = '-pod-image', description = 'Alias for -head-image (deprecated)')
String podImage
@Parameter(names = '-head-cpus', description = 'Specify number of CPUs requested for the Nextflow driver pod')
int headCpus
@Parameter(names = '-head-memory', description = 'Specify amount of memory requested for the Nextflow driver pod')
String headMemory
@Parameter(names = '-head-prescript', description = 'Specify script to be run before nextflow run starts')
String headPreScript
@Parameter(names= '-remoteConfig', description = 'Add the specified file from the K8s cluster to configuration set', hidden = true )
List<String> runRemoteConfig
@Parameter(names=['-remoteProfile'], description = 'Choose a configuration profile in the remoteConfig')
String remoteProfile
@Override
String getName() { 'kuberun' }
@Override
protected void checkRunName() {
if( runName && !runName.matches(POD_NAME) )
throw new AbortOperationException("Not a valid K8s pod name -- It can only contain lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character")
super.checkRunName()
runName = runName.replace('_','-')
}
boolean background() { launcher.options.background }
protected hasAnsiLogFlag() { launcher.options.hasAnsiLogFlag() }
@Override
void run() {
final scriptArgs = (args?.size()>1 ? args[1..-1] : []) as List<String>
final pipeline = stdin ? '-' : ( args ? args[0] : null )
if( !pipeline )
throw new AbortOperationException("No project name was specified")
if( hasAnsiLogFlag() )
log.warn "Ansi logging not supported by kuberun command"
if( podImage ) {
log.warn "-pod-image is deprecated (use -head-image instead)"
headImage = podImage
}
checkRunName()
Plugins.init()
Plugins.start('nf-k8s')
// load the command operations
final command = Plugins.getExtension(KubeCommand)
final status = command.run(this, pipeline, scriptArgs)
System.exit(status)
}
}

View File

@@ -0,0 +1,207 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
import com.beust.jcommander.DynamicParameter
import com.beust.jcommander.Parameter
import com.beust.jcommander.Parameters
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.exception.AbortOperationException
import nextflow.plugin.Plugins
import org.pf4j.ExtensionPoint
/**
* Implements the 'nextflow launch' command
*
* @author Phil Ewels <phil.ewels@seqera.io>
*/
@Slf4j
@CompileStatic
@Parameters(commandDescription = "Launch a workflow in Seqera Platform")
class CmdLaunch extends CmdBase implements UsageAware {
interface LaunchCommand extends ExtensionPoint {
void launch(LaunchOptions options)
}
static final public String NAME = 'launch'
@Override
String getName() { NAME }
@Parameter(description = 'Pipeline repository URL')
List<String> args
@Parameter(names = ['-workspace'], description = 'Workspace name')
String workspace
@Parameter(names = ['-compute-env'], description = 'Compute environment name')
String computeEnv
@Parameter(names = ['-name'], description = 'Assign a mnemonic name to the pipeline run')
String runName
@Parameter(names = ['-w', '-work-dir'], description = 'Directory where intermediate result files are stored')
String workDir
@Parameter(names = ['-r', '-revision'], description = 'Revision of the project to run (either a git branch, tag or commit SHA number)')
String revision
@Parameter(names = ['-profile'], description = 'Choose a configuration profile')
String profile
@Parameter(names = ['-c', '-config'], description = 'Add the specified file to configuration set')
List<String> configFiles
@Parameter(names = ['-params-file'], description = 'Load script parameters from a JSON/YAML file')
String paramsFile
@Parameter(names = ['-entry'], description = 'Entry workflow name to be executed')
String entryName
@Parameter(names = ['-resume'], description = 'Execute the script using the cached results')
String resume
@Parameter(names = ['-latest'], description = 'Pull latest changes before run')
boolean latest
@Parameter(names = ['-stub-run', '-stub'], description = 'Execute the workflow replacing process scripts with command stubs')
boolean stubRun
@Parameter(names = ['-main-script'], description = 'The script file to be executed when launching a project directory or repository')
String mainScript
@Parameter(names = ['-user-secret'], description = 'Specify a user secret name to use in the pipeline (can be specified multiple times)')
List<String> userSecrets = []
@Parameter(names = ['-workspace-secret'], description = 'Specify a workspace secret name to use in the pipeline (can be specified multiple times)')
List<String> workspaceSecrets = []
/**
* Defines the parameters to be passed to the pipeline script
*/
@DynamicParameter(names = '--', description = 'Set a parameter used by the pipeline', hidden = true)
Map<String, String> params = new LinkedHashMap<>()
private LaunchCommand operation
@Override
void run() {
// Validate required parameters
if (!args || args.isEmpty()) {
log.debug "No pipeline repository URL provided"
throw new AbortOperationException("Pipeline repository URL is required")
}
// Load the Launch command implementation
this.operation = loadOperation()
if (!operation)
throw new IllegalStateException("Unable to load launch extension.")
// Create options object
final options = new LaunchOptions(
pipeline: args[0],
workspace: workspace,
computeEnv: computeEnv,
runName: runName,
workDir: workDir,
revision: revision,
profile: profile,
configFiles: configFiles,
paramsFile: paramsFile,
entryName: entryName,
resume: resume,
latest: latest,
stubRun: stubRun,
mainScript: mainScript,
params: params,
launcher: launcher,
userSecrets: userSecrets,
workspaceSecrets: workspaceSecrets
)
// Execute launch
operation.launch(options)
}
@Override
void usage() {
usage(null)
}
@Override
void usage(List<String> args) {
def result = []
result << this.getClass().getAnnotation(Parameters).commandDescription()
result << 'Usage: nextflow launch <pipeline> [options]'
result << ''
result << 'Options:'
result << ' -workspace <name> Workspace name'
result << ' -compute-env <name> Compute environment name (default: primary)'
result << ' -name <name> Assign a mnemonic name to the pipeline run'
result << ' -w, -work-dir <path> Directory where intermediate result files are stored'
result << ' -r, -revision <revision> Revision of the project to run (git branch, tag or commit SHA)'
result << ' -profile <profile> Choose a configuration profile'
result << ' -c, -config <file> Add the specified file to configuration set'
result << ' -params-file <file> Load script parameters from a JSON/YAML file'
result << ' -entry <name> Entry workflow name to be executed'
result << ' -resume [session] Execute the script using the cached results'
result << ' -latest Pull latest changes before run'
result << ' -stub-run, -stub Execute the workflow replacing process scripts with command stubs'
result << ' -main-script <file> The script file to be executed when launching a project'
result << ' -user-secret <name> User secret name to use in the pipeline (can be specified multiple times)'
result << ' -workspace-secret <name> Workspace secret name to use in the pipeline (can be specified multiple times)'
result << ' --<param>=<value> Set a parameter used by the pipeline'
result << ''
println result.join('\n').toString()
}
protected LaunchCommand loadOperation() {
// Setup the plugins system and load the launch provider
Plugins.init()
// Load the config
Plugins.start('nf-tower')
// Get Launch command operations implementation from plugins
return Plugins.getExtension(LaunchCommand)
}
/**
* Data class to hold launch options
*/
@CompileStatic
static class LaunchOptions {
String pipeline
String workspace
String computeEnv
String runName
String workDir
String revision
String profile
List<String> configFiles
String paramsFile
String entryName
String resume
boolean latest
boolean stubRun
String mainScript
Map<String, String> params
Launcher launcher
List<String> userSecrets
List<String> workspaceSecrets
}
}

View File

@@ -0,0 +1,313 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
import java.nio.file.Paths
import com.beust.jcommander.Parameter
import com.beust.jcommander.Parameters
import groovy.transform.CompileStatic
import nextflow.config.ConfigBuilder
import nextflow.config.ConfigMap
import nextflow.exception.AbortOperationException
import nextflow.plugin.Plugins
import org.pf4j.ExtensionPoint
/**
* CID command line interface
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
@Parameters(commandDescription = "Explore workflows lineage metadata", commandNames = ['li'])
class CmdLineage extends CmdBase implements UsageAware {
private static final String NAME = 'lineage'
interface LinCommand extends ExtensionPoint {
void list(ConfigMap config)
void view(ConfigMap config, List<String> args)
void render(ConfigMap config, List<String> args)
void diff(ConfigMap config, List<String> args)
void find(ConfigMap config, List<String> args)
void check(ConfigMap config, List<String> args)
}
interface SubCmd {
String getName()
String getDescription()
void apply(List<String> args)
void usage()
}
private List<SubCmd> commands = new ArrayList<>()
private LinCommand operation
private ConfigMap config
CmdLineage() {
commands << new CmdList()
commands << new CmdView()
commands << new CmdRender()
commands << new CmdDiff()
commands << new CmdFind()
commands << new CmdCheck()
}
@Parameter(hidden = true)
List<String> args
@Override
String getName() {
return NAME
}
@Override
void run() {
if( !args ) {
usage(List.of())
return
}
// setup the plugins system and load the secrets provider
Plugins.init()
// load the config
this.config = new ConfigBuilder()
.setOptions(launcher.options)
.setBaseDir(Paths.get('.'))
.build()
// init plugins
Plugins.load(config)
// load the command operations
this.operation = Plugins.getExtension(LinCommand)
if( !operation )
throw new IllegalStateException("Unable to load lineage extensions.")
// consume the first argument
getCmd(args).apply(args.drop(1))
}
/**
* Print the command usage help
*/
void usage() {
usage(args)
}
/**
* Print the command usage help
*
* @param args The arguments as entered by the user
*/
void usage(List<String> args) {
if( !args ) {
List<String> result = []
result << this.getClass().getAnnotation(Parameters).commandDescription()
result << "Usage: nextflow $NAME <sub-command> [options]".toString()
result << ''
result << 'Commands:'
int len = 0
commands.forEach {len = it.name.size() > len ? it.name.size() : len }
commands.sort(){it.name}.each { result << " ${it.name.padRight(len)}\t${it.description}".toString() }
result << ''
println result.join('\n').toString()
}
else {
def sub = commands.find { it.name == args[0] }
if( sub )
sub.usage()
else {
throw new AbortOperationException("Unknown $NAME sub-command: ${args[0]}")
}
}
}
protected SubCmd getCmd(List<String> args) {
def cmd = commands.find { it.name == args[0] }
if( cmd ) {
return cmd
}
def matches = commands.collect{ it.name }.closest(args[0])
def msg = "Unknown cloud sub-command: ${args[0]}"
if( matches )
msg += " -- Did you mean one of these?\n" + matches.collect { " $it"}.join('\n')
throw new AbortOperationException(msg)
}
class CmdList implements SubCmd {
@Override
String getName() {
return 'list'
}
@Override
String getDescription() {
return 'List the executions with lineage enabled'
}
@Override
void apply(List<String> args) {
if (args.size() != 0) {
println("ERROR: Incorrect number of parameters")
return
}
operation.list(config)
}
@Override
void usage() {
println description
println "Usage: nextflow $NAME $name"
}
}
class CmdView implements SubCmd{
@Override
String getName() {
return 'view'
}
@Override
String getDescription() {
return 'Print the description of a Lineage ID (lid)'
}
void apply(List<String> args) {
if (args.size() != 1) {
println("ERROR: Incorrect number of parameters")
return
}
operation.view(config, args)
}
@Override
void usage() {
println description
println "Usage: nextflow $NAME $name <lid> "
}
}
class CmdRender implements SubCmd {
@Override
String getName() { 'render' }
@Override
String getDescription() {
return 'Render the lineage graph for a workflow output'
}
void apply(List<String> args) {
if (args.size() < 1 || args.size() > 2) {
println("ERROR: Incorrect number of parameters")
return
}
operation.render(config, args)
}
@Override
void usage() {
println description
println "Usage: nextflow $NAME $name <workflow output lid> [<html output file>]"
}
}
class CmdDiff implements SubCmd {
@Override
String getName() { 'diff' }
@Override
String getDescription() {
return 'Show differences between two lineage descriptions'
}
void apply(List<String> args) {
if (args.size() != 2) {
println("ERROR: Incorrect number of parameters")
return
}
operation.diff(config, args)
}
@Override
void usage() {
println description
println "Usage: nextflow $NAME $name <lid-1> <lid-2>"
}
}
class CmdFind implements SubCmd {
@Override
String getName() { 'find' }
@Override
String getDescription() {
return 'Find lineage metadata descriptions matching with a query'
}
void apply(List<String> args) {
if (args.size() < 1) {
println("ERROR: Incorrect number of parameters")
return
}
operation.find(config, args)
}
@Override
void usage() {
println description
println "Usage: nextflow $NAME $name <query>"
}
}
class CmdCheck implements SubCmd {
@Override
String getName() { 'check' }
@Override
String getDescription() {
return 'Checks the integrity of an lineage file path'
}
void apply(List<String> args) {
if (args.size() != 1) {
println("ERROR: Incorrect number of parameters")
return
}
operation.check(config, args)
}
@Override
void usage() {
println description
println "Usage: nextflow $NAME $name <lid-file>"
}
}
}

View File

@@ -0,0 +1,541 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
import java.nio.file.Path
import java.time.Instant
import java.time.ZoneOffset
import java.time.format.DateTimeFormatter
import com.beust.jcommander.IParameterValidator
import com.beust.jcommander.Parameter
import com.beust.jcommander.Parameters
import com.beust.jcommander.ParameterException
import groovy.json.JsonOutput
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.BuildInfo
import nextflow.config.control.ConfigParser
import nextflow.config.formatter.ConfigFormattingVisitor
import nextflow.exception.AbortOperationException
import nextflow.script.control.Compiler
import nextflow.script.control.ParanoidWarning
import nextflow.script.control.ScriptParser
import nextflow.script.formatter.FormattingOptions
import nextflow.script.formatter.ScriptFormattingVisitor
import nextflow.script.parser.v2.ErrorListener
import nextflow.script.parser.v2.ErrorSummary
import nextflow.script.parser.v2.StandardErrorListener
import nextflow.util.ClassLoaderFactory
import nextflow.util.PathUtils
import org.codehaus.groovy.control.SourceUnit
import org.codehaus.groovy.control.messages.SyntaxErrorMessage
import org.codehaus.groovy.control.messages.WarningMessage
import org.codehaus.groovy.syntax.SyntaxException
/**
* CLI sub-command LINT
*
* @author Ben Sherman <bentshermann@gmail.com>
*/
@Slf4j
@CompileStatic
@Parameters(commandDescription = "Lint Nextflow scripts and config files")
class CmdLint extends CmdBase {
@Parameter(description = 'List of paths to lint')
List<String> args = []
@Parameter(
names = ['-exclude'],
description = 'File pattern to exclude from error checking (can be specified multiple times)'
)
List<String> excludePatterns = ['.git', '.lineage', '.nextflow', '.nf-test', 'nf-test.config', 'work']
@Parameter(
names = ['-files-from'],
description = 'Read list of paths to lint from a text file (one per line, use - for stdin)'
)
String filesFrom
@Parameter(
names = ['-o', '-output'],
description = 'Output mode for reporting errors: full, extended, concise, json, markdown',
validateWith = OutputModeValidator
)
String outputMode = 'full'
static class OutputModeValidator implements IParameterValidator {
private static final List<String> MODES = List.of('full', 'extended', 'concise', 'json', 'markdown')
@Override
void validate(String name, String value) {
if( !MODES.contains(value) )
throw new ParameterException("Output mode must be one of $MODES (found: $value)")
}
}
@Parameter(
names = ['-project-dir'],
description = 'Path to project directory (default: .)'
)
String projectDir = '.'
@Parameter(names = ['-format'], description = 'Format scripts and config files that have no errors')
boolean formatting
@Parameter(names = ['-harshil-alignment'], description = 'Use Harshil alignment')
boolean harhsilAlignment
@Parameter(names = ['-sort-declarations'], description = 'Sort script declarations in Nextflow scripts')
boolean sortDeclarations
@Parameter(names=['-spaces'], description = 'Number of spaces to indent')
int spaces
@Parameter(names = ['-tabs'], description = 'Indent with tabs')
boolean tabs
private ScriptParser scriptParser
private ConfigParser configParser
private ErrorListener errorListener
private FormattingOptions formattingOptions
private ErrorSummary summary = new ErrorSummary()
@Override
String getName() { 'lint' }
@Override
void run() {
// read input files from positional args and -files-from option
final inputs = getInputs(args, filesFrom)
if( !inputs )
throw new AbortOperationException("Error: No input files were specified")
if( spaces && tabs )
throw new AbortOperationException("Error: Cannot specify both `-spaces` and `-tabs`")
if( !spaces && !tabs )
spaces = 4
final baseDir = Path.of(projectDir)
final libDir = baseDir.resolve('lib')
final classLoader = ClassLoaderFactory.create([ libDir ])
scriptParser = new ScriptParser(baseDir, classLoader)
configParser = new ConfigParser()
errorListener = switch( outputMode ) {
case 'json' -> new JsonErrorListener()
case 'markdown' -> new MarkdownErrorListener()
default -> new StandardErrorListener(outputMode, launcher.options.ansiLog, launcher.options.quiet)
}
formattingOptions = new FormattingOptions(spaces, !tabs, harhsilAlignment, false, sortDeclarations)
errorListener.beforeAll()
// collect files to lint
final List<File> files = []
for( final input : inputs ) {
PathUtils.visitFiles(
Path.of(input),
(path) -> !PathUtils.isExcluded(path, excludePatterns),
(path) -> files.add(path.toFile()))
}
// parse and analyze files
for( final file : files )
parse(file)
scriptParser.analyze()
configParser.analyze()
// report errors
checkErrors(scriptParser.compiler())
checkErrors(configParser.compiler())
// format files if specified
if( formatting ) {
for( final file : files )
format(file)
}
// print summary
errorListener.afterAll(summary)
// If there were errors, throw an exception to return a non-zero exit code
if( summary.errors > 0 )
throw new AbortOperationException()
}
private static List<String> getInputs(List<String> args, String filesFrom) {
final List<String> result = []
result.addAll(args)
if( filesFrom ) {
final lines = filesFrom == '-'
? System.in.readLines()
: Path.of(filesFrom).readLines()
for( final line : lines ) {
final trimmed = line.trim()
if( trimmed )
result.add(trimmed)
}
}
return result
}
private void parse(File file) {
final name = file.getName()
if( name.endsWith('.nf') )
parseScript(file)
else if( name.endsWith('.config') )
parseConfig(file)
}
private void parseScript(File file) {
log.debug "Linting script ${file}"
errorListener.beforeFile(file)
scriptParser.parse(file)
}
private void parseConfig(File file) {
log.debug "Linting config ${file}"
errorListener.beforeFile(file)
configParser.parse(file)
}
private void checkErrors(Compiler compiler) {
compiler.getSources()
.values()
.stream()
.sorted(Comparator.comparing((SourceUnit source) -> source.getSource().getURI()))
.forEach((source) -> {
final errorCollector = source.getErrorCollector()
final hasWarnings = (errorCollector.getWarnings() ?: []).stream()
.anyMatch(warning -> warning !instanceof ParanoidWarning)
if( errorCollector.hasErrors() || hasWarnings )
printErrors(source)
if( errorCollector.hasErrors() )
summary.filesWithErrors += 1
else
summary.filesWithoutErrors += 1
if( hasWarnings )
summary.filesWithWarnings += 1
else
summary.filesWithoutWarnings += 1
})
}
private void printErrors(SourceUnit source) {
errorListener.beforeErrors()
final errors = source.getErrorCollector().getErrors() ?: []
errors.stream()
.filter(message -> message instanceof SyntaxErrorMessage)
.map(message -> ((SyntaxErrorMessage) message).getCause())
.sorted(ERROR_COMPARATOR)
.forEach((cause) -> {
errorListener.onError(cause, source.getName(), source)
summary.errors += 1
})
final warnings = source.getErrorCollector().getWarnings() ?: []
warnings.stream()
.filter(warning -> warning !instanceof ParanoidWarning)
.sorted(WARNING_COMPARATOR)
.forEach((warning) -> {
errorListener.onWarning(warning, source.getName(), source)
summary.warnings += 1
})
errorListener.afterErrors()
}
private static final Comparator<SyntaxException> ERROR_COMPARATOR = (SyntaxException a, SyntaxException b) -> {
return a.getStartLine() != b.getStartLine()
? a.getStartLine() - b.getStartLine()
: a.getStartColumn() - b.getStartColumn()
}
private static final Comparator<WarningMessage> WARNING_COMPARATOR = (WarningMessage w1, WarningMessage w2) -> {
final a = w1.getContext()
final b = w2.getContext()
return a.getStartLine() != b.getStartLine()
? a.getStartLine() - b.getStartLine()
: a.getStartColumn() - b.getStartColumn()
}
private void format(File file) {
final name = file.getName()
final result =
name.endsWith('.nf') ? formatScript(file) :
name.endsWith('.config') ? formatConfig(file) :
null
if( result != null && file.text != result ) {
summary.filesFormatted += 1
file.text = result
}
}
private String formatScript(File file) {
final source = scriptParser.compiler().getSource(file.toURI())
if( source.getErrorCollector().hasErrors() ) {
printErrors(source)
return null
}
log.debug "Formatting script ${file}"
errorListener.beforeFormat(file)
final formatter = new ScriptFormattingVisitor(source, formattingOptions)
formatter.visit()
return formatter.toString()
}
private String formatConfig(File file) {
final source = configParser.compiler().getSource(file.toURI())
if( source.getErrorCollector().hasErrors() ) {
printErrors(source)
return null
}
log.debug "Formatting config ${file}"
errorListener.beforeFormat(file)
final formatter = new ConfigFormattingVisitor(source, formattingOptions)
formatter.visit()
return formatter.toString()
}
}
@CompileStatic
class JsonErrorListener implements ErrorListener {
private List<Map> errors = []
private List<Map> warnings = []
@Override
void beforeAll() {
}
@Override
void beforeFile(File file) {
}
@Override
void beforeErrors() {
}
@Override
void onError(SyntaxException error, String filename, SourceUnit source) {
errors.add([
filename: filename,
startLine: error.getStartLine(),
startColumn: error.getStartColumn(),
message: error.getOriginalMessage()
])
}
@Override
void onWarning(WarningMessage warning, String filename, SourceUnit source) {
final token = warning.getContext().getRoot()
warnings.add([
filename: filename,
startLine: token.getStartLine(),
startColumn: token.getStartColumn(),
message: warning.getMessage()
])
}
@Override
void afterErrors() {
}
@Override
void beforeFormat(File file) {
}
@Override
void afterAll(ErrorSummary summary) {
final result = [
date: Instant.now().toString(),
summary: summary,
errors: errors,
warnings: warnings
]
println JsonOutput.prettyPrint(JsonOutput.toJson(result))
}
}
@CompileStatic
class MarkdownErrorListener implements ErrorListener {
private static class LintEntry {
String filename
int startLine
int startColumn
int endLine
int endColumn
String message
SourceUnit source
}
private List<LintEntry> errors = []
private List<LintEntry> warnings = []
@Override
void beforeAll() {
}
@Override
void beforeFile(File file) {
}
@Override
void beforeErrors() {
}
@Override
void onError(SyntaxException error, String filename, SourceUnit source) {
errors.add(new LintEntry(
filename: filename,
startLine: error.getStartLine(),
startColumn: error.getStartColumn(),
endLine: error.getEndLine(),
endColumn: error.getEndColumn(),
message: error.getOriginalMessage(),
source: source
))
}
@Override
void onWarning(WarningMessage warning, String filename, SourceUnit source) {
final token = warning.getContext().getRoot()
warnings.add(new LintEntry(
filename: filename,
startLine: token.getStartLine(),
startColumn: token.getStartColumn(),
endLine: token.getStartLine(),
endColumn: token.getStartColumn() + token.getText().length(),
message: warning.getMessage(),
source: source
))
}
@Override
void afterErrors() {
}
@Override
void beforeFormat(File file) {
}
@Override
void afterAll(ErrorSummary summary) {
final sb = new StringBuilder()
// Header
sb.append('# Nextflow lint results\n\n')
// Metadata
final timestamp = DateTimeFormatter.ISO_INSTANT.format(Instant.now().atOffset(ZoneOffset.UTC))
sb.append("- Generated: ${timestamp}\n")
sb.append("- Nextflow version: ${BuildInfo.version}\n")
// Summary line
final parts = []
if( summary.errors > 0 )
parts.add("${summary.errors} error${summary.errors == 1 ? '' : 's'}")
if( summary.warnings > 0 )
parts.add("${summary.warnings} warning${summary.warnings == 1 ? '' : 's'}")
if( parts.size() > 0 )
sb.append("- Summary: ${parts.join(', ')}\n")
else
sb.append("- Summary: No issues found\n")
// Sort entries by filename then position
final sortedErrors = errors.sort { a, b ->
final cmp = a.filename <=> b.filename
if( cmp != 0 ) return cmp
final lineCmp = a.startLine <=> b.startLine
if( lineCmp != 0 ) return lineCmp
return a.startColumn <=> b.startColumn
}
final sortedWarnings = warnings.sort { a, b ->
final cmp = a.filename <=> b.filename
if( cmp != 0 ) return cmp
final lineCmp = a.startLine <=> b.startLine
if( lineCmp != 0 ) return lineCmp
return a.startColumn <=> b.startColumn
}
// Errors section
if( sortedErrors.size() > 0 ) {
sb.append('\n## :x: Errors\n\n')
for( final entry : sortedErrors ) {
sb.append(formatEntry('Error', entry))
}
}
// Warnings section
if( sortedWarnings.size() > 0 ) {
sb.append('\n## :warning: Warnings\n\n')
for( final entry : sortedWarnings ) {
sb.append(formatEntry('Warning', entry))
}
}
println sb.toString().trim()
}
private String formatEntry(String type, LintEntry entry) {
final sb = new StringBuilder()
sb.append("- ${type}: `${entry.filename}:${entry.startLine}:${entry.startColumn}`: ${entry.message}\n\n")
// Add code context
final lines = entry.source.getSource().getReader().readLines()
if( entry.startLine > 0 && entry.startLine <= lines.size() ) {
final line = lines[entry.startLine - 1]
final startCol = Math.max(0, entry.startColumn - 1)
final endCol = Math.min(line.length(), Math.max(startCol + 1, entry.endColumn - 1))
sb.append(" ```nextflow\n")
sb.append(" ${line}\n")
// Add caret markers
final caretCount = Math.max(1, endCol - startCol)
sb.append(" ${' ' * startCol}${'^' * caretCount}\n")
sb.append(" ```\n\n")
}
return sb.toString()
}
}

View File

@@ -0,0 +1,51 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
import com.beust.jcommander.Parameters
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.scm.AssetManager
/**
* CLI sub-command LIST. Prints a list of locally installed pipelines
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
@Parameters(commandDescription = "List all downloaded projects")
class CmdList extends CmdBase {
static final public NAME = 'list'
@Override
final String getName() { NAME }
@Override
void run() {
def all = AssetManager.list()
if( !all ) {
log.info '(none)'
return
}
all.each { println it }
}
}

View File

@@ -0,0 +1,316 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
import java.nio.file.Path
import ch.artecat.grengine.Grengine
import com.beust.jcommander.Parameter
import com.beust.jcommander.Parameters
import com.google.common.hash.HashCode
import groovy.text.Template
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.exception.AbortOperationException
import nextflow.file.FileHelper
import nextflow.plugin.Plugins
import nextflow.processor.TaskRun
import nextflow.processor.TaskTemplateEngine
import nextflow.trace.TraceRecord
import nextflow.ui.TableBuilder
import static nextflow.cli.CmdHelper.fixEqualsOp
/**
* Implements the `log` command to print tasks runtime information of an execute pipeline
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
@Parameters(commandDescription = "Print executions log and runtime info")
class CmdLog extends CmdBase implements CacheBase {
static private List<String> ALL_FIELDS
static private DEFAULT_FIELDS = 'workdir'
static {
ALL_FIELDS = []
ALL_FIELDS.addAll( TraceRecord.FIELDS.keySet().collect { it.startsWith('%') ? 'p'+it.substring(1) : it } )
ALL_FIELDS << 'stdout'
ALL_FIELDS << 'stderr'
ALL_FIELDS << 'log'
ALL_FIELDS.sort(true)
}
static final public NAME = 'log'
@Parameter(names = ['-s'], description='Character used to separate column values')
String sep = '\\t'
@Parameter(names=['-f','-fields'], description = 'Comma separated list of fields to include in the printed log -- Use the `-l` option to show the list of available fields')
String fields
@Parameter(names = ['-t','-template'], description = 'Text template used to each record in the log ')
String templateStr
@Parameter(names=['-l','-list-fields'], description = 'Show all available fields', arity = 0)
boolean listFields
@Parameter(names=['-F','-filter'], description = "Filter log entries by a custom expression e.g. process =~ /foo.*/ && status == 'COMPLETED'")
String filterStr
@Parameter(names='-after', description = 'Show log entries for runs executed after the specified one')
String after
@Parameter(names='-before', description = 'Show log entries for runs executed before the specified one')
String before
@Parameter(names='-but', description = 'Show log entries of all runs except the specified one')
String but
@Parameter(names=['-q','-quiet'], description = 'Show only run names', arity = 0)
boolean quiet
@Parameter(description = 'Run name or session id')
List<String> args
private Script filterScript
private boolean showHistory
private Template templateScript
private Map<HashCode,Boolean> printed = new HashMap<>()
@Override
final String getName() { NAME }
void init() {
CacheBase.super.init()
//
// validate input options
//
if( fields && templateStr )
throw new AbortOperationException("Options `-f` and `-t` cannot be used in the same command")
//
// when no CLI options have been specified, just show the history log
showHistory = !args && !before && !after && !but
//
// initialise filter engine
//
if( filterStr ) {
filterScript = new Grengine().create("{ it -> ${fixEqualsOp(filterStr)} }")
}
//
// initialize the template engine
//
if( !templateStr ) {
if( !fields ) fields = DEFAULT_FIELDS
templateStr = fields.tokenize(', \n').collect { '$'+it } .join(sep)
}
else if( new File(templateStr).exists() ) {
templateStr = new File(templateStr).text
}
templateScript = new TaskTemplateEngine().createTemplate(templateStr)
}
/**
* Implements the `log` command
*/
@Override
void run() {
Plugins.init()
init()
// -- show the list of expected fields and exit
if( listFields ) {
ALL_FIELDS.each { println " $it" }
return
}
// -- show the current history and exit
if( showHistory ) {
quiet ? printQuiet() : printHistory()
return
}
// -- main
listIds().each { entry ->
cacheFor(entry)
.openForRead()
.eachRecord(this.&printRecord)
.close()
}
}
/**
* Print a log {@link TraceRecord} the the standard output by using the specified {@link #templateStr}
*
* @param record A {@link TraceRecord} instance representing a task runtime information
*/
protected void printRecord(HashCode hash, TraceRecord record) {
if( printed.containsKey(hash) )
return
else
printed.put(hash,Boolean.TRUE)
final adaptor = new TraceAdaptor(record)
if( filterScript ) {
filterScript.setBinding(adaptor)
// dynamic execution of the filter statement
// the `run` method interprets the statement groovy closure
// then the `call` method invokes the closure which returns a bool value
// if `false` skip this record
if( !((Closure)filterScript.run()).call() ) {
return
}
}
println templateScript.make(adaptor).toString()
}
private void printHistory() {
def table = new TableBuilder(cellSeparator: '\t')
.head('TIMESTAMP')
.head('DURATION')
.head('RUN NAME')
.head('STATUS')
.head('REVISION ID')
.head('SESSION ID')
.head('COMMAND')
history.eachRow { List<String> row ->
row[4] = row[4].size()>10 ? row[4].substring(0,10) : row[4]
table.append(row)
}
println table.toString()
}
private void printQuiet() {
history.eachRow { List row -> println(row[2]) }
}
/**
* Wrap a {@link TraceRecord} instance as a {@link Map} or a {@link Binding} object
*/
private static class TraceAdaptor extends Binding {
static private int MAX_LINES = 100
private TraceRecord record
@Delegate
private Map<String,Object> delegate = [:]
TraceAdaptor(TraceRecord record) {
this.record = record
}
@Override
boolean containsKey(Object key) {
delegate.containsKey(key.toString()) || record.containsKey(key.toString())
}
@Override
Object get(Object key) {
if( delegate.containsKey(key) ) {
return delegate.get(key)
}
if( key == 'stdout' ) {
return fetch(getWorkDir().resolve(TaskRun.CMD_OUTFILE))
}
if( key == 'stderr' ) {
return fetch(getWorkDir().resolve(TaskRun.CMD_ERRFILE))
}
if( key == 'log' ) {
return fetch(getWorkDir().resolve(TaskRun.CMD_LOG))
}
if( key == 'pcpu' )
return record.getFmtStr('%cpu')
if( key == 'pmem' )
return record.getFmtStr('%mem')
return record.getFmtStr(normaliseKey(key))
}
String normaliseKey(key) {
key .toString() .toLowerCase()
}
Object getVariable(String name) {
if( name == 'pcpu' )
return record.store.get('%cpu')
if( name == 'pmem' )
return record.store.get('%mem')
if( record.containsKey(name) )
return record.store.get(name)
throw new MissingPropertyException(name)
}
Map getVariables() {
new HashMap(record.store)
}
private Path getWorkDir() {
def folder = (String)record.get('workdir')
folder ? FileHelper.asPath(folder) : null
}
private String fetch(Path path) {
try {
int c=0
def result = new StringBuilder()
path.withReader { reader ->
String line
while( (line=reader.readLine()) && c++<MAX_LINES ) {
result << line << '\n'
}
}
result.toString() ?: TraceRecord.NA
}
catch( IOError e ) {
log.debug "Failed to fetch content for file: $path -- Cause: ${e.message ?: e}"
return TraceRecord.NA
}
}
}
}

View File

@@ -0,0 +1,158 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
import com.beust.jcommander.JCommander
import com.beust.jcommander.Parameter
import com.beust.jcommander.ParameterException
import com.beust.jcommander.Parameters
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.cli.module.CmdModuleCreate
import nextflow.cli.module.CmdModuleView
import nextflow.cli.module.CmdModuleInstall
import nextflow.cli.module.CmdModuleList
import nextflow.cli.module.CmdModulePublish
import nextflow.cli.module.CmdModuleRemove
import nextflow.cli.module.CmdModuleRun
import nextflow.cli.module.CmdModuleSearch
import nextflow.cli.module.CmdModuleSpec
import nextflow.cli.module.CmdModuleValidate
import nextflow.exception.AbortOperationException
/**
* Implements `module` command
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
@CompileStatic
@Slf4j
@Parameters(commandDescription = "Manage Nextflow modules")
class CmdModule extends CmdBase implements UsageAware {
static final public String NAME = 'module'
private JCommander jCommander
static final List<CmdBase> commands = new ArrayList<>()
static {
commands << new CmdModuleCreate()
commands << new CmdModuleInstall()
commands << new CmdModuleRun()
commands << new CmdModuleList()
commands << new CmdModuleRemove()
commands << new CmdModuleSearch()
commands << new CmdModuleView()
commands << new CmdModulePublish()
commands << new CmdModuleSpec()
commands << new CmdModuleValidate()
}
protected JCommander commander() {
if( !this.jCommander ) {
this.jCommander = new JCommander(this)
this.jCommander.setProgramName('nextflow module')
// Register all subcommands
commands.each { cmd ->
cmd.launcher = this.launcher
this.jCommander.addCommand(cmd.getName(), cmd, cmd.getAliases() as String[])
}
}
return jCommander
}
@Parameter
List<String> args
@Override
String getName() {
return NAME
}
@Override
void run() {
try {
if( !args ) {
usage()
return
}
final jc = commander()
final moduleArgs = args + unknownOptions
jc.parse(moduleArgs as String[])
final parsedCommand = jc.getParsedCommand()
if( !parsedCommand ) {
jc.usage()
return
}
// Get the parsed subcommand instance
final subcommand = jc.getCommands()
.get(parsedCommand)
.getObjects()[0] as CmdBase
// Execute with fields already populated by JCommander
subcommand.run()
} catch( ParameterException e ) {
throw new AbortOperationException("${e.getMessage()} -- Check the available commands and options and syntax with 'nextflow module -h'")
}
}
private CmdBase findCmd(String name) {
commands.find { it.name == name || name in it.aliases }
}
/**
* Print the command usage help
*/
@Override
void usage() {
usage(args)
}
/**
* Print the command usage help
*
* @param args The arguments as entered by the user
*/
@Override
void usage(List<String> args) {
def result = []
if( !args ) {
result << 'Usage: nextflow module <command> [options]'
result << ''
result << 'Commands:'
commands.each {
def description = it.getClass().getAnnotation(Parameters)?.commandDescription()
result << " ${it.name.padRight(12)}${description}"
}
result << ''
println result.join('\n').toString()
} else {
final sub = findCmd(args[0])
if( sub ) {
commander().usage(args[0])
} else {
throw new AbortOperationException("Unknown module sub-command: ${args[0]}")
}
}
}
}

View File

@@ -0,0 +1,151 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
import com.beust.jcommander.DynamicParameter
import com.beust.jcommander.Parameter
import com.beust.jcommander.Parameters
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.config.ConfigBuilder
import nextflow.daemon.DaemonLauncher
import nextflow.plugin.Plugins
import nextflow.util.ServiceName
import nextflow.util.ServiceDiscover
/**
* CLI-command NODE
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
@Parameters
class CmdNode extends CmdBase {
static final public NAME = 'node'
@Override
final String getName() { NAME }
@DynamicParameter(names ='-cluster.', description='Define cluster config options')
Map<String,String> clusterOptions = [:]
@Parameter(names = ['-bg'], arity = 0, description = 'Start the cluster node daemon in background')
void setBackground(boolean value) {
launcher.options.background = value
}
@Parameter
List<String> provider
@Override
void run() {
System.setProperty('nxf.node.daemon', 'true')
launchDaemon(provider ? provider[0] : null)
}
/**
* Launch the daemon service
*
* @param config The nextflow configuration map
*/
protected launchDaemon(String name = null) {
// create the config object
def config = new ConfigBuilder()
.setOptions(launcher.options)
.setCmdNode(this)
.build()
DaemonLauncher instance
if( name ) {
if( name.contains('.') ) {
instance = loadDaemonByClass(name)
}
else {
instance = loadDaemonByName(name)
}
}
else {
instance = loadDaemonFirst()
}
// launch it
instance.launch(config)
}
/**
* Load a {@code DaemonLauncher} instance of the its *friendly* name i.e. the name provided
* by using the {@code ServiceName} annotation on the daemon class definition
*
* @param name The executor name e.g. {@code gridgain}
* @return The daemon launcher instance
* @throws IllegalStateException if the class does not exist or it cannot be instantiated
*/
static DaemonLauncher loadDaemonByName( String name ) {
Class<DaemonLauncher> clazz = null
for( Class<DaemonLauncher> item : ServiceDiscover.load(DaemonLauncher) ) {
log.debug "Discovered daemon class: ${item.name}"
ServiceName annotation = item.getAnnotation(ServiceName)
if( annotation && annotation.value() == name ) {
clazz = item
break
}
}
if( !clazz )
throw new IllegalStateException("Unknown daemon name: $name")
try {
clazz.newInstance()
}
catch( Exception e ) {
throw new IllegalStateException("Unable to launch executor: $name", e)
}
}
/**
* Load a class implementing the {@code DaemonLauncher} interface by the specified class name
*
* @param name The fully qualified class name e.g. {@code nextflow.executor.local.LocalExecutor}
* @return The daemon launcher instance
* @throws IllegalStateException if the class does not exist or it cannot be instantiated
*/
static DaemonLauncher loadDaemonByClass( String name ) {
try {
return (DaemonLauncher)Class.forName(name).newInstance()
}
catch( Exception e ) {
throw new IllegalStateException("Cannot load daemon: ${name}")
}
}
/**
* @return The first available instance of a class implementing {@code DaemonLauncher}
* @throws IllegalStateException when no class implementing {@code DaemonLauncher} is available
*/
static DaemonLauncher loadDaemonFirst() {
Plugins.init()
final loader = Plugins.getExtension(DaemonLauncher)
if( !loader )
throw new IllegalStateException("No cluster services are available -- Cannot launch Nextflow in cluster mode")
return loader
}
}

View File

@@ -0,0 +1,185 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
import static nextflow.cli.PluginExecAware.CMD_SEP
import java.nio.file.Path
import com.beust.jcommander.DynamicParameter
import com.beust.jcommander.Parameter
import com.beust.jcommander.Parameters
import groovy.transform.CompileStatic
import nextflow.exception.AbortOperationException
import nextflow.plugin.Plugins
import nextflow.plugin.util.PluginRefactor
import org.eclipse.jgit.api.Git
/**
* Plugin manager command
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
@Parameters(commandDescription = "Execute plugin-specific commands")
class CmdPlugin extends CmdBase {
@Override
String getName() {
return 'plugin'
}
@DynamicParameter(names = "--", description = "Custom plugin parameters go here", hidden = true)
private Map<String, String> params = new HashMap<>();
@Parameter(hidden = true)
List<String> args
@Parameter(names = ['-template'], description = 'Plugin template version to use', hidden = true)
String templateVersion = 'v0.3.0'
@Override
void run() {
if( !args )
throw new AbortOperationException("Missing plugin command - usage: nextflow plugin install <pluginId,..>")
// setup plugins system
Plugins.init()
Runtime.addShutdownHook((it)-> Plugins.stop())
// check for the plugins install
if( args[0] == 'install' ) {
if( args.size()!=2 )
throw new AbortOperationException("Missing plugin install target - usage: nextflow plugin install <pluginId,..>")
Plugins.pull(args[1].tokenize(','))
}
else if( args[0] == 'create' ) {
createPlugin(args, templateVersion)
}
// plugin run command
else if( args[0].contains(CMD_SEP) ) {
final head = args.pop()
final items = head.tokenize(CMD_SEP)
final target = items[0]
final cmd = items[1] ? items[1..-1].join(CMD_SEP) : null
// push back the command as the first item
Plugins.start(target)
final wrapper = Plugins.manager.getPlugin(target)
if( !wrapper )
throw new AbortOperationException("Cannot find target plugin: $target")
final plugin = wrapper.getPlugin()
if( plugin instanceof PluginExecAware ) {
def mapped = [] as List<String>
params.entrySet().each{
mapped << "--$it.key".toString()
mapped << "$it.value".toString()
}
args.addAll(mapped)
final ret = plugin.exec(getLauncher(), target, cmd, args)
// use explicit exit to invoke the system shutdown hooks
System.exit(ret)
}
else
throw new AbortOperationException("Invalid target plugin: $target")
}
else {
throw new AbortOperationException("Invalid plugin command: ${args[0]}")
}
}
static createPlugin(List<String> args, String templateVersion) {
if( args != ['create'] && (args[0] != 'create' || !(args.size() in [3, 4])) )
throw new AbortOperationException("Invalid create parameters - usage: nextflow plugin create <Plugin name> <Provider name>")
final refactor = new PluginRefactor()
if( args.size()>1 ) {
refactor.withPluginName(args[1])
refactor.withProviderName(args[2])
refactor.withPluginDir(Path.of(args[3] ?: refactor.pluginName).toFile())
}
else {
// Prompt for plugin name
print "Enter plugin name: "
refactor.withPluginName(readLine())
// Prompt for provider name
print "Enter provider name: "
refactor.withProviderName(readLine())
// Prompt for project path (default to the normalised plugin name)
print "Enter project path [${refactor.pluginName}]: "
refactor.withPluginDir(Path.of(readLine() ?: refactor.pluginName).toFile())
// confirm and proceed
print "All good, are you OK to continue [y/N]? "
final confirm = readLine()
if( confirm?.toLowerCase()!='y' ) {
println "Plugin creation aborted."
return
}
}
// the final directory where the plugin is created
final File targetDir = refactor.getPluginDir()
// clone the template repo
clonePluginTemplate(targetDir, templateVersion)
// now refactor the template code
refactor.apply()
// remove git plat
cleanup(targetDir)
// done
println "Plugin created successfully at path: $targetDir"
}
static private String readLine() {
final console = System.console()
return console != null
? console.readLine()
: new BufferedReader(new InputStreamReader(System.in)).readLine()
}
static private void clonePluginTemplate(File targetDir, String templateVersion) {
final templateUri = "https://github.com/nextflow-io/nf-plugin-template.git"
final isTag = templateVersion.startsWith('v')
final refSpec = isTag ? "refs/tags/$templateVersion".toString() : templateVersion
try {
final gitCmd = Git.cloneRepository()
.setURI(templateUri)
.setDirectory(targetDir)
if (isTag) {
gitCmd.setBranchesToClone([refSpec])
gitCmd.setBranch(refSpec)
} else {
// For branches, let Git handle the default behavior
gitCmd.setBranch(templateVersion)
}
gitCmd.call()
}
catch (Exception e) {
throw new AbortOperationException("Unable to clone pluging template repository - cause: ${e.message}")
}
}
static private void cleanup(File targetDir) {
new File(targetDir, '.git').deleteDir()
new File(targetDir, '.github').deleteDir()
new File(targetDir, 'validation').deleteDir()
}
}

View File

@@ -0,0 +1,123 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
import com.beust.jcommander.Parameter
import com.beust.jcommander.Parameters
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.exception.AbortOperationException
import nextflow.plugin.Plugins
import nextflow.scm.AssetManager
import nextflow.util.TestOnly
/**
* CLI sub-command PULL
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
@Parameters(commandDescription = "Download or update a project")
class CmdPull extends CmdBase implements HubOptions {
static final public NAME = 'pull'
@Parameter(description = 'project name or repository url to pull', arity = 1)
List<String> args
@Parameter(names=['-a','-all'], description = 'Update all downloaded projects', arity = 0)
boolean all
@Parameter(names=['-r','-revision'], description = 'Revision of the project to pull (either a git branch, tag or commit SHA number)')
String revision
@Parameter(names=['-d','-deep'], description = 'Create a shallow clone of the specified depth')
Integer deep
@Parameter(names=['-m','-migrate'], description = 'Migrate projects to multi-revision strategy', arity = 0)
boolean migrate
@Override
final String getName() { NAME }
@TestOnly
protected File root
@Override
void run() {
if( !all && !args )
throw new AbortOperationException('Missing argument')
if( all && args )
throw new AbortOperationException('Option `all` requires no arguments')
if( all && revision )
throw new AbortOperationException('Option `all` is not compatible with `revision`')
def list = all ? AssetManager.list() : args.toList()
if( !list ) {
log.info "(nothing to do)"
return
}
if( root ) {
AssetManager.root = root
}
// init plugin system
Plugins.init()
for( String proj : list ) {
if( all ) {
try (def mgr = new AssetManager(proj)) {
def branches = mgr.getBranchesAndTags(false).pulled as List<String>
branches.each { rev -> pullProjectRevision(proj, rev) }
}
} else {
pullProjectRevision(proj, revision)
}
}
}
private pullProjectRevision(String project, String revision) {
try (final manager = new AssetManager(project, this)) {
if( manager.isUsingLegacyStrategy() ) {
if( migrate ) {
log.info "Migrating ${project} revision ${revision} to multi-revision strategy"
manager.setStrategyType(AssetManager.RepositoryStrategyType.MULTI_REVISION)
} else {
log.warn "The local asset for ${project} does not support multi-revision - Pulling with legacy strategy\n" +
"Consider updating the project ${project} using '-migrate' option"
}
}
if( revision )
manager.setRevision(revision)
log.info "Checking ${manager.getProjectWithRevision()} ..."
def result = manager.download(revision, deep)
manager.updateModules()
def scriptFile = manager.getScriptFile()
String message = !result ? " done" : " $result"
message += " - revision: ${scriptFile.revisionInfo}"
log.info message
}
}
}

View File

@@ -0,0 +1,864 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
import static org.fusesource.jansi.Ansi.*
import java.nio.file.NoSuchFileException
import java.nio.file.Path
import java.util.regex.Pattern
import com.beust.jcommander.DynamicParameter
import com.beust.jcommander.IStringConverter
import com.beust.jcommander.Parameter
import com.beust.jcommander.Parameters
import groovy.json.JsonSlurper
import groovy.transform.CompileStatic
import groovy.transform.Memoized
import groovy.util.logging.Slf4j
import groovyx.gpars.GParsConfig
import nextflow.BuildInfo
import nextflow.NF
import nextflow.NextflowMeta
import nextflow.SysEnv
import nextflow.config.ConfigBuilder
import nextflow.config.ConfigMap
import nextflow.config.ConfigValidator
import nextflow.config.Manifest
import nextflow.exception.AbortOperationException
import nextflow.file.FileHelper
import nextflow.plugin.Plugins
import nextflow.scm.AssetManager
import nextflow.script.ScriptFile
import nextflow.script.ScriptRunner
import nextflow.secret.EmptySecretProvider
import nextflow.secret.SecretsLoader
import nextflow.util.CustomPoolFactory
import nextflow.util.Duration
import nextflow.util.HistoryFile
import nextflow.util.VersionNumber
import org.apache.commons.lang3.StringUtils
import org.fusesource.jansi.AnsiConsole
import org.yaml.snakeyaml.Yaml
/**
* CLI sub-command RUN
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
@Parameters(commandDescription = "Execute a pipeline project")
class CmdRun extends CmdBase implements HubOptions {
static final public Pattern RUN_NAME_PATTERN = Pattern.compile(/^[a-z](?:[a-z\d]|[-_](?=[a-z\d])){0,79}$/, Pattern.CASE_INSENSITIVE)
static final public List<String> VALID_PARAMS_FILE = ['json', 'yml', 'yaml']
static final public String DSL2 = '2'
static final public String DSL1 = '1'
static {
// install the custom pool factory for GPars threads
GParsConfig.poolFactory = new CustomPoolFactory()
}
static class DurationConverter implements IStringConverter<Long> {
@Override
Long convert(String value) {
if( !value ) throw new IllegalArgumentException()
if( value.isLong() ) { return value.toLong() }
return Duration.of(value).toMillis()
}
}
static final public String NAME = 'run'
private Map<String,String> sysEnv = System.getenv()
@Parameter(names=['-name'], description = 'Assign a mnemonic name to the a pipeline run')
String runName
@Parameter(names=['-lib'], description = 'Library extension path')
String libPath
@Parameter(names=['-cache'], description = 'Enable/disable processes caching', arity = 1)
Boolean cacheable
@Parameter(names=['-resume'], description = 'Execute the script using the cached results, useful to continue executions that was stopped by an error')
String resume
@Parameter(names=['-ps','-pool-size'], description = 'Number of threads in the execution pool', hidden = true)
Integer poolSize
@Parameter(names=['-pi','-poll-interval'], description = 'Executor poll interval (duration string ending with ms|s|m)', converter = DurationConverter, hidden = true)
long pollInterval
@Parameter(names=['-qs','-queue-size'], description = 'Max number of processes that can be executed in parallel by each executor')
Integer queueSize
@Parameter(names=['-test'], description = 'Test a script function with the name specified')
String test
@Parameter(names=['-o', '-output-dir'], description = 'Directory where workflow outputs are stored')
String outputDir
@Parameter(names=['-output-format'], description = 'Output format for printing workflow outputs. Options: `text` (default), `json`, `none`')
String outputFormat = 'text'
@Parameter(names=['-w', '-work-dir'], description = 'Directory where intermediate result files are stored')
String workDir
@Parameter(names=['-bucket-dir'], description = 'Remote bucket where intermediate result files are stored')
String bucketDir
@Parameter(names=['-with-cloudcache'], description = 'Enable the use of object storage bucket as storage for cache meta-data')
String cloudCachePath
/**
* Defines the parameters to be passed to the pipeline script
*/
@DynamicParameter(names = '--', description = 'Set a parameter used by the pipeline', hidden = true)
Map<String,String> params = new LinkedHashMap<>()
@Parameter(names='-params-file', description = 'Load script parameters from a JSON/YAML file')
String paramsFile
@DynamicParameter(names = ['-process.'], description = 'Set process options' )
Map<String,String> process = [:]
@DynamicParameter(names = ['-e.'], description = 'Add the specified variable to execution environment')
Map<String,String> env = [:]
@Parameter(names = ['-E'], description = 'Exports all current system environment')
boolean exportSysEnv
@DynamicParameter(names = ['-executor.'], description = 'Set executor options', hidden = true )
Map<String,String> executorOptions = [:]
@Parameter(description = 'Project name or repository url')
List<String> args
@Parameter(names=['-r','-revision'], description = 'Revision of the project to run (either a git branch, tag or commit SHA number)')
String revision
@Parameter(names=['-d','-deep'], description = 'Create a shallow clone of the specified depth')
Integer deep
@Parameter(names=['-latest'], description = 'Pull latest changes before run')
boolean latest
@Parameter(names='-stdin', hidden = true)
boolean stdin
@Parameter(names = ['-ansi'], hidden = true, arity = 0)
void setAnsi(boolean value) {
launcher.options.ansiLog = value
}
@Parameter(names = ['-ansi-log'], description = 'Enable/disable ANSI console logging', arity = 1)
void setAnsiLog(boolean value) {
launcher.options.ansiLog = value
}
@Parameter(names = ['-with-tower'], description = 'Monitor workflow execution with Seqera Platform (formerly Tower Cloud)')
String withTower
@Parameter(names = ['-with-wave'], hidden = true)
String withWave
@Parameter(names = ['-with-fusion'], hidden = true)
String withFusion
@Parameter(names = ['-with-weblog'], description = 'Send workflow status messages via HTTP to target URL')
String withWebLog
@Parameter(names = ['-with-trace'], description = 'Create processes execution tracing file')
String withTrace
@Parameter(names = ['-with-report'], description = 'Create processes execution html report')
String withReport
@Parameter(names = ['-with-timeline'], description = 'Create processes execution timeline file')
String withTimeline
@Parameter(names = '-with-charliecloud', description = 'Enable process execution in a Charliecloud container runtime')
def withCharliecloud
@Parameter(names = '-with-singularity', description = 'Enable process execution in a Singularity container')
def withSingularity
@Parameter(names = '-with-apptainer', description = 'Enable process execution in a Apptainer container')
def withApptainer
@Parameter(names = '-with-podman', description = 'Enable process execution in a Podman container')
def withPodman
@Parameter(names = '-without-podman', description = 'Disable process execution in a Podman container')
def withoutPodman
@Parameter(names = '-with-docker', description = 'Enable process execution in a Docker container')
def withDocker
@Parameter(names = '-without-docker', description = 'Disable process execution with Docker', arity = 0)
boolean withoutDocker
@Parameter(names = '-with-mpi', hidden = true)
boolean withMpi
@Parameter(names = '-with-dag', description = 'Create pipeline DAG file')
String withDag
@Parameter(names = ['-bg'], arity = 0, hidden = true)
void setBackground(boolean value) {
launcher.options.background = value
}
@Parameter(names=['-c','-config'], hidden = true )
List<String> runConfig
@DynamicParameter(names = ['-cluster.'], description = 'Set cluster options', hidden = true )
Map<String,String> clusterOptions = [:]
@Parameter(names=['-profile'], description = 'Choose a configuration profile')
String profile
@Parameter(names=['-dump-hashes'], description = 'Dump task hash keys for debugging purpose')
String dumpHashes
@Parameter(names=['-dump-channels'], description = 'Dump channels for debugging purpose')
String dumpChannels
@Parameter(names=['-N','-with-notification'], description = 'Send a notification email on workflow completion to the specified recipients')
String withNotification
@Parameter(names=['-with-conda'], description = 'Use the specified Conda environment package or file (must end with .yml|.yaml suffix)')
String withConda
@Parameter(names=['-without-conda'], description = 'Disable the use of Conda environments')
Boolean withoutConda
@Parameter(names=['-with-spack'], description = 'Use the specified Spack environment package or file (must end with .yaml suffix)')
String withSpack
@Parameter(names=['-without-spack'], description = 'Disable the use of Spack environments')
Boolean withoutSpack
@Parameter(names=['-offline'], description = 'Do not check for remote project updates')
boolean offline = System.getenv('NXF_OFFLINE')=='true'
@Parameter(names=['-entry'], description = 'Entry workflow name to be executed', arity = 1)
String entryName
@Parameter(names=['-main-script'], description = 'The script file to be executed when launching a project directory or repository' )
String mainScript
@Parameter(names=['-stub-run','-stub'], description = 'Execute the workflow replacing process scripts with command stubs')
boolean stubRun
@Parameter(names=['-preview'], description = "Run the workflow script skipping the execution of all processes")
boolean preview
@Parameter(names=['-plugins'], description = 'Specify the plugins to be applied for this run e.g. nf-amazon,nf-tower')
String plugins
@Parameter(names=['-disable-jobs-cancellation'], description = 'Prevent the cancellation of child jobs on execution termination')
Boolean disableJobsCancellation
Boolean skipHistoryFile
Boolean getDisableJobsCancellation() {
return disableJobsCancellation!=null
? disableJobsCancellation
: sysEnv.get('NXF_DISABLE_JOBS_CANCELLATION') as boolean
}
/**
* Optional closure modelling an action to be invoked when the preview mode is enabled
*/
Closure<Void> previewAction
@Override
String getName() { NAME }
String getParamsFile() {
return paramsFile ?: sysEnv.get('NXF_PARAMS_FILE')
}
boolean hasParams() {
return params || getParamsFile()
}
@Override
void run() {
final scriptArgs = (args?.size()>1 ? args[1..-1] : []) as List<String>
final pipeline = stdin ? '-' : ( args ? args[0] : null )
if( !pipeline )
throw new AbortOperationException("No project name was specified")
if( withPodman && withoutPodman )
throw new AbortOperationException("Command line options `-with-podman` and `-without-podman` cannot be specified at the same time")
if( withDocker && withoutDocker )
throw new AbortOperationException("Command line options `-with-docker` and `-without-docker` cannot be specified at the same time")
if( withConda && withoutConda )
throw new AbortOperationException("Command line options `-with-conda` and `-without-conda` cannot be specified at the same time")
if( withSpack && withoutSpack )
throw new AbortOperationException("Command line options `-with-spack` and `-without-spack` cannot be specified at the same time")
if( offline && latest )
throw new AbortOperationException("Command line options `-latest` and `-offline` cannot be specified at the same time")
if( outputFormat !in ['text', 'json', 'none'] )
throw new AbortOperationException("Command line option `-output-format` should be either `text`, `json`, or `none`")
checkRunName()
printBanner()
Plugins.init()
// -- resolve main script
final scriptFile = getScriptFile(pipeline)
// -- load command line params
final baseDir = scriptFile.parent
final cliParams = parsedParams(ConfigBuilder.getConfigVars(baseDir, null))
/*
* 2-PHASE CONFIGURATION LOADING STRATEGY
*
* Problem: Configuration files may reference secrets provided by plugins (e.g., AWS secrets),
* but plugins are loaded AFTER configuration parsing. This creates a chicken-and-egg problem:
* - Config parsing needs secret values to complete
* - Plugin loading needs config to determine which plugins to load
* - Secret providers are registered by plugins
*
* Solution: Parse configuration twice when secrets are referenced
*
* PHASE 1: Parse config with EmptySecretProvider (returns "" for all secrets)
* - Configuration must use defensive patterns: secrets.FOO ? "value-${secrets.FOO}" : "fallback"
* - Config parses successfully with fallback values
* - EmptySecretProvider tracks if ANY secrets were accessed
*/
// -- PHASE 1: Load config with mock secrets provider
final secretsProvider = new EmptySecretProvider()
ConfigBuilder builder = new ConfigBuilder()
.setOptions(launcher.options)
.setCmdRun(this)
.setBaseDir(scriptFile.parent)
.setCliParams(cliParams)
.setSecretsProvider(secretsProvider) // Mock provider returns empty strings
ConfigMap config = builder.build()
Map configParams = builder.getConfigParams()
// -- Check Nextflow version
checkVersion(config)
// -- Load plugins (may register secret providers)
Plugins.load(config)
// -- Initialize real secrets system
SecretsLoader.getInstance().load()
/*
* PHASE 2: Conditionally reload config with real secrets
* - Only reload if Phase 1 actually accessed any secrets
* - This time, real secret providers are available (including plugin-provided ones)
* - Same config expressions now resolve with actual secret values
*/
// -- PHASE 2: Reload config if secrets were used in Phase 1
if( secretsProvider.usedSecrets() ) {
log.debug "Config file used secrets -- reloading config with secrets provider"
builder = new ConfigBuilder()
.setOptions(launcher.options)
.setCmdRun(this)
.setBaseDir(scriptFile.parent)
.setCliParams(cliParams)
// No .setSecretsProvider() - uses real secrets system now
config = builder.build()
configParams = builder.getConfigParams()
}
// check DSL syntax in the config
launchInfo(config, scriptFile)
// -- validate config options
if( NF.isSyntaxParserV2() )
new ConfigValidator().validate(config)
// -- create a new runner instance
final runner = new ScriptRunner(config)
runner.setScript(scriptFile)
runner.setPreview(this.preview, previewAction)
runner.session.profile = profile
runner.session.commandLine = launcher.cliString
runner.session.ansiLog = launcher.options.ansiLog && !SysEnv.isAgentMode()
runner.session.agentLog = SysEnv.isAgentMode()
runner.session.debug = launcher.options.remoteDebug
runner.session.disableJobsCancellation = getDisableJobsCancellation()
final isTowerEnabled = config.navigate('tower.enabled') as Boolean
final isDataEnabled = config.navigate("lineage.enabled") as Boolean
if( isTowerEnabled || isDataEnabled || log.isTraceEnabled() )
runner.session.resolvedConfig = ConfigBuilder.resolveConfig(scriptFile.parent, this, cliParams)
// note config files are collected during the build process
// this line should be after `ConfigBuilder#build`
runner.session.configFiles = builder.parsedConfigFiles
// set the commit id (if any)
runner.session.commitId = scriptFile.commitId
if( this.test ) {
runner.test(this.test, scriptArgs)
return
}
def info = CmdInfo.status( log.isTraceEnabled() )
log.debug( '\n'+info )
// -- add this run to the local history
if( !skipHistoryFile ) {
runner.verifyAndTrackHistory(launcher.cliString, runName)
}
// -- run it!
runner.execute(scriptArgs, cliParams, configParams, this.entryName)
}
protected void printBanner() {
// Suppress banner in agent mode for minimal output
if( SysEnv.isAgentMode() ) {
return
}
if( launcher.options.ansiLog ){
// Plain header for verbose log
log.debug "N E X T F L O W ~ version ${BuildInfo.version}"
// Fancy coloured header for the ANSI console output
def fmt = ansi()
fmt.a("\n")
// Use exact colour codes so that they render the same on every terminal,
// irrespective of terminal colour scheme.
// Nextflow green RGB (13, 192, 157) and exact black text (0,0,0),
// Apple Terminal only supports 256 colours, so use the closest match:
// light sea green | #20B2AA | 38;5;0
// Don't use black for text as terminals mess with this in their colour schemes.
// Use very dark grey, which is more reliable.
// Jansi library bundled in Jline can't do exact RGBs,
// so just do the ANSI codes manually
final BACKGROUND = "\033[1m\033[38;5;232m\033[48;5;43m"
fmt.a("$BACKGROUND N E X T F L O W ").reset()
// Show Nextflow version
fmt.a(Attribute.INTENSITY_FAINT).a(" ~ ").reset().a("version " + BuildInfo.version).reset()
fmt.a("\n")
AnsiConsole.out.println(fmt.eraseLine())
}
else {
// Plain header to the console if ANSI is disabled
log.info "N E X T F L O W ~ version ${BuildInfo.version}"
}
}
protected void launchInfo(ConfigMap config, ScriptFile scriptFile) {
// -- determine strict mode
detectStrictFeature(config, sysEnv)
// -- determine moduleBinary
detectModuleBinaryFeature(config)
// -- show launch info
final repo = scriptFile.repository ?: scriptFile.source.toString()
final head = preview ? "* PREVIEW * $scriptFile.repository" : "Launching `$repo`"
final revision = scriptFile.repository
? scriptFile.revisionInfo.toString()
: scriptFile.getScriptId()?.substring(0,10)
printLaunchInfo(repo, head, revision)
}
static void detectModuleBinaryFeature(ConfigMap config) {
final moduleBinaries = config.navigate('nextflow.enable.moduleBinaries', false)
if( moduleBinaries ) {
log.debug "Enabling module binaries"
NextflowMeta.instance.moduleBinaries(true)
}
}
static void detectStrictFeature(ConfigMap config, Map sysEnv) {
if( NF.isSyntaxParserV2() )
return
final defStrict = sysEnv.get('NXF_ENABLE_STRICT') ?: false
final strictMode = config.navigate('nextflow.enable.strict', defStrict)
if( strictMode ) {
log.debug "Enabling nextflow strict mode"
NextflowMeta.instance.strictMode(true)
}
}
protected void printLaunchInfo(String repo, String head, String revision) {
// Agent mode output is handled by AgentLogObserver
if( SysEnv.isAgentMode() ) {
return
}
if( launcher.options.ansiLog ){
log.debug "${head} [$runName] - revision: ${revision}"
def fmt = ansi()
fmt.a("Launching").fg(Color.MAGENTA).a(" `$repo` ").reset()
fmt.a(Attribute.INTENSITY_FAINT).a("[").reset()
fmt.bold().fg(Color.CYAN).a(runName).reset()
fmt.a(Attribute.INTENSITY_FAINT).a("] ").reset()
fmt.fg(Color.CYAN).a("revision: ").reset()
fmt.fg(Color.CYAN).a(revision).reset()
fmt.a("\n")
AnsiConsole.out().println(fmt.eraseLine())
}
else {
log.info "${head} [$runName] - revision: ${revision}"
}
}
protected void checkRunName() {
if( runName == 'last' )
throw new AbortOperationException("Not a valid run name: `last`")
if( runName && !matchRunName(runName) )
throw new AbortOperationException("Not a valid run name: `$runName` -- It must match the pattern $RUN_NAME_PATTERN")
if( !runName ) {
if( HistoryFile.disabled() )
throw new AbortOperationException("Missing workflow run name")
// -- make sure the generated name does not exist already
runName = HistoryFile.DEFAULT.generateNextName()
}
else if( !HistoryFile.disabled() && HistoryFile.DEFAULT.checkExistsByName(runName) )
throw new AbortOperationException("Run name `$runName` has been already used -- Specify a different one")
}
static protected boolean matchRunName(String name) {
RUN_NAME_PATTERN.matcher(name).matches()
}
protected ScriptFile getScriptFile(String pipelineName) {
try {
getScriptFile0(pipelineName)
}
catch (IllegalArgumentException | AbortOperationException e) {
if( e.message.startsWith("Not a valid project name:") && !guessIsRepo(pipelineName)) {
throw new AbortOperationException("Cannot find script file: $pipelineName")
}
else
throw e
}
}
static protected boolean guessIsRepo(String name) {
if( FileHelper.getUrlProtocol(name) != null )
return true
if( name.startsWith('/') )
return false
if( name.startsWith('./') || name.startsWith('../') )
return false
if( name.endsWith('.nf') )
return false
if( name.count('/') != 1 )
return false
return true
}
protected ScriptFile getScriptFile0(String pipelineName) {
assert pipelineName
/*
* read from the stdin
*/
if( pipelineName == '-' ) {
final file = tryReadFromStdin()
if( !file )
throw new AbortOperationException("Cannot access `stdin` stream")
if( revision )
throw new AbortOperationException("Revision option cannot be used when running a script from stdin")
return new ScriptFile(file)
}
/*
* look for a file with the specified pipeline name
*/
def script = new File(pipelineName)
if( script.isDirectory() ) {
script = mainScript ? new File(mainScript) : new AssetManager().setLocalPath(script).getMainScriptFile()
}
if( script.exists() ) {
if( revision )
throw new AbortOperationException("Revision option cannot be used when running a local script")
return new ScriptFile(script)
}
/*
* try to look for a pipeline in the repository
*/
try( final manager = new AssetManager(pipelineName, revision, mainScript, this) ) {
final repo = manager.getProjectWithRevision()
final remoteSource = !manager.isLocalScmSource()
boolean checkForUpdate = true
if( !manager.isRunnable() || latest ) {
if( offline && remoteSource )
throw new AbortOperationException("Unknown project `$repo` -- NOTE: automatic download from remote repositories is disabled")
log.info "Pulling $repo ..."
final result = manager.download(revision,deep)
if( result )
log.info " $result"
checkForUpdate = false
}
// Warn if using legacy
if( manager.isUsingLegacyStrategy() ) {
log.warn1 "This Nextflow version supports a new Multi-revision strategy for managing the SCM repositories, " +
"but '${repo}' is single-revision legacy strategy - Please consider to update the repository with the 'nextflow pull -migrate' command."
}
// post download operations
try {
manager.checkout(revision)
manager.updateModules()
final scriptFile = manager.getScriptFile(mainScript)
if( checkForUpdate && !(offline && remoteSource) )
manager.checkRemoteStatus(scriptFile.revisionInfo)
// return the script file
return scriptFile
}
catch( AbortOperationException e ) {
throw e
}
catch( Exception e ) {
throw new AbortOperationException("Unknown error accessing project `$repo` -- Repository may be corrupted: ${manager.localPath}", e)
}
}
}
static protected File tryReadFromStdin() {
if( !System.in.available() )
return null
getScriptFromStream(System.in)
}
static protected File getScriptFromStream( InputStream input, String name = 'nextflow' ) {
input != null
File result = File.createTempFile(name, null)
result.deleteOnExit()
input.withReader { Reader reader -> result << reader }
return result
}
/**
* Check the Nextflow version against the version required by
* the pipeline via `manifest.nextflowVersion`.
*
* When the version spec is prefixed with '!', the run will fail
* if the Nextflow version does not match.
*
* @param config
*/
protected void checkVersion(Map config) {
final manifest = new Manifest(config.manifest as Map ?: Collections.emptyMap())
String version = manifest.nextflowVersion?.trim()
if( !version )
return
final important = version.startsWith('!')
if( important )
version = version.substring(1).trim()
if( !getCurrentVersion().matches(version) ) {
if( important )
showVersionError(version)
else
showVersionWarning(version)
}
}
protected VersionNumber getCurrentVersion() {
return new VersionNumber(BuildInfo.version)
}
protected void showVersionError(String ver) {
throw new AbortOperationException("Nextflow version ${BuildInfo.version} does not match version required by pipeline: ${ver}")
}
protected void showVersionWarning(String ver) {
log.warn "Nextflow version ${BuildInfo.version} does not match version required by pipeline: ${ver} -- execution will continue, but things might break!"
}
@Memoized // <-- avoid parse multiple times the same file and params
Map parsedParams(Map configVars) {
final result = [:]
// apply params file
final file = getParamsFile()
if( file ) {
def path = validateParamsFile(file)
def type = path.extension.toLowerCase() ?: null
if( type == 'json' )
readJsonFile(path, configVars, result)
else if( type == 'yml' || type == 'yaml' )
readYamlFile(path, configVars, result)
}
// apply CLI params
if( !params )
return result
for( Map.Entry<String,String> entry : params ) {
addParam( result, entry.key, entry.value )
}
return result
}
static final private Pattern DOT_ESCAPED = ~/\\\./
static final private Pattern DOT_NOT_ESCAPED = ~/(?<!\\)\./
static protected void addParam(Map params, String key, String value, List path=[], String fullKey=null) {
if( !fullKey )
fullKey = key
final m = DOT_NOT_ESCAPED.matcher(key)
if( m.find() ) {
final p = m.start()
final root = key.substring(0, p)
if( !root ) throw new AbortOperationException("Invalid parameter name: $fullKey")
path.add(root)
def nested = params.get(root)
if( nested == null ) {
nested = new LinkedHashMap<>()
params.put(root, nested)
}
else if( nested !instanceof Map ) {
log.warn "Command line parameter --${path.join('.')} is overwritten by --${fullKey}"
nested = new LinkedHashMap<>()
params.put(root, nested)
}
addParam((Map)nested, key.substring(p+1), value, path, fullKey)
}
else {
addParam0(params, key.replaceAll(DOT_ESCAPED,'.'), parseParamValue(value))
}
}
static protected void addParam0(Map params, String key, Object value) {
if( key.contains('-') )
key = kebabToCamelCase(key)
params.put(key, value)
}
static protected String kebabToCamelCase(String str) {
final result = new StringBuilder()
str.split('-').eachWithIndex { String entry, int i ->
result << (i>0 ? StringUtils.capitalize(entry) : entry )
}
return result.toString()
}
static protected parseParamValue(String str) {
if ( SysEnv.get('NXF_DISABLE_PARAMS_TYPE_DETECTION') || NF.isSyntaxParserV2() )
return str
if ( str == null ) return null
if ( str.toLowerCase() == 'true') return Boolean.TRUE
if ( str.toLowerCase() == 'false' ) return Boolean.FALSE
if ( str==~/-?\d+(\.\d+)?/ && str.isInteger() ) return str.toInteger()
if ( str==~/-?\d+(\.\d+)?/ && str.isLong() ) return str.toLong()
if ( str==~/-?\d+(\.\d+)?/ && str.isDouble() ) return str.toDouble()
return str
}
private Path validateParamsFile(String file) {
def result = FileHelper.asPath(file)
def ext = result.getExtension()
if( !VALID_PARAMS_FILE.contains(ext) )
throw new AbortOperationException("Not a valid params file extension: $file -- It must be one of the following: ${VALID_PARAMS_FILE.join(',')}")
return result
}
static private Pattern PARAMS_VAR = ~/(?m)\$\{(\p{javaJavaIdentifierStart}\p{javaJavaIdentifierPart}*)}/
protected String replaceVars0(String content, Map binding) {
content.replaceAll(PARAMS_VAR) { List<String> matcher ->
// - the regex matcher is represented as list
// - the first element is the matching string ie. `${something}`
// - the second element is the group content ie. `something`
// - make sure the regex contains at least a group otherwise the closure
// parameter is a string instead of a list of the call fail
final placeholder = matcher.get(0)
final key = matcher.get(1)
if( !binding.containsKey(key) ) {
final msg = "Missing params file variable: $placeholder"
if(NF.strictMode)
throw new AbortOperationException(msg)
log.warn msg
return placeholder
}
return binding.get(key)
}
}
private void readJsonFile(Path file, Map configVars, Map result) {
try {
def text = configVars ? replaceVars0(file.text, configVars) : file.text
def json = (Map<String,Object>) new JsonSlurper().parseText(text)
json.forEach((name, value) -> {
addParam0(result, name, value)
})
}
catch (NoSuchFileException | FileNotFoundException e) {
throw new AbortOperationException("Specified params file does not exist: ${file.toUriString()}")
}
catch( Exception e ) {
throw new AbortOperationException("Cannot parse params file: ${file.toUriString()} - Cause: ${e.message}", e)
}
}
private void readYamlFile(Path file, Map configVars, Map result) {
try {
def text = configVars ? replaceVars0(file.text, configVars) : file.text
def yaml = (Map<String,Object>) new Yaml().load(text)
yaml.forEach((name, value) -> {
addParam0(result, name, value)
})
}
catch (NoSuchFileException | FileNotFoundException e) {
throw new AbortOperationException("Specified params file does not exist: ${file.toUriString()}")
}
catch( Exception e ) {
throw new AbortOperationException("Cannot parse params file: ${file.toUriString()}", e)
}
}
}

View File

@@ -0,0 +1,256 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
import com.beust.jcommander.Parameter
import com.beust.jcommander.Parameters
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.exception.AbortOperationException
import nextflow.plugin.Plugins
import nextflow.secret.SecretsLoader
import nextflow.secret.SecretsProvider
/**
* Implements the {@code secret} command
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
@Parameters(commandDescription = "Manage pipeline secrets")
class CmdSecret extends CmdBase implements UsageAware {
interface SubCmd {
String getName()
void apply(List<String> result)
void usage(List<String> result)
}
static public final String NAME = 'secrets'
private List<SubCmd> commands = []
String getName() {
return NAME
}
@Parameter(hidden = true)
List<String> args
private SecretsProvider provider
CmdSecret() {
commands.add( new GetCmd() )
commands.add( new SetCmd() )
commands.add( new ListCmd() )
commands.add( new DeleteCmd() )
}
/**
* Print the command usage help
*/
void usage() {
usage(args)
}
/**
* Print the command usage help
*
* @param args The arguments as entered by the user
*/
void usage(List<String> args) {
List<String> result = []
if( !args ) {
result << this.getClass().getAnnotation(Parameters).commandDescription()
result << 'Usage: nextflow secrets <sub-command> [options]'
result << ''
result << 'Commands:'
commands.collect{ it.name }.sort().each { result << " $it".toString() }
result << ''
}
else {
def sub = commands.find { it.name == args[0] }
if( sub )
sub.usage(result)
else {
throw new AbortOperationException("Unknown secrets sub-command: ${args[0]}")
}
}
println result.join('\n').toString()
}
/**
* Main command entry point
*/
@Override
void run() {
if( !args ) {
usage()
return
}
// setup the plugins system and load the secrets provider
Plugins.init()
provider = SecretsLoader.instance.load()
// run the command
try {
getCmd(args).apply(args.drop(1))
}
finally {
// close the provider
provider?.close()
}
}
protected SubCmd getCmd(List<String> args) {
def cmd = commands.find { it.name == args[0] }
if( cmd ) {
return cmd
}
def matches = commands.collect{ it.name }.closest(args[0])
def msg = "Unknown cloud sub-command: ${args[0]}"
if( matches )
msg += " -- Did you mean one of these?\n" + matches.collect { " $it"}.join('\n')
throw new AbortOperationException(msg)
}
private void addOption(String fieldName, List<String> result) {
def annot = this.class.getDeclaredField(fieldName)?.getAnnotation(Parameter)
if( annot ) {
result << ' ' + annot.names().join(', ')
result << ' ' + annot.description()
}
else {
log.debug "Unknown help field: $fieldName"
}
}
class SetCmd implements SubCmd {
@Override
String getName() { 'set' }
@Override
void apply(List<String> result) {
if( result.size() < 1 )
throw new AbortOperationException("Missing secret name")
if( result.size() < 2 )
throw new AbortOperationException("Missing secret value")
String secretName = result.first()
String secretValue = result.last()
provider.putSecret(secretName, secretValue)
}
@Override
void usage(List<String> result) {
result << 'Set a key-pair in the secrets store'
result << "Usage: nextflow secrets $name <NAME> <VALUE>".toString()
result << ''
result << ''
}
}
class GetCmd implements SubCmd {
@Override
String getName() { 'get' }
@Override
void apply(List<String> result) {
if( result.size() != 1 )
throw new AbortOperationException("Wrong number of arguments")
String secretName = result.first()
if( !secretName )
throw new AbortOperationException("Missing secret name")
println provider.getSecret(secretName)?.value
}
@Override
void usage(List<String> result) {
result << 'Get a secret value with the name'
result << "Usage: nextflow secrets $name <NAME>".toString()
result << ''
}
}
/**
* Implements the secret `list` sub-command
*/
class ListCmd implements SubCmd {
@Override
String getName() { 'list' }
@Override
void apply(List<String> result) {
if( result.size() )
throw new AbortOperationException("Wrong number of arguments")
final names = new ArrayList(provider.listSecretsNames()).sort()
if( names ) {
for( String it : names ) {
println it
}
}
else {
println "no secrets available"
}
}
@Override
void usage(List<String> result) {
result << 'List all names in the secrets store'
result << "Usage: nextflow secrets $name".toString()
result << ''
}
}
/**
* Implements the secret `remove` sub-command
*/
class DeleteCmd implements SubCmd {
@Override
String getName() { 'delete' }
@Override
void apply(List<String> result) {
if( result.size() != 1 )
throw new AbortOperationException("Wrong number of arguments")
String secretName = result.first()
if( !secretName )
throw new AbortOperationException("Missing secret name")
provider.removeSecret(secretName)
}
@Override
void usage(List<String> result) {
result << 'Delete an entry from the secrets store'
result << "Usage: nextflow secrets $name".toString()
result << ''
addOption('secretName', result)
result << ''
}
}
}

View File

@@ -0,0 +1,36 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
import com.beust.jcommander.Parameters
/**
* Self-update command
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Parameters(commandDescription = "Update nextflow runtime to the latest available version")
class CmdSelfUpdate extends CmdBase {
@Override
String getName() { 'self-update' }
@Override
void run() {
// actually it's doing nothing, the update process is managed by the external launcher script
// this class in only necessary to print the command line the usage output
}
}

View File

@@ -0,0 +1,88 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
import com.beust.jcommander.Parameter
import com.beust.jcommander.Parameters
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.exception.AbortOperationException
import nextflow.plugin.Plugins
import nextflow.scm.AssetManager
/**
* CLI sub-command VIEW -- Print a pipeline script to console
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
@Parameters(commandDescription = "View project script file(s)")
class CmdView extends CmdBase {
static final public NAME = 'view'
@Override
String getName() { NAME }
@Parameter(description = 'project name', required = true)
List<String> args = []
@Parameter(names=['-r','-revision'], description = 'Revision of the project (either a git branch, tag or commit SHA number)')
String revision
@Parameter(names = '-q', description = 'Hide header line', arity = 0)
boolean quiet
@Parameter(names = '-l', description = 'List repository content', arity = 0)
boolean all
@Override
void run() {
Plugins.init()
try (final manager = new AssetManager(args[0], revision)) {
if( !manager.isLocal() )
throw new AbortOperationException("Unknown project `${manager.getProjectWithRevision()}`")
if( revision && manager.isUsingLegacyStrategy()){
log.warn("The local asset ${args[0]} does not support multi-revision - 'revision' option is ignored\n" +
"Consider updating the asset using 'nextflow pull ${args[0]} -r $revision -migrate'")
}
if( all ) {
if( !quiet )
println "== content of path: ${manager.localPath}"
manager.localPath.eachFile { File it ->
println it.name
}
}
else {
/*
* prints the script main file
*/
final script = manager.getMainScriptFile()
if( !script.exists() )
throw new AbortOperationException("Missing script file: '${script}'")
if( !quiet )
println "== content of file: $script"
script.readLines().each { println it }
}
}
}
}

View File

@@ -0,0 +1,68 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
import com.beust.jcommander.Parameter
import groovy.transform.CompileStatic
/**
* Defines the command line parameters for command that need to interact with a pipeline service hub i.e. GitHub or BitBucket
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
trait HubOptions {
@Parameter(names=['-hub'], description = "Service hub where the project is hosted")
String hubProvider
@Parameter(names='-user', description = 'Private repository user name')
String hubUser
/**
* Return the password provided on the command line or stop allowing the user to enter it on the console
*
* @return The password entered or {@code null} if no user has been entered
*/
String getHubPassword() {
if( !hubUser )
return null
def p = hubUser.indexOf(':')
if( p != -1 )
return hubUser.substring(p+1)
def console = System.console()
if( !console )
return null
print "Enter your $hubProvider password: "
char[] pwd = console.readPassword()
new String(pwd)
}
String getHubUser() {
if(!hubUser) {
return hubUser
}
def p = hubUser.indexOf(':')
return p != -1 ? hubUser.substring(0,p) : hubUser
}
}

View File

@@ -0,0 +1,724 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
import static nextflow.Const.*
import java.lang.reflect.Field
import com.beust.jcommander.DynamicParameter
import com.beust.jcommander.JCommander
import com.beust.jcommander.Parameter
import com.beust.jcommander.ParameterException
import com.beust.jcommander.Parameters
import groovy.transform.CompileDynamic
import groovy.transform.CompileStatic
import groovy.transform.PackageScope
import groovy.util.logging.Slf4j
import nextflow.BuildInfo
import nextflow.NF
import nextflow.exception.AbortOperationException
import nextflow.exception.AbortRunException
import nextflow.exception.ConfigParseException
import nextflow.exception.ScriptCompilationException
import nextflow.exception.ScriptRuntimeException
import nextflow.secret.SecretsLoader
import nextflow.util.Escape
import nextflow.util.LoggerHelper
import nextflow.util.ProxyConfig
import nextflow.util.SpuriousDeps
import org.eclipse.jgit.api.errors.GitAPIException
import static nextflow.util.SysHelper.dumpThreads
/**
* Main application entry point. It parses the command line and
* launch the pipeline execution.
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
class Launcher {
/**
* Create the application command line parser
*
* @return An instance of {@code CliBuilder}
*/
private JCommander jcommander
private CliOptions options
private boolean fullVersion
private CmdBase command
private String cliString
private List<CmdBase> allCommands
private List<String> normalizedArgs
private boolean daemonMode
private String colsString
/**
* Create a launcher object and parse the command line parameters
*
* @param args The command line arguments provided by the user
*/
Launcher() {
init()
}
protected void init() {
allCommands = (List<CmdBase>)[
new CmdAuth(),
new CmdClean(),
new CmdClone(),
new CmdConsole(),
new CmdFs(),
new CmdInfo(),
new CmdLaunch(),
new CmdList(),
new CmdLog(),
new CmdPull(),
new CmdRun(),
new CmdKubeRun(),
new CmdDrop(),
new CmdConfig(),
new CmdNode(),
new CmdView(),
new CmdHelp(),
new CmdSelfUpdate(),
new CmdPlugin(),
new CmdInspect(),
new CmdLint(),
new CmdLineage(),
new CmdModule()
]
if(SecretsLoader.isEnabled())
allCommands.add(new CmdSecret())
// legacy command
final cmdCloud = SpuriousDeps.cmdCloud()
if( cmdCloud )
allCommands.add(cmdCloud)
options = new CliOptions()
jcommander = new JCommander(options)
for( CmdBase cmd : allCommands ) {
cmd.launcher = this;
jcommander.addCommand(cmd.name, cmd, aliases(cmd))
}
jcommander.setProgramName( APP_NAME )
//Allow unknown options for module command
jcommander.getCommands().get(CmdModule.NAME)?.setAcceptUnknownOptions(true)
}
private static final String[] EMPTY = new String[0]
private static String[] aliases(CmdBase cmd) {
final aliases = cmd.getClass().getAnnotation(Parameters)?.commandNames()
return aliases ?: EMPTY
}
/**
* Create the Jcommander 'interpreter' and parse the command line arguments
*/
@PackageScope
Launcher parseMainArgs(String... args) {
this.cliString = makeCli(System.getenv('NXF_CLI'), args)
this.colsString = System.getenv('COLUMNS')
def cols = getColumns()
if( cols )
jcommander.setColumnSize(cols)
normalizedArgs = normalizeArgs(args)
jcommander.parse( normalizedArgs as String[] )
fullVersion = '-version' in normalizedArgs
command = allCommands.find { it.name == jcommander.getParsedCommand() }
//Attach unknown options to command in case of needed
if (command) {
final unknownOptions = jcommander.commands.get(jcommander.getParsedCommand())?.getUnknownOptions() ?: []
command.setUnknownOptions(unknownOptions)
}
// whether is running a daemon
daemonMode = command instanceof CmdNode
// set the log file name
checkLogFileName()
return this
}
protected String makeCli(String cli, String... args) {
if( !cli )
cli = 'nextflow'
if( !args )
return cli
def cmd = ' ' + args[0]
int p = cli.indexOf(cmd)
if( p!=-1 )
cli = cli.substring(0,p)
if( cli.endsWith('nextflow') )
cli = 'nextflow'
cli += ' ' + Escape.cli(args)
return cli
}
private void checkLogFileName() {
if( !options.logFile ) {
if( isDaemon() )
options.logFile = System.getenv('NXF_LOG_FILE') ?: '.node-nextflow.log'
else if( command instanceof CmdModule && (command as CmdModule).args?.first() == 'run' )
options.logFile = System.getenv('NXF_LOG_FILE') ?: ".module.log"
else if( command instanceof CmdRun || command instanceof CmdLaunch || command instanceof CmdAuth || options.debug || options.trace )
options.logFile = System.getenv('NXF_LOG_FILE') ?: ".nextflow.log"
}
}
private short getColumns() {
if( !colsString ) {
return 0
}
try {
colsString.toShort()
}
catch( Exception e ) {
log.debug "Unexpected terminal \$COLUMNS value: $colsString"
return 0
}
}
CliOptions getOptions() { options }
List<String> getNormalizedArgs() { normalizedArgs }
String getCliString() { cliString }
boolean isDaemon() { daemonMode }
/**
* normalize the command line arguments to handle some corner cases
*/
@PackageScope
List<String> normalizeArgs( String ... args ) {
List<String> normalized = []
int i=0
while( true ) {
if( i==args.size() ) { break }
def current = args[i++]
normalized << current
// when the first argument is a file, it's supposed to be a script to be executed
if( i==1 && !allCommands.find { it.name == current } && new File(current).isFile() ) {
normalized.add(0,CmdRun.NAME)
}
else if( current == '-resume' ) {
if( i<args.size() && !args[i].startsWith('-') && (args[i]=='last' || args[i] =~~ /[0-9a-f]{8}\-[0-9a-f]{4}\-[0-9a-f]{4}\-[0-9a-f]{4}\-[0-9a-f]{8}/) ) {
normalized << args[i++]
}
else {
normalized << 'last'
}
}
else if( current == '-test' && (i==args.size() || args[i].startsWith('-'))) {
normalized << '%all'
}
else if( current == '-dump-hashes' && (i==args.size() || args[i].startsWith('-'))) {
normalized << '-'
}
else if( current == '-with-cloudcache' && (i==args.size() || args[i].startsWith('-'))) {
normalized << '-'
}
else if( current == '-with-trace' && (i==args.size() || args[i].startsWith('-'))) {
normalized << '-'
}
else if( current == '-with-report' && (i==args.size() || args[i].startsWith('-'))) {
normalized << '-'
}
else if( current == '-with-timeline' && (i==args.size() || args[i].startsWith('-'))) {
normalized << '-'
}
else if( current == '-with-dag' && (i==args.size() || args[i].startsWith('-'))) {
normalized << '-'
}
else if( current == '-with-docker' && (i==args.size() || args[i].startsWith('-'))) {
normalized << '-'
}
else if( current == '-with-podman' && (i==args.size() || args[i].startsWith('-'))) {
normalized << '-'
}
else if( current == '-with-singularity' && (i==args.size() || args[i].startsWith('-'))) {
normalized << '-'
}
else if( current == '-with-apptainer' && (i==args.size() || args[i].startsWith('-'))) {
normalized << '-'
}
else if( current == '-with-charliecloud' && (i==args.size() || args[i].startsWith('-'))) {
normalized << '-'
}
else if( current == '-with-conda' && (i==args.size() || args[i].startsWith('-'))) {
normalized << '-'
}
else if( current == '-with-spack' && (i==args.size() || args[i].startsWith('-'))) {
normalized << '-'
}
else if( current == '-with-weblog' && (i==args.size() || args[i].startsWith('-'))) {
normalized << '-'
}
else if( current == '-with-tower' && (i==args.size() || args[i].startsWith('-'))) {
normalized << '-'
}
else if( current == '-with-wave' && (i==args.size() || args[i].startsWith('-'))) {
normalized << '-'
}
else if( current == '-ansi-log' && (i==args.size() || args[i].startsWith('-'))) {
normalized << 'true'
}
else if( (current == '-stub' || current == '-stub-run') && (i==args.size() || args[i].startsWith('-'))) {
normalized << 'true'
}
else if( (current == '-N' || current == '-with-notification') && (i==args.size() || args[i].startsWith('-'))) {
normalized << 'true'
}
else if( current == '-with-fusion' && (i==args.size() || args[i].startsWith('-'))) {
normalized << 'true'
}
else if( current == '-syslog' && (i==args.size() || args[i].startsWith('-') || allCommands.find { it.name == args[i] } )) {
normalized << 'localhost'
}
else if( current == '-dump-channels' && (i==args.size() || args[i].startsWith('-'))) {
normalized << '*'
}
else if( current ==~ /^\-\-[a-zA-Z\d].*/ && !current.contains('=') ) {
current += '='
current += ( i<args.size() && isValue(args[i]) ? args[i++] : 'true' )
normalized[-1] = current
}
else if( current ==~ /^\-process\..+/ && !current.contains('=')) {
current += '='
current += ( i<args.size() && isValue(args[i]) ? args[i++] : 'true' )
normalized[-1] = current
}
else if( current ==~ /^\-cluster\..+/ && !current.contains('=')) {
current += '='
current += ( i<args.size() && isValue(args[i]) ? args[i++] : 'true' )
normalized[-1] = current
}
else if( current ==~ /^\-executor\..+/ && !current.contains('=')) {
current += '='
current += ( i<args.size() && isValue(args[i]) ? args[i++] : 'true' )
normalized[-1] = current
}
else if( current == 'run' && i<args.size() && args[i] == '-' ) {
i++
normalized << '-stdin'
}
}
return normalized
}
static private boolean isValue( String x ) {
if( !x ) return false // an empty string -> not a value
if( x.size() == 1 ) return true // a single char is not an option -> value true
!x.startsWith('-') || x.isNumber() || x.contains(' ')
}
CmdBase findCommand( String cmdName ) {
allCommands.find { it.name == cmdName }
}
/**
* Print the usage string for the given command - or -
* the main program usage string if not command is specified
*
* @param command The command for which get help or {@code null}
* @return The usage string
*/
void usage(String command = null ) {
if( command ) {
def exists = allCommands.find { it.name == command } != null
if( !exists ) {
println "Asking help for unknown command: $command"
return
}
jcommander.usage(command)
return
}
println "Usage: nextflow [options] COMMAND [arg...]\n"
printOptions(CliOptions)
printCommands(allCommands)
}
@CompileDynamic
protected void printOptions(Class clazz) {
List params = []
for( Field f : clazz.getDeclaredFields() ) {
def p = f.getAnnotation(Parameter)
if(!p)
p = f.getAnnotation(DynamicParameter)
if( p && !p.hidden() && p.description() && p.names() )
params.add(p)
}
params.sort(true) { it -> it.names()[0] }
println "Options:"
for( def p : params ) {
println " ${p.names().join(', ')}"
println " ${p.description()}"
}
}
protected void printCommands(List<CmdBase> commands) {
println "\nCommands:"
int len = 0
def all = new TreeMap<String,String>()
new ArrayList<CmdBase>(commands).each {
def description = it.getClass().getAnnotation(Parameters)?.commandDescription()
if( description ) {
all[it.name] = description
if( it.name.size()>len ) len = it.name.size()
}
}
all.each { String name, String desc ->
print ' '
print name.padRight(len)
print ' '
println desc
}
println ''
}
Launcher command( String[] args ) {
/*
* CLI argument parsing
*/
try {
parseMainArgs(args)
LoggerHelper.configureLogger(this)
}
catch( ParameterException e ) {
// print command line parsing errors
// note: use system.err.println since if an exception is raised
// parsing the cli params the logging is not configured
System.err.println "${e.getMessage()} -- Check the available commands and options and syntax with 'help'"
System.exit(1)
}
catch ( AbortOperationException e ) {
final msg = e.message ?: "Unknown abort reason"
System.err.println(LoggerHelper.formatErrMessage(msg, e))
System.exit(1)
}
catch( Throwable e ) {
e.printStackTrace(System.err)
System.exit(1)
}
return this
}
protected void checkForHelp() {
if( options.help || !command || command.help ) {
if( command instanceof UsageAware ) {
(command as UsageAware).usage()
// reset command to null to skip default execution
command = null
return
}
// replace the current command with the `help` command
def target = command?.name
command = allCommands.find { it instanceof CmdHelp }
if( target ) {
(command as CmdHelp).args = [target]
}
}
}
/**
* Launch the pipeline execution
*/
int run() {
/*
* setup environment
*/
setupEnvironment()
/*
* Real execution starts here
*/
try {
log.debug '$> ' + cliString
// -- print out the version number, then exit
if ( options.version ) {
println getVersion(fullVersion)
return 0
}
// -- print out the program help, then exit
checkForHelp()
// launch the command
command?.run()
if( log.isTraceEnabled())
log.trace "Exit\n" + dumpThreads()
return 0
}
catch( AbortRunException e ) {
return(1)
}
catch ( AbortOperationException e ) {
def message = e.getMessage()
if( message )
System.err.println(LoggerHelper.formatErrMessage(message,e))
log.debug ("Operation aborted", e.cause ?: e)
return(1)
}
catch ( GitAPIException e ) {
System.err.println e.getMessage() ?: e.toString()
log.debug ("Operation aborted", e.cause ?: e)
return(1)
}
catch( ConfigParseException e ) {
if( NF.isSyntaxParserV2() ) {
log.error(e.message, e)
}
else {
def message = e.message
if( e.cause?.message ) {
message += "\n\n${e.cause.message.toString().indent(' ')}"
}
log.error(message, e.cause ?: e)
}
return(1)
}
catch( ScriptCompilationException e ) {
log.error(e.message, e)
return(1)
}
catch ( ScriptRuntimeException | IllegalArgumentException e) {
log.error(e.message, e)
return(1)
}
catch( IOException e ) {
log.error(e.message, e)
return(1)
}
catch( Throwable fail ) {
log.error("@unknown", fail)
return(1)
}
}
/**
* set up environment and system properties. It checks the following
* environment variables:
* <li>http_proxy</li>
* <li>https_proxy</li>
* <li>ftp_proxy</li>
* <li>HTTP_PROXY</li>
* <li>HTTPS_PROXY</li>
* <li>FTP_PROXY</li>
* <li>NO_PROXY</li>
*/
private void setupEnvironment() {
final env = System.getenv()
setProxy('HTTP',env)
setProxy('HTTPS',env)
setProxy('FTP',env)
setProxy('http',env)
setProxy('https',env)
setProxy('ftp',env)
setNoProxy(env)
setHttpClientProperties(env)
}
static void setHttpClientProperties(Map<String,String> env) {
// Set the httpclient connection pool timeout to 10 seconds.
// This required because the default is 20 minutes, which cause the error
// "HTTP/1.1 header parser received no bytes" when in some circumstances
// https://github.com/nextflow-io/nextflow/issues/3983#issuecomment-1702305137
System.setProperty("jdk.httpclient.keepalive.timeout", env.getOrDefault("NXF_JDK_HTTPCLIENT_KEEPALIVE_TIMEOUT","10"))
if( env.get("NXF_JDK_HTTPCLIENT_CONNECTIONPOOLSIZE") )
System.setProperty("jdk.httpclient.connectionPoolSize", env.get("NXF_JDK_HTTPCLIENT_CONNECTIONPOOLSIZE"))
}
/**
* Set no proxy property if defined in the launching env
*
* See for details
* https://docs.oracle.com/javase/8/docs/technotes/guides/net/proxies.html
*
* @param env
*/
@PackageScope
static void setNoProxy(Map<String,String> env) {
final noProxy = env.get('NO_PROXY') ?: env.get('no_proxy')
if(noProxy) {
System.setProperty('http.nonProxyHosts', noProxy.tokenize(',').join('|'))
}
}
/**
* Setup proxy system properties and optionally configure the network authenticator
*
* See:
* http://docs.oracle.com/javase/6/docs/technotes/guides/net/proxies.html
* https://github.com/nextflow-io/nextflow/issues/24
*
* @param qualifier Either {@code http/HTTP} or {@code https/HTTPS}.
* @param env The environment variables system map
*/
@PackageScope
static void setProxy(String qualifier, Map<String,String> env ) {
assert qualifier in ['http','https','ftp','HTTP','HTTPS','FTP']
def str = null
def var = "${qualifier}_" + (qualifier.isLowerCase() ? 'proxy' : 'PROXY')
// -- setup HTTP proxy
try {
final proxy = ProxyConfig.parse(str = env.get(var.toString()))
if( proxy ) {
// set the expected protocol
proxy.protocol = qualifier.toLowerCase()
log.debug "Setting $qualifier proxy: $proxy"
System.setProperty("${qualifier.toLowerCase()}.proxyHost", proxy.host)
if( proxy.port )
System.setProperty("${qualifier.toLowerCase()}.proxyPort", proxy.port)
if( proxy.authenticator() ) {
log.debug "Setting $qualifier proxy authenticator"
Authenticator.setDefault(proxy.authenticator())
}
}
}
catch ( MalformedURLException e ) {
log.warn "Not a valid $qualifier proxy: '$str' -- Check the value of variable `$var` in your environment"
}
}
/**
* Hey .. Nextflow starts here!
*
* @param args The program options as specified by the user on the CLI
*/
static void main(String... args) {
LoggerHelper.bootstrapLogger()
final status = new Launcher() .command(args) .run()
if( status )
System.exit(status)
}
/**
* Print the application version number
* @param full When {@code true} prints full version number including build timestamp
* @return The version number string
*/
static String getVersion(boolean full = false) {
if ( full ) {
SPLASH
}
else {
"${APP_NAME} version ${BuildInfo.version}.${BuildInfo.buildNum}"
}
}
/*
* The application 'logo'
*/
/*
* The application 'logo'
*/
static public final String SPLASH =
"""
N E X T F L O W
version ${BuildInfo.version} build ${BuildInfo.buildNum}
created ${BuildInfo.timestampUTC} ${BuildInfo.timestampDelta}
cite doi:10.1038/nbt.3820
http://nextflow.io
"""
}

View File

@@ -0,0 +1,83 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
import java.nio.file.Paths
import groovy.transform.CompileStatic
import nextflow.Session
import nextflow.config.ConfigBuilder
import nextflow.exception.AbortOperationException
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import static PluginExecAware.CMD_SEP
/**
* Abstract implementation for plugin commands
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
trait PluginAbstractExec implements PluginExecAware {
private static Logger log = LoggerFactory.getLogger(PluginAbstractExec)
private Session session
private Launcher launcher
Session getSession() { session }
Launcher getLauncher() { launcher }
abstract List<String> getCommands()
@Override
final int exec(Launcher launcher1, String pluginId, String cmd, List<String> args) {
this.launcher = launcher1
// create the config
final config = new ConfigBuilder()
.setOptions(launcher1.options)
.setBaseDir(Paths.get('.'))
.build()
if( !cmd || cmd !in getCommands() ) {
def msg = cmd ? "Invalid command '$pluginId:$cmd' - usage: nextflow plugin ${pluginId}${CMD_SEP}<command>" : "Usage: nextflow plugin ${pluginId}${CMD_SEP}<command>"
msg += "\nAvailable commands:"
for( String it : getCommands() )
msg += "\n $it"
throw new AbortOperationException(msg)
}
// create the session object
this.session = new Session(config)
// invoke the command
try {
exec(cmd, args)
}
catch (Throwable e) {
log.error("Unexpected error on command: $cmd - cause: ${e.message}", e)
}
finally {
session.destroy()
}
return 0
}
abstract int exec(String cmd, List<String> args)
}

View File

@@ -0,0 +1,30 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
/**
* Define the interface for plugin commands
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
interface PluginExecAware {
static final String CMD_SEP = ':'
int exec(Launcher launcher, String pluginId, String cmd, List<String> args)
}

View File

@@ -0,0 +1,31 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli
/**
* Command can implement this interface to provide a
* custom usage description
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
interface UsageAware {
void usage()
void usage(List<String> args)
}

View File

@@ -0,0 +1,203 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli.module
import java.nio.file.Files
import java.nio.file.Path
import com.beust.jcommander.Parameter
import com.beust.jcommander.Parameters
import groovy.transform.CompileStatic
import nextflow.cli.CmdBase
import nextflow.exception.AbortOperationException
import nextflow.module.ModuleInfo
/**
* Module create subcommand -- creates a new module skeleton
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
@Parameters(commandDescription = "Create a new module skeleton")
class CmdModuleCreate extends CmdBase {
@Parameter(description = "[namespace/name]")
List<String> args
@Override
String getName() {
return 'create'
}
@Override
void run() {
String namespace
String name
if( args && args.size() == 1 && args[0].contains('/') ) {
// non-interactive: namespace/name passed as argument
final slash = args[0].indexOf('/')
namespace = args[0].substring(0, slash)
name = args[0].substring(slash + 1)
if( !namespace || !name )
throw new AbortOperationException("Invalid module identifier -- expected format: namespace/name")
}
else if( args ) {
throw new AbortOperationException("Invalid arguments -- usage: nextflow module create [namespace/name]")
}
else {
// interactive mode
print "Enter module namespace: "
namespace = readLine()?.trim()
if( !namespace )
throw new AbortOperationException("Module namespace cannot be empty")
print "Enter module name: "
name = readLine()?.trim()
if( !name )
throw new AbortOperationException("Module name cannot be empty")
println ""
println " Module namespace : $namespace"
println " Module name : $name"
println " Directory : ./modules/$namespace/$name"
println ""
print "Are you OK to continue [y/N]? "
final confirm = readLine()
if( confirm?.toLowerCase() != 'y' ) {
println "Module creation aborted."
return
}
}
validateSegment('namespace', namespace)
validateSegments('name', name)
createModule(namespace, name)
}
static private void validateSegment(String field, String value) {
if( !value.matches('[a-zA-Z0-9][a-zA-Z0-9._\\-]*') )
throw new AbortOperationException("Invalid module $field '${value}' -- only alphanumeric characters, hyphens, underscores and dots are allowed, and must start with an alphanumeric character")
}
static private void validateSegments(String field, String value) {
for( String segment : value.tokenize('/') ) {
validateSegment(field, segment)
}
}
protected Path modulesBase() {
return Path.of('modules')
}
protected void createModule(String namespace, String name) {
final moduleDir = modulesBase().resolve(namespace).resolve(name)
if( Files.exists(moduleDir) )
throw new AbortOperationException("Module directory already exists: $moduleDir")
// create directory structure
Files.createDirectories(moduleDir)
// create main.nf
moduleDir.resolve('main.nf').text = mainNf(namespace, name)
// create README.md
moduleDir.resolve('README.md').text = readmeMd(namespace, name)
// create meta.yml
moduleDir.resolve('meta.yml').text = metaYml(namespace, name)
// create .module-info so it's recognised as a Nextflow managed module
Files.createFile(moduleDir.resolve(ModuleInfo.MODULE_INFO_FILE))
println "Module created successfully at path: $moduleDir"
println ""
println "To run the module:"
println ""
println " nextflow module run $namespace/$name --greeting 'Hello world!'"
}
static private String readLine() {
final console = System.console()
return console != null
? console.readLine()
: new BufferedReader(new InputStreamReader(System.in)).readLine()
}
static String mainNf(String namespace, String name) {
"""\
/*
* Module: ${namespace}/${name}
*/
process ${name.replaceAll('[^a-zA-Z0-9_]', '_').toUpperCase()} {
input:
val greeting
output:
stdout
script:
\"\"\"
echo '\${greeting}'
\"\"\"
}
""".stripIndent()
}
static String metaYml(String namespace, String name) {
"""\
name: ${namespace}/${name}
version: 1.0.0
description: A brief description of the ${namespace}/${name} module
license: Apache-2.0
input:
- name: greeting
type: string
description: A greeting string
output:
- name: stdout
type: string
description: The greeting message
""".stripIndent()
}
static String readmeMd(String namespace, String name) {
"""\
# ${namespace}/${name}
## Summary
A brief description of the `${namespace}/${name}` module.
## Get started
Include this module in your Nextflow pipeline:
```nextflow
include { ${name.replaceAll('[^a-zA-Z0-9_]', '_').toUpperCase()} } from '${namespace}/${name}'
```
## Dependencies
None.
## License
Apache-2.0
""".stripIndent()
}
}

View File

@@ -0,0 +1,101 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli.module
import com.beust.jcommander.Parameter
import com.beust.jcommander.Parameters
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import io.seqera.npr.client.RegistryClient
import nextflow.cli.CmdBase
import nextflow.config.ConfigBuilder
import nextflow.config.RegistryConfig
import nextflow.exception.AbortOperationException
import nextflow.module.ModuleReference
import nextflow.module.RegistryClientFactory
import nextflow.module.ModuleResolver
import nextflow.module.ModuleSpecFactory
import nextflow.util.TestOnly
import java.nio.file.Path
import java.nio.file.Paths
/**
* Module install subcommand
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
@Slf4j
@Parameters(commandDescription = "Install a module from the registry")
@CompileStatic
class CmdModuleInstall extends CmdBase {
@Parameter(names = ["-version"], description = "Module version")
String version
@Parameter(names = ["-force"], description = "Force reinstall even if already installed", arity = 0)
boolean force = false
@Parameter(description = "[scope/name]", required = true)
List<String> args
@TestOnly
protected Path root
@TestOnly
protected RegistryClient client
@Override
String getName() {
return 'install'
}
@Override
void run() {
if( !args || args.size() != 1 ) {
throw new AbortOperationException("Incorrect number of arguments")
}
def reference = ModuleReference.parse(args[0])
// Get config
def baseDir = root ?: Paths.get('.').toAbsolutePath().normalize()
def config = new ConfigBuilder()
.setOptions(launcher.options)
.setBaseDir(baseDir)
.build()
final registryConfig = config.navigate('registry') as RegistryConfig ?: new RegistryConfig()
// Create resolver and install
def resolver = new ModuleResolver(baseDir, client ?: RegistryClientFactory.forConfig(registryConfig))
try {
def installedMainFile = resolver.installModule(reference, version, force)
// Read the installed version from meta.yml to avoid a redundant registry call
def installedVersion = ModuleSpecFactory.fromYaml(installedMainFile.parent.resolve('meta.yml')).version
println "Module ${reference}@${installedVersion} installed and configured successfully"
}
catch( AbortOperationException e ) {
throw e
}
catch( Exception e ) {
throw new AbortOperationException("Installation failed: ${e.message}", e)
}
}
}

View File

@@ -0,0 +1,148 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli.module
import com.beust.jcommander.IParameterValidator
import com.beust.jcommander.Parameter
import com.beust.jcommander.ParameterException
import com.beust.jcommander.Parameters
import groovy.json.JsonOutput
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.cli.CmdBase
import nextflow.exception.AbortOperationException
import nextflow.module.InstalledModule
import nextflow.module.ModuleIntegrity
import nextflow.module.ModuleStorage
import nextflow.util.TestOnly
import java.nio.file.Path
import java.nio.file.Paths
/**
* Module list subcommand
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
@Slf4j
@CompileStatic
@Parameters(commandDescription = "List all installed modules")
class CmdModuleList extends CmdBase {
@Parameter(
names = ['-o', '-output'],
description = 'Output mode for reporting search results: table, json',
validateWith = OutputModeValidator
)
String output = 'table'
static class OutputModeValidator implements IParameterValidator {
private static final List<String> MODES = List.of('table', 'json')
@Override
void validate(String name, String value) {
if( !MODES.contains(value) )
throw new ParameterException("Output mode must be one of $MODES (found: $value)")
}
}
@TestOnly
protected Path root
@Override
String getName() {
return 'list'
}
@Override
void run() {
// Get config
def baseDir = root ?: Paths.get('.').toAbsolutePath().normalize()
// Create resolver and list modules
def storage = new ModuleStorage(baseDir)
try {
def installed = storage.listInstalled()
if( installed.isEmpty() ) {
println "No modules installed"
return
}
if( !output || output == 'table' ) {
printFormattedList(installed)
} else if( output == 'json' ) {
printJsonList(installed)
} else {
throw new AbortOperationException("Not implemented output mode $output)")
}
}
catch( Exception e ) {
log.error("Failed to list modules", e)
throw new AbortOperationException("List failed: ${e.message}", e)
}
}
private void printFormattedList(List<InstalledModule> installed) {
println ""
println "Installed modules:"
println ""
println "Module".padRight(40) + "Version".padRight(15) + "Status"
println("-" * 70)
installed.each { module ->
def status = getStatusString(module.integrity)
println "${module.reference.toString().padRight(40)}${(module.installedVersion ?: 'unknown').padRight(15)}${status}"
}
println ""
}
private void printJsonList(List<InstalledModule> installed) {
def modules = installed.collect { module ->
[
name : module.reference.toString(),
version : module.installedVersion ?: 'unknown',
integrity: module.integrity.toString(),
directory: module.directory.toString(),
registry : module.registryUrl ?: 'unknown'
]
}
// Simple JSON output (could use groovy.json.JsonOutput for better formatting)
println JsonOutput.prettyPrint(JsonOutput.toJson(modules: modules))
}
private String getStatusString(ModuleIntegrity integrity) {
switch( integrity ) {
case ModuleIntegrity.VALID:
return 'OK'
case ModuleIntegrity.MODIFIED:
return 'MODIFIED'
case ModuleIntegrity.NO_REMOTE_MODULE:
return 'LOCAL'
case ModuleIntegrity.CORRUPTED:
return 'CORRUPTED'
default:
return 'UNKNOWN'
}
}
}

View File

@@ -0,0 +1,215 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli.module
import com.beust.jcommander.Parameter
import com.beust.jcommander.Parameters
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import io.seqera.npr.client.RegistryClient
import nextflow.cli.CmdBase
import nextflow.config.ConfigBuilder
import nextflow.config.RegistryConfig
import nextflow.exception.AbortOperationException
import nextflow.module.ModuleChecksum
import nextflow.module.ModuleInfo
import nextflow.module.ModuleSpec
import nextflow.module.ModuleSpecFactory
import nextflow.module.ModuleReference
import nextflow.module.ModuleValidator
import nextflow.module.RegistryClientFactory
import nextflow.module.ModuleStorage
import nextflow.util.TestOnly
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
/**
* Module publish subcommand
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
@Slf4j
@CompileStatic
@Parameters(commandDescription = "Publish a module to the registry")
class CmdModulePublish extends CmdBase {
@Parameter(names = ["-dry-run"], description = "Validate without uploading", arity=0)
boolean dryRun = false
@Parameter(names = ["-registry"], description = "Target registry URL.")
String registryUrl
@Parameter(description = "Module directory path or scope/name")
List<String> args
@TestOnly
protected Path root
@TestOnly
protected RegistryClient client
//Flag if publish is invoked from a scope/name. In this case we should create/update the .module-info with the correct checksum
private boolean useModuleReference = false
@Override
String getName() {
return 'publish'
}
@Override
void run() {
if (!args || args.size() != 1) {
throw new AbortOperationException("Incorrect number of arguments")
}
Path moduleDir = determineModuleDir(args[0])
log.info "Publishing module from: ${moduleDir}"
// Step 1: Validate module structure and spec
def validationErrors = ModuleValidator.validate(moduleDir)
if (!validationErrors.isEmpty()) {
throw new AbortOperationException(
"Module validation failed:\n" + validationErrors.collect { " - ${it}" }.join('\n')
)
}
// Step 2: Load spec for publish metadata
def manifestPath = moduleDir.resolve(ModuleStorage.MODULE_MANIFEST_FILE)
def spec = ModuleSpecFactory.fromYaml(manifestPath)
log.info "Module validated: ${spec.name}@${spec.version}"
if (dryRun) {
printDryRunInfo(spec)
return
}
// Step 3: Get authentication token
def config = new ConfigBuilder()
.setOptions(launcher.options)
.setBaseDir(moduleDir)
.build()
def registryConfig = config.navigate('registry') as RegistryConfig ?: new RegistryConfig()
publishModule(moduleDir, registryConfig, spec)
}
private void publishModule(Path moduleDir, RegistryConfig registryConfig, ModuleSpec spec){
log.info "Creating module bundle..."
def tempBundleFile = Files.createTempFile("nf-module-publish-", ".tar.gz")
try {
def checksum = ModuleStorage.createBundle(moduleDir, tempBundleFile)
log.info "Bundle checksum: ${checksum}"
// Read bundle content as bytes
def bundleBytes = Files.readAllBytes(tempBundleFile)
// Create publish request as a map (npr-api will serialize it)
def request = [
version: spec.version,
bundle: bundleBytes
]
// Publish to registry
final registry = registryUrl ?: registryConfig.url
log.info "Publishing module to registry: ${registryUrl ?: registryConfig.url}"
def registryClient = RegistryClientFactory.forConfig(registryConfig)
def response = registryClient.publishModuleRelease(spec.name, request, registry)
if (useModuleReference) {
// If publish is performed using the module reference we should create/update the .module-info with the correct checksum
try {
ModuleInfo.save(moduleDir, [checksum: ModuleChecksum.compute(moduleDir), registryUrl: registry] )
}catch (Exception e){
log.warn("Unable to save the checksum - ${e.message}")
}
}
println "✓ Module published successfully!"
println ""
println "Module details:"
println " Name: ${spec.name}"
println " Version: ${spec.version}"
println " DownloadUrl: ${response.downloadUrl}"
println ""
println "Others can now install this module using:"
println " nextflow module install ${spec.name}"
} finally {
// Clean up temporary bundle file
if (Files.exists(tempBundleFile)) {
try {
Files.delete(tempBundleFile)
} catch (Exception e) {
log.warn "Failed to clean up temporary bundle file: ${e.message}"
}
}
}
}
private void printDryRunInfo(ModuleSpec spec) {
println "✓ Module structure is valid"
println ""
println "Module details:"
println " Name: ${spec.name}"
println " Version: ${spec.version}"
println " Description: ${spec.description}"
println " License: ${spec.license}"
if( spec.authors ) {
println " Authors: ${spec.authors.join(', ')}"
}
if( spec.keywords ) {
println " Keywords: ${spec.keywords.join(', ')}"
}
if( spec.requires ) {
println " Requires:"
spec.requires.each { name, version ->
println " - ${name}: ${version}"
}
}
println ""
println "Dry run complete. Module is ready to publish."
println "Run without -dry-run to publish to the registry."
}
/**
* Determine if the specified module is a local path or a reference
* @param module
* @return
*/
private Path determineModuleDir(String module) {
//If local path exists return this path as module dir
if (Paths.get(module).exists()){
return Paths.get(module).toAbsolutePath().normalize()
}
final ref = ModuleReference.parse(module)
final localStorage = new ModuleStorage(root ?: Paths.get('.').toAbsolutePath().normalize())
if (!localStorage.isInstalled(ref)){
throw new AbortOperationException("No module directory found for $module")
}
useModuleReference = true
return localStorage.getModuleDir(ref)
}
}

View File

@@ -0,0 +1,105 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli.module
import com.beust.jcommander.Parameter
import com.beust.jcommander.Parameters
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.cli.CmdBase
import nextflow.exception.AbortOperationException
import nextflow.module.ModuleReference
import nextflow.module.ModuleStorage
import nextflow.util.TestOnly
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
/**
* Module remove subcommand
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
@Slf4j
@CompileStatic
@Parameters(commandDescription = "Remove an installed module")
class CmdModuleRemove extends CmdBase {
@Parameter(description = "<module>", required = true)
List<String> args
@Parameter(names = ["-keep-files"], description = "Remove only .module-info keeping the local files", arity = 0)
boolean keepFiles = false
@Parameter(names = ["-force"], description = "Force remove", arity = 0)
boolean force = false
@TestOnly
protected Path root
@Override
String getName() {
return 'remove'
}
@Override
void run() {
if( !args || args.size() != 1 ) {
throw new AbortOperationException("Incorrect number of arguments")
}
if( keepFiles && force ) {
throw new AbortOperationException("Cannot use both -keep-files and -force options")
}
def reference = ModuleReference.parse(args[0])
// Get config
def baseDir = root ?: Paths.get('.').toAbsolutePath().normalize()
// Create resolver and spec file manager
def storage = new ModuleStorage(baseDir)
try {
def filesRemoved = false
// Remove local files unless -keep-files is set
if( !keepFiles ) {
filesRemoved = storage.removeModule(reference, force)
if( filesRemoved ) {
println "Module ${reference} files removed successfully"
} else {
println "Module ${reference} not found locally"
}
} else {
println "Keeping module files for ${reference} (-keep-files flag)"
final moduleInfo = storage.getModuleInfo(reference)
if( Files.exists(moduleInfo) ) {
Files.delete(moduleInfo)
}
}
}
catch( AbortOperationException e ) {
throw e
}
catch( Exception e ) {
throw new AbortOperationException("Failed to remove module $reference: ${e.message}", e)
}
}
}

View File

@@ -0,0 +1,109 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli.module
import java.nio.file.Path
import com.beust.jcommander.Parameter
import com.beust.jcommander.Parameters
import groovy.transform.CompileStatic
import io.seqera.npr.client.RegistryClient
import nextflow.Const
import nextflow.cli.CmdRun
import nextflow.config.ConfigBuilder
import nextflow.config.RegistryConfig
import nextflow.exception.AbortOperationException
import nextflow.module.ModuleReference
import nextflow.module.ModuleResolver
import nextflow.module.RegistryClientFactory
import nextflow.util.TestOnly
/**
* Module run subcommand
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
@CompileStatic
@Parameters(commandDescription = "Run a module directly from the registry")
class CmdModuleRun extends CmdRun {
@Parameter(names = ["-version"], description = "Module version")
String version
@TestOnly
protected Path root
@TestOnly
protected RegistryClient client
@Override
String getName() {
return 'run'
}
@Override
void run() {
if( !args ) {
throw new AbortOperationException("Module name/path not provided")
}
final moduleFile = isLocalModule(args[0])
? resolveLocalModule(args[0])
: resolveRemoteModule(args[0], version)
if( moduleFile ) {
args[0] = moduleFile.toAbsolutePath().toString()
super.run()
}
}
private boolean isLocalModule(String str) {
return str.startsWith('/') || str.startsWith('./') || str.startsWith('../')
}
private Path resolveLocalModule(String str) {
final module = Path.of(str).toAbsolutePath().normalize()
final path = module.isDirectory() ? module.resolve(Const.DEFAULT_MAIN_FILE_NAME) : module
if( !path.exists() )
throw new AbortOperationException("Invalid module path: ${str}")
return path
}
private Path resolveRemoteModule(String name, String version) {
// Parse and validate module reference
ModuleReference reference
try {
reference = ModuleReference.parse(name)
} catch( Exception e ) {
throw new AbortOperationException("Invalid module reference: ${name}", e)
}
// Get config
final baseDir = root ?: Path.of('.').toAbsolutePath().normalize()
final config = new ConfigBuilder()
.setOptions(launcher.options)
.setBaseDir(baseDir)
.build()
final registryConfig = new RegistryConfig(config.registry as Map ?: Collections.emptyMap())
try {
final resolver = new ModuleResolver(baseDir, client ?: RegistryClientFactory.forConfig(registryConfig))
return resolver.installModule(reference, version)
} catch( Exception e ) {
throw new AbortOperationException("Unable to install module: ${name}", e)
}
}
}

View File

@@ -0,0 +1,158 @@
/*
* Copyright 2013-2024, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli.module
import com.beust.jcommander.IParameterValidator
import com.beust.jcommander.Parameter
import com.beust.jcommander.ParameterException
import com.beust.jcommander.Parameters
import groovy.json.JsonOutput
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import io.seqera.npr.api.schema.v1.ModuleSearchResult
import io.seqera.npr.api.schema.v1.SearchModulesResponse
import io.seqera.npr.client.RegistryClient
import nextflow.cli.CmdBase
import nextflow.config.ConfigBuilder
import nextflow.config.RegistryConfig
import nextflow.exception.AbortOperationException
import nextflow.module.RegistryClientFactory
import nextflow.util.TestOnly
import java.nio.file.Paths
/**
* Module search subcommand
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
@Slf4j
@CompileStatic
@Parameters(commandDescription = "Search for modules in the registry")
class CmdModuleSearch extends CmdBase {
@Parameter(names = ["-limit"], description = "Maximum number of results")
int limit = 20
@Parameter(
names = ['-o', '-output'],
description = 'Output mode for reporting search results: simple, json',
validateWith = OutputModeValidator
)
String output = 'simple'
static class OutputModeValidator implements IParameterValidator {
private static final List<String> MODES = List.of('simple', 'json')
@Override
void validate(String name, String value) {
if( !MODES.contains(value) )
throw new ParameterException("Output mode must be one of $MODES (found: $value)")
}
}
@Parameter(description = "<query>", required = true)
List<String> args
@TestOnly
protected RegistryClient client
@Override
String getName() {
return 'search'
}
@Override
void run() {
if( !args || args.size() != 1 ) {
throw new AbortOperationException("Unexpected number of parameters")
}
String query = args[0]
// Get config
def baseDir = Paths.get('.').toAbsolutePath().normalize()
def config = new ConfigBuilder()
.setOptions(launcher.options)
.setBaseDir(baseDir)
.build()
final registryConfig = config.navigate('registry') as RegistryConfig ?: new RegistryConfig()
// Create client to search
final client = this.client ?: RegistryClientFactory.forConfig(registryConfig)
try {
println "Searching for '${query}'..."
final results = client.searchModules(query, limit)
if( !results || results.totalResults == 0 || !results.results || results.results.isEmpty() ) {
println "No modules found"
return
}
if( !output || output == 'simple' ) {
printFormattedResults(results)
} else if( output == 'json' ) {
printJsonResults(results)
} else {
throw new AbortOperationException("Not implemented output mode $output)")
}
}
catch( AbortOperationException e ) {
throw e
}
catch( Exception e ) {
log.error("Failed to search modules", e)
throw new AbortOperationException("Search failed: ${e.message}", e)
}
}
private void printFormattedResults(SearchModulesResponse response) {
println ""
println "Top ${response.totalResults} matching module(s):"
println ""
response.results.each { ModuleSearchResult result ->
println " ${result.name}"
if( result.description ) {
println " Description: ${result.description}"
}
println ""
}
}
private void printJsonResults(SearchModulesResponse response) {
final modules = response.results.collect { ModuleSearchResult result ->
[
name : result.name,
repositoryPath: result.repositoryPath,
description : result.description,
relevanceScore: result.relevanceScore,
keywords : result.keywords,
tools : result.tools,
revoked : result.revoked
]
}
println JsonOutput.prettyPrint(JsonOutput.toJson(
query: response.query,
totalResults: response.totalResults,
results: modules
))
}
}

View File

@@ -0,0 +1,149 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli.module
import java.nio.file.Files
import java.nio.file.Path
import com.beust.jcommander.Parameter
import com.beust.jcommander.Parameters
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.cli.CmdBase
import nextflow.exception.AbortOperationException
import nextflow.module.ModuleReference
import nextflow.module.ModuleSpec
import nextflow.module.ModuleSpecFactory
import nextflow.module.ModuleStorage
import nextflow.util.TestOnly
/**
* Implements the {@code nextflow module spec} subcommand.
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
@Slf4j
@CompileStatic
@Parameters(commandDescription = "Generate a meta.yml spec for a local module")
class CmdModuleSpec extends CmdBase {
@Parameter(description = "Reference to local module (namespace/name) or module directory path")
List<String> args
@Parameter(names = ["-dry-run"], description = "Print generated YAML to stdout without writing any file", arity = 0)
boolean dryRun = false
@Parameter(names = ["-namespace"], description = "Module namespace")
String moduleScope
@Parameter(names = ["-version"], description = "Module version (e.g. 1.0.0)")
String moduleVersion
@Parameter(names = ["-description"], description = "Short description of what the module does")
String description
@Parameter(names = ["-license"], description = "Module license identifier (e.g. MIT, Apache-2.0)")
String license
@Parameter(names = ["-author"], description = "Module author; may be specified multiple times")
List<String> authors
@TestOnly
protected Path root
private String moduleName
@Override
String getName() { 'spec' }
@Override
void run() {
if( !args ){
throw new AbortOperationException("Reference to local module (namespace/name) or path is required")
}
final baseDir = root ?: Path.of('.').toAbsolutePath().normalize()
def moduleDir = resolveAsModuleReference(baseDir, args[0])
if( moduleDir ) {
moduleName = args[0]
if( !dryRun )
log.info "Module name inferred as ${moduleName} ${moduleScope ? '(-namespace option is ignored)' : ''}"
}
else {
moduleDir = resolveAsPath(baseDir, args[0])
if( !moduleScope )
throw new AbortOperationException("The -namespace option is required when referencing a module by path")
}
final mainNfPath = moduleDir.resolve('main.nf')
if( !Files.exists(mainNfPath) )
throw new AbortOperationException("Missing module script (main.nf) in ${moduleDir}")
final metaYmlPath = moduleDir.resolve('meta.yml')
if( Files.exists(metaYmlPath) && !dryRun )
log.info "Using existing module spec: ${metaYmlPath}"
final yamlContent = generateSpec(mainNfPath, metaYmlPath).toYaml()
if( dryRun ) {
println yamlContent
return
}
Files.writeString(metaYmlPath, yamlContent)
println "Saved module spec to: ${metaYmlPath}"
}
private Path resolveAsModuleReference(Path baseDir, String module) {
try {
final ref = ModuleReference.parse(module)
final localStorage = new ModuleStorage(baseDir)
if( localStorage.isInstalled(ref) ) {
log.debug("Argument $module refers to a local module reference")
return localStorage.getModuleDir(ref)
}
log.debug("Argument $module is not a local module reference")
return null
} catch( AbortOperationException e ) {
log.debug("Argument $module is not a correct module reference")
return null
}
}
private Path resolveAsPath(Path baseDir, String module) {
final path = Path.of(module)
final moduleDir = path.isAbsolute() ? path : baseDir.resolve(module)
if( !Files.isDirectory(moduleDir) )
throw new AbortOperationException("Not a directory: ${moduleDir}")
return moduleDir
}
private ModuleSpec generateSpec(Path mainNfPath, Path metaYmlPath) {
final oldSpec = Files.exists(metaYmlPath)
? ModuleSpecFactory.fromYaml(metaYmlPath)
: new ModuleSpec()
return ModuleSpecFactory.fromScript(
mainNfPath,
oldSpec,
namespace: moduleScope,
name: moduleName,
version: moduleVersion,
description: description,
license: license,
authors: authors
)
}
}

View File

@@ -0,0 +1,87 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli.module
import java.nio.file.Path
import java.nio.file.Paths
import com.beust.jcommander.Parameter
import com.beust.jcommander.Parameters
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.cli.CmdBase
import nextflow.exception.AbortOperationException
import nextflow.module.ModuleReference
import nextflow.module.ModuleStorage
import nextflow.module.ModuleValidator
import nextflow.util.TestOnly
/**
* Module validate subcommand -- validates module structure and meta.yml consistency
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
@Parameters(commandDescription = "Validate a module structure and metadata")
class CmdModuleValidate extends CmdBase {
@Parameter(description = "[namespace/name or path]", required = true)
List<String> args
@TestOnly
protected Path root
@Override
String getName() {
return 'validate'
}
@Override
void run() {
if( !args || args.size() != 1 )
throw new AbortOperationException("Incorrect number of arguments -- usage: nextflow module validate <namespace/name>")
final moduleDir = determineModuleDir(args[0])
final errors = ModuleValidator.validate(moduleDir)
if( errors ) {
throw new AbortOperationException(
"Module validation failed:\n" + errors.collect { " - ${it}" }.join('\n')
)
}
println "Module validation passed."
}
/**
* Resolve module path from argument (directory path or namespace/name reference)
*/
protected Path determineModuleDir(String module) {
final path = Paths.get(module)
if( path.exists() )
return path.toAbsolutePath().normalize()
final ref = ModuleReference.parse(module)
final storage = new ModuleStorage(root ?: Paths.get('.').toAbsolutePath().normalize())
if( !storage.isInstalled(ref) )
throw new AbortOperationException("No module directory found for $module")
return storage.getModuleDir(ref)
}
}

View File

@@ -0,0 +1,347 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cli.module
import com.beust.jcommander.IParameterValidator
import com.beust.jcommander.Parameter
import com.beust.jcommander.ParameterException
import com.beust.jcommander.Parameters
import groovy.json.JsonOutput
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import io.seqera.npr.client.RegistryClient
import io.seqera.npr.api.schema.v1.ModuleChannel
import io.seqera.npr.api.schema.v1.ModuleChannelItem
import io.seqera.npr.api.schema.v1.ModuleMetadata
import io.seqera.npr.api.schema.v1.ModuleRelease
import io.seqera.npr.api.schema.v1.ModuleTool
import nextflow.cli.CmdBase
import nextflow.config.ConfigBuilder
import nextflow.config.RegistryConfig
import nextflow.exception.AbortOperationException
import nextflow.module.ModuleReference
import nextflow.module.RegistryClientFactory
import nextflow.util.TestOnly
import java.nio.file.Path
import java.nio.file.Paths
/**
* Module view subcommand - displays module info and usage template
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
@Slf4j
@CompileStatic
@Parameters(commandDescription = "Show module information and usage template")
class CmdModuleView extends CmdBase {
@Parameter(names = ["-version"], description = "Module version")
String version
@Parameter(
names = ['-o', '-output'],
description = 'Output mode for reporting search results: text, json',
validateWith = OutputModeValidator
)
String output = 'text'
static class OutputModeValidator implements IParameterValidator {
private static final List<String> MODES = List.of('text', 'json')
@Override
void validate(String name, String value) {
if( !MODES.contains(value) )
throw new ParameterException("Output mode must be one of $MODES (found: $value)")
}
}
@Parameter(description = "[scope/name]", required = true)
List<String> args
@TestOnly
protected Path root
@TestOnly
protected RegistryClient client
@Override
String getName() {
return 'view'
}
@Override
List<String> getAliases() {
return List.of('info')
}
@Override
void run() {
if( !args || args.size() != 1 ) {
throw new AbortOperationException("Incorrect number of arguments")
}
def reference = ModuleReference.parse(args[0])
// Get config
def baseDir = root ?: Paths.get('.').toAbsolutePath().normalize()
def config = new ConfigBuilder()
.setOptions(launcher.options)
.setBaseDir(baseDir)
.build()
final registryConfig = config.navigate('registry') as RegistryConfig ?: new RegistryConfig()
// Fetch full metadata from registry to get input/output parameters
def registryClient = this.client ?: RegistryClientFactory.forConfig(registryConfig)
ModuleRelease release = null
try {
if( version ) {
release = registryClient.getModuleRelease(reference.fullName, version)
} else {
release = registryClient.getModule(reference.fullName).latest
}
} catch( Exception e ) {
log.warn "Failed to fetch metadata from registry: ${e.message}"
}
if( !release ) {
throw new AbortOperationException("No release information available for ${reference}")
}
if( !release.metadata ) {
throw new AbortOperationException("No metadata found for ${reference}${release.version ? " (${release.version})" : ''}")
}
def moduleUrl = buildModuleUrl(registryConfig.url, reference, release.version)
if( !output || output == 'text' ) {
printFormattedInfo(reference, release, moduleUrl)
} else if( output == 'json' ) {
printJsonInfo(reference, release, moduleUrl)
} else {
throw new AbortOperationException("Not implemented output mode $output)")
}
}
private void printFormattedInfo(ModuleReference reference, ModuleRelease release, String moduleUrl) {
ModuleMetadata metadata = release.metadata
println ""
println "Module: ${reference}"
println "Version: ${release.version}"
println "URL: ${moduleUrl}"
println "Description: ${metadata.description ?: release.description ?: 'N/A'}"
if( metadata.authors ) {
println "Authors: ${metadata.authors.join(', ')}"
}
if( metadata.maintainers ) {
println "Maintainers: ${metadata.maintainers.join(', ')}"
}
if( metadata.keywords ) {
println "Keywords: ${metadata.keywords.join(', ')}"
}
printToolsInfo(metadata?.tools ?: [])
printInputsInfo(metadata.input ?: [])
printOutputsInfo(metadata.output ?: [:])
// Generate and display usage template
println ""
println "Usage Template:"
println "---------------"
println generateUsageTemplate(reference, metadata).join(" \\\n ")
println ""
}
private void printOutputsInfo(Map<String, ModuleChannel> outputs) {
if( outputs ) {
println ""
println "Output:"
outputs.each { name, output ->
println "- ${name} ${output.tuple ? '(tuple)' : ''}"
displayChannel("\t", output)
}
}
}
private void printInputsInfo(List<ModuleChannel> inputs) {
if( inputs ) {
println ""
println "Input:"
inputs.each { input ->
if( input.tuple ) {
println "- (tuple)"
displayChannel("\t", input)
} else
displayChannel("", input)
}
}
}
private void printToolsInfo(List<ModuleTool> toolsList) {
if( toolsList ) {
println ""
println "Tools:"
toolsList.each { tool ->
println " - ${tool.name}${tool.version ? ' v' + tool.version : ''}"
if( tool.homepage ) {
println " Homepage: ${tool.homepage}"
}
}
}
}
private void displayChannel(String prefix, ModuleChannel channel) {
channel.items.each { ModuleChannelItem item ->
println "${prefix}- ${item.name}${item.type ? ' (' + item.type + ')' : ''}"
if( item.description ) {
println "${prefix}\t${item.description.replaceAll(/\R/, ' ')}"
}
if( item.pattern ) {
println "${prefix}\tPattern: ${item.pattern}"
}
}
}
private List<String> generateUsageTemplate(ModuleReference reference, ModuleMetadata metadata) {
def template = new ArrayList<String>()
template.add("nextflow module run ${reference}".toString())
if( version )
template.add(" -version $version".toString())
def inputs = metadata?.input ?: []
inputs.each { input ->
input.items.each { ModuleChannelItem item ->
template.add(reference.scope == 'nf-core'
? inferNfCoreParam(item.name, item.type)
: inferNormalParam(item.name, item.type))
}
}
if( reference.scope == 'nf-core' ) {
template.add('--outdir <OUTPUT_DIRECTORY>')
}
return template
}
private static String inferNfCoreParam(String paramName, String type) {
if( type?.equalsIgnoreCase("map") && paramName.equalsIgnoreCase("meta") ) {
return "--${paramName}.id <ID>"
}
if( type?.equalsIgnoreCase("file") || type?.equalsIgnoreCase("path") ) {
return "--${paramName} ${inferBioFilePlaceholder(paramName)}"
}
return inferNormalParam(paramName, type)
}
private static String inferBioFilePlaceholder(String paramName) {
final String lower = paramName.toLowerCase()
if( lower.contains("fasta") ) return "<FASTA_FILE>"
if( lower.contains("bam") ) return "<BAM_FILE>"
if( lower.contains("fastq") || lower.equals("reads") ) return "<FASTQ_FILE>"
if( lower.contains("vcf") ) return "<VCF_FILE>"
if( lower.contains("ref") ) return "<REFERENCE_FILE>"
if( lower.contains("bed") ) return "<BED_FILE>"
if( lower.contains("gff") || lower.contains("gtf") ) return "<ANNOTATION_FILE>"
return "<${paramName.toUpperCase().replaceAll(/[^A-Z0-9]/, '_')}_PATH>"
}
private static String inferNormalParam(String paramName, String type) {
final paramPlaceholder = paramName.toUpperCase().replaceAll(/[^A-Z0-9]/, '_')
if( type?.equalsIgnoreCase("map") ) {
return "--${paramName}.<KEY> <${paramPlaceholder}_KEY_VALUE>"
}
if( type?.equalsIgnoreCase("file") || type?.equalsIgnoreCase("path") ) {
return "--${paramName} <${paramPlaceholder}_PATH>"
}
return "--${paramName} <${paramPlaceholder}>"
}
private static String buildModuleUrl(String registryUrl, ModuleReference reference, String version) {
// Strip /api suffix to get the base UI URL
def baseUrl = registryUrl.endsWith('/api') ? registryUrl[0..-5] : registryUrl
return "${baseUrl}/modules/${reference.fullName}@${version}"
}
private void printJsonInfo(ModuleReference reference, ModuleRelease release, String moduleUrl) {
def metadata = release?.metadata
def info = [
name : reference.toString(),
fullName : reference.fullName,
version : release.version,
url : moduleUrl,
description: metadata.description ?: release.description,
authors : metadata.authors,
keywords : metadata.keywords,
]
def toolsList = metadata.tools ?: []
if( toolsList ) {
info.tools = toolsList.collect { tool ->
return [
name : tool.name,
version : tool.version,
homepage : tool.homepage,
documentation: tool.documentation
]
}
}
def inputs = metadata.input ?: []
if( inputs ) {
info.input = inputs.collect { input ->
return [
tuple: input.tuple,
items: input.items?.collect { item ->
[
name : item.name,
type : item.type,
description: item.description,
pattern : item.pattern
]
}
]
}
}
def outputs = metadata.output ?: [:]
if( outputs ) {
info.output = outputs.collectEntries { name, output ->
return [name, [
tuple: output.tuple,
items: output.items?.collect { item ->
[
name : item.name,
type : item.type,
description: item.description,
pattern : item.pattern
]
}
]]
}
}
info.usageTemplate = generateUsageTemplate(reference, metadata).join(" ")
println JsonOutput.prettyPrint(JsonOutput.toJson(info))
}
}

View File

@@ -0,0 +1,36 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud
import groovy.transform.CompileStatic
/**
* Exception thrown when a spot instance is retired by the
* provider
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Deprecated
@CompileStatic
class CloudSpotTerminationException extends RuntimeException {
String termination
CloudSpotTerminationException(String termination) {
super("Cloud spot termination notice detected: $termination")
this.termination = termination
}
}

View File

@@ -0,0 +1,38 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud
import nextflow.util.Duration
/**
* Define common cloud data transfer options
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
interface CloudTransferOptions {
static final public int MAX_TRANSFER = 4
static final public int MAX_TRANSFER_ATTEMPTS = 1
static final public Duration DEFAULT_DELAY_BETWEEN_ATTEMPTS = Duration.of('10sec')
int getMaxParallelTransfers()
int getMaxTransferAttempts()
Duration getDelayBetweenAttempts()
}

View File

@@ -0,0 +1,81 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.types
import groovy.transform.CompileStatic
import groovy.transform.Immutable
/**
* Model a cloud instance
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Immutable
@CompileStatic
class CloudInstance implements Serializable, Cloneable {
/**
* The instance unique identifier as assigned by the provider
*/
String id
/**
* The instance current state
*/
String state
/**
* The instance public DNS name
*/
String publicDnsName
/**
* The instance private DNS name
*/
String privateDnsName
/**
* The instance private DNS name
*/
String publicIpAddress
/**
* The instance private IP address
*/
String privateIpAddress
/**
* The instance role in the cluster, either {@link nextflow.Const#ROLE_MASTER} or {@link nextflow.Const#ROLE_WORKER}
*/
String role
/**
* The name of the cluster to which this instance belong to
*/
String clusterName
/**
* @return The instance public address, falling back on the private when the public is not available
*/
String getAddress() {
publicDnsName ?: publicIpAddress ?: privateDnsName ?: privateIpAddress
}
boolean hasPublicAddress() {
publicDnsName || publicIpAddress
}
}

View File

@@ -0,0 +1,41 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.types
/**
* Model a cloud instance status
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
enum CloudInstanceStatus implements Serializable, Cloneable {
/**
* The instance is started, bootstrap/initialisation may be on-going
*/
STARTED,
/**
* The instance is running and available for use
*/
READY,
/**
* The instance ws terminated
*/
TERMINATED
}

View File

@@ -0,0 +1,58 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.types
import groovy.transform.Immutable
import nextflow.util.MemoryUnit
/**
* Models a cloud instance type
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Immutable
class CloudInstanceType implements Serializable, Cloneable {
/**
* Instance type ID as assigned by the cloud provider
*/
String id
/**
* Number of CPUs
*/
int cpus
/**
* Amount of memory (RAM)
*/
MemoryUnit memory
/**
* Amount of local storage
*/
MemoryUnit disk
/**
* Number of disks
*/
int numOfDisks
String toString() {
"id=$id; cpus=$cpus; mem=$memory; disk=$disk x $numOfDisks"
}
}

View File

@@ -0,0 +1,31 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.types
import groovy.transform.Canonical
import groovy.transform.CompileStatic
/**
* Model cloud job metadata
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
@Canonical
class CloudMachineInfo {
String type
String zone
PriceModel priceModel
}

View File

@@ -0,0 +1,55 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.types
import groovy.transform.CompileStatic
import groovy.transform.Immutable
/**
* Models an instance spot price record
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Immutable
@CompileStatic
class CloudSpotPrice implements Serializable, Cloneable {
/**
* The instance type identifier e.g. {@code m4.xlarge}
*/
String type
/**
* The spot price in USD
*/
String price
/**
* The instance availability zone
*/
String zone
/**
* The instance description
*/
String description
Date timestamp
}

View File

@@ -0,0 +1,26 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.types
/**
* Model different cloud price models
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
enum PriceModel {
standard, spot;
}

View File

@@ -0,0 +1,434 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.conda
import java.nio.file.FileSystems
import java.nio.file.NoSuchFileException
import java.nio.file.Path
import java.nio.file.Paths
import java.util.concurrent.ConcurrentHashMap
import groovy.transform.CompileStatic
import groovy.transform.Memoized
import groovy.transform.PackageScope
import groovy.util.logging.Slf4j
import groovyx.gpars.dataflow.DataflowVariable
import groovyx.gpars.dataflow.LazyDataflowVariable
import nextflow.Global
import nextflow.SysEnv
import nextflow.file.FileMutex
import nextflow.util.CacheHelper
import nextflow.util.Duration
import nextflow.util.Escape
import nextflow.util.TestOnly
/**
* Handle Conda environment creation and caching
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
class CondaCache {
static final private Object condaLock = new Object()
/**
* Cache the prefix path for each Conda environment
*/
static final private Map<String,DataflowVariable<Path>> condaPrefixPaths = new ConcurrentHashMap<>()
/**
* The Conda settings defined in the nextflow config file
*/
private CondaConfig config
/**
* Timeout after which the environment creation is aborted
*/
private Duration createTimeout
private String createOptions
private boolean useMamba
private boolean useMicromamba
private Path configCacheDir0
private List<String> channels = Collections.emptyList()
@PackageScope String getCreateOptions() { createOptions }
@PackageScope Duration getCreateTimeout() { createTimeout }
@PackageScope Map<String,String> getEnv() { SysEnv.get() }
@PackageScope Path getConfigCacheDir0() { configCacheDir0 }
@PackageScope List<String> getChannels() { channels }
@PackageScope String getBinaryName() {
if (useMamba)
return "mamba"
if (useMicromamba)
return "micromamba"
return "conda"
}
@TestOnly
protected CondaCache() {}
/**
* Create a Conda env cache object
*
* @param config A {@link Map} object
*/
CondaCache(CondaConfig config) {
this.config = config
if( config.createTimeout() )
createTimeout = config.createTimeout()
if( config.createOptions() )
createOptions = config.createOptions()
if( config.cacheDir() )
configCacheDir0 = config.cacheDir().toAbsolutePath()
if( config.useMamba() && config.useMicromamba() )
throw new IllegalArgumentException("Both conda.useMamba and conda.useMicromamba were enabled -- Please choose only one")
if( config.useMamba() ) {
useMamba = config.useMamba()
}
if( config.useMicromamba() )
useMicromamba = config.useMicromamba()
if( config.getChannels() )
channels = config.getChannels()
}
/**
* Retrieve the directory where store the conda environment.
*
* If tries these setting in the following order:
* 1) {@code conda.cacheDir} setting in the nextflow config file;
* 2) the {@code $workDir/conda} path
*
* @return
* the {@code Path} where store the conda envs
*/
@PackageScope
Path getCacheDir() {
def cacheDir = configCacheDir0
if( !cacheDir && getEnv().NXF_CONDA_CACHEDIR )
cacheDir = getEnv().NXF_CONDA_CACHEDIR as Path
if( !cacheDir )
cacheDir = getSessionWorkDir().resolve('conda')
if( cacheDir.fileSystem != FileSystems.default ) {
throw new IOException("Cannot store Conda environments to a remote work directory -- Use a POSIX compatible work directory or specify an alternative path with the `conda.cacheDir` config setting")
}
if( !cacheDir.exists() && !cacheDir.mkdirs() ) {
throw new IOException("Failed to create Conda cache directory: $cacheDir -- Make sure a file with the same name does not exist and you have write permission")
}
return cacheDir
}
@PackageScope Path getSessionWorkDir() {
Global.session.workDir
}
@PackageScope
boolean isYamlFilePath(String str) {
(str.endsWith('.yml') || str.endsWith('.yaml')) && !str.contains('\n')
}
/**
* Check if the given string is a path to a conda explicit file
* by verifying it contains the @EXPLICIT marker in the first 20 lines
*
* @param str The conda environment string
* @return {@code true} if it's a path to an explicit file, {@code false} otherwise
*/
@PackageScope
@Memoized // <-- annotate as "Memoized" to avoid parsing multiple time the same file
boolean isExplicitFile(String str) {
if( str.contains('\n') )
return false
try {
final path = str as Path
if( !path.exists() )
return false
return containsExplicitMarker(path)
}
catch( Exception e ) {
log.debug "Failed to check explicit file: $str - ${e.message}"
return false
}
}
/**
* Check if a file contains the @EXPLICIT marker in the first 20 lines
*
* @param path The file path to check
* @return {@code true} if the marker is found, {@code false} otherwise
*/
private boolean containsExplicitMarker(Path path) {
try {
try(final reader = path.newReader()) {
for( int i = 0; i < 20; i++ ) {
final line = reader.readLine()
if( line == null )
break
if( line.trim() == '@EXPLICIT' )
return true
}
return false
}
}
catch( Exception e ) {
log.debug "Failed to check for @EXPLICIT marker in file: $path - ${e.message}"
return false
}
}
/**
* Get the path on the file system where store a Conda environment
*
* @param condaEnv The conda environment
* @return the conda unique prefix {@link Path} where the env is created
*/
@PackageScope
Path condaPrefixPath(String condaEnv) {
assert condaEnv
String content
String name = 'env'
// check if it's a remote uri
if( isYamlUriPath(condaEnv) ) {
content = condaEnv
}
// check if it's a YAML file
else if( isYamlFilePath(condaEnv) ) {
try {
final path = condaEnv as Path
content = path.text
}
catch( NoSuchFileException e ) {
throw new IllegalArgumentException("Conda environment file does not exist: $condaEnv")
}
catch( Exception e ) {
throw new IllegalArgumentException("Error parsing Conda environment YAML file: $condaEnv -- Check the log file for details", e)
}
}
// check if it's a conda explicit file (contains @EXPLICIT marker)
else if( isExplicitFile(condaEnv) ) {
try {
final path = condaEnv as Path
content = path.text
}
catch( Exception e ) {
throw new IllegalArgumentException("Error reading Conda explicit file: $condaEnv -- Check the log file for details", e)
}
}
// it's interpreted as user provided prefix directory
else if( condaEnv.contains('/') ) {
final prefix = condaEnv as Path
if( !prefix.isDirectory() )
throw new IllegalArgumentException("Conda prefix path does not exist or is not a directory: $prefix")
if( prefix.fileSystem != FileSystems.default )
throw new IllegalArgumentException("Conda prefix path must be a POSIX file path: $prefix")
return prefix
}
else if( condaEnv.contains('\n') ) {
throw new IllegalArgumentException("Invalid Conda environment definition: $condaEnv")
}
else {
content = condaEnv
}
final hash = CacheHelper.hasher(content).hash().toString()
getCacheDir().resolve("$name-$hash")
}
/**
* Run the conda tool to create an environment in the file system.
*
* @param condaEnv The conda environment definition
* @return the conda environment prefix {@link Path}
*/
@PackageScope
Path createLocalCondaEnv(String condaEnv, Path prefixPath) {
if( prefixPath.isDirectory() ) {
log.debug "${binaryName} found local env for environment=$condaEnv; path=$prefixPath"
return prefixPath
}
final file = new File("${prefixPath.parent}/.${prefixPath.name}.lock")
final wait = "Another Nextflow instance is creating the conda environment $condaEnv -- please wait till it completes"
final err = "Unable to acquire exclusive lock after $createTimeout on file: $file"
final mutex = new FileMutex(target: file, timeout: createTimeout, waitMessage: wait, errorMessage: err)
try {
mutex .lock { createLocalCondaEnv0(condaEnv, prefixPath) }
}
finally {
file.delete()
}
return prefixPath
}
@PackageScope
Path makeAbsolute( String envFile ) {
Paths.get(envFile).toAbsolutePath()
}
@PackageScope boolean isYamlUriPath(String env) {
env.startsWith('http://') || env.startsWith('https://')
}
@PackageScope
Path createLocalCondaEnv0(String condaEnv, Path prefixPath) {
if( prefixPath.isDirectory() ) {
log.debug "${binaryName} found local env for environment=$condaEnv; path=$prefixPath"
return prefixPath
}
log.info "Creating env using ${binaryName}: $condaEnv [cache $prefixPath]"
String opts = createOptions ? "$createOptions " : ''
def cmd
if( isYamlFilePath(condaEnv) ) {
final target = isYamlUriPath(condaEnv) ? condaEnv : Escape.path(makeAbsolute(condaEnv))
final yesOpt = binaryName=="mamba" || binaryName == "micromamba" ? '--yes ' : ''
cmd = "${binaryName} env create ${yesOpt}--prefix ${Escape.path(prefixPath)} --file ${target}"
}
else if( isExplicitFile(condaEnv) ) {
cmd = "${binaryName} create ${opts}--yes --quiet --prefix ${Escape.path(prefixPath)} --file ${Escape.path(makeAbsolute(condaEnv))}"
}
else {
final channelsOpt = channels.collect(it -> "-c $it ").join('')
cmd = "${binaryName} create ${opts}--yes --quiet --prefix ${Escape.path(prefixPath)} ${channelsOpt}$condaEnv"
}
try {
// Parallel execution of conda causes data and package corruption.
// https://github.com/nextflow-io/nextflow/issues/4233
// https://github.com/conda/conda/issues/13037
// Should be removed as soon as the upstream bug is fixed and released.
synchronized(condaLock) {
runCommand( cmd )
}
log.debug "'${binaryName}' create complete env=$condaEnv path=$prefixPath"
}
catch( Exception e ){
// clean-up to avoid to keep eventually corrupted image file
prefixPath.delete()
throw e
}
return prefixPath
}
@PackageScope
int runCommand( String cmd ) {
log.trace """${binaryName} create
command: $cmd
timeout: $createTimeout""".stripIndent(true)
final max = createTimeout.toMillis()
final builder = new ProcessBuilder(['bash','-c',cmd])
final proc = builder.redirectErrorStream(true).start()
final err = new StringBuilder()
final consumer = proc.consumeProcessOutputStream(err)
proc.waitForOrKill(max)
def status = proc.exitValue()
if( status != 0 ) {
consumer.join()
def msg = "Failed to create Conda environment\n command: $cmd\n status : $status\n message:\n"
msg += err.toString().trim().indent(' ')
throw new IllegalStateException(msg)
}
return status
}
/**
* Given a remote image URL returns a {@link DataflowVariable} which holds
* the local image path.
*
* This method synchronise multiple concurrent requests so that only one
* image download is actually executed.
*
* @param condaEnv
* Conda environment string
* @return
* The {@link DataflowVariable} which hold (and pull) the local image file
*/
@PackageScope
DataflowVariable<Path> getLazyImagePath(String condaEnv) {
final prefixPath = condaPrefixPath(condaEnv)
final condaEnvPath = prefixPath.toString()
if( condaEnvPath in condaPrefixPaths ) {
log.trace "${binaryName} found local environment `$condaEnv`"
return condaPrefixPaths[condaEnvPath]
}
synchronized (condaPrefixPaths) {
def result = condaPrefixPaths[condaEnvPath]
if( result == null ) {
result = new LazyDataflowVariable<Path>({ createLocalCondaEnv(condaEnv, prefixPath) })
condaPrefixPaths[condaEnvPath] = result
}
else {
log.trace "${binaryName} found local cache for environment `$condaEnv` (2)"
}
return result
}
}
/**
* Create a conda environment caching it in the file system.
*
* This method synchronise multiple concurrent requests so that only one
* environment is actually created.
*
* @param condaEnv The conda environment string
* @return the local environment path prefix {@link Path}
*/
Path getCachePathFor(String condaEnv) {
def promise = getLazyImagePath(condaEnv)
def result = promise.getVal()
if( promise.isError() )
throw new IllegalStateException(promise.getError())
if( !result )
throw new IllegalStateException("Cannot create Conda environment `$condaEnv`")
log.trace "Conda cache for env `$condaEnv` path=$result"
return result
}
}

View File

@@ -0,0 +1,129 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.conda
import java.nio.file.Path
import groovy.transform.CompileStatic
import nextflow.config.spec.ConfigOption
import nextflow.config.spec.ConfigScope
import nextflow.config.spec.ScopeName
import nextflow.script.dsl.Description
import nextflow.util.Duration
/**
* Model Conda configuration
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@ScopeName("conda")
@Description("""
The `conda` scope controls the creation of Conda environments by the Conda package manager.
""")
@CompileStatic
class CondaConfig implements ConfigScope {
@ConfigOption
@Description("""
Execute tasks with Conda environments (default: `false`).
""")
final boolean enabled
@ConfigOption
@Description("""
The path where Conda environments are stored. It should be accessible from all compute nodes when using a shared file system.
""")
final String cacheDir
@ConfigOption
@Description("""
The list of Conda channels that can be used to resolve Conda packages. Channel priority decreases from left to right.
""")
final List<String> channels
@ConfigOption
@Description("""
Extra command line options for the `conda create` command. See the [Conda documentation](https://docs.conda.io/projects/conda/en/latest/commands/create.html) for more information.
""")
final String createOptions
@ConfigOption
@Description("""
The amount of time to wait for the Conda environment to be created before failing (default: `20 min`).
""")
final Duration createTimeout
@ConfigOption
@Description("""
Use [Mamba](https://github.com/mamba-org/mamba) instead of `conda` to create Conda environments (default: `false`).
""")
final boolean useMamba
@ConfigOption
@Description("""
Use [Micromamba](https://mamba.readthedocs.io/en/latest/user_guide/micromamba.html) instead of `conda` to create Conda environments (default: `false`).
""")
final boolean useMicromamba
/* required by extension point -- do not remove */
CondaConfig() {}
CondaConfig(Map opts, Map<String, String> env) {
enabled = opts.enabled != null
? opts.enabled as boolean
: (env.NXF_CONDA_ENABLED?.toString() == 'true')
cacheDir = opts.cacheDir
channels = parseChannels(opts.channels)
createOptions = opts.createOptions
createTimeout = opts.createTimeout as Duration ?: Duration.of('20min')
useMamba = opts.useMamba as boolean
useMicromamba = opts.useMicromamba as boolean
if( useMamba && useMicromamba )
throw new IllegalArgumentException("Both conda.useMamba and conda.useMicromamba were enabled -- Please choose only one")
}
private List<String> parseChannels(Object value) {
if( !value )
return ['conda-forge','bioconda']
if( value instanceof List )
return value
if( value instanceof CharSequence )
return value.tokenize(',').collect(it -> it.trim())
throw new IllegalArgumentException("Unexpected conda.channels value: $value")
}
Duration createTimeout() {
createTimeout
}
String createOptions() {
createOptions
}
Path cacheDir() {
cacheDir as Path
}
boolean useMamba() {
useMamba
}
boolean useMicromamba() {
useMicromamba
}
}

View File

@@ -0,0 +1,150 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.config
import java.lang.reflect.Method
import groovy.transform.CompileDynamic
import groovy.transform.CompileStatic
import groovy.transform.Memoized
/**
* Configuration object which fallback to a parent {@link CascadingConfig} object
* when an attribute value is missing.
*
* @see ConfigField
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
abstract class CascadingConfig<K,V> {
protected Map<K,V> config = [:]
protected CascadingConfig<K,V> parent
CascadingConfig() {}
CascadingConfig(Map<K,V> config, CascadingConfig<K,V> parent) {
this.config = config
this.parent = parent
config.keySet().each { K it -> checkField(it) }
}
@CompileDynamic
private Set<String> discoverFields(Closure<Boolean> accept) {
def result = new HashSet<String>()
def clazz = this.getClass()
while( clazz != null ) {
def methods = clazz.getMethods()
def names = (methods as List<Method>)?.findResults { field ->
def annotation = field.getAnnotation(ConfigField)
if( annotation && accept(annotation) )
return annotation.value() ?: field.getName().replaceFirst('^get','').uncapitalize()
return null
}
result.addAll(names)
clazz = clazz.getSuperclass()
}
return result ?: Collections.<String>emptySet()
}
@Memoized
protected Set<String> validFields() {
return discoverFields({ true })
}
protected Set<String> privateFields() {
return discoverFields({ ConfigField field -> field._private() })
}
final protected void checkField(K name) throws IllegalArgumentException {
def fields = validFields()
if( !fields.contains(name.toString()) )
throw new IllegalArgumentException("Not a valid config attribute: `$name`")
}
protected Map<String,Object> copyPublicAttributes() {
def skip = privateFields()
def copy = new LinkedHashMap()
this.config.each { k,v -> if(!skip.contains(k)) copy[k]=v }
return copy
}
boolean isEmpty() { config.isEmpty() }
boolean containsAttribute( K key ) {
config.containsKey(key) ?: ( parent ? parent.containsAttribute(key) : false )
}
V getAttribute(K key) {
getAttribute(key, null)
}
V getAttribute(K key, V defValue) {
checkField(key)
return config.containsKey(key) ? config.get(key) : ( parent && parent.containsAttribute(key) ? parent.getAttribute(key) : defValue )
}
V getOrCreateAttribute(K key, Closure<V> missing) {
def result = getAttribute(key)
if( result == null ) {
result = missing.call()
config.put(key, result)
}
return result
}
void setAttribute( K key, V value ) {
checkField(key)
config.put(key,value)
}
Set<K> getAttributeNames() {
new HashSet<K>(config.keySet())
}
/**
* Convert this object to an equivalent {@link ConfigObject}
*
* @return A {@link ConfigObject} holding the same data
*/
ConfigObject toConfigObject() {
toConfigObject0(this.config)
}
@CompileDynamic
protected ConfigObject toConfigObject0( Map map ) {
def result = new ConfigObject()
map.each { key, value ->
if( value instanceof Map ) {
result.put( key, toConfigObject0((Map)value) )
}
else if( value instanceof CascadingConfig ) {
result.put( key, value.toConfigObject() )
}
else {
result.put( key, value )
}
}
return result
}
}

View File

@@ -0,0 +1,916 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.config
import static nextflow.util.ConfigHelper.*
import java.nio.file.Path
import java.nio.file.Paths
import groovy.transform.Memoized
import groovy.transform.PackageScope
import groovy.util.logging.Slf4j
import nextflow.Const
import nextflow.NF
import nextflow.SysEnv
import nextflow.cli.CliOptions
import nextflow.cli.CmdConfig
import nextflow.cli.CmdNode
import nextflow.cli.CmdRun
import nextflow.exception.AbortOperationException
import nextflow.exception.ConfigParseException
import nextflow.secret.SecretsLoader
import nextflow.secret.SecretsProvider
import nextflow.util.HistoryFile
import nextflow.util.SecretHelper
/**
* Builds up the Nextflow configuration object
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
class ConfigBuilder {
static final public String DEFAULT_PROFILE = 'standard'
CliOptions options
CmdRun cmdRun
CmdNode cmdNode
Path baseDir
Path homeDir
Path currentDir
Map<String,?> cliParams
boolean showAllProfiles
String profile = DEFAULT_PROFILE
boolean validateProfile
List<Path> userConfigFiles = []
List<Path> parsedConfigFiles = []
boolean showClosures
boolean stripSecrets
boolean showMissingVariables
SecretsProvider secretsProvider
Map<ConfigObject, String> emptyVariables = new LinkedHashMap<>(10)
Map<String,String> env = new HashMap<>(SysEnv.get())
List<String> warnings = new ArrayList<>(10)
Map<String,Object> declaredParams
ConfigBuilder() {
setHomeDir(Const.APP_HOME_DIR)
setCurrentDir(Paths.get('.'))
}
ConfigBuilder setShowClosures(boolean value) {
this.showClosures = value
return this
}
ConfigBuilder setStripSecrets(boolean value) {
this.stripSecrets = value
return this
}
ConfigBuilder showMissingVariables(boolean value) {
this.showMissingVariables = value
return this
}
ConfigBuilder setSecretsProvider(SecretsProvider value) {
this.secretsProvider = value
return this
}
ConfigBuilder setOptions( CliOptions options ) {
this.options = options
return this
}
ConfigBuilder setCmdRun( CmdRun cmdRun ) {
this.cmdRun = cmdRun
setProfile(cmdRun.profile)
return this
}
ConfigBuilder setCliParams( Map<String,?> cliParams ) {
this.cliParams = cliParams
return this
}
ConfigBuilder setBaseDir( Path path ) {
this.baseDir = path.complete()
return this
}
ConfigBuilder setCurrentDir( Path path ) {
this.currentDir = path.complete()
return this
}
ConfigBuilder setHomeDir( Path path ) {
this.homeDir = path.complete()
return this
}
ConfigBuilder setCmdNode( CmdNode node ) {
this.cmdNode = node
return this
}
ConfigBuilder setCmdConfig( CmdConfig cmdConfig ) {
showAllProfiles = cmdConfig.showAllProfiles
setProfile(cmdConfig.profile)
return this
}
ConfigBuilder setProfile( String value ) {
profile = value ?: DEFAULT_PROFILE
validateProfile = value as boolean
return this
}
ConfigBuilder setShowAllProfiles(boolean value) {
this.showAllProfiles = value
return this
}
ConfigBuilder setUserConfigFiles( Path... files ) {
setUserConfigFiles(files as List)
return this
}
ConfigBuilder setUserConfigFiles( List<Path> files ) {
if( files )
userConfigFiles.addAll(files)
return this
}
Map<String,Object> getConfigParams() {
return declaredParams
}
static private wrapValue( value ) {
if( !value )
return ''
value = value.toString().trim()
if( value == 'true' || value == 'false')
return value
if( value.isNumber() )
return value
return "'$value'"
}
/**
* Transform the specified list of string to a list of files, verifying their existence.
* <p>
* If a file in the list does not exist an exception of type {@code CliArgumentException} is thrown.
* <p>
* If the specified list is empty it tries to return of default configuration files located at:
* <li>$HOME/.nextflow/taskConfig
* <li>$PWD/nextflow.taskConfig
*
* @param files
* @return
*/
@PackageScope
List<Path> validateConfigFiles( List<String> files ) {
def result = []
if ( files ) {
for( String fileName : files ) {
def thisFile = currentDir.resolve(fileName)
if(!thisFile.exists()) {
throw new AbortOperationException("The specified configuration file does not exist: $thisFile -- check the name or choose another file")
}
result << thisFile
}
return result
}
/*
* config file in the nextflow home
*/
def home = homeDir.resolve('config')
if( home.exists() ) {
log.debug "Found config home: $home"
result << home
}
/**
* Config file in the pipeline base dir
* This config file name should be predictable, therefore cannot be overridden
*/
def base = null
if( baseDir && baseDir != currentDir ) {
base = baseDir.resolve('nextflow.config')
if( base.exists() ) {
log.debug "Found config base: $base"
result << base
}
}
/**
* Local or user provided file
* Default config file name can be overridden with `NXF_CONFIG_FILE` env variable
*/
def configFileName = env.get('NXF_CONFIG_FILE') ?: 'nextflow.config'
def local = currentDir.resolve(configFileName)
if( local.exists() && local != base ) {
log.debug "Found config local: $local"
result << local
}
def customConfigs = []
if( userConfigFiles ) customConfigs.addAll(userConfigFiles)
if( options?.userConfig ) customConfigs.addAll(options.userConfig)
if( cmdRun?.runConfig ) customConfigs.addAll(cmdRun.runConfig)
if( customConfigs ) {
for( def item : customConfigs ) {
def configFile = item instanceof Path ? item : currentDir.resolve(item.toString())
if(!configFile.exists()) {
throw new AbortOperationException("The specified configuration file does not exist: $configFile -- check the name or choose another file")
}
log.debug "User config file: $configFile"
result << configFile
}
}
return result
}
/**
* Create the nextflow configuration {@link ConfigObject} given a one or more
* config files
*
* @param files A list of config files {@link Path}
* @return The resulting {@link ConfigObject} instance
*/
@PackageScope
ConfigObject buildGivenFiles(List<Path> files) {
final Map<String,String> vars = cmdRun?.env
final boolean exportSysEnv = cmdRun?.exportSysEnv
def items = []
if( files ) for( Path file : files ) {
log.debug "Parsing config file: ${file.complete()}"
if (!file.exists()) {
log.warn "The specified configuration file cannot be found: $file"
}
else {
items << file
}
}
Map env = [:]
if( exportSysEnv ) {
log.debug "Adding current system environment to session environment"
env.putAll(SysEnv.get())
}
if( vars ) {
log.debug "Adding the following variables to session environment: $vars"
env.putAll(vars)
}
// set the cluster options for the node command
if( cmdNode?.clusterOptions ) {
def str = new StringBuilder()
cmdNode.clusterOptions.each { k, v ->
str << "cluster." << k << '=' << wrapValue(v) << '\n'
}
items << str
}
// -- add the executor obj from the command line args
if( cmdRun?.clusterOptions ) {
def str = new StringBuilder()
cmdRun.clusterOptions.each { k, v ->
str << "cluster." << k << '=' << wrapValue(v) << '\n'
}
items << str
}
if( cmdRun?.executorOptions ) {
def str = new StringBuilder()
cmdRun.executorOptions.each { k, v ->
str << "executor." << k << '=' << wrapValue(v) << '\n'
}
items << str
}
buildConfig0( env, items )
}
@PackageScope
ConfigObject buildGivenFiles(Path... files) {
buildGivenFiles(files as List<Path>)
}
protected Map configVars() {
// this is needed to make sure to reuse the same
// instance of the config vars across different instances of the ConfigBuilder
// and prevent multiple parsing of the same params file (which can even be remote resource)
final secretContext = secretsProvider
? SecretsLoader.secretContext(secretsProvider)
: SecretsLoader.secretContext()
return getConfigVars(baseDir, secretContext)
}
@Memoized
static Map getConfigVars(Path base, Object secretContext) {
final binding = new HashMap(10)
binding.put('baseDir', base)
binding.put('projectDir', base)
binding.put('launchDir', Paths.get('.').toRealPath())
binding.put('outputDir', Paths.get('results').complete())
binding.put('secrets', secretContext)
return binding
}
protected ConfigObject buildConfig0( Map env, List configEntries ) {
assert env != null
final ignoreIncludes = options ? options.ignoreConfigIncludes : false
final ansiLog = options ? options.ansiLog : false
final parser = ConfigParserFactory.create()
.setRenderClosureAsString(showClosures)
.setStripSecrets(stripSecrets)
.setIgnoreIncludes(ignoreIncludes)
.setAnsiLog(ansiLog)
ConfigObject result = new ConfigObject()
if( cliParams )
parser.setParams(cliParams)
// add the user specified environment to the session env
env.sort().each { name, value -> result.env.put(name,value) }
if( configEntries ) {
// the configuration object binds always the current environment
// so that in the configuration file may be referenced any variable
// in the current environment
final binding = new HashMap(SysEnv.get())
binding.putAll(env)
binding.putAll(configVars())
parser.setBinding(binding)
// merge of the provided configuration files
for( def entry : configEntries ) {
try {
merge0(result, parser, entry)
}
catch( ConfigParseException e ) {
throw e
}
catch( Exception e ) {
def message = (entry instanceof Path ? "Unable to parse config file: '$entry'" : "Unable to parse configuration ")
throw new ConfigParseException(message,e)
}
}
if( validateProfile ) {
checkValidProfile(parser.getDeclaredProfiles())
}
this.declaredParams = parser.getDeclaredParams()
}
// guarantee top scopes
for( String name : ['env','session','params','process','executor']) {
if( !result.isSet(name) ) result.put(name, new ConfigObject())
}
return result
}
/**
* Merge the main config with a separate config file
*
* @param result The main {@link ConfigObject}
* @param parser The {@ConfigParser} instance
* @param entry The next config snippet/file to be parsed
* @return
*/
protected void merge0(ConfigObject result, ConfigParser parser, entry) {
if( !entry )
return
// select the profile
if( !showAllProfiles ) {
log.debug "Applying config profile: `${profile}`"
parser.setProfiles(profile.tokenize(','))
}
final config = parse0(parser, entry)
if( NF.getSyntaxParserVersion() == 'v1' )
validate(config, entry)
result.merge(config)
}
protected ConfigObject parse0(ConfigParser parser, entry) {
if( entry instanceof File ) {
final path = entry.toPath()
parsedConfigFiles << path
return parser.parse(path)
}
if( entry instanceof Path ) {
parsedConfigFiles << entry
return parser.parse(entry)
}
if( entry instanceof CharSequence ) {
return parser.parse(entry.toString())
}
throw new IllegalStateException("Unexpected config entry: ${entry}")
}
/**
* Validate a config object verifying is does not contains unresolved attributes
*
* @param config The {@link ConfigObject} to verify
* @param file The source config file/snippet
* @return
*/
protected void validate(ConfigObject config, file, String parent=null, List stack = new ArrayList()) {
for( String key : new ArrayList<>(config.keySet()) ) {
final value = config.get(key)
if( value instanceof ConfigObject ) {
final fqKey = parent ? "${parent}.${key}": key as String
if( value.isEmpty() ) {
final msg = "Unknown config attribute `$fqKey` -- check config file: $file".toString()
if( showMissingVariables ) {
emptyVariables.put(value, key)
warnings.add(msg)
}
else {
log.debug("In the following config snippet the attribute `$fqKey` is empty:\n${->config.prettyPrint().indent(' ')}")
throw new ConfigParseException(msg)
}
}
else {
stack.push(config)
try {
if( !stack.contains(value)) {
validate(value, file, fqKey, stack)
}
else {
log.debug("Found a recursive config property: `$fqKey`")
}
}
finally {
stack.pop()
}
}
}
else if( value instanceof GString && showMissingVariables ) {
final str = (GString) value
for( int i=0; i<str.values.length; i++ ) {
// try replace empty interpolated strings with variable handle
final arg = str.values[i]
final name = emptyVariables.get(arg)
if( name )
str.values[i] = '$' + name
}
}
}
}
protected void checkValidProfile(Collection<String> validNames) {
if( !profile || profile == DEFAULT_PROFILE ) {
return
}
log.debug "Available config profiles: $validNames"
for( String name : profile.tokenize(',') ) {
if( name in validNames )
continue
def message = "Unknown configuration profile: '${name}'"
def choices = validNames.closest(name)
if( choices ) {
message += "\n\nDid you mean one of these?\n"
choices.each { message += " ${it}\n" }
message += '\n'
}
throw new AbortOperationException(message)
}
}
private String normalizeResumeId( String uniqueId ) {
if( !uniqueId )
return null
if( uniqueId == 'last' || uniqueId == 'true' ) {
if( HistoryFile.disabled() )
throw new AbortOperationException("The resume session id should be specified via `-resume` option when history file tracking is disabled")
uniqueId = HistoryFile.DEFAULT.getLast()?.sessionId
if( !uniqueId ) {
log.warn "It appears you have never run this project before -- Option `-resume` is ignored"
}
}
return uniqueId
}
@PackageScope
void configRunOptions(ConfigObject config, Map env, CmdRun cmdRun) {
// -- set config options
if( cmdRun.cacheable != null )
config.cacheable = cmdRun.cacheable
// -- set the run name
if( cmdRun.runName )
config.runName = cmdRun.runName
if( cmdRun.stubRun )
config.stubRun = cmdRun.stubRun
// -- set the output directory
if( cmdRun.outputDir )
config.outputDir = cmdRun.outputDir
if( cmdRun.outputFormat )
config.outputFormat = cmdRun.outputFormat
if( cmdRun.preview )
config.preview = cmdRun.preview
if( cmdRun.plugins )
config.plugins = cmdRun.plugins.tokenize(',')
// -- sets the working directory
if( cmdRun.workDir )
config.workDir = cmdRun.workDir
else if( !config.workDir )
config.workDir = env.get('NXF_WORK') ?: 'work'
if( cmdRun.bucketDir )
config.bucketDir = cmdRun.bucketDir
// -- sets the library path
if( cmdRun.libPath )
config.libDir = cmdRun.libPath
else if ( !config.isSet('libDir') && env.get('NXF_LIB') )
config.libDir = env.get('NXF_LIB')
// -- override 'process' parameters defined on the cmd line
cmdRun.process.each { name, value ->
config.process[name] = parseValue(value)
}
if( cmdRun.withoutConda && config.conda instanceof Map ) {
// disable conda execution
log.debug "Disabling execution with Conda as requested by command-line option `-without-conda`"
config.conda.enabled = false
}
// -- apply the conda environment
if( cmdRun.withConda ) {
if( cmdRun.withConda != '-' )
config.process.conda = cmdRun.withConda
config.conda.enabled = true
}
if( cmdRun.withoutSpack && config.spack instanceof Map ) {
// disable spack execution
log.debug "Disabling execution with Spack as requested by command-line option `-without-spack`"
config.spack.enabled = false
}
// -- apply the spack environment
if( cmdRun.withSpack ) {
if( cmdRun.withSpack != '-' )
config.process.spack = cmdRun.withSpack
config.spack.enabled = true
}
// -- sets the resume option
if( cmdRun.resume )
config.resume = cmdRun.resume
if( config.isSet('resume') )
config.resume = normalizeResumeId(config.resume as String)
// -- sets `dumpHashes` option
if( cmdRun.dumpHashes ) {
config.dumpHashes = cmdRun.dumpHashes != '-' ? cmdRun.dumpHashes : 'default'
}
if( cmdRun.dumpChannels )
config.dumpChannels = cmdRun.dumpChannels.tokenize(',')
// -- other configuration parameters
if( cmdRun.poolSize ) {
config.poolSize = cmdRun.poolSize
}
if( cmdRun.queueSize ) {
config.executor.queueSize = cmdRun.queueSize
}
if( cmdRun.pollInterval ) {
config.executor.pollInterval = cmdRun.pollInterval
}
// -- sets trace file options
if( cmdRun.withTrace ) {
if( !(config.trace instanceof Map) )
config.trace = [:]
config.trace.enabled = true
if( cmdRun.withTrace != '-' )
config.trace.file = cmdRun.withTrace
}
// -- sets report report options
if( cmdRun.withReport ) {
if( !(config.report instanceof Map) )
config.report = [:]
config.report.enabled = true
if( cmdRun.withReport != '-' )
config.report.file = cmdRun.withReport
}
// -- sets timeline report options
if( cmdRun.withTimeline ) {
if( !(config.timeline instanceof Map) )
config.timeline = [:]
config.timeline.enabled = true
if( cmdRun.withTimeline != '-' )
config.timeline.file = cmdRun.withTimeline
}
// -- sets DAG report options
if( cmdRun.withDag ) {
if( !(config.dag instanceof Map) )
config.dag = [:]
config.dag.enabled = true
if( cmdRun.withDag != '-' )
config.dag.file = cmdRun.withDag
}
if( cmdRun.withNotification ) {
if( !(config.notification instanceof Map) )
config.notification = [:]
if( cmdRun.withNotification in ['true','false']) {
config.notification.enabled = cmdRun.withNotification == 'true'
}
else {
config.notification.enabled = true
config.notification.to = cmdRun.withNotification
}
}
// -- sets the messages options
if( cmdRun.withWebLog ) {
log.warn "The command line option '-with-weblog' is deprecated - consider enabling this feature by setting 'weblog.enabled=true' in your configuration file"
if( !(config.weblog instanceof Map) )
config.weblog = [:]
config.weblog.enabled = true
if( cmdRun.withWebLog != '-' )
config.weblog.url = cmdRun.withWebLog
else if( !config.weblog.url )
config.weblog.url = 'http://localhost'
}
// -- sets tower options
if( cmdRun.withTower ) {
if( !(config.tower instanceof Map) )
config.tower = [:]
config.tower.enabled = true
if( cmdRun.withTower != '-' )
config.tower.endpoint = cmdRun.withTower
}
// -- set wave options
if( cmdRun.withWave ) {
if( !(config.wave instanceof Map) )
config.wave = [:]
config.wave.enabled = true
if( cmdRun.withWave != '-' )
config.wave.endpoint = cmdRun.withWave
}
// -- set fusion options
if( cmdRun.withFusion ) {
if( !(config.fusion instanceof Map) )
config.fusion = [:]
config.fusion.enabled = cmdRun.withFusion == 'true'
}
// -- set cloudcache options
final envCloudPath = env.get('NXF_CLOUDCACHE_PATH')
if( cmdRun.cloudCachePath || envCloudPath ) {
if( !(config.cloudcache instanceof Map) )
config.cloudcache = [:]
if( !config.cloudcache.isSet('enabled') )
config.cloudcache.enabled = true
if( cmdRun.cloudCachePath && cmdRun.cloudCachePath != '-' )
config.cloudcache.path = cmdRun.cloudCachePath
else if( !config.cloudcache.isSet('path') && envCloudPath )
config.cloudcache.path = envCloudPath
}
// -- add the command line parameters to the 'taskConfig' object
if( cliParams )
config.params = mergeMaps( (Map)config.params, cliParams, NF.strictMode )
if( cmdRun.withoutDocker && config.docker instanceof Map ) {
// disable docker execution
log.debug "Disabling execution in Docker container as requested by command-line option `-without-docker`"
config.docker.enabled = false
}
if( cmdRun.withDocker ) {
configContainer(config, 'docker', cmdRun.withDocker)
}
if( cmdRun.withPodman ) {
configContainer(config, 'podman', cmdRun.withPodman)
}
if( cmdRun.withSingularity ) {
configContainer(config, 'singularity', cmdRun.withSingularity)
}
if( cmdRun.withApptainer ) {
configContainer(config, 'apptainer', cmdRun.withApptainer)
}
if( cmdRun.withCharliecloud ) {
configContainer(config, 'charliecloud', cmdRun.withCharliecloud)
}
}
private void configContainer(ConfigObject config, String engine, def cli) {
log.debug "Enabling execution in ${engine.capitalize()} container as requested by command-line option `-with-$engine ${cmdRun.withDocker}`"
if( !config.containsKey(engine) )
config.put(engine, [:])
if( !(config.get(engine) instanceof Map) )
throw new AbortOperationException("Invalid `$engine` definition in the config file")
def containerConfig = (Map)config.get(engine)
containerConfig.enabled = true
if( cli != '-' ) {
// this is supposed to be a docker image name
config.process.container = cli
}
else if( containerConfig.image ) {
config.process.container = containerConfig.image
}
}
ConfigObject buildConfigObject() {
// -- configuration file(s)
def configFiles = validateConfigFiles(options?.config)
def config = buildGivenFiles(configFiles)
if( cmdRun )
configRunOptions(config, SysEnv.get(), cmdRun)
return config
}
/**
* @return A the application options hold in a {@code ConfigObject} instance
*/
ConfigMap build() {
toConfigMap(buildConfigObject())
}
protected static ConfigMap toConfigMap(ConfigObject config) {
assert config != null
(ConfigMap)normalize0((Map)config)
}
static private normalize0( config ) {
if( config instanceof Map ) {
ConfigMap result = new ConfigMap(config.size())
for( String name : config.keySet() ) {
def value = (config as Map).get(name)
result.put(name, normalize0(value))
}
return result
}
else if( config instanceof Collection ) {
List result = new ArrayList(config.size())
for( entry in config ) {
result << normalize0(entry)
}
return result
}
else {
return config
}
}
/**
* Merge two maps recursively avoiding keys to be overwritten
*
* @param config
* @param params
* @return a map resulting of merging result and right maps
*/
protected Map mergeMaps(Map config, Map params, boolean strict, List keys=[]) {
if( config==null )
config = new LinkedHashMap()
for( Map.Entry entry : params ) {
final key = entry.key.toString()
final value = entry.value
final previous = getConfigVal0(config, key)
keys << entry.key
if( previous==null ) {
config[key] = value
}
else if( previous instanceof Map && value instanceof Map ) {
mergeMaps(previous, value, strict, keys)
}
else {
if( previous instanceof Map || value instanceof Map ) {
final msg = "Configuration setting type with key '${keys.join('.')}' does not match the parameter with the same key - Config value=$previous; parameter value=$value"
if(strict)
throw new AbortOperationException(msg)
log.warn(msg)
}
config[key] = value
}
}
return config
}
private Object getConfigVal0(Map config, String key) {
if( config instanceof ConfigObject ) {
return config.isSet(key) ? config.get(key) : null
}
else {
return config.get(key)
}
}
static String resolveConfig(Path baseDir, CmdRun cmdRun, Map cliParams) {
final config = new ConfigBuilder()
.setShowClosures(true)
.setStripSecrets(true)
.setOptions(cmdRun.launcher.options)
.setCmdRun(cmdRun)
.setCliParams(cliParams)
.setBaseDir(baseDir)
.buildConfigObject()
// strip secrets
SecretHelper.hideSecrets(config)
// compute config
final result = toCanonicalString(config, false)
// dump config for debugging
log.trace "Resolved config:\n${result.indent('\t')}"
return result
}
}

View File

@@ -0,0 +1,38 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.config
import groovy.transform.CompileStatic
import groovy.transform.EqualsAndHashCode
/**
* Placeholder class that replacing closure definitions in the nextflow configuration
* file in order to print the closure content itself
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@EqualsAndHashCode
@CompileStatic
class ConfigClosurePlaceholder {
private String str
ConfigClosurePlaceholder(String str) {
this.str = str
}
@Override String toString() { str }
}

View File

@@ -0,0 +1,45 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.config
import java.lang.annotation.ElementType
import java.lang.annotation.Retention
import java.lang.annotation.RetentionPolicy
import java.lang.annotation.Target
/**
* Annotation used to marked fields of {@link CascadingConfig} subclasses
*
* @see CascadingConfig
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.METHOD)
@interface ConfigField {
/**
* @return The field name (default: empty string)
*/
String value() default ''
/**
* Mark the configuration field as private
*
* @return {@code true} when the field is private (default: false)
*/
boolean _private() default false
}

View File

@@ -0,0 +1,79 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.config
import groovy.transform.CompileStatic
import nextflow.config.spec.ConfigOption
import nextflow.config.spec.ConfigScope
import nextflow.config.spec.ScopeName
import nextflow.script.dsl.Description
/**
* Represent Nextflow config as Map
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@ScopeName('')
@CompileStatic
class ConfigMap extends LinkedHashMap implements ConfigScope {
@ConfigOption
@Description("""
The remote work directory used by hybrid workflows. Equivalent to the `-bucket-dir` option of the `run` command.
""")
final String bucketDir
@ConfigOption
@Description("""
Delete all files associated with a run in the work directory when the run completes successfully (default: `false`).
""")
final boolean cleanup
@ConfigOption
@Description("""
The pipeline output directory. Equivalent to the `-output-dir` option of the `run` command.
""")
final String outputDir
@ConfigOption
@Description("""
Enable the use of previously cached task executions. Equivalent to the `-resume` option of the `run` command.
""")
final boolean resume
@ConfigOption
@Description("""
The pipeline work directory. Equivalent to the `-work-dir` option of the `run` command.
""")
final String workDir
ConfigMap() {
}
ConfigMap(int initialCapacity) {
super(initialCapacity)
}
ConfigMap(Map opts) {
super(opts)
bucketDir = opts.bucketDir
cleanup = opts.cleanup as boolean
outputDir = opts.outputDir
resume = opts.resume as boolean
workDir = opts.workDir
}
}

View File

@@ -0,0 +1,101 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.config
import java.nio.file.Path
/**
* Interface for Nextflow config parsers.
*
* @author Ben Sherman <bentshermann@gmail.com>
*/
interface ConfigParser {
/**
* Toggle whether config include statements should be ignored.
*
* @param value
*/
ConfigParser setIgnoreIncludes(boolean value)
/**
* Toggle whether to strip secrets when rendering the config.
*
* @param value
*/
ConfigParser setStripSecrets(boolean value)
/**
* Toggle whether to render the source code of closures.
*
* @param value
*/
ConfigParser setRenderClosureAsString(boolean value)
/**
* Toggle whether to raise an error if a missing property is accessed.
*
* @param value
*/
ConfigParser setStrict(boolean value)
/**
* Define variables which will be made available to the config script.
*
* @param vars
*/
ConfigParser setBinding(Map vars)
/**
* Define pipeline parameters which will be made available to the config script.
*
* @param vars
*/
ConfigParser setParams(Map vars)
/**
* Set the profiles that should be applied.
*/
ConfigParser setProfiles(List<String> profiles)
/**
* Toggle whether to render compilation errors with ANSI colors.
*
* @param value
*/
default ConfigParser setAnsiLog(boolean value) {
return this
}
/**
* Parse a config object from the given source.
*/
ConfigObject parse(String text)
ConfigObject parse(File file)
ConfigObject parse(Path path)
/**
* Get the set of declared profiles.
*/
Set<String> getDeclaredProfiles()
/**
* Get the map of declared params.
*/
Map<String,Object> getDeclaredParams()
}

View File

@@ -0,0 +1,46 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.config
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.NF
import nextflow.config.parser.v1.ConfigParserV1
import nextflow.config.parser.v2.ConfigParserV2
/**
* Factory for creating an instance of {@link ConfigParser}.
*
* @author Ben Sherman <bentshermann@gmail.com>
*/
@Slf4j
@CompileStatic
class ConfigParserFactory {
static ConfigParser create() {
final parser = NF.getSyntaxParserVersion()
if( parser == 'v1' ) {
return new ConfigParserV1()
}
if( parser == 'v2' ) {
log.debug "Using config parser v2"
return new ConfigParserV2()
}
throw new IllegalStateException("Invalid NXF_SYNTAX_PARSER setting -- should be either 'v1' or 'v2'")
}
}

View File

@@ -0,0 +1,210 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.config
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.config.spec.ConfigScope
import nextflow.config.spec.SpecNode
import nextflow.config.spec.ScopeName
import nextflow.plugin.Plugins
import nextflow.script.dsl.Description
/**
* Validate the Nextflow configuration
*
* @author Ben Sherman <bentshermann@gmail.com>
*/
@Slf4j
@CompileStatic
class ConfigValidator {
/**
* Hidden options added by ConfigBuilder
*/
private static final List<String> HIDDEN_OPTIONS = List.of(
'cacheable',
'dumpChannels',
'dumpHashes',
'libDir',
'outputFormat',
'poolSize',
'preview',
'runName',
'stubRun',
);
/**
* Core plugin scopes which can only be validated when
* the plugin is loaded.
*/
private static final List<String> CORE_PLUGIN_SCOPES = List.of(
'aws',
'azure',
'cloudcache',
'google',
'k8s',
'tower',
'wave'
)
/**
* Additional config scopes added by third-party plugins
*/
private SpecNode.Scope pluginScopes
ConfigValidator() {
loadPluginScopes()
}
private void loadPluginScopes() {
final children = new HashMap<String, SpecNode>()
for( final scope : Plugins.getExtensions(ConfigScope) ) {
final clazz = scope.getClass()
final name = clazz.getAnnotation(ScopeName)?.value()
final description = clazz.getAnnotation(Description)?.value()
if( name == '' ) {
children.putAll(SpecNode.Scope.of(clazz, '').children())
continue
}
if( !name )
continue
if( name in children ) {
log.warn "Plugin config scope `${clazz.name}` conflicts with existing scope: `${name}`"
continue
}
children.put(name, SpecNode.Scope.of(clazz, description))
}
pluginScopes = new SpecNode.Scope('', children)
}
/**
* Validate a config block within the given scope.
*
* @param config
* @param scopes
*/
void validate(Map<String,?> config, List<String> scopes=[]) {
for( final entry : config.entrySet() ) {
final key = entry.key
final value = entry.value
final names = scopes + [key]
if( names.size() == 2 && names.first() == 'profiles' )
names.clear()
if( value instanceof Map ) {
if( isSelector(key) )
names.removeLast()
if( isMapOption(names) )
continue
validate(value, names)
}
else {
validateOption(names)
}
}
}
/**
* Determine whether a scope name is a process selector.
*
* @param name
*/
private boolean isSelector(String name) {
return name.startsWith('withLabel:') || name.startsWith('withName:')
}
/**
* Validate a config option given by the list of names.
*
* For example, the option 'process.resourceLimits' is represented
* as ['process', 'resourceLimits'].
*
* @param names
*/
void validateOption(List<String> names) {
final scope = names.first()
if( scope == 'env' ) {
checkEnv(names.last())
return
}
if( scope == 'params' )
return
if( isMissingCorePluginScope(scope) )
return
if( isValid(names) )
return
log.warn1 "Unrecognized config option '${names.join('.')}'"
}
/**
* Determine whether a config option is defined in the spec.
*
* @param names
*/
boolean isValid(List<String> names) {
if( names.size() == 1 && names.first() in HIDDEN_OPTIONS )
return true
final child = SpecNode.ROOT.getChild(names)
if( child instanceof SpecNode.Option || child instanceof SpecNode.DslOption )
return true
if( pluginScopes.getOption(names) )
return true
return false
}
/**
* Determine whether a config scope is from a core plugin
* which is not currently loaded.
*
* @param name
*/
private boolean isMissingCorePluginScope(String name) {
return name in CORE_PLUGIN_SCOPES
&& !pluginScopes.children().containsKey(name)
}
/**
* Determine whether a config option is a map option.
*
* This method is needed to distinguish between config scopes
* and config options that happen to be maps, since this distinction
* is lost when the config is resolved.
*
* @param names
*/
private boolean isMapOption(List<String> names) {
return isMapOption0(SpecNode.ROOT, names)
|| isMapOption0(pluginScopes, names)
}
private static boolean isMapOption0(SpecNode.Scope scope, List<String> names) {
final node = scope.getOption(names)
return node != null && node.types().contains(Map.class)
}
/**
* Warn about setting `NXF_*` environment variables in the config.
*
* @param name
*/
private void checkEnv(String name) {
if( name.startsWith('NXF_') && name!='NXF_DEBUG' )
log.warn "Nextflow environment variables must be defined in the launch environment -- the following environment variable in the config will be ignored: '$name'"
}
}

View File

@@ -0,0 +1,251 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.config
import java.util.stream.Collectors
import groovy.transform.CompileStatic
import groovy.transform.EqualsAndHashCode
import nextflow.config.spec.ConfigOption
import nextflow.config.spec.ConfigScope
import nextflow.config.spec.ScopeName
import nextflow.exception.AbortOperationException
import nextflow.script.dsl.Description
import static nextflow.Const.DEFAULT_MAIN_FILE_NAME
/**
* Models the nextflow config manifest settings
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@ScopeName("manifest")
@Description("""
The `manifest` scope allows you to define some metadata that is useful when publishing or running your pipeline.
""")
@CompileStatic
class Manifest implements ConfigScope {
@Deprecated
@ConfigOption
@Description("""
Project author name (use a comma to separate multiple names).
""")
final String author
@ConfigOption
@Description("""
List of project contributors. Should be a list of maps.
""")
final List<Contributor> contributors
@Deprecated
@ConfigOption
@Description("""
Git repository default branch (default: `master`).
""")
final String defaultBranch
@ConfigOption
@Description("""
Free text describing the workflow project.
""")
final String description
@ConfigOption
@Description("""
Project documentation URL.
""")
final String docsUrl
@ConfigOption
@Description("""
Project related publication DOI identifier.
""")
final String doi
@ConfigOption
@Description("""
Controls whether git sub-modules should be cloned with the main repository.
Can be either a boolean value, a list of submodule names, or a comma-separated string of submodule names.
""")
final Object gitmodules
@ConfigOption
@Description("""
Project home page URL.
""")
final String homePage
@ConfigOption
@Description("""
Project related icon location (Relative path or URL).
""")
final String icon
@ConfigOption
@Description("""
Project license.
""")
final String license
@ConfigOption
@Description("""
Project main script (default: `main.nf`).
""")
final String mainScript
@ConfigOption
@Description("""
Project short name.
""")
final String name
@ConfigOption
@Description("""
Minimum required Nextflow version.
""")
final String nextflowVersion
@ConfigOption
@Description("""
Project organization.
""")
final String organization
@ConfigOption
@Description("""
Pull submodules recursively from the Git repository.
""")
final boolean recurseSubmodules
@ConfigOption
@Description("""
Project version number.
""")
final String version
/* required by extension point -- do not remove */
Manifest() {}
Manifest(Map opts) {
author = opts.author as String
contributors = parseContributors(opts.contributors)
defaultBranch = opts.defaultBranch as String
description = opts.description as String
docsUrl = opts.docsUrl as String
doi = opts.doi as String
gitmodules = opts.gitmodules
homePage = opts.homePage as String
icon = opts.icon as String
license = opts.license as String
mainScript = opts.mainScript as String ?: DEFAULT_MAIN_FILE_NAME
name = opts.name as String
nextflowVersion = opts.nextflowVersion as String
organization = opts.organization as String
recurseSubmodules = opts.recurseSubmodules as boolean
version = opts.version as String
}
private List<Contributor> parseContributors(Object value) {
if( !value )
return Collections.emptyList()
try {
final contributors = value as List<Map>
return contributors.stream()
.map(opts -> new Contributor(opts))
.toList()
}
catch( IllegalArgumentException e ){
throw new AbortOperationException(e.message)
}
catch( ClassCastException e ){
throw new AbortOperationException("Invalid setting for `manifest.contributors` config option -- should be a list of maps")
}
}
Map toMap() {
return [
author: author,
contributors: contributors.stream().map(c -> c.toMap()).toList(),
defaultBranch: defaultBranch,
description: description,
homePage: homePage,
gitmodules: gitmodules,
mainScript: mainScript,
version: version,
nextflowVersion: nextflowVersion,
doi: doi,
docsUrl: docsUrl,
icon: icon,
organization: organization,
license: license,
]
}
@EqualsAndHashCode
static class Contributor {
final String name
final String affiliation
final String email
final String github
final List<ContributionType> contribution
final String orcid
Contributor(Map opts) {
name = opts.name as String
affiliation = opts.affiliation as String
email = opts.email as String
github = opts.github as String
contribution = parseContributionTypes(opts.contribution as List<String>)
orcid = opts.orcid as String
}
private List<ContributionType> parseContributionTypes(List<String> values) {
if( values == null )
return []
final result = new LinkedList<ContributionType>()
for( final value : values ) {
try {
result.add(ContributionType.valueOf(value.toUpperCase()))
}
catch( IllegalArgumentException e ) {
throw new IllegalArgumentException("Invalid contribution type '$value' in `manifest.contributors` config option")
}
}
return result.toSorted()
}
Map toMap() {
return [
name: name,
affiliation: affiliation,
email: email,
github: github,
contribution: contribution.stream().map(c -> c.toString().toLowerCase()).toList(),
orcid: orcid,
]
}
}
static enum ContributionType {
AUTHOR,
MAINTAINER,
CONTRIBUTOR
}
}

View File

@@ -0,0 +1,54 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.config
import java.lang.annotation.ElementType
import java.lang.annotation.Retention
import java.lang.annotation.RetentionPolicy
import java.lang.annotation.Target
import groovy.transform.CompileStatic
import nextflow.config.control.StripSecretsVisitor
import org.codehaus.groovy.ast.ASTNode
import org.codehaus.groovy.ast.ClassNode
import org.codehaus.groovy.control.CompilePhase
import org.codehaus.groovy.control.SourceUnit
import org.codehaus.groovy.transform.ASTTransformation
import org.codehaus.groovy.transform.GroovyASTTransformation
import org.codehaus.groovy.transform.GroovyASTTransformationClass
/**
* AST transformation that replaces properties prefixed with `secrets.`
* with a static string
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Retention(RetentionPolicy.SOURCE)
@Target(ElementType.METHOD)
@GroovyASTTransformationClass(classes = [StripSecretsXformImpl])
@interface StripSecretsXform {
@CompileStatic
@GroovyASTTransformation(phase = CompilePhase.CONVERSION)
class StripSecretsXformImpl implements ASTTransformation {
@Override
void visit(ASTNode[] nodes, SourceUnit source) {
new StripSecretsVisitor(source).visitClass((ClassNode)nodes[1])
}
}
}

View File

@@ -0,0 +1,141 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.config
import groovy.transform.CompileStatic
import nextflow.config.spec.ConfigOption
import nextflow.config.spec.ConfigScope
import nextflow.config.spec.ScopeName
import nextflow.script.dsl.Description
@ScopeName("workflow")
@Description("""
The `workflow` scope provides workflow execution options.
""")
@CompileStatic
class WorkflowConfig implements ConfigScope {
@ConfigOption
@Description("""
When `true`, the pipeline will exit with a non-zero exit code if any failed tasks are ignored using the `ignore` error strategy (default: `false`).
""")
final boolean failOnIgnore
@ConfigOption
@Description("""
Specify a closure that will be invoked at the end of a workflow run (including failed runs).
""")
final Closure onComplete
@ConfigOption
@Description("""
Specify a closure that will be invoked if a workflow run is terminated.
""")
final Closure onError
@Description("""
The `workflow.output` scope provides options for publishing workflow outputs.
[Read more](https://nextflow.io/docs/latest/reference/config.html#workflow)
""")
final WorkflowOutputConfig output
/* required by extension point -- do not remove */
WorkflowConfig() {}
WorkflowConfig(Map opts) {
failOnIgnore = opts.failOnIgnore as boolean
onComplete = opts.onComplete as Closure
onError = opts.onError as Closure
output = new WorkflowOutputConfig(opts.output as Map ?: Collections.emptyMap())
}
}
@CompileStatic
class WorkflowOutputConfig implements ConfigScope {
@ConfigOption
@Description("""
*Currently only supported for S3.*
Specify the media type, also known as [MIME type](https://developer.mozilla.org/en-US/docs/Web/HTTP/MIME_types), of published files (default: `false`). Can be a string (e.g. `'text/html'`), or `true` to infer the content type from the file extension.
""")
final Object contentType
@ConfigOption
@Description("""
*Currently only supported for local and shared filesystems.*
Copy file attributes (such as the last modified timestamp) to the published file (default: `false`).
""")
final boolean copyAttributes
@ConfigOption
@Description("""
Enable or disable publishing (default: `true`).
""")
final boolean enabled
@ConfigOption
@Description("""
When `true`, the workflow will not fail if a file can't be published for some reason (default: `false`).
""")
final boolean ignoreErrors
@ConfigOption
@Description("""
The file publishing method (default: `'symlink'`).
""")
final String mode
@ConfigOption
@Description("""
When `true` any existing file in the specified folder will be overwritten (default: `'standard'`).
""")
final Object overwrite
@ConfigOption
@Description("""
*Currently only supported for S3.*
Specify the storage class for published files.
""")
final String storageClass
@ConfigOption
@Description("""
*Currently only supported for S3.*
Specify arbitrary tags for published files.
""")
final Map tags
/* required by extension point -- do not remove */
WorkflowOutputConfig() {}
WorkflowOutputConfig(Map opts) {
contentType = opts.contentType
copyAttributes = opts.copyAttributes as boolean
enabled = opts.enabled as boolean
ignoreErrors = opts.ignoreErrors as boolean
mode = opts.mode
overwrite = opts.overwrite
storageClass = opts.storageClass
tags = opts.tags as Map
}
}

View File

@@ -0,0 +1,174 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.config.parser.v1
import java.nio.file.NoSuchFileException
import java.nio.file.Path
import ch.artecat.grengine.Grengine
import groovy.transform.Memoized
import nextflow.SysEnv
import nextflow.config.StripSecretsXform
import nextflow.exception.IllegalConfigException
import nextflow.file.FileHelper
import nextflow.util.Duration
import nextflow.util.MemoryUnit
import org.codehaus.groovy.control.CompilerConfiguration
import org.codehaus.groovy.control.customizers.ASTTransformationCustomizer
import org.codehaus.groovy.control.customizers.ImportCustomizer
import org.slf4j.Logger
import org.slf4j.LoggerFactory
/**
* Extends a {@link Script} class adding a method that allows a configuration
* file to include other configuration files.
* <p>
* This class is used to base class when parsing the groovy config object
* <p>
*
* Based on
* http://naleid.com/blog/2009/07/30/modularizing-groovy-config-files-with-a-dash-of-meta-programming
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
abstract class ConfigBase extends Script {
private static final Logger log = LoggerFactory.getLogger(ConfigBase)
private Stack<Path> configStack
private boolean ignoreIncludes
private boolean renderClosureAsString
private boolean stripSecrets
protected void setStripSecrets( boolean value ) {
this.stripSecrets = value
}
protected void setIgnoreIncludes( boolean value ) {
this.ignoreIncludes = value
}
protected void setRenderClosureAsString( boolean value ) {
this.renderClosureAsString = value
}
protected void setConfigPath(Path path) {
if( configStack == null )
configStack = new Stack<>()
configStack.push(path)
}
protected void setConfigStack(Stack<Path> stack) {
this.configStack = stack
}
/**
* Get the value of an environment variable from the launch environment.
*
* @param name
* The environment variable name to be referenced
* @return
* The value associate with the specified variable name or {@code null} if the variable does not exist.
*/
String env(String name) {
return SysEnv.get(name)
}
/**
* Implements the config file include
*/
def includeConfig( includeFile ) {
if( !includeFile )
throw new IllegalConfigException("includeConfig argument cannot be empty")
if( ignoreIncludes )
return
if( configStack == null )
configStack = new Stack<>()
Path owner = configStack ? this.configStack.peek() : null
Path includePath = FileHelper.asPath(includeFile.toString())
log.trace "Include config file: $includeFile [parent: $owner]"
if( !includePath.isAbsolute() && owner ) {
includePath = owner.resolveSibling(includeFile.toString())
}
def configText = readConfigFile(includePath)
// -- set the required base script
def config = new CompilerConfiguration()
config.scriptBaseClass = ConfigBase.class.name
if( stripSecrets )
config.addCompilationCustomizers(new ASTTransformationCustomizer(StripSecretsXform))
def params = [:]
if( renderClosureAsString )
params.put('renderClosureAsString', true)
config.addCompilationCustomizers(new ASTTransformationCustomizer(params, ConfigTransform))
// -- add implicit types
def importCustomizer = new ImportCustomizer()
importCustomizer.addImports( Duration.name )
importCustomizer.addImports( MemoryUnit.name )
config.addCompilationCustomizers(importCustomizer)
// -- setup the grengine instance
def engine = new Grengine(this.class.classLoader,config)
def clazz = engine.load(configText)
// -- push this file on the stack
this.configStack.push(includePath)
/*
* here it is the magic code
*/
def script = (ConfigBase)clazz.newInstance()
script.setConfigStack(this.configStack)
script.setBinding(this.getBinding())
script.metaClass.getProperty = { String name -> this.metaClass.getProperty(this, name) }
script.metaClass.invokeMethod = { String name, args -> this.metaClass.invokeMethod(this, name, args)}
script.&run.call()
// remove the path from the stack
this.configStack.pop()
}
/**
* Reads the content of a config file. The result is cached to
* avoid multiple reads.
*
* @param includePath The config file path
* @return The file content
*/
@Memoized
protected static String readConfigFile(Path includePath) {
def configText
try {
configText = includePath.getText()
}
catch (NoSuchFileException | FileNotFoundException ignored) {
throw new NoSuchFileException("Config file does not exist: ${includePath.toUriString()}")
}
catch (IOException e) {
throw new IOException("Cannot read config file include: ${includePath.toUriString()}", e)
}
return configText
}
}

View File

@@ -0,0 +1,491 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.config.parser.v1
import java.nio.file.Path
import ch.artecat.grengine.Grengine
import com.google.common.hash.Hashing
import groovy.transform.PackageScope
import nextflow.ast.NextflowXform
import nextflow.config.ConfigParser
import nextflow.config.StripSecretsXform
import nextflow.exception.ConfigParseException
import nextflow.extension.Bolts
import nextflow.file.FileHelper
import nextflow.util.Duration
import nextflow.util.MemoryUnit
import org.codehaus.groovy.control.CompilerConfiguration
import org.codehaus.groovy.control.customizers.ASTTransformationCustomizer
import org.codehaus.groovy.control.customizers.ImportCustomizer
import org.codehaus.groovy.runtime.InvokerHelper
/**
* A ConfigSlurper that allows to include a config file into another. For example:
*
* <pre>
* process {
* foo = 1
* that = 2
*
* includeConfig( 'path/to/another/config/file' )
*
* }
*
* </pre>
*
* See http://naleid.com/blog/2009/07/30/modularizing-groovy-config-files-with-a-dash-of-meta-programming
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*
*/
/**
* <p>
* ConfigSlurper is a utility class for reading configuration files defined in the form of Groovy
* scripts. Configuration settings can be defined using dot notation or scoped using closures
*
* <pre><code>
* grails.webflow.stateless = true
* smtp {
* mail.host = 'smtp.myisp.com'
* mail.auth.user = 'server'
* }
* resources.URL = "http://localhost:80/resources"
* </pre></code>
*
* <p>Settings can either be bound into nested maps or onto a specified JavaBean instance. In the case
* of the latter an error will be thrown if a property cannot be bound.
*
* @author Graeme Rocher
* @author Andres Almiray
* @since 1.5
*/
class ConfigParserV1 implements ConfigParser {
private Map bindingVars = [:]
private Map paramVars = [:]
private final Map<String, List<String>> conditionValues = [:]
private final Stack<Map<String, ConfigObject>> conditionalBlocks = new Stack<Map<String,ConfigObject>>()
private final Set<String> conditionalNames = new HashSet<>()
private final Set<String> profileNames = new HashSet<>()
private boolean ignoreIncludes
private boolean renderClosureAsString
private boolean stripSecrets
private Grengine grengine
@Override
ConfigParser setProfiles(List<String> profiles) {
final blockName = 'profiles'
if (!profiles) {
conditionValues.remove(blockName)
}
else {
conditionValues[blockName] = profiles
}
return this
}
@Override
Set<String> getDeclaredProfiles() {
Collections.unmodifiableSet(conditionalNames)
}
@Override
Map<String,Object> getDeclaredParams() {
[:]
}
private Grengine getGrengine() {
if( grengine ) {
return grengine
}
// set the required base script
def config = new CompilerConfiguration()
config.scriptBaseClass = ConfigBase.class.name
if( stripSecrets )
config.addCompilationCustomizers(new ASTTransformationCustomizer(StripSecretsXform))
def params = [:]
if( renderClosureAsString )
params.put('renderClosureAsString', true)
config.addCompilationCustomizers(new ASTTransformationCustomizer(params, ConfigTransform))
config.addCompilationCustomizers(new ASTTransformationCustomizer(NextflowXform))
// add implicit types
def importCustomizer = new ImportCustomizer()
importCustomizer.addImports( Duration.name )
importCustomizer.addImports( MemoryUnit.name )
config.addCompilationCustomizers(importCustomizer)
grengine = new Grengine(config)
}
@Override
ConfigParser setRenderClosureAsString(boolean value) {
this.renderClosureAsString = value
return this
}
@Override
ConfigParser setStrict(boolean value) {
// not supported
return this
}
@Override
ConfigParser setStripSecrets(boolean value) {
this.stripSecrets = value
return this
}
/**
* Sets any additional variables that should be placed into the binding when evaluating Config scripts
*/
@Override
ConfigParser setBinding(Map vars) {
this.bindingVars = vars
return this
}
@Override
ConfigParser setParams(Map vars) {
// deep clone the map to prevent side-effect
// see https://github.com/nextflow-io/nextflow/issues/1923
this.paramVars = Bolts.deepClone(vars)
return this
}
/**
* Creates a unique name for the config class in order to avoid collision
* with top level configuration scopes
*
* @param text
* @return
*/
private String createUniqueName(String text) {
def hash = Hashing
.murmur3_32()
.newHasher()
.putUnencodedChars(text)
.hash()
return "_nf_config_$hash"
}
private Script loadScript(String text) {
(Script)getGrengine().load(text, createUniqueName(text)).newInstance()
}
/**
* Parses a ConfigObject instances from an instance of java.util.Properties
* @param The java.util.Properties instance
*/
ConfigObject parse(Properties properties) {
ConfigObject config = new ConfigObject()
for (key in properties.keySet()) {
def tokens = key.split(/\./)
def current = config
def last
def lastToken
def foundBase = false
for (token in tokens) {
if (foundBase) {
// handle not properly nested tokens by ignoring
// hierarchy below this point
lastToken += "." + token
current = last
} else {
last = current
lastToken = token
current = current."${token}"
if (!(current instanceof ConfigObject)) foundBase = true
}
}
if (current instanceof ConfigObject) {
if (last[lastToken]) {
def flattened = last.flatten()
last.clear()
flattened.each { k2, v2 -> last[k2] = v2 }
last[lastToken] = properties.get(key)
}
else {
last[lastToken] = properties.get(key)
}
}
current = config
}
return config
}
/**
* Parse the given script as a string and return the configuration object
*
* @see ConfigParser#parse(groovy.lang.Script)
*/
@Override
ConfigObject parse(String text) {
return parse(loadScript(text))
}
/**
* Parse the given script into a configuration object (a Map)
* (This method creates a new class to parse the script each time it is called.)
* @param script The script to parse
* @return A Map of maps that can be navigating with dot de-referencing syntax to obtain configuration entries
*/
@Deprecated
ConfigObject parse(Script script) {
return parse(script, null)
}
/**
* Parses a Script represented by the given URL into a ConfigObject
*
* @param location The location of the script to parse
* @return The ConfigObject instance
*/
@Deprecated
ConfigObject parse(URL location) {
return parse(loadScript(location.text), FileHelper.asPath(location.toURI()))
}
@Override
ConfigObject parse(File file) {
return parse(file.toPath())
}
@Override
ConfigObject parse(Path path) {
return parse(loadScript(path.text), path)
}
/**
* Parses the passed groovy.lang.Script instance using the second argument to allow the ConfigObject
* to retain an reference to the original location other Groovy script
*
* @param script The groovy.lang.Script instance
* @param location The original location of the Script as a URL
* @return The ConfigObject instance
*/
ConfigObject parse(Script _script, Path location) {
final script = (ConfigBase)_script
Stack<String> currentConditionalBlock = new Stack<String>()
def config = location ? new ConfigObject(location.toUri().toURL()) : new ConfigObject()
GroovySystem.metaClassRegistry.removeMetaClass(script.class)
def mc = script.class.metaClass
def prefix = ""
LinkedList stack = new LinkedList()
LinkedList profileStack = new LinkedList()
stack << [config: config, scope: [:]]
boolean withinProfile = false
def pushStack = { co ->
stack << [config: co, scope: stack.last.scope.clone()]
}
def assignName = { name, co ->
def current = stack.last
current.config[name] = co
current.scope[name] = co
}
mc.getProperty = { String name ->
def current = stack.last
def result
if (current.config.get(name)) {
result = current.config.get(name)
} else if (current.scope.get(name)) {
result = current.scope[name]
} else {
try {
result = InvokerHelper.getProperty(this, name)
} catch (GroovyRuntimeException e) {
result = new ConfigObject()
assignName.call(name, result)
}
}
if( name=='params' && result instanceof Map && paramVars ) {
result.putAll(Bolts.deepMerge(result, paramVars))
}
return result
}
ConfigObject overrides = new ConfigObject()
mc.invokeMethod = { String name, args ->
def result
if (args.length == 1 && args[0] instanceof Closure) {
if( profileStack && profileStack.last == 'profiles' )
profileNames.add(name)
if (name in conditionValues.keySet()) {
try {
if( name == 'profiles' ){
withinProfile=true
}
currentConditionalBlock.push(name)
conditionalBlocks.push([:])
args[0].call()
} finally {
currentConditionalBlock.pop()
for (entry in conditionalBlocks.pop().entrySet()) {
def c = stack.last.config
(c != config? c : overrides).merge(entry.value)
}
if( name == 'profiles' ){
withinProfile=false
}
}
} else if (currentConditionalBlock.size() > 0) {
String conditionalBlockKey = currentConditionalBlock.peek()
conditionalNames.add(name)
if (name in conditionValues[conditionalBlockKey]) {
def co = conditionalBlocks.peek()[conditionalBlockKey]
if( co == null ) {
co = new ConfigObject()
conditionalBlocks.peek()[conditionalBlockKey] = co
}
pushStack.call(co)
try {
currentConditionalBlock.pop()
args[0].call()
} finally {
currentConditionalBlock.push(conditionalBlockKey)
}
stack.removeLast()
}
}
else if( name == 'plugins' ) {
if( stack.size()>1 )
throw new ConfigParseException("Plugins definition is only allowed in config top-most scope")
// Implements `plugins` mini-dsl for plugins definition
def dsl = new PluginsDsl()
def clo = args[0] as Closure
clo.delegate = dsl
clo.resolveStrategy = Closure.DELEGATE_ONLY
clo.call()
assignName.call(name, dsl.plugins)
}
else {
def current = name=='profiles' || withinProfile ? stack.first : stack.last
def co
if (current.config.containsKey(name) && current.config.get(name) instanceof ConfigObject) {
co = current.config.get(name)
}
else if (current.scope.containsKey(name) && current.scope.get(name) instanceof ConfigObject) {
co = current.scope.get(name).clone()
}
else {
co = new ConfigObject()
}
profileStack.add(name)
assignName.call(name, co)
pushStack.call(co)
args[0].call()
stack.removeLast()
profileStack.removeLast()
if (current.scope.containsKey(name) && current.scope.get(name) instanceof ConfigObject) {
if( current.scope.get(name) != co) {
current.scope.get(name).merge(co)
}
} else {
current.scope.put(name,co)
}
}
} else if (args.length == 2 && args[1] instanceof Closure) {
try {
prefix = name + '.'
assignName.call(name, args[0])
args[1].call()
} finally { prefix = "" }
} else {
MetaMethod mm = mc.getMetaMethod(name, args)
if (mm) {
result = mm.invoke(delegate, args)
} else {
throw new MissingMethodException(name, getClass(), args)
}
}
result
}
script.metaClass = mc
def setProperty = { String name, value ->
assignName.call(prefix + name, value)
}
def binding = new ConfigBinding(setProperty)
if (this.bindingVars) {
binding.getVariables().putAll(this.bindingVars)
}
// add the script file location into the binding
if( location ) {
script.setConfigPath(location)
}
// disable include parsing when required
script.setIgnoreIncludes(ignoreIncludes)
script.setRenderClosureAsString(renderClosureAsString)
script.setStripSecrets(stripSecrets)
// -- set the binding and run
script.binding = binding
script.run()
config.merge(overrides)
return config
}
/**
* Disable parsing of {@code includeConfig} directive
*
* @param value A boolean value, when {@code true} includes are disabled
* @return The {@link ConfigParser} object itself
*/
@Override
ConfigParser setIgnoreIncludes(boolean value) {
this.ignoreIncludes = value
return this
}
/**
* Since Groovy Script doesn't support overriding setProperty, we have to using a trick with the Binding to provide this
* functionality
*/
@PackageScope
static class ConfigBinding extends Binding {
Closure callable
ConfigBinding(Closure c) {
this.callable = c
}
void setVariable(String name, Object value) {
callable(name, value)
}
}
}

View File

@@ -0,0 +1,40 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.config.parser.v1
import java.lang.annotation.ElementType
import java.lang.annotation.Retention
import java.lang.annotation.RetentionPolicy
import java.lang.annotation.Target
import org.codehaus.groovy.transform.GroovyASTTransformationClass
/**
* Nextflow configuration file AST xform marker interface
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Retention(RetentionPolicy.SOURCE)
@Target(ElementType.METHOD)
@GroovyASTTransformationClass(classes = [ConfigTransformImpl])
@interface ConfigTransform {
/**
* hack to pass a parameter in the {@link ConfigTransformImpl} class -- do not remove
*
* See {@link ConfigTransformImpl#renderClosureAsString}
*/
boolean renderClosureAsString()
}

View File

@@ -0,0 +1,208 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.config.parser.v1
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.config.ConfigClosurePlaceholder
import org.codehaus.groovy.ast.ASTNode
import org.codehaus.groovy.ast.AnnotationNode
import org.codehaus.groovy.ast.ClassCodeVisitorSupport
import org.codehaus.groovy.ast.ClassNode
import org.codehaus.groovy.ast.expr.ArgumentListExpression
import org.codehaus.groovy.ast.expr.BinaryExpression
import org.codehaus.groovy.ast.expr.ClosureExpression
import org.codehaus.groovy.ast.expr.ConstantExpression
import org.codehaus.groovy.ast.expr.ConstructorCallExpression
import org.codehaus.groovy.ast.expr.Expression
import org.codehaus.groovy.ast.expr.MapEntryExpression
import org.codehaus.groovy.ast.expr.MapExpression
import org.codehaus.groovy.ast.expr.MethodCallExpression
import org.codehaus.groovy.ast.stmt.ExpressionStatement
import org.codehaus.groovy.control.CompilePhase
import org.codehaus.groovy.control.SourceUnit
import org.codehaus.groovy.transform.ASTTransformation
import org.codehaus.groovy.transform.GroovyASTTransformation
/**
* Implements Nextflow configuration file AST xform
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
@GroovyASTTransformation(phase = CompilePhase.CONVERSION)
class ConfigTransformImpl implements ASTTransformation {
private boolean renderClosureAsString
@Override
void visit(ASTNode[] astNodes, SourceUnit unit) {
final annot = (AnnotationNode)astNodes[0]
final clazz = (ClassNode)astNodes[1]
// the following line is mostly an hack to pass a parameter to this xform instance
this.renderClosureAsString = annot.getMember('renderClosureAsString') != null
createVisitor(unit).visitClass(clazz)
}
protected ClassCodeVisitorSupport createVisitor(SourceUnit unit) {
return (renderClosureAsString
? new RetainClosureSourceCodeVisitorSupport(unit: unit)
: new DefaultConfigCodeVisitor(unit: unit) )
}
/**
* Nextflow config file visitor support class. Apply default transformations to
* the config object to implements config syntax sugars
*/
@CompileStatic
static class DefaultConfigCodeVisitor extends ClassCodeVisitorSupport {
protected SourceUnit unit
@Override
protected SourceUnit getSourceUnit() { unit }
@Override
void visitExpressionStatement(ExpressionStatement stm) {
if( stm.expression instanceof MethodCallExpression && stm.getStatementLabel() == 'withLabel' ) {
replaceMethodName( stm.expression as MethodCallExpression, 'withLabel' )
}
else if( stm.expression instanceof MethodCallExpression && stm.getStatementLabel() == 'withName' ) {
replaceMethodName( stm.expression as MethodCallExpression, 'withName' )
}
super.visitExpressionStatement(stm)
}
/**
* Replace the name of the invoked method pre-pending with the specified string
*
* @param call A object representing a method call expression
* @param prefix A string to prepend to the name of the invoked method
*/
protected void replaceMethodName(MethodCallExpression call, String prefix) {
call.setMethod( new ConstantExpression(prefix + ":" + call.method.text) )
}
}
/**
* This visitor is only used to render the closure source code
* when is required to visualise the nextflow config content
*/
@CompileStatic
static class RetainClosureSourceCodeVisitorSupport extends DefaultConfigCodeVisitor {
/**
* Visit expression statements replacing a binary assignment such as:
*
* foo = { closure }
*
* with an equivalent assignment replacing the closure with placeholder
* class holding the closure source code
*
* @param stm
*
* @see ConfigClosurePlaceholder
*/
@Override
void visitExpressionStatement(ExpressionStatement stm) {
if( stm.expression instanceof BinaryExpression ) {
replaceClosureAssignment(stm, stm.expression as BinaryExpression)
}
super.visitExpressionStatement(stm)
}
protected void replaceClosureAssignment(ExpressionStatement stm, BinaryExpression expr ) {
if( expr.operation.text == '=' && expr.rightExpression instanceof ClosureExpression ) {
final value = closureToString(expr.rightExpression)
final replace = new BinaryExpression(expr.leftExpression, expr.getOperation(), value)
stm.setExpression(replace)
}
}
/**
* Visit Map expressions replacing closure values such as :
*
* [key: { closure }]
*
* with an equivalent Map in which the closure is replaced with a placeholder
* class holding the closure source code
*
* @param expr
*
* @see ConfigClosurePlaceholder
*/
@Override
void visitMapExpression(MapExpression expr) {
for( int i=0; i<expr.mapEntryExpressions.size(); i++ ) {
def entry = expr.mapEntryExpressions[i]
if( entry.valueExpression instanceof ClosureExpression ) {
expr.mapEntryExpressions[i] = replaceMapEntryAssignment(entry)
}
}
super.visitMapExpression(expr)
}
protected MapEntryExpression replaceMapEntryAssignment(MapEntryExpression expr) {
if( expr.valueExpression instanceof ClosureExpression ) {
final value = closureToString(expr.valueExpression)
new MapEntryExpression(expr.keyExpression, value)
}
else
return expr
}
protected Expression closureToString( Expression closure ) {
def buffer = new StringBuilder()
readSource(closure, buffer)
def str = new ConstantExpression(buffer.toString())
def type = new ClassNode(ConfigClosurePlaceholder)
def args = new ArgumentListExpression( [str] as List<Expression> )
return new ConstructorCallExpression(type,args)
}
/**
* Read the user provided script source string
*
* @param expr
* @param buffer
* @param unit
*/
protected void readSource(Expression expr, StringBuilder buffer) {
final colBegin = Math.max(expr.getColumnNumber()-1, 0)
final colEnd = Math.max(expr.getLastColumnNumber()-1, 0)
final lineFirst = expr.getLineNumber()
final lineLast = expr.getLastLineNumber()
for( int i=lineFirst; i<=lineLast; i++ ) {
def line = unit.source.getLine(i, null)
if( i==lineFirst ) {
def str = i==lineLast ? line.substring(colBegin,colEnd) : line.substring(colBegin)
buffer.append(str)
}
else {
def str = i==lineLast ? line.substring(0, colEnd) : line
buffer.append('\n')
buffer.append(str)
}
}
}
}
}

View File

@@ -0,0 +1,38 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.config.parser.v1
import groovy.transform.CompileStatic
/**
* Model a mini-dsl for plugins configuration
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class PluginsDsl {
private Set<String> plugins = []
Set<String> getPlugins() { plugins }
void id( String plg ) {
if( !plg )
throw new IllegalArgumentException("Plugin id cannot be empty or null")
plugins << plg
}
}

View File

@@ -0,0 +1,100 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.config.parser.v2;
import nextflow.config.ConfigClosurePlaceholder;
import org.codehaus.groovy.ast.ClassCodeVisitorSupport;
import org.codehaus.groovy.ast.ClassNode;
import org.codehaus.groovy.ast.expr.ArgumentListExpression;
import org.codehaus.groovy.ast.expr.ClosureExpression;
import org.codehaus.groovy.ast.expr.Expression;
import org.codehaus.groovy.ast.expr.MapExpression;
import org.codehaus.groovy.ast.expr.MethodCallExpression;
import org.codehaus.groovy.control.SourceUnit;
import static org.codehaus.groovy.ast.tools.GeneralUtils.*;
/**
* AST transformation to render closure source text
*
* @author Ben Sherman <bentshermann@gmail.com>
*/
class ClosureToStringVisitor extends ClassCodeVisitorSupport {
protected SourceUnit sourceUnit;
public ClosureToStringVisitor(SourceUnit sourceUnit) {
this.sourceUnit = sourceUnit;
}
@Override
protected SourceUnit getSourceUnit() {
return sourceUnit;
}
@Override
public void visitMethodCallExpression(MethodCallExpression node) {
var name = node.getMethodAsString();
if( !"assign".equals(name) ) {
super.visitMethodCallExpression(node);
return;
}
var arguments = (ArgumentListExpression) node.getArguments();
if( arguments.getExpressions().size() != 2 )
return;
var secondArg = arguments.getExpression(1);
if( secondArg instanceof MapExpression me ) {
for( var entry : me.getMapEntryExpressions() ) {
if( entry.getValueExpression() instanceof ClosureExpression ce )
entry.setValueExpression(wrapExprAsPlaceholder(ce));
}
}
else if( secondArg instanceof ClosureExpression ce ) {
node.setArguments(args(arguments.getExpression(0), wrapExprAsPlaceholder(ce)));
}
}
protected Expression wrapExprAsPlaceholder(Expression node) {
var type = new ClassNode(ConfigClosurePlaceholder.class);
var text = constX(getSourceText(node));
return ctorX(type, args(text));
}
protected String getSourceText(Expression node) {
var builder = new StringBuilder();
var colBegin = Math.max(node.getColumnNumber()-1, 0);
var colEnd = Math.max(node.getLastColumnNumber()-1, 0);
var lineFirst = node.getLineNumber();
var lineLast = node.getLastLineNumber();
for( int i = lineFirst; i <= lineLast; i++ ) {
var line = sourceUnit.getSource().getLine(i, null);
if( i == lineFirst ) {
var str = i == lineLast ? line.substring(colBegin,colEnd) : line.substring(colBegin);
builder.append(str);
}
else {
var str = i == lineLast ? line.substring(0, colEnd) : line;
builder.append('\n');
builder.append(str);
}
}
return builder.toString();
}
}

View File

@@ -0,0 +1,214 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.config.parser.v2;
import java.io.IOException;
import java.nio.file.Path;
import java.util.List;
import com.google.common.hash.Hashing;
import groovy.lang.Binding;
import groovy.lang.GroovyClassLoader;
import groovy.lang.Script;
import nextflow.config.control.ConfigResolveVisitor;
import nextflow.config.control.ConfigToGroovyVisitor;
import nextflow.config.control.ResolveIncludeVisitor;
import nextflow.config.control.StringReaderSourceWithURI;
import nextflow.config.control.StripSecretsVisitor;
import nextflow.config.parser.ConfigParserPluginFactory;
import nextflow.script.control.Compiler;
import nextflow.script.control.GStringToStringVisitor;
import nextflow.script.control.PathCompareVisitor;
import org.codehaus.groovy.ast.ASTNode;
import org.codehaus.groovy.ast.ClassHelper;
import org.codehaus.groovy.ast.ClassNode;
import org.codehaus.groovy.control.CompilationUnit;
import org.codehaus.groovy.control.CompilerConfiguration;
import org.codehaus.groovy.control.Phases;
import org.codehaus.groovy.control.SourceUnit;
import org.codehaus.groovy.control.customizers.ImportCustomizer;
import org.codehaus.groovy.control.io.StringReaderSource;
import org.codehaus.groovy.control.messages.SyntaxErrorMessage;
import org.codehaus.groovy.runtime.InvokerHelper;
/**
* Compile a Nextflow config file into a Groovy class.
*
* @see groovy.lang.GroovyShell::parse()
* @see groovy.lang.GroovyClassLoader::doParseClass()
*
* @author Ben Sherman <bentshermann@gmail.com>
*/
public class ConfigCompiler {
private static final List<String> DEFAULT_IMPORT_NAMES = List.of(
"nextflow.util.Duration",
"nextflow.util.MemoryUnit"
);
private static final String BASE_CLASS_NAME = "nextflow.config.parser.v2.ConfigDsl";
private final CompilerConfiguration config;
private final GroovyClassLoader loader;
private boolean renderClosureAsString;
private boolean stripSecrets;
private Compiler compiler;
private SourceUnit sourceUnit;
public ConfigCompiler(boolean renderClosureAsString, boolean stripSecrets) {
this.config = getConfig();
this.loader = new GroovyClassLoader(new GroovyClassLoader(), config);
this.renderClosureAsString = renderClosureAsString;
this.stripSecrets = stripSecrets;
}
private static CompilerConfiguration getConfig() {
var importCustomizer = new ImportCustomizer();
for ( var name : DEFAULT_IMPORT_NAMES )
importCustomizer.addImports(name);
var config = new CompilerConfiguration();
config.addCompilationCustomizers(importCustomizer);
config.setScriptBaseClass(BASE_CLASS_NAME);
config.setPluginFactory(new ConfigParserPluginFactory());
return config;
}
public Script compile(String text, Path path) throws IOException {
return compile0(text, path);
}
private Script compile0(String text, Path path) throws IOException {
// prepare compiler and source unit
var unit = new ConfigCompilationUnit(config, loader);
sourceUnit = unit.createSourceUnit(text, path);
var collector = new ConfigClassLoader(loader).createCollector(unit, sourceUnit);
compiler = new Compiler(unit);
compiler.addSource(sourceUnit);
// compile config file
unit.addSource(sourceUnit);
unit.setClassgenCallback(collector);
unit.compile(Phases.CLASS_GENERATION);
// return compiled script class
var clazz = (Class) collector.getLoadedClasses().stream().findFirst().orElse(null);
return InvokerHelper.createScript(clazz, new Binding());
}
public SourceUnit getSource() {
return sourceUnit;
}
public List<SyntaxErrorMessage> getErrors() {
if( sourceUnit == null )
return null;
return sourceUnit
.getErrorCollector()
.getErrors()
.stream()
.map(e -> e instanceof SyntaxErrorMessage sem ? sem : null)
.filter(sem -> sem != null)
.toList();
}
private static class ConfigClassLoader extends GroovyClassLoader {
ConfigClassLoader(GroovyClassLoader parent) {
super(parent);
}
@Override
protected ClassCollector createCollector(CompilationUnit unit, SourceUnit su) {
return super.createCollector(unit, su);
}
}
private class ConfigCompilationUnit extends CompilationUnit {
private static final List<ClassNode> DEFAULT_IMPORTS = defaultImports();
private static List<ClassNode> defaultImports() {
return DEFAULT_IMPORT_NAMES.stream()
.map(name -> ClassHelper.makeWithoutCaching(name))
.toList();
}
ConfigCompilationUnit(CompilerConfiguration configuration, GroovyClassLoader loader) {
super(configuration, null, loader);
super.addPhaseOperation(source -> analyze(source), Phases.CONVERSION);
}
private void analyze(SourceUnit source) {
// initialize script class
var cn = source.getAST().getClasses().get(0);
// perform strict syntax checking
if( source.getSource() instanceof StringReaderSourceWithURI ) {
var includeResolver = new ResolveIncludeVisitor(source);
includeResolver.visit();
for( var error : includeResolver.getErrors() )
source.getErrorCollector().addErrorAndContinue(error);
}
new ConfigResolveVisitor(source, this, DEFAULT_IMPORTS).visit();
if( source.getErrorCollector().hasErrors() )
return;
// convert to Groovy
new ConfigToGroovyVisitor(source).visit();
new PathCompareVisitor(source).visitClass(cn);
if( stripSecrets )
new StripSecretsVisitor(source).visitClass(cn);
if( renderClosureAsString )
new ClosureToStringVisitor(source).visitClass(cn);
new GStringToStringVisitor(source).visitClass(cn);
}
SourceUnit createSourceUnit(String source, Path path) {
var readerSource = path != null
? new StringReaderSourceWithURI(source, path.toUri(), getConfiguration())
: new StringReaderSource(source, getConfiguration());
return new SourceUnit(
uniqueClassName(source),
readerSource,
getConfiguration(),
getClassLoader(),
getErrorCollector());
}
/**
* Creates a unique name for the config class in order to avoid collision
* with config DSL
*
* @param text
*/
private String uniqueClassName(String text) {
var hash = Hashing
.sipHash24()
.newHasher()
.putUnencodedChars(text)
.hash();
return "_nf_config_" + hash;
}
}
}

View File

@@ -0,0 +1,396 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.config.parser.v2
import java.nio.file.NoSuchFileException
import java.nio.file.Path
import groovy.transform.CompileDynamic
import groovy.transform.CompileStatic
import groovy.transform.Memoized
import groovy.util.logging.Slf4j
import nextflow.SysEnv
import nextflow.exception.ConfigParseException
import nextflow.extension.Bolts
import nextflow.file.FileHelper
/**
* Builder DSL for Nextflow config files.
*
* @author Ben Sherman <bentshermann@gmail.com>
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
class ConfigDsl extends Script {
private boolean ignoreIncludes
private boolean renderClosureAsString
private boolean strict
private boolean stripSecrets
private Path configPath
private Map cliParams
private List<String> profiles
private Map target = [params: [:]]
private Set<String> declaredProfiles = []
private Map<String,Object> declaredParams = [:]
void setIgnoreIncludes(boolean value) {
this.ignoreIncludes = value
}
void setRenderClosureAsString(boolean value) {
this.renderClosureAsString = value
}
void setStrict(boolean value) {
this.strict = value
}
void setStripSecrets(boolean value) {
this.stripSecrets = value
}
void setConfigPath(Path path) {
this.configPath = path
}
void setParams(Map params) {
this.cliParams = params
(target.params as Map).putAll(params)
}
void setConfigParams(Map params) {
(target.params as Map).putAll(params)
}
void setProfiles(List<String> profiles) {
this.profiles = profiles
}
void declareProfile(String profile) {
declaredProfiles.add(profile)
}
Set<String> getDeclaredProfiles() {
return declaredProfiles
}
void declareParam(String name, Object value) {
declaredParams.put(name, value)
}
Map<String,Object> getDeclaredParams() {
return declaredParams
}
Map getTarget() {
return target
}
Object run() {}
@Override
def getProperty(String name) {
if( name == 'params' )
return target.params
try {
return super.getProperty(name)
}
catch( MissingPropertyException e ) {
if( strict )
throw e
else
return null
}
}
/**
* Assign a value to a config option.
*
* When assigning a param, if the param was specified
* on the command line, then the command line value takes
* precedence. CLI params are applied here in order to ensure
* that if the param is referenced later in the config file,
* the command line value is used.
*
* @param names
* @param value
*/
void assign(List<String> names, Object value) {
if( names.size() == 2 && names.first() == 'params' ) {
final name = names.last()
declareParam(name, value)
if( cliParams.containsKey(name) )
value = asDeclaredType(cliParams[name], value)
}
navigate(names.init()).put(names.last(), value)
}
/**
* Convert a CLI param override to an appropriate type based
* on the default param value in the config file.
*
* Note: this applies only to numbers and booleans.
*
* @param value
* @param declValue
*/
private Object asDeclaredType(Object value, Object declValue) {
if( value == null )
return null
if( value !instanceof CharSequence )
return value
final str = value.toString()
if( declValue instanceof Boolean ) {
if( str.toLowerCase() == 'true' ) return Boolean.TRUE
if( str.toLowerCase() == 'false' ) return Boolean.FALSE
}
if( declValue instanceof Number ) {
if( str.isInteger() ) return str.toInteger()
if( str.isLong() ) return str.toLong()
if( str.isBigInteger() ) return str.toBigInteger()
if( str.isFloat() ) return str.toFloat()
if( str.isDouble() ) return str.toDouble()
if( str.isBigDecimal() ) return str.toBigDecimal()
}
return value
}
private Map navigate(List<String> names) {
Map ctx = target
for( final name : names ) {
if( name !in ctx ) ctx[name] = [:]
ctx = ctx[name] as Map
}
return ctx
}
void block(String name, Closure closure) {
block([name], closure)
}
void block(List<String> names, Closure closure) {
final dsl = blockDsl(names)
final cl = (Closure)closure.clone()
cl.setResolveStrategy(Closure.DELEGATE_FIRST)
cl.setDelegate(dsl)
cl.call()
dsl.apply()
}
private ConfigBlockDsl blockDsl(List<String> names) {
if( names.size() == 1 && names.first() == 'plugins' )
return new PluginsDsl(this)
final relativeNames = names.size() == 3 && names.first() == 'profiles'
? List.of(names.last())
: names
if( relativeNames.size() == 1 && relativeNames.last() == 'process' )
return new ProcessDsl(this, names)
if( names.size() == 1 && names.first() == 'profiles' )
return new ProfilesDsl(this, profiles)
return new ConfigBlockDsl(this, names)
}
/**
* Get the value of an environment variable from the launch environment.
*
* @param name
*/
String env(String name) {
return SysEnv.get(name)
}
void includeConfig(String includeFile) {
includeConfig([], includeFile)
}
void includeConfig(List<String> names, String includeFile) {
assert includeFile
if( ignoreIncludes )
return
Path includePath = FileHelper.asPath(includeFile)
log.trace "Include config file: $includeFile [parent: $configPath]"
if( !includePath.isAbsolute() && configPath )
includePath = configPath.resolveSibling(includeFile)
final configText = readConfigFile(includePath)
final parser = new ConfigParserV2()
.setIgnoreIncludes(ignoreIncludes)
.setRenderClosureAsString(renderClosureAsString)
.setStrict(strict)
.setStripSecrets(stripSecrets)
.setBinding(binding.getVariables())
.setParams(cliParams)
.setConfigParams(target.params as Map)
.setProfiles(profiles)
final config = parser.parse(configText, includePath)
declaredProfiles.addAll(parser.getDeclaredProfiles())
declaredParams.putAll(parser.getDeclaredParams())
final ctx = navigate(names)
ctx.putAll(Bolts.deepMerge(ctx, config))
}
/**
* Read the content of a config file. The result is cached to
* avoid multiple reads.
*
* @param includePath
*/
@Memoized
@CompileDynamic // required to support ProviderPath::getText() over NioExtensions::getText()
protected static String readConfigFile(Path includePath) {
try {
return includePath.getText()
}
catch (NoSuchFileException | FileNotFoundException ignored) {
throw new NoSuchFileException("Config file does not exist: ${includePath.toUriString()}")
}
catch (IOException e) {
throw new IOException("Cannot read config file include: ${includePath.toUriString()}", e)
}
}
private static class ConfigBlockDsl {
protected ConfigDsl dsl
protected List<String> scope
ConfigBlockDsl(ConfigDsl dsl, List<String> scope) {
this.dsl = dsl
this.scope = scope
}
void assign(List<String> names, Object value) {
dsl.assign(scope + names, value)
}
void block(String name, Closure closure) {
dsl.block(scope + [name], closure)
}
void withLabel(String label, Closure closure) {
throw new ConfigParseException("Process selectors are only allowed in the `process` scope (offending scope: `${scope.join('.')}`)")
}
void withName(String selector, Closure closure) {
throw new ConfigParseException("Process selectors are only allowed in the `process` scope (offending scope: `${scope.join('.')}`)")
}
void includeConfig(String includeFile) {
dsl.includeConfig(scope, includeFile)
}
void apply() {
}
}
private static class PluginsDsl extends ConfigBlockDsl {
PluginsDsl(ConfigDsl dsl) {
super(dsl, Collections.<String>emptyList())
}
void id(String value) {
final target = dsl.getTarget()
final plugins = (Set) target.computeIfAbsent('plugins', (k) -> new HashSet<>())
plugins.add(value)
}
}
private static class ProcessDsl extends ConfigBlockDsl {
ProcessDsl(ConfigDsl dsl, List<String> scope) {
super(dsl, scope)
}
@Override
void withLabel(String label, Closure closure) {
dsl.block(scope + ["withLabel:${label}".toString()], closure)
}
@Override
void withName(String selector, Closure closure) {
dsl.block(scope + ["withName:${selector}".toString()], closure)
}
}
private static class ProfilesDsl extends ConfigBlockDsl {
private List<String> profiles
private Map<String,Closure> blocks = [:]
ProfilesDsl(ConfigDsl dsl, List<String> profiles) {
super(dsl, Collections.<String>emptyList())
this.profiles = profiles
}
@Override
void assign(List<String> names, Object value) {
throw new ConfigParseException("Only profile blocks are allowed in the `profiles` scope")
}
@Override
void block(String name, Closure closure) {
blocks[name] = closure
dsl.declareProfile(name)
}
@Override
void includeConfig(String includeFile) {
throw new ConfigParseException("Only profile blocks are allowed in the `profiles` scope")
}
@Override
void apply() {
if( profiles != null ) {
// apply profiles in the order they were specified
for( final name : profiles ) {
final closure = blocks[name]
if( closure )
dsl.block(scope, closure)
}
}
else {
// append all profiles to the config map
for( final name : blocks.keySet() ) {
final closure = blocks[name]
dsl.block(['profiles', name], closure)
}
}
}
}
}

View File

@@ -0,0 +1,203 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.config.parser.v2
import java.nio.file.Path
import groovy.transform.CompileDynamic
import groovy.transform.CompileStatic
import nextflow.config.ConfigParser
import nextflow.exception.ConfigParseException
import nextflow.extension.Bolts
import nextflow.script.parser.v2.StandardErrorListener
import org.codehaus.groovy.control.CompilationFailedException
import org.codehaus.groovy.control.SourceUnit
/**
* The parser for Nextflow config files.
*
* @author Ben Sherman <bentshermann@gmail.com>
*/
@CompileStatic
class ConfigParserV2 implements ConfigParser {
private Map bindingVars = [:]
private Map cliParams = [:]
private Map configParams = [:]
private boolean ignoreIncludes = false
private boolean renderClosureAsString = false
private boolean strict = true
private boolean stripSecrets
private boolean ansiLog
private List<String> appliedProfiles
private Set<String> declaredProfiles = []
private Map<String,Object> declaredParams = [:]
private GroovyShell groovyShell
@Override
ConfigParserV2 setProfiles(List<String> profiles) {
this.appliedProfiles = profiles
return this
}
@Override
ConfigParserV2 setIgnoreIncludes(boolean value) {
this.ignoreIncludes = value
return this
}
@Override
ConfigParserV2 setRenderClosureAsString(boolean value) {
this.renderClosureAsString = value
return this
}
@Override
ConfigParserV2 setStrict(boolean value) {
this.strict = value
return this
}
@Override
ConfigParserV2 setStripSecrets(boolean value) {
this.stripSecrets = value
return this
}
@Override
ConfigParserV2 setAnsiLog(boolean value) {
this.ansiLog = value
return this
}
@Override
ConfigParserV2 setBinding(Map vars) {
this.bindingVars = vars
return this
}
@Override
ConfigParserV2 setParams(Map params) {
// deep clone the map to prevent side effects with nested params
// see https://github.com/nextflow-io/nextflow/issues/1923
this.cliParams = Bolts.deepClone(params)
return this
}
ConfigParserV2 setConfigParams(Map params) {
this.configParams = params
return this
}
@Override
Set<String> getDeclaredProfiles() {
return declaredProfiles
}
@Override
Map<String,Object> getDeclaredParams() {
return declaredParams
}
/**
* Parse the given script as a string and return the configuration object
*
* @param text
* @param path
*/
@Override
ConfigObject parse(String text) {
parse(text, null)
}
ConfigObject parse(String text, Path path) {
final compiler = getCompiler()
try {
final script = (ConfigDsl) compiler.compile(text, path)
script.setBinding(new Binding(bindingVars))
if( path )
script.setConfigPath(path)
script.setIgnoreIncludes(ignoreIncludes)
script.setRenderClosureAsString(renderClosureAsString)
script.setStrict(strict)
script.setStripSecrets(stripSecrets)
script.setParams(cliParams)
script.setConfigParams(configParams)
script.setProfiles(appliedProfiles)
script.run()
final target = script.getTarget()
declaredProfiles.addAll(script.getDeclaredProfiles())
declaredParams.putAll(script.getDeclaredParams())
return Bolts.toConfigObject(target)
}
catch( CompilationFailedException e ) {
if( path )
printErrors(path)
throw new ConfigParseException("Config parsing failed", e)
}
}
private void printErrors(Path path) {
final source = compiler.getSource()
final errorListener = new StandardErrorListener('full', ansiLog)
println()
errorListener.beforeErrors()
for( final message : compiler.getErrors() ) {
final cause = message.getCause()
final filename = getRelativePath(source, path)
errorListener.onError(cause, filename, source)
}
errorListener.afterErrors()
}
private String getRelativePath(SourceUnit source, Path path) {
final uri = source.getSource().getURI()
return path.getParent().relativize(Path.of(uri)).toString()
}
@Override
ConfigObject parse(File file) {
return parse(file.toPath())
}
@Override
@CompileDynamic // required to support ProviderPath::getText() over NioExtensions::getText()
ConfigObject parse(Path path) {
return parse(path.getText(), path)
}
private ConfigCompiler compiler
private ConfigCompiler getCompiler() {
if( !compiler )
compiler = new ConfigCompiler(renderClosureAsString, stripSecrets)
return compiler
}
}

View File

@@ -0,0 +1,103 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.config.spec
import groovy.transform.TypeChecked
import nextflow.plugin.Plugins
import nextflow.script.dsl.Description
@TypeChecked
class MarkdownRenderer {
String render() {
final entries = getSpec().entrySet().sort { entry -> entry.key }
final result = new StringBuilder()
entries.each { entry ->
final scopeName = entry.key
final anchor = scopeName == ''
? '(config-unscoped)='
: "(config-$scopeName)="
result.append("\n$anchor\n")
final title = scopeName == ''
? 'Unscoped options'
: "`$scopeName`"
result.append("\n## $title\n")
final scope = entry.value
final description = scope.description()
if( description )
result.append("\n${fromDescription(description)}\n")
result.append("\nThe following settings are available:\n")
final options = scope.children().findAll { name, node -> node instanceof SpecNode.Option }
renderOptions(options, scopeName, result)
final scopes = scope.children().findAll { name, node -> node instanceof SpecNode.Scope }
renderOptions(scopes, scopeName, result)
}
return result.toString()
}
private static Map<String,SpecNode.Scope> getSpec() {
final result = new HashMap<String,SpecNode.Scope>()
for( final scope : Plugins.getExtensions(ConfigScope) ) {
final clazz = scope.getClass()
final scopeName = clazz.getAnnotation(ScopeName)?.value()
final description = clazz.getAnnotation(Description)?.value()
if( scopeName == null )
continue
final node = SpecNode.Scope.of(clazz, description)
result.put(scopeName, node)
}
return result
}
private static String fromDescription(String description) {
return description.stripIndent(true).trim()
}
private static void renderOptions(Map<String,SpecNode> nodes, String scopeName, StringBuilder result) {
final prefix = scopeName ? scopeName + '.' : ''
final entries = nodes.entrySet().sort { entry -> entry.key }
entries.each { entry ->
final name = entry.key
final node = entry.value
if( node instanceof SpecNode.Option )
renderOption("${prefix}${name}", node, result)
else if( node instanceof SpecNode.Placeholder )
renderOptions(node.scope().children(), "${prefix}${name}.${node.placeholderName()}", result)
else if( node instanceof SpecNode.Scope )
renderOptions(node.children(), "${prefix}${name}", result)
else
throw new IllegalStateException()
}
}
private static void renderOption(String name, SpecNode.Option node, StringBuilder result) {
final description = fromDescription(node.description())
if( !description )
return
result.append("\n`${name}`")
description.eachLine { line ->
if( line )
result.append("\n: ${line}")
}
result.append('\n')
}
}

View File

@@ -0,0 +1,158 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.container
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
/**
* Wrap a task execution inside an Apple container (apple/container) runtime.
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
class AppleContainerBuilder extends ContainerBuilder<AppleContainerBuilder> {
private boolean remove
private boolean tty
private String name
private String capAdd
private String removeCommand
private String killCommand
private kill = true
AppleContainerBuilder(String name, AppleContainerConfig config) {
this.image = name
if( config.engineOptions )
addEngineOptions(config.engineOptions)
if( config.runOptions )
addRunOptions(config.runOptions)
if( config.temp )
this.temp = config.temp
this.remove = config.remove
this.tty = config.tty
if( !config.writableInputMounts )
this.readOnlyInputs = true
}
AppleContainerBuilder(String name) {
this(name, new AppleContainerConfig([:]))
}
@Override
AppleContainerBuilder params( Map params ) {
if( !params ) return this
if( params.containsKey('entry') )
this.entryPoint = params.entry
if( params.containsKey('kill') )
this.kill = params.kill
if( params.containsKey('capAdd') )
this.capAdd = params.capAdd
return this
}
@Override
AppleContainerBuilder setName( String name ) {
this.name = name
return this
}
@Override
AppleContainerBuilder build(StringBuilder result) {
assert image
result << 'container '
if( engineOptions )
result << engineOptions.join(' ') << ' '
result << 'run -i '
if( tty )
result << '-t '
if( cpus )
result << "--cpus ${cpus} "
if( memory )
result << "-m ${memory} "
if( platform )
result << "--platform ${platform} "
// environment variables
appendEnv(result)
if( temp )
result << "-v $temp:/tmp "
// volume mounts
result << makeVolumes(mounts)
result << '-w "$NXF_TASK_WORKDIR" '
if( entryPoint )
result << '--entrypoint ' << entryPoint << ' '
if( runOptions )
result << runOptions.join(' ') << ' '
if( capAdd )
result << '--cap-add ' << capAdd << ' '
if( name )
result << '--name ' << name << ' '
// image is the final positional argument
result << image
runCommand = result.toString()
if( remove && name ) {
removeCommand = 'container rm ' + name
}
if( kill ) {
killCommand = 'container stop '
if( kill instanceof String ) killCommand = "container kill -s $kill "
killCommand += name
}
return this
}
@Override
String getRemoveCommand() { removeCommand }
@Override
String getKillCommand() { killCommand }
}

View File

@@ -0,0 +1,119 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.container
import groovy.transform.CompileStatic
import groovy.transform.EqualsAndHashCode
import groovy.util.logging.Slf4j
import nextflow.config.spec.ConfigOption
import nextflow.config.spec.ConfigScope
import nextflow.config.spec.ScopeName
import nextflow.script.dsl.Description
@ScopeName("appleContainer")
@Description("""
The `appleContainer` scope controls how [Apple container](https://github.com/apple/container) runtime is used by Nextflow.
""")
@Slf4j
@CompileStatic
@EqualsAndHashCode
class AppleContainerConfig implements ConfigScope, ContainerConfig {
@ConfigOption
@Description("""
Enable Apple container execution (default: `false`).
""")
boolean enabled
@ConfigOption
@Description("""
Specify additional options supported by the `container` CLI i.e. `container [OPTIONS] run`.
""")
final String engineOptions
@ConfigOption
@Description("""
Comma separated list of environment variable names to be included in the container environment.
""")
final List<String> envWhitelist
@ConfigOption(types=[String,Boolean])
@Description("""
""")
final Object kill
@ConfigOption
@Description("""
The registry from where container images are pulled. It should NOT include the protocol prefix i.e. `http://`.
""")
final String registry
@ConfigOption
@Description("""
Clean up the container after the execution (default: `true`).
""")
final boolean remove
@ConfigOption
@Description("""
Specify extra command line options supported by the `container run` command.
""")
final String runOptions
@ConfigOption
@Description("""
Mounts a path of your choice as the `/tmp` directory in the container.
""")
final String temp
@ConfigOption
@Description("""
Allocates a pseudo-tty (default: `false`).
""")
final boolean tty
@ConfigOption
@Description("""
When `false`, mount input directories as read-only (default: `true`).
""")
final boolean writableInputMounts
/* required by extension point -- do not remove */
AppleContainerConfig() {}
AppleContainerConfig(Map opts) {
enabled = opts.enabled as boolean
engineOptions = opts.engineOptions
envWhitelist = ContainerHelper.parseEnvWhitelist(opts.envWhitelist)
kill = opts.kill != null ? opts.kill : true
registry = opts.registry
remove = opts.remove != null ? opts.remove as boolean : true
runOptions = opts.runOptions
temp = opts.temp
tty = opts.tty as boolean
writableInputMounts = opts.writableInputMounts != null ? opts.writableInputMounts as boolean : true
}
@Override
String getEngine() {
return 'apple-container'
}
@Override
boolean canRunOciImage() {
return true
}
}

View File

@@ -0,0 +1,54 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.container
import groovy.transform.CompileStatic
/**
* Implements a builder for Apptainer containerisation
*
* see https://apptainer.org
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class ApptainerBuilder extends SingularityBuilder {
ApptainerBuilder(String name, ApptainerConfig config) {
super(name)
applyConfig(config)
}
ApptainerBuilder(String name) {
this(name, new ApptainerConfig([:]))
}
@Override
protected String getBinaryName() { 'apptainer' }
protected void applyConfig(ApptainerConfig config) {
if( config.autoMounts != null )
this.autoMounts = config.autoMounts
if( config.engineOptions )
this.addEngineOptions(config.engineOptions)
if( config.runOptions )
this.addRunOptions(config.runOptions)
}
}

View File

@@ -0,0 +1,35 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.container
import groovy.transform.CompileStatic
/**
* Handle caching of remote Apptainer images
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class ApptainerCache extends SingularityCache {
ApptainerCache(ApptainerConfig config, Map<String,String> env=null) {
super(config.cacheDir, config.libraryDir, config.noHttps, config.pullTimeout, env)
}
@Override
protected String getBinaryName() { 'apptainer' }
}

View File

@@ -0,0 +1,125 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.container
import groovy.transform.CompileStatic
import nextflow.config.spec.ConfigOption
import nextflow.config.spec.ConfigScope
import nextflow.config.spec.ScopeName
import nextflow.script.dsl.Description
import nextflow.util.Duration
@ScopeName("apptainer")
@Description("""
The `apptainer` scope controls how [Apptainer](https://apptainer.org) containers are executed by Nextflow.
""")
@CompileStatic
class ApptainerConfig implements ConfigScope, ContainerConfig {
@ConfigOption
@Description("""
Automatically mount host paths in the executed container (default: `true`). It requires the `user bind control` feature to be enabled in your Apptainer installation.
""")
final Boolean autoMounts
@ConfigOption
@Description("""
The directory where remote Apptainer images are stored. When using a computing cluster it must be a shared folder accessible to all compute nodes.
""")
final String cacheDir
@ConfigOption
@Description("""
Execute tasks with Apptainer containers (default: `false`).
""")
final boolean enabled
@ConfigOption
@Description("""
Specify additional options supported by the Apptainer engine i.e. `apptainer [OPTIONS]`.
""")
final String engineOptions
@ConfigOption
@Description("""
Comma separated list of environment variable names to be included in the container environment.
""")
final List<String> envWhitelist
@ConfigOption
@Description("""
Directory where remote Apptainer images are retrieved. When using a computing cluster it must be a shared folder accessible to all compute nodes.
""")
final String libraryDir
@ConfigOption
@Description("""
Pull the Apptainer image with http protocol (default: `false`).
""")
final boolean noHttps
@ConfigOption
@Description("""
When enabled, OCI (and Docker) container images are pulled and converted to the SIF format by the Apptainer run command, instead of Nextflow (default: `false`).
""")
final boolean ociAutoPull
@ConfigOption
@Description("""
The amount of time the Apptainer pull can last, exceeding which the process is terminated (default: `20 min`).
""")
final Duration pullTimeout
@ConfigOption
@Description("""
The registry from where Docker images are pulled. It should be only used to specify a private registry server. It should NOT include the protocol prefix i.e. `http://`.
""")
final String registry
@ConfigOption
@Description("""
Specify extra command line options supported by `apptainer exec`.
""")
final String runOptions
/* required by extension point -- do not remove */
ApptainerConfig() {}
ApptainerConfig(Map opts) {
autoMounts = opts.autoMounts as Boolean
cacheDir = opts.cacheDir
enabled = opts.enabled as boolean
engineOptions = opts.engineOptions
envWhitelist = ContainerHelper.parseEnvWhitelist(opts.envWhitelist)
libraryDir = opts.libraryDir
noHttps = opts.noHttps as boolean
ociAutoPull = opts.ociAutoPull as boolean
pullTimeout = opts.pullTimeout as Duration ?: Duration.of('20min')
registry = opts.registry
runOptions = opts.runOptions
}
@Override
String getEngine() {
return 'apptainer'
}
@Override
boolean canRunOciImage() {
return ociAutoPull
}
}

View File

@@ -0,0 +1,157 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.container
import java.nio.file.Path
import java.nio.file.Paths
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.Global
/**
* Implements a builder for Charliecloud containerisation
*
* see https://hpc.github.io/charliecloud/
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
* @author Patrick Hüther <patrick.huether@gmail.com>
* @author Laurent Modolo <laurent.modolo@ens-lyon.fr>
* @author Niklas Schandry <niklas@bio.lmu.de>
*/
@CompileStatic
@Slf4j
class CharliecloudBuilder extends ContainerBuilder<CharliecloudBuilder> {
private boolean writeFake
CharliecloudBuilder(String name, CharliecloudConfig config) {
this.image = name
if( config.runOptions )
addRunOptions(config.runOptions)
if( config.temp )
this.temp = config.temp
if( !config.writableInputMounts )
this.readOnlyInputs = true
this.writeFake = config.writeFake
}
CharliecloudBuilder(String name) {
this(name, new CharliecloudConfig([:]))
}
@Override
CharliecloudBuilder params(Map params) {
if( params.containsKey('entry') )
this.entryPoint = params.entry
return this
}
CharliecloudBuilder addRunOptions(String str) {
runOptions.add(str)
return this
}
@Override
CharliecloudBuilder build(StringBuilder result) {
assert image
def imageStorage = Paths.get(image).parent.parent
def imageName = image.split('/')[-1]
result << 'ch-run --unset-env="*" -c "$NXF_TASK_WORKDIR" --set-env '
if ( writeFake )
result << '--write-fake '
if ( !writeFake && !readOnlyInputs ) {
// -w and CH_IMAGE_STORAGE are incompatible.
if(System.getenv('CH_IMAGE_STORAGE') == imageStorage)
throw new Exception('It is not possible to run writeable images from `$CH_IMAGE_STORAGE`')
result << '-w '
}
appendEnv(result)
if( temp )
result << "-b $temp:/tmp "
makeVolumes(mounts, result)
if( runOptions )
result << runOptions.join(' ') << ' '
if( writeFake && System.getenv('CH_IMAGE_STORAGE') ) {
// Run by name if writeFake is true and CH_IMAGE_STORAGE is set
result << imageName
} else {
// Otherwise run by path
result << image
}
result << ' --'
runCommand = result.toString()
return this
}
protected String getRoot(String path) {
def rootPath = path.tokenize("/")
if (rootPath.size() >= 1 && path[0] == '/')
rootPath = "/${rootPath[0]}"
else
throw new IllegalArgumentException("Not a valid working directory value (absolute path?): ${path}")
return rootPath
}
@Override
protected String composeVolumePath(String path, boolean readOnlyInputs = false) {
def mountCmd = "-b ${escape(path)}"
if (readOnlyInputs)
mountCmd = "-b ${getRoot(escape(path))}"
return mountCmd
}
@Override
protected StringBuilder makeEnv( env, StringBuilder result = new StringBuilder() ) {
if( env instanceof Map ) {
short index = 0
for( Map.Entry entry : env.entrySet() ) {
if( index++ ) result << ' '
result << ("--set-env=${entry.key}=${entry.value}")
}
}
else if( env instanceof String && env.contains('=') ) {
result << "--set-env=" << env
}
else if( env instanceof String ) {
result << "\${$env:+--set-env=$env=\$$env}"
}
else if( env ) {
throw new IllegalArgumentException("Not a valid environment value: $env [${env.class.name}]")
}
return result
}
}

Some files were not shown because too many files have changed in this diff Show More