add nextflow d30e48d

This commit is contained in:
2026-04-29 23:01:54 +02:00
parent d0b12d668d
commit 97cc9058d3
2840 changed files with 730250 additions and 0 deletions

View File

@@ -0,0 +1,50 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
apply plugin: 'groovy'
sourceSets {
main.java.srcDirs = []
main.groovy.srcDirs = ['src/main']
main.resources.srcDirs = ['src/resources']
test.groovy.srcDirs = ['src/test']
test.java.srcDirs = []
test.resources.srcDirs = ['src/testResources']
}
dependencies {
api(project(':nf-lang'))
api "ch.qos.logback:logback-classic:1.5.26"
api "org.apache.groovy:groovy:4.0.31"
api "org.apache.groovy:groovy-nio:4.0.31"
api "org.apache.commons:commons-lang3:3.18.0"
api 'com.google.guava:guava:33.0.0-jre'
api 'org.pf4j:pf4j:3.14.1'
api 'org.pf4j:pf4j-update:2.3.0'
api 'dev.failsafe:failsafe:3.1.0'
api 'io.seqera:lib-httpx:2.1.0'
api 'io.seqera:lib-retry:2.0.0'
// patch gson dependency required by pf4j
api 'com.google.code.gson:gson:2.13.1'
api 'io.seqera:npr-api:0.22.0'
/* testImplementation inherited from top gradle build file */
testImplementation(testFixtures(project(":nextflow")))
testFixturesImplementation(project(":nextflow"))
testImplementation "org.apache.groovy:groovy-json:4.0.31" // needed by wiremock
testImplementation ('org.wiremock:wiremock:3.13.1') { exclude module: 'groovy-all' }
}

View File

@@ -0,0 +1,87 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow
import static nextflow.extension.Bolts.*
import java.text.SimpleDateFormat
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
/**
* Model app build information
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
class BuildInfo {
private static Properties properties
static {
final BUILD_INFO = '/META-INF/build-info.properties'
properties = new Properties()
try {
properties.load( BuildInfo.getResourceAsStream(BUILD_INFO) )
}
catch( Exception e ) {
log.warn "Unable to parse $BUILD_INFO - Cause ${e.message ?: e}"
}
}
static Properties getProperties() { properties }
static String getVersion() { properties.getProperty('version') }
static String getCommitId() { properties.getProperty('commitId')}
static String getBuildNum() { properties.getProperty('build') }
static long getTimestampMillis() { properties.getProperty('timestamp') as long }
static String getTimestampUTC() {
def tz = TimeZone.getTimeZone('UTC')
def fmt = new SimpleDateFormat(DATETIME_FORMAT)
fmt.setTimeZone(tz)
fmt.format(new Date(getTimestampMillis())) + ' ' + tz.getDisplayName( true, TimeZone.SHORT )
}
static String getTimestampLocal() {
def tz = TimeZone.getDefault()
def fmt = new SimpleDateFormat(DATETIME_FORMAT)
fmt.setTimeZone(tz)
fmt.format(new Date(getTimestampMillis())) + ' ' + tz.getDisplayName( true, TimeZone.SHORT )
}
static String getTimestampDelta() {
if( getTimestampUTC() == getTimestampLocal() ) {
return ''
}
final utc = getTimestampUTC().tokenize(' ')
final loc = getTimestampLocal().tokenize(' ')
final result = utc[0] == loc[0] ? loc[1,-1].join(' ') : loc.join(' ')
return "($result)"
}
static String getFullVersion() {
"${version}_${commitId}"
}
}

View File

@@ -0,0 +1,91 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow
import java.nio.file.Path
import java.nio.file.Paths
/**
* Application main constants
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
class Const {
static final public String ISO_8601_DATETIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ss'Z'"
static final public transient BOOL_YES = ['true','yes','on']
static final public transient BOOL_NO = ['false','no','off']
/**
* The application main package name
*/
static public final String MAIN_PACKAGE = Const.name.split('\\.')[0]
/**
* The application main name
*/
static public final String APP_NAME = MAIN_PACKAGE
/**
* The application home folder
*/
static public final Path APP_HOME_DIR = getHomeDir(APP_NAME)
static Path sysHome() {
def home = System.getProperty("user.home")
if( !home || home=='?' )
home = SysEnv.get('HOME')
if( !home )
throw new IllegalStateException("Unable to detect system home path - Make sure the variable HOME or NXF_HOME is defined in your environment")
return Path.of(home)
}
private static Path getHomeDir(String appname) {
final home = SysEnv.get('NXF_HOME')
final result = home ? Paths.get(home) : sysHome().resolve(".$appname")
if( !result.exists() && !result.mkdir() ) {
throw new IllegalStateException("Cannot create path '${result}' -- check file system access permission")
}
return result
}
static final Path getAppCacheDir() {
return Path.of(SysEnv.get('NXF_CACHE_DIR', '.nextflow'))
}
static public final String ROLE_WORKER = 'worker'
static public final String ROLE_MASTER = 'master'
static public final String MANIFEST_FILE_NAME = 'nextflow.config'
static public final String DEFAULT_MAIN_FILE_NAME = 'main.nf'
static public final String DEFAULT_ORGANIZATION = SysEnv.get('NXF_ORG') ?: 'nextflow-io'
static public final String DEFAULT_HUB = SysEnv.get('NXF_HUB') ?: 'github'
static public final File DEFAULT_ROOT = SysEnv.get('NXF_ASSETS') ? new File(SysEnv.get('NXF_ASSETS')) : APP_HOME_DIR.resolve('assets').toFile()
static public final String DEFAULT_BRANCH = 'master'
static public final String SCOPE_SEP = ':'
}

View File

@@ -0,0 +1,120 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow
import java.util.function.Consumer
import groovy.util.logging.Slf4j
import nextflow.util.Duration
import nextflow.util.IniFile
import nextflow.util.MemoryUnit
import nextflow.util.TestOnly
import org.apache.commons.lang3.StringUtils
import org.apache.commons.lang3.exception.ExceptionUtils
/**
* Hold global variables
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
class Global {
/**
* When {@code true}, Fusion trace metrics replace the bash command-trace wrapper
*/
static boolean isFusionTraceEnabled() {
return SysEnv.get('NXF_FUSION_TRACE', 'false') == 'true'
}
/**
* The pipeline session instance
*/
static private ISession session
/**
* Allow to load session in a lazy manner
*/
static private Closure<ISession> loader
/**
* The main configuration object
*/
static Map config
/**
* @return The object instance representing the current session
*/
static ISession getSession() {
if( session != null )
return session
if( loader != null )
session = loader.call()
return session
}
/**
* Set the application session object
*
* @param value An object instance representing the current session
*/
static void setSession( ISession value ) {
session = value
}
/**
* Set a session lazy loader
*
* @param loader
*/
static void setLazySession( Closure<ISession> loader ) {
Global.loader = loader
}
/**
* Run the specified closure on application termination
*
* @param callback A closure to be executed on application shutdown
*/
static void onCleanup(Consumer<ISession> callback) {
if( callback==null ) {
log.warn "Cleanup consumer cannot be null\n${ExceptionUtils.getStackTrace(new Exception())}"
return
}
hooks.add(callback)
}
static final private List<Consumer<ISession>> hooks = []
static synchronized cleanUp() {
for( Consumer<ISession> c : hooks ) {
try {
c.accept(session)
}
catch( Exception e ) {
log.debug("Error during on cleanup", e )
}
}
}
@TestOnly
static void reset() {
session = null
config = null
hooks.clear()
}
}

View File

@@ -0,0 +1,69 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow
import java.nio.file.Path
/**
* Nextflow session interface
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
interface ISession {
/**
* The folder where tasks temporary files are stored
*/
Path getWorkDir()
/**
* The folder where the main script is contained
*/
Path getBaseDir()
/**
* Holds the configuration object
*/
Map getConfig()
/**
* The pipeline script name (without parent path)
*/
String getScriptName()
/**
* @return List of path added to application classpath at runtime
*/
List<Path> getLibDir()
/**
* The unique identifier of this session
*/
UUID getUniqueId()
/**
* @return The global script binding object
*/
Binding getBinding()
boolean isCacheable()
boolean isResumeMode()
String getCommitId()
}

View File

@@ -0,0 +1,99 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow
import groovy.transform.CompileStatic
import groovy.transform.PackageScope
/**
* Helper class that holds a reference system environment and
* allow to swap to a different one for testing purposes
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class SysEnv {
private static Holder holder = new Holder(System.getenv())
private static final List<Map<String,String>> history = new ArrayList<Map<String,String>>()
static boolean containsKey(String key) {
return holder.containsKey(key)
}
static Map<String,String> get() {
return holder
}
static String get(String name) {
return holder.get(name)
}
static String get(String name, String defValue) {
return holder.containsKey(name) ? holder.get(name) : defValue
}
static boolean getBool(String name, boolean defValue) {
final result = get(name,String.valueOf(defValue))
return Boolean.parseBoolean(result) || result == '1'
}
static Integer getInteger(String name, Integer defValue) {
final result = get(name, defValue!=null ? String.valueOf(defValue) : null)
return result!=null ? Integer.valueOf(result) : null
}
static Long getLong(String name, Long defValue) {
final result = get(name, defValue!=null ? String.valueOf(defValue) : null)
return result!=null ? Long.valueOf(result) : null
}
/**
* Check if agent output mode is enabled via environment variables.
* When enabled, Nextflow replaces interactive ANSI logging with minimal,
* structured output optimized for AI agents.
*
* Supported variables (any truthy value activates the mode):
* {@code NXF_AGENT_MODE}, {@code AGENT}, {@code CLAUDECODE}.
*
* @return {@code true} if agent mode is enabled
*/
static boolean isAgentMode() {
return getBool('NXF_AGENT_MODE', false) ||
getBool('AGENT', false) ||
getBool('CLAUDECODE', false)
}
static void push(Map<String,String> env) {
history.push(holder.getTarget())
holder.setTarget(env)
}
static void pop() {
holder.setTarget(history.pop())
}
}
@PackageScope
class Holder implements Map<String,String> {
@Delegate Map<String,String> target
Holder(Map<String,String> target) { this.target = target }
void setTarget(Map<String,String> target) { this.target=target }
}

View File

@@ -0,0 +1,91 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.config
import groovy.transform.CompileStatic
import nextflow.SysEnv
import nextflow.config.spec.ConfigOption
import nextflow.config.spec.ConfigScope
import nextflow.config.spec.ScopeName
import nextflow.script.dsl.Description
/**
* Configuration scope for module registry settings
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
@ScopeName("registry")
@Description("""
The `registry` scope provides configuration for the Nextflow module registry.
This includes the registry URL(s) and API key for authentication.
""")
@CompileStatic
class RegistryConfig implements ConfigScope {
public static final String DEFAULT_REGISTRY_URL = 'https://registry.nextflow.io/api'
@ConfigOption
@Description("Registry URL or list of registry URLs in priority order (primary URL first)")
private final Collection<String> url
@ConfigOption
@Description("API key for authenticating with the primary registry")
private final String apiKey
/* required by extension point -- do not remove */
RegistryConfig() {
this.url = [DEFAULT_REGISTRY_URL]
this.apiKey = null
}
RegistryConfig(Map opts) {
final urlObject = opts.url ?: [DEFAULT_REGISTRY_URL]
if (urlObject instanceof Collection<String>)
this.url = (urlObject as Collection<String>).collect { it.replaceAll(/\/+$/, '') }
else
this.url = [urlObject.toString().replaceAll(/\/+$/, '')]
this.apiKey = opts.apiKey as String
}
/**
* Get the primary (first) registry URL
*
* @return The primary registry URL
*/
String getUrl() {
return this.url ? url[0] as String : DEFAULT_REGISTRY_URL
}
/**
* Get all registry URLs (primary first, fallbacks after)
*
* @return Collection of registry URLs
*/
Collection<String> getAllUrls() {
return this.url ?: [DEFAULT_REGISTRY_URL]
}
/**
* Get the API key for the primary registry.
* Authentication is only supported for the primary registry.
*
* @return The API key, or 'NXF_REGISTRY_TOKEN' env value if not configured
*/
String getApiKey() {
return apiKey ?: SysEnv.get('NXF_REGISTRY_TOKEN')
}
}

View File

@@ -0,0 +1,29 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.exception
import groovy.transform.CompileStatic
import groovy.transform.InheritConstructors
/**
* Abort the current operation execution
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@InheritConstructors
@CompileStatic
class AbortOperationException extends RuntimeException { }

View File

@@ -0,0 +1,981 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.extension
import java.nio.file.Path
import java.text.DateFormat
import java.text.SimpleDateFormat
import java.time.OffsetDateTime
import java.time.format.DateTimeFormatter
import java.util.concurrent.locks.Lock
import java.util.regex.Pattern
import groovy.transform.CompileDynamic
import groovy.transform.CompileStatic
import groovy.transform.Memoized
import groovy.transform.stc.ClosureParams
import groovy.transform.stc.FirstParam
import nextflow.file.FileHelper
import nextflow.file.FileMutex
import nextflow.util.CheckHelper
import nextflow.util.Duration
import nextflow.util.MemoryUnit
import nextflow.util.RateUnit
import org.apache.commons.lang3.StringUtils
import org.codehaus.groovy.runtime.DefaultGroovyMethods
import org.codehaus.groovy.runtime.GStringImpl
import org.codehaus.groovy.runtime.ResourceGroovyMethods
import org.codehaus.groovy.runtime.StringGroovyMethods
import org.slf4j.Logger
/**
* Generic extensions
*
* See more about extension methods
* http://docs.codehaus.org/display/GROOVY/Creating+an+extension+module
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class Bolts {
static public final String DATETIME_FORMAT = 'dd-MM-yyyy HH:mm'
static private Pattern PATTERN_RIGHT_TRIM = ~/\s+$/
static private Pattern PATTERN_LEFT_TRIM = ~/^\s+/
@Memoized
static private ThreadLocal<DateFormat> getLocalDateFormat(String fmt, TimeZone tz) {
return new ThreadLocal<DateFormat>() {
@Override
protected DateFormat initialValue() {
def result = new SimpleDateFormat(fmt, Locale.ENGLISH)
if(tz) result.setTimeZone(tz)
return result
}
}
}
/**
* Format a {@link Date} object
*
* @param self The {@link Date} object to format
* @param format The date format to use eg. {@code dd-MM-yyyy HH:mm}.
* @param tz The timezone to be used eg. {@code UTC}. If {@code null} the current timezone is used.
* @return The date-time formatted as a string
*/
static String format(Date self, String format=null, String tz=null) {
TimeZone zone = tz ? TimeZone.getTimeZone(tz) : null
getLocalDateFormat(format ?: DATETIME_FORMAT, zone).get().format(self)
}
static String format(OffsetDateTime self, String format) {
return self.format(DateTimeFormatter.ofPattern(format).withLocale(Locale.ENGLISH))
}
/**
* Format a {@link Date} object
*
* @param self The {@link Date} object to format
* @param format The date format to use eg. {@code dd-MM-yyyy HH:mm}
* @param tz The timezone to be used. If {@code null} the current timezone is used.
* @return The date-time formatted as a string
*/
static String format(Date self, String format, TimeZone tz) {
getLocalDateFormat(format ?: DATETIME_FORMAT, tz).get().format(self)
}
static List pairs(Map self, Map opts=null) {
def flat = opts?.flat == true
def result = []
for( Map.Entry entry : self.entrySet() ) {
if( flat && entry.value instanceof Collection )
entry.value.iterator().each { result << [entry.key, it] }
else
result << [entry.key, entry.value]
}
return result
}
/**
* Remove the left side after a dot (including it) e.g.
* <pre>
* 0.10 => 0
* 10000.00 => 10000
* </pre>
*
* @param self
* @return
*/
static String trimDotZero(String self) {
int p = self?.indexOf('.')
p!=-1 ? self.substring(0,p) : self
}
/**
* Remove blank chars at the end of the string
*
* @param self The string itself
* @return The string with blanks removed
*/
static String rightTrim(String self) {
self.replaceAll(PATTERN_RIGHT_TRIM,'')
}
/**
* Remove blank chars at string beginning
*
* @param self The string itself
* @return The string with blanks removed
*/
static String leftTrim( String self ) {
self.replaceAll(PATTERN_LEFT_TRIM,'')
}
/**
* <p>Strips any of a set of characters from the start and end of a String.
* This is similar to {@link String#trim()} but allows the characters
* to be stripped to be controlled.</p>
*
* <p>A <code>null</code> input String returns <code>null</code>.
* An empty string ("") input returns the empty string.</p>
*
* <p>If the stripChars String is <code>null</code>, whitespace is
* stripped as defined by {@link Character#isWhitespace(char)}.
* Alternatively use {@link #strip(String)}.</p>
*
* <pre>
* StringUtils.strip(null, *) = null
* StringUtils.strip("", *) = ""
* StringUtils.strip("abc", null) = "abc"
* StringUtils.strip(" abc", null) = "abc"
* StringUtils.strip("abc ", null) = "abc"
* StringUtils.strip(" abc ", null) = "abc"
* StringUtils.strip(" abcyx", "xyz") = " abc"
* </pre>
*
* @param str the String to remove characters from, may be null
* @param stripChars the characters to remove, null treated as whitespace
* @return the stripped String, <code>null</code> if null String input
*/
static String strip( String self, String stripChars = null ) {
StringUtils.strip(self, stripChars)
}
/**
* <p>Strips any of a set of characters from the start of a String.</p>
*
* <p>A <code>null</code> input String returns <code>null</code>.
* An empty string ("") input returns the empty string.</p>
*
* <p>If the stripChars String is <code>null</code>, whitespace is
* stripped as defined by {@link Character#isWhitespace(char)}.</p>
*
* <pre>
* StringUtils.stripStart(null, *) = null
* StringUtils.stripStart("", *) = ""
* StringUtils.stripStart("abc", "") = "abc"
* StringUtils.stripStart("abc", null) = "abc"
* StringUtils.stripStart(" abc", null) = "abc"
* StringUtils.stripStart("abc ", null) = "abc "
* StringUtils.stripStart(" abc ", null) = "abc "
* StringUtils.stripStart("yxabc ", "xyz") = "abc "
* </pre>
*
* @param str the String to remove characters from, may be null
* @param stripChars the characters to remove, null treated as whitespace
* @return the stripped String, <code>null</code> if null String input
*/
static String stripStart( String self, String stripChars = null ) {
StringUtils.stripStart(self, stripChars)
}
/**
* <p>Strips any of a set of characters from the end of a String.</p>
*
* <p>A <code>null</code> input String returns <code>null</code>.
* An empty string ("") input returns the empty string.</p>
*
* <p>If the stripChars String is <code>null</code>, whitespace is
* stripped as defined by {@link Character#isWhitespace(char)}.</p>
*
* <pre>
* StringUtils.stripEnd(null, *) = null
* StringUtils.stripEnd("", *) = ""
* StringUtils.stripEnd("abc", "") = "abc"
* StringUtils.stripEnd("abc", null) = "abc"
* StringUtils.stripEnd(" abc", null) = " abc"
* StringUtils.stripEnd("abc ", null) = "abc"
* StringUtils.stripEnd(" abc ", null) = " abc"
* StringUtils.stripEnd(" abcyx", "xyz") = " abc"
* StringUtils.stripEnd("120.00", ".0") = "12"
* </pre>
*
* @param str the String to remove characters from, may be null
* @param stripChars the set of characters to remove, null treated as whitespace
* @return the stripped String, <code>null</code> if null String input
*/
static String stripEnd( String self, String stripChars = null ) {
StringUtils.stripEnd(self, stripChars)
}
/**
* <p>Capitalizes a String changing the first letter to title case as
* per {@link Character#toTitleCase(char)}. No other letters are changed.</p>
*
* <p>For a word based algorithm, see {@link org.apache.commons.lang.WordUtils#capitalize(String)}.
* A <code>null</code> input String returns <code>null</code>.</p>
*
* <pre>
* StringUtils.capitalize(null) = null
* StringUtils.capitalize("") = ""
* StringUtils.capitalize("cat") = "Cat"
* StringUtils.capitalize("cAt") = "CAt"
* </pre>
*
*/
static String capitalize(String self) {
StringUtils.capitalize(self)
}
/**
* <p>Uncapitalizes a String changing the first letter to title case as
* per {@link Character#toLowerCase(char)}. No other letters are changed.</p>
*
* <p>For a word based algorithm, see {@link org.apache.commons.lang.WordUtils#uncapitalize(String)}.
* A <code>null</code> input String returns <code>null</code>.</p>
*
* <pre>
* StringUtils.uncapitalize(null) = null
* StringUtils.uncapitalize("") = ""
* StringUtils.uncapitalize("Cat") = "cat"
* StringUtils.uncapitalize("CAT") = "cAT"
* </pre>
*
*/
static String uncapitalize(String self) {
StringUtils.uncapitalize(self)
}
/**
* Check if a alphabetic characters in a string are lowercase. Non alphabetic characters are ignored
* @param self The string to check
* @return {@true} if the string contains no uppercase characters, {@code false} otherwise
*/
static boolean isLowerCase(String self) {
if( self ) for( int i=0; i<self.size(); i++ ) {
if( Character.isUpperCase(self.charAt(i)))
return false
}
return true
}
/**
* Check if a alphabetic characters in a string are uppercase. Non alphabetic characters are ignored
* @param self The string to check
* @return {@true} if the string contains no lowercase characters, {@code false} otherwise
*/
static boolean isUpperCase(String self) {
if( self ) for( int i=0; i<self.size(); i++ ) {
if( Character.isLowerCase(self.charAt(i)))
return false
}
return true
}
/**
* Check if ALL characters in a string are lowercase.
* @param self The string to check
* @return {@true} when all characters are uppercase, {@code false} otherwise
*/
static boolean isAllLowerCase(String self) {
StringUtils.isAllLowerCase(self)
}
/**
* Check if ALL characters in a string are uppercase.
* @param self The string to check
* @return {@true} when all characters are uppercase, {@code false} otherwise
*/
static boolean isAllUpperCase(String self) {
StringUtils.isAllUpperCase(self)
}
/**
* Invokes the specify closure including it with a lock/unlock calls pair
*
* @param self
* @param interruptible
* @param closure
* @return the closure result
*/
static <T> T withLock( Lock self, boolean interruptible = false, Closure<T> closure ) {
// acquire the lock
if( interruptible )
self.lockInterruptibly()
else
self.lock()
try {
return closure.call()
}
finally {
self.unlock();
}
}
/**
* Invokes the specify closure only if it is able to acquire a lock
*
* @param self
* @param interruptible
* @param closure
* @return the closure result
*/
static boolean tryLock( Lock self, Closure closure ) {
if( !self.tryLock() )
return false
try {
closure.call()
}
finally {
self.unlock()
return true
}
}
/**
* Creates a file system wide lock that prevent two or more JVM instances/process
* to work on the same file
*
* Note: this does not protected against multiple-thread accessing the file in a
* concurrent manner.
*
* @param
* self The file over which define the lock
* @param
* timeout An option timeout elapsed which the a {@link InterruptedException} is thrown
* @param
* closure The action to apply during the lock file spawn
* @return
* The user provided {@code closure} result
*
* @throws
* InterruptedException if the lock cannot be acquired within the specified {@code timeout}
*/
static withLock(File self, Duration timeout = null, Closure closure) {
def locker = new FileMutex(self)
if( timeout )
locker.setTimeout(timeout)
locker.lock(closure)
}
/**
* Creates a file system wide lock that prevent two or more JVM instances/process
* to work on the same file
*
* Note: this does not protected against multiple-thread accessing the file in a
* concurrent manner.
*
* @param
* self The file over which define the lock
* @param
* timeout An option timeout elapsed which the a {@link InterruptedException} is thrown
* @param
* closure The action to apply during the lock file spawn
* @return
* The user provided {@code closure} result
*
* @throws
* InterruptedException if the lock cannot be acquired within the specified {@code timeout}
*/
static withLock( Path self, Duration timeout, Closure closure ) {
def locker = new FileMutex(self.toFile())
if( timeout )
locker.setTimeout(timeout)
locker.lock(closure)
}
static Duration toDuration(Number number) {
new Duration(number.toLong())
}
static MemoryUnit toMemory(Number number) {
new MemoryUnit(number.toLong())
}
/**
* Converts a {@code String} to a {@code Duration} object
*
* @param self
* @param type
* @return
*/
static Object asType( String self, Class<Object> type ) {
if( type == Duration ) {
return new Duration(self)
}
else if( Path.isAssignableFrom(type) ) {
return FileHelper.asPath(self)
}
else if( type == MemoryUnit ) {
return new MemoryUnit(self)
}
else if( type == RateUnit ) {
return new RateUnit(self)
}
else if ( type == URL ) {
return new URL(self)
}
else if ( type == URI ) {
return URI.create(self)
}
StringGroovyMethods.asType(self, type);
}
/**
* Converts a {@code GString} to a {@code Duration} object
*
* @param self
* @param type
* @return
*/
static Object asType( GString self, Class<Object> type ) {
if( type == Duration ) {
return new Duration(self.toString())
}
else if( Path.isAssignableFrom(type) ) {
return FileHelper.asPath(self)
}
else if( type == MemoryUnit ) {
return new MemoryUnit(self.toString())
}
else if ( type == URL ) {
return new URL(self.toString())
}
else if ( type == URI ) {
return URI.create(self.toString())
}
StringGroovyMethods.asType(self, type);
}
/**
* Converts a {@code Number} to a {@code Duration} object
*
* @param self
* @param type
* @return
*/
static Object asType( Number self, Class type ) {
if( type == Duration ) {
return new Duration(self.longValue())
}
if( type == MemoryUnit ) {
return new MemoryUnit(self.longValue())
}
DefaultGroovyMethods.asType(self, type);
}
/**
* Converts a {@code File} to a {@code Path} object
*
* @param self
* @param type
* @return
*/
static Object asType( File self, Class<Object> type ) {
if( Path.isAssignableFrom(type) ) {
return self.toPath()
}
ResourceGroovyMethods.asType(self, type);
}
static <K,V> V getOrCreate( Map<K,V> self, K key, @ClosureParams(FirstParam.FirstGenericType) Closure <V> value ) {
getOrCreate0(self,key,value)
}
static <K,V> V getOrCreate( Map<K,V> self, K key, V value ) {
getOrCreate0(self,key,value)
}
static private <K,V> V getOrCreate0(Map<K,V> self, K key, value) {
if( self.containsKey(key) )
return self.get(key)
synchronized (self) {
if( self.containsKey(key) )
return self.get(key)
final result = (V)(value instanceof Closure ? value.call(key) : value)
self.put(key,result)
return result
}
}
/**
* Navigate a map of maps traversing multiple attribute using dot notation. For example:
* {@code x.y.z }
*
* @param self The root map object
* @param key A dot separated list of keys
* @param closure An optional closure to be applied. Only if all keys exists
* @return The value associated to the specified key(s) or null on first missing entry
*/
static def navigate( Map self, String key, Closure closure = null ) {
assert key
def items = key.split(/\./)
def current = self.get(items[0])
for( int i=1; i<items.length; i++ ) {
if( current instanceof Map ) {
if( current.containsKey(items[i]))
current = current.get(items[i])
else
return null
}
else if( !current ) {
return null
}
else {
throw new IllegalArgumentException("Cannot navigate map attribute: '$key' -- Content: $self")
}
}
return closure ? closure(current) : current
}
static def navigate(Map self, String key, defValue) {
def result = navigate(self,key)
return result!=null ? result : defValue
}
/**
* Converts {@code ConfigObject}s to a plain {@code Map}
*
* @param config
* @return A normalized config object
*/
static Map toMap( ConfigObject config ) {
assert config != null
(Map)normalize0((Map)config)
}
static ConfigObject toConfigObject(Map self) {
def result = new ConfigObject()
self.each { key, value ->
if( value instanceof Map ) {
result.put( key, toConfigObject((Map)value) )
}
else {
result.put( key, value )
}
}
return result
}
static private normalize0( config ) {
if( config instanceof Map ) {
Map result = new LinkedHashMap(config.size())
config.keySet().each { name ->
def value = (config as Map).get(name)
result.put(name, normalize0(value))
}
return result
}
else if( config instanceof Collection ) {
List result = new ArrayList(config.size())
for( entry in config ) {
result << normalize0(entry)
}
return result
}
else if( config instanceof GString ) {
return config.toString()
}
else {
return config
}
}
/**
* Indent each line in the given test by a specified prefix
*
* @param text
* @param prefix
* @return The string indented
*/
static String indent( String text, String prefix = ' ' ) {
def result = new StringBuilder()
def lines = text ? text.readLines() : Collections.emptyList()
for( int i=0; i<lines.size(); i++ ) {
result << prefix
result << lines.get(i)
result << '\n'
}
return result.toString()
}
/**
* Find all the best matches for the given example string in a list of values
*
* @param options A list of string
* @param sample The example string -- cannot be empty
* @return The list of options that best matches to the specified example -- return an empty list if none match
*/
static List<String> bestMatches( Collection<String> options, String sample ) {
assert sample
assert options
// Otherwise look for the most similar
Map<String,Integer> diffs = [:]
options.each {
diffs[it] = StringUtils.getLevenshteinDistance(sample, it)
}
// sort the Levenshtein Distance and get the fist entry
def sorted = diffs.sort { Map.Entry<String,Integer> it -> it.value }
def nearest = (Map.Entry<String,Integer>)sorted.find()
int min = nearest.value
int len = sample.length()
int threshold = len<=3 ? 1 : ( len > 10 ? 5 : Math.floorDiv(len,2))
List<String> result
if( min <= threshold ) {
result = (List<String>)sorted.findAll { it.value==min } .collect { it.key }
}
else {
result = []
}
return result
}
static boolean isCamelCase(String str) {
if( !str ) return false
for( int i=0; i<str.size()-1; i++ )
if( Character.getType(str.charAt(i)) == Character.LOWERCASE_LETTER && Character.getType(str.charAt(i+1)) == Character.UPPERCASE_LETTER)
return true
return false
}
/**
* Clone a {@link Closure} object and set the specified map as delegate object
* in the resulting closure
*
* @param self The {@link Closure} object to be cloned
* @param binding The delegate object to set in the new closure
* @return The cloned {@link Closure} object
*/
static <T extends Closure> T cloneWith( T self, binding ) {
def copy = (Closure)self.clone()
if( binding != null ) {
copy.setDelegate(binding)
copy.setResolveStrategy( Closure.DELEGATE_FIRST )
}
return (T)copy
}
/**
* Create a copy of a {@link GString} object cloning all values that are instance of {@link Closure}
*
* @param self The {@link GString} itself
* @param binding A {@link Map} object that is set as delegate object in the cloned closure.
* @return The cloned {@link GString} instance
*/
static GString cloneAsLazy(GString self, binding) {
def values = new Object[ self.valueCount ]
// clone the GString setting the delegate for each closure argument
for( int i=0; i<self.valueCount; i++ ) {
values[i] = ( self.values[i] instanceof Closure
? cloneWith(self.values[i] as Closure, binding)
: self.values[i] )
}
new GStringImpl(values, self.strings)
}
/**
* Find all the best matches for the given example string in a list of values
*
* @param sample The example string -- cannot be empty
* @param options A list of string
* @return The list of options that best matches to the specified example -- return an empty list if none match
*/
@CompileDynamic
static List<String> closest(Collection<String> options, String sample ) {
assert sample
if( !options )
return Collections.emptyList()
// Otherwise look for the most similar
def diffs = [:]
options.each {
diffs[it] = StringUtils.getLevenshteinDistance(sample, it)
}
// sort the Levenshtein Distance and get the fist entry
def sorted = diffs.sort { it.value }
def nearest = sorted.find()
def min = nearest.value
def len = sample.length()
def threshold = len<=3 ? 1 : ( len > 10 ? 5 : Math.floor(len/2))
def result
if( min <= threshold ) {
result = sorted.findAll { it.value==min } .collect { it.key }
}
else {
result = []
}
return result
}
private static HashMap<Object,Long> LOGGER_CACHE = new LinkedHashMap<Object,Long>() {
protected boolean removeEldestEntry(Map.Entry<Object, Long> eldest) {
return size() > 10_000
}
}
private static final Duration LOG_DFLT_THROTTLE = Duration.of('1min')
static synchronized private checkLogCache( Object msg, Map params, Closure action ) {
// -- check if this message has already been printed
final String str = msg.toString()
final Throwable error = params?.causedBy as Throwable
final Duration throttle = params?.throttle as Duration ?: LOG_DFLT_THROTTLE
final firstOnly = params?.firstOnly == true
final key = params?.cacheKey ?: str
long now = System.currentTimeMillis()
Long ts = LOGGER_CACHE.get(key)
if( ts && (now - ts <= throttle.toMillis() || firstOnly) ) {
return
}
LOGGER_CACHE.put(key, now)
action.call(str, error)
}
private static Map<String,?> LOGGER_PARAMS = [ cacheKey: Object, causedBy: Throwable, throttle: [String, Number, Duration], firstOnly: Boolean ]
/**
* Append a `trace` level entry in the application log.
*
* @param log
* The {@link Logger} object
* @param params
* Optional named parameters
* - `causedBy`: A {@link Throwable} object that raised the error
* - `throttle`: When specified suppress identical logs within the specified time {@link Duration}
* @param msg
* The message to print
*
*/
static void trace1(Logger log, Map params=null, Object msg ) {
CheckHelper.checkParams('trace1', params, LOGGER_PARAMS)
if( !log.isTraceEnabled() || msg==null ) return
checkLogCache(msg,params) { String str, Throwable t -> t ? log.trace(str,t) : log.trace(str) }
}
/**
* Append a `debug` level entry in the application log.
*
* @param log
* The {@link Logger} object
* @param params
* Optional named parameters
* - `causedBy`: A {@link Throwable} object that raised the error
* - `throttle`: When specified suppress identical logs within the specified time {@link Duration}
* @param msg
* The message to print
*
*/
static void debug1(Logger log, Map params=null, Object msg ) {
CheckHelper.checkParams('debug1', params, LOGGER_PARAMS)
if( !log.isDebugEnabled() || msg==null ) return
checkLogCache(msg,params) { String str, Throwable t -> t ? log.debug(str,t) : log.debug(str) }
}
/**
* Append a `info` level entry in the application log.
*
* @param log
* The {@link Logger} object
* @param params
* Optional named parameters
* - `causedBy`: A {@link Throwable} object that raised the error
* - `throttle`: When specified suppress identical logs within the specified time {@link Duration}
* @param msg
* The message to print
*
*/
static void info1(Logger log, Map params=null, Object msg ) {
CheckHelper.checkParams('info1', params, LOGGER_PARAMS)
if( !log.isInfoEnabled() || msg==null ) return
checkLogCache(msg,params) { String str, Throwable t -> t ? log.info(str,t) : log.info(str) }
}
/**
* Append a `warn` level entry in the application log.
*
* @param log
* The {@link Logger} object
* @param params
* Optional named parameters
* - `causedBy`: A {@link Throwable} object that raised the error
* - `throttle`: When specified suppress identical logs within the specified time {@link Duration}
* @param msg
* The message to print
*
*/
static void warn1(Logger log, Map params=null, Object msg ) {
CheckHelper.checkParams('warn1', params, LOGGER_PARAMS)
if( !log.isWarnEnabled() || msg==null ) return
checkLogCache(msg,params) { String str, Throwable t -> t ? log.warn(str,t) : log.warn(str) }
}
/**
* Append a `error` level entry in the application log.
*
* @param log
* The {@link Logger} object
* @param params
* Optional named parameters
* - `causedBy`: A {@link Throwable} object that raised the error
* - `throttle`: When specified suppress identical logs within the specified time {@link Duration}
* @param msg
* The message to print
*
*/
static void error1(Logger log, Map params=null, Object msg ) {
CheckHelper.checkParams('error1', params, LOGGER_PARAMS)
if( !log.isErrorEnabled() || msg==null ) return
checkLogCache(msg,params) { String str, Throwable t -> t ? log.error(str,t) : log.error(str) }
}
static void trace(Logger log, Object msg) {
if( log.isTraceEnabled() ) {
log.trace(msg.toString())
}
}
static void trace(Logger log, Object msg, Throwable e) {
if( log.isTraceEnabled() ) {
log.trace(msg.toString(),e)
}
}
@Deprecated
static redact(String self, int max=5, String suffix='...') {
if( !self )
return self
if( self.size()<max )
return suffix
else
return self.substring(0,Math.min(self.size()-max, max)) + suffix
}
protected static <T extends Serializable> T deepClone0(T obj) {
final buffer = new ByteArrayOutputStream()
final oos = new ObjectOutputStream(buffer)
oos.writeObject(obj)
oos.flush()
final inputStream = new ByteArrayInputStream(buffer.toByteArray())
return (T) new ObjectInputStream(inputStream).readObject()
}
static <T extends Map> T deepClone(T map) {
if( map == null)
return null
final result = map instanceof LinkedHashMap ? new LinkedHashMap<>(map) : new HashMap<>(map)
for( def key : map.keySet() ) {
def value = map.get(key)
if( value instanceof Map ) {
result.put(key, deepClone(value))
}
}
return (T)result
}
static Map deepMerge(Map target, Map source) {
final result = cloneMap0(target)
for (Object name : source.keySet()) {
// to prevent side-effects with ConfigObject object (which creates a value on-fly
// when getting it, always use `containsKey` before
if( result.containsKey(name) && result.get(name) instanceof Map && source.containsKey(name) && source.get(name) instanceof Map ) {
result.put(name, deepMerge( (Map)result.get(name), (Map)source.get(name)))
}
else {
result.put(name, source.get(name))
}
}
return result
}
static private Map cloneMap0(Map map) {
if( map instanceof ConfigObject )
return ((ConfigObject)map).clone()
if( map instanceof LinkedHashMap )
return new LinkedHashMap<>(map)
else
return new HashMap<>(map)
}
/**
* Resolve a lazy expression (e.g. closure, gstring) against
* a delegate (i.e. binding).
*
* @param binding
* @param value
*/
static Object resolveLazy(Object binding, Object value) {
if( value instanceof Closure )
return cloneWith(value, binding).call()
if( value instanceof GString )
return cloneAsLazy(value, binding).toString()
return value
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,194 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.file;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.AtomicMoveNotSupportedException;
import java.nio.file.CopyOption;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.FileVisitOption;
import java.nio.file.FileVisitResult;
import java.nio.file.FileVisitor;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.StandardCopyOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.EnumSet;
import nextflow.extension.FilesEx;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Helper class to handle copy/move files and directories
*/
public class CopyMoveHelper {
/**
* True if currently performing a copy of a foreign file.
*/
public static final ThreadLocal<Boolean> IN_FOREIGN_COPY = new ThreadLocal<>();
private static Logger log = LoggerFactory.getLogger(CopyMoveHelper.class);
private CopyMoveHelper() { }
/**
* Converts the given array of options for moving a file to options suitable
* for copying the file when a move is implemented as copy + delete.
*/
private static CopyOption[] convertMoveToCopyOptions(CopyOption... options)
throws AtomicMoveNotSupportedException
{
int len = options.length;
CopyOption[] newOptions = new CopyOption[len+1];
for (int i=0; i<len; i++) {
CopyOption option = options[i];
if (option == StandardCopyOption.ATOMIC_MOVE) {
throw new AtomicMoveNotSupportedException(null, null,
"Atomic move between providers is not supported");
}
newOptions[i] = option;
}
newOptions[len] = LinkOption.NOFOLLOW_LINKS;
return newOptions;
}
/**
* Copy a file to a local or foreign file system
*
* @param source The source file path
* @param target The target file path
* @param foreign When {@code true} create a copy on the remote file system
* @param options Copy options
* @throws IOException
*/
private static void copyFile(Path source, Path target, boolean foreign, CopyOption... options)
throws IOException
{
if( !foreign ) {
source.getFileSystem().provider().copy(source, target, options);
return;
}
IN_FOREIGN_COPY.set(true);
try (InputStream in = Files.newInputStream(source)) {
Files.copy(in, target);
} finally {
IN_FOREIGN_COPY.set(false);
}
}
/**
* Create a directory content copy
*
* @param source The source directory to copy
* @param target The target path where directory will be copied
* @param options Copy options
* @throws IOException
*/
static void copyDirectory( final Path source, final Path target, final CopyOption... options )
throws IOException
{
final boolean foreign = source.getFileSystem().provider() != target.getFileSystem().provider();
FileVisitor<Path> visitor = new SimpleFileVisitor<Path>() {
public FileVisitResult preVisitDirectory(Path current, BasicFileAttributes attr)
throws IOException
{
// get the *delta* path against the source path
Path rel = source.relativize(current);
String delta = rel != null ? rel.toString() : null;
Path newFolder = delta != null ? target.resolve(delta) : target;
if(log.isTraceEnabled())
log.trace("Copy DIR: " + current + " -> " + newFolder);
// this `copy` creates the new folder, but does not copy the contained files
Files.createDirectory(newFolder);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(Path current, BasicFileAttributes attr)
throws IOException
{
// get the *delta* path against the source path
Path rel = source.relativize(current);
String delta = rel != null ? rel.toString() : null;
Path newFile = delta != null ? target.resolve(delta) : target;
if( log.isTraceEnabled())
log.trace("Copy file: " + current + " -> "+ FilesEx.toUriString(newFile));
copyFile(current, newFile, foreign, options);
return FileVisitResult.CONTINUE;
}
};
Files.walkFileTree(source, EnumSet.of(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, visitor);
}
/**
* Simple copy for use when source and target are associated with different
* providers
*/
static void copyToForeignTarget(Path source, Path target, CopyOption... options)
throws IOException
{
CopyOptions opts = CopyOptions.parse(options);
LinkOption[] linkOptions = (opts.followLinks()) ? new LinkOption[0] : new LinkOption[] { LinkOption.NOFOLLOW_LINKS };
// attributes of source file
BasicFileAttributes attrs = Files.readAttributes(source, BasicFileAttributes.class, linkOptions);
if (attrs.isSymbolicLink())
throw new IOException("Copying of symbolic links not supported");
// delete target if it exists and REPLACE_EXISTING is specified
if (opts.replaceExisting()) {
FileHelper.deletePath(target);
}
else if (Files.exists(target))
throw new FileAlreadyExistsException(target.toString());
// create directory or copy file
if (attrs.isDirectory()) {
copyDirectory(source, target);
}
else {
copyFile(source, target, true);
}
}
/**
* Simple move implements as copy+delete for use when source and target are
* associated with different providers
*/
static void moveToForeignTarget(Path source, Path target, CopyOption... options)
throws IOException
{
copyToForeignTarget(source, target, convertMoveToCopyOptions(options));
FileHelper.deletePath(source);
}
}

View File

@@ -0,0 +1,64 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.file;
import java.nio.file.CopyOption;
import java.nio.file.LinkOption;
import java.nio.file.StandardCopyOption
import groovy.transform.Canonical
import groovy.transform.CompileStatic;
/**
* Parses the arguments for a file copy operation.
*/
@CompileStatic
@Canonical
class CopyOptions {
private boolean replaceExisting = false;
private boolean copyAttributes = false;
private boolean followLinks = true;
boolean replaceExisting() { replaceExisting }
boolean copyAttributes() { copyAttributes }
boolean followLinks() { followLinks }
private CopyOptions() { }
static public CopyOptions parse(CopyOption... options) {
CopyOptions result = new CopyOptions();
for (CopyOption option: options) {
if (option == StandardCopyOption.REPLACE_EXISTING) {
result.replaceExisting = true;
continue;
}
if (option == LinkOption.NOFOLLOW_LINKS) {
result.followLinks = false;
continue;
}
if (option == StandardCopyOption.COPY_ATTRIBUTES) {
result.copyAttributes = true;
continue;
}
if (option == null)
throw new NullPointerException();
throw new UnsupportedOperationException("'" + option +
"' is not a recognized copy option");
}
return result;
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,113 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.file
import java.nio.file.FileSystems
import java.nio.file.Path
import com.google.common.hash.Hasher
import groovy.transform.CompileStatic
import groovy.transform.EqualsAndHashCode
import groovy.transform.PackageScope
import groovy.transform.ToString
import groovy.util.logging.Slf4j
import nextflow.util.CacheFunnel
import nextflow.util.CacheHelper
import nextflow.util.CacheHelper.HashMode
/**
* Implements a special {@code Path} used to stage files in the work area
*/
@Slf4j
@ToString(includePackage = false, includeNames = true)
@EqualsAndHashCode
@CompileStatic
class FileHolder implements CacheFunnel {
final def sourceObj
final Path storePath
final String stageName
FileHolder( Path path ) {
this.sourceObj = path
this.storePath = real(path)
this.stageName = norm(path.getFileName())
}
FileHolder( def origin, Path path ) {
this.sourceObj = origin
this.storePath = real(path)
this.stageName = norm(path.getFileName())
}
protected FileHolder( def source, Path store, def stageName ) {
this.sourceObj = source
this.storePath = store
this.stageName = norm(stageName)
}
FileHolder withName( def stageName ) {
new FileHolder( this.sourceObj, this.storePath, stageName )
}
Path getSourcePath() {
sourceObj instanceof Path ? sourceObj : null
}
Path getStorePath() { storePath }
String getStageName() { stageName }
@Override
Hasher funnel(Hasher hasher, HashMode mode) {
return CacheHelper.hasher(hasher, sourceObj, mode)
}
@PackageScope
static FileHolder get( def path, def name = null ) {
Path storePath = path as Path
def target = name ? name : storePath.getFileName()
new FileHolder( path, storePath, target )
}
/**
* Make sure the stage name does not start with a slash character
*
* @param path
* @return The normalised path
*/
static private String norm(path) {
def result = path.toString()
return result.startsWith('/') ? result.substring(1) : result
}
static private Path real( Path path ) {
try {
// main reason for this is to resolve symlinks to real file location
// hence apply only for default file system
// note: also for Google Cloud storage path it may convert to relative path
// it may return invalid (relative) paths therefore do not apply it
return path.getFileSystem() == FileSystems.default ? path.toRealPath() : path
}
catch( Exception e ) {
log.trace "Unable to get real path for: $path"
return path
}
}
}

View File

@@ -0,0 +1,117 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.file
import java.nio.channels.FileLock
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.util.Duration
/**
* Implements a mutual exclusive file lock strategy to prevent
* two or more JVMs/process to access the same file or resource
*
* NOTE: it should NOT be used to synchronise concurrent from different
* threads in the same JVM
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
class FileMutex {
String waitMessage
String errorMessage
File target
long delayMillis = 100
private long timeoutMillis
FileMutex() { }
FileMutex(File target) {
this.target = target
}
FileMutex setTimeout( value ) {
if( value instanceof Duration )
timeoutMillis = value.millis
else if( value instanceof CharSequence )
timeoutMillis = Duration.of(value.toString()).millis
else if( value instanceof Number )
timeoutMillis = value.longValue()
else if( value != null )
throw new IllegalArgumentException("Not a valid Duration value: $value [${value.class.name}]")
return this
}
FileMutex setWaitMessage( String str ) {
this.waitMessage = str
return this
}
FileMutex setErrorMessage( String str ) {
this.errorMessage = str
return this
}
def <T> T lock(Closure<T> closure) {
assert target
final file = new RandomAccessFile(target, "rw")
try {
/*
* wait to acquire a lock
*/
boolean showed=0
long max = timeoutMillis ? timeoutMillis : Long.MAX_VALUE
long begin = System.currentTimeMillis()
FileLock lock
while( !(lock=file.getChannel().tryLock()) ) {
if( waitMessage && !showed ) {
log.info waitMessage
showed = true
}
if( System.currentTimeMillis()-begin > max )
throw new InterruptedException("Cannot acquire FileLock on $target")
sleep delayMillis
}
/*
* now it can do the job
*/
try {
return closure.call()
}
finally {
lock.close()
}
}
finally {
file.close()
}
}
}

View File

@@ -0,0 +1,255 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.file
import java.nio.file.Path
import java.util.regex.Pattern
import groovy.transform.CompileStatic
/**
* Parse a file path to isolate the parent, file-name and whenever it contains a glob|regex pattern
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class FilePatternSplitter {
static final public FilePatternSplitter GLOB = glob()
static enum Syntax { GLOB, REGEX }
static final public Pattern QUESTION_MARK_REGEX = ~/(?<!\\)\?/
static final public Pattern STAR_CHAR_REGEX = ~/(?<!\\)\*/
static final public Pattern GLOB_CURLY_BRACKETS = Pattern.compile(/(.*)(?<!\\)(\{.*,.*\})(.*)/)
static final public Pattern GLOB_SQUARE_BRACKETS = Pattern.compile(/(.*)(?<!\\)(\[.+\])(.*)/)
static final private char BACK_SLASH = '\\' as char
static final private String GLOB_CHARS = '*?[]{}'
static final private String REGEX_CHARS = '.^$+{}[]|()'
private boolean pattern
private final Syntax syntax
private String scheme
private String parent
private String fileName
boolean isPattern() { pattern }
String getFileName() { fileName }
String getParent() { parent }
String getScheme() { scheme }
static FilePatternSplitter glob() { new FilePatternSplitter(Syntax.GLOB) }
static FilePatternSplitter regex() { new FilePatternSplitter(Syntax.REGEX) }
static boolean isMatchingPattern(pattern) {
if( !pattern )
return false
if( pattern instanceof Pattern )
return true
def str = pattern.toString()
if( STAR_CHAR_REGEX.matcher(str).find() )
return true
if( QUESTION_MARK_REGEX.matcher(str).find() )
return true
if( GLOB_CURLY_BRACKETS.matcher(str).find() )
return true
if( GLOB_SQUARE_BRACKETS.matcher(str).find() )
return true
return false
}
FilePatternSplitter( Syntax syntax ) {
this.syntax = syntax
}
private String metaChars() {
syntax == Syntax.GLOB ? GLOB_CHARS : REGEX_CHARS
}
private boolean containsMetaChars(String str) {
final meta = metaChars()
for( int i=0; i<str.length(); i++ ) {
if( meta.contains(str[i]) )
return true
}
return false
}
/**
* Parse a file path detecting the schema, parent folder, file name and pattern components
*
* @param filePath The file path string to parse
* @return
*/
FilePatternSplitter parse(String filePath) {
// -- detected the file scheme if any
int p = filePath.indexOf('://')
if( p != -1 ) {
scheme = filePath.substring(0, p)
filePath = filePath.substring(p+3)
}
else {
scheme = null
}
//
// split the path in two components
// - folder: the part not containing meta characters
// - pattern: the part containing a meta character
boolean found
String norm = replaceMetaChars(filePath)
p = firstMetaIndex(norm)
if( p == -1 ) {
// find the last SLASH
p = filePath.lastIndexOf('/')
found = false
}
else {
found = true
// walk back to the first SLASH char
int i = p
p = -1
while( --i >= 0 ) {
if( filePath[i] == '/' ) {
p = i
break
}
}
}
if( p == -1 ) {
parent = './'
fileName = filePath
pattern = found && pairedBrackets(norm)
}
else {
parent = strip(filePath.substring(0,p+1))
fileName = filePath.substring(p+1)
pattern = found && pairedBrackets(norm)
}
return this
}
private boolean pairedBrackets(String str) {
if( syntax == Syntax.REGEX )
return true
if( str.contains('{') || str.contains('}') )
return GLOB_CURLY_BRACKETS.matcher(str).matches()
if( str.contains('[') || str.contains(']') )
return GLOB_SQUARE_BRACKETS.matcher(str).matches()
return true
}
protected String replaceMetaChars( String str, char marker = 0x0 ) {
// create a version replacing escape meta chars with an 0x0
final meta = metaChars()
final result = new StringBuilder()
int i=0;
while( i<str.length() ) {
def ch = str.charAt(i++)
if( ch == BACK_SLASH && i<str.length() && meta.contains(str[i])) {
result.append(ch).append(marker)
i++
}
else {
result.append(ch)
}
}
return result
}
protected int firstMetaIndex(String str) {
// find the index of the first meta chars
final meta = metaChars()
def min = Integer.MAX_VALUE
for( int i=0; i<meta.length(); i++ ) {
def p = str.indexOf(meta[i])
if( p!=-1 && p<min )
min = p
}
return min != Integer.MAX_VALUE ? min : -1
}
/**
* Strips backslash characters from a path
*/
String strip( String str ) {
int p = str.indexOf('\\')
if( p == -1 )
return str
final meta = metaChars()
final result = new StringBuilder()
int i=0;
while( i<str.length() ) {
def ch = str.charAt(i++)
if( ch != BACK_SLASH || i==str.length() || !meta.contains(str[i])) {
result.append(ch)
}
}
return result.toString()
}
/**
* Escape pattern meta-characters with a backslash
*/
String escape( Path path ) {
escape(path.toString())
}
/**
* Escape pattern meta-characters with a backslash
*/
String escape( String str ) {
final meta = metaChars()
def result = str
for( int i=0; i<meta.length(); i++ ) {
result = result.replace( meta[i], '\\' + meta[i] )
}
return result
}
}

View File

@@ -0,0 +1,111 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.file
import java.nio.file.Path
import groovy.transform.CompileStatic
import groovy.transform.stc.ClosureParams
import groovy.transform.stc.SimpleType
import groovy.util.logging.Slf4j
import nextflow.plugin.Plugins
import org.pf4j.ExtensionPoint
/**
* Define extension methods for supporting pluggable file remote file systems e.g. AWS S3 or Google Storage
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
abstract class FileSystemPathFactory implements ExtensionPoint {
/**
* Converts path uri string to the corresponding {@link Path} object
*
* @param uri
* A fully qualified path uri including the protocol scheme
* @return
* A {@link Path} for the given path or {@code null} if protocol is unknown
*/
abstract protected Path parseUri(String uri)
/**
* Converts a {@link Path} object to a fully qualified uri string
*
* @param path
* The {@link Path} object to be converted
* @return
* The uri string corresponding to the specified path or {@code null}
* if the no provider is found
*/
abstract protected String toUriString(Path path)
/**
* Bash helper library that implements the support for third-party storage
* to be included in the job command wrapper script
*
* @return The Bash snippet implementing the support for third-party such as AWS S3
* or {@code null} if not supported
*/
abstract protected String getBashLib(Path target)
/**
* The name of a Bash helper function to upload a file to a remote file storage
*
* @return The name of the upload function or {@code null} if not supported
*/
abstract protected String getUploadCmd(String source, Path target)
static Path parse(String uri) {
lookup0 { it.parseUri(uri) }
}
static String getUriString(Path path) {
lookup0 { it.toUriString(path) }
}
static String bashLib(Path target) {
lookup0 { it.getBashLib(target) }
}
static String uploadCmd(String source, Path target) {
lookup0 { it.getUploadCmd(source, target) }
}
private static List<FileSystemPathFactory> factories0() {
final factories = new ArrayList(10)
final itr = Plugins.getPriorityExtensions(FileSystemPathFactory).iterator()
while( itr.hasNext() )
factories.add(itr.next())
log.trace "File system path factories: ${factories}"
return factories
}
private static <T> T lookup0( @ClosureParams(value = SimpleType, options = ['nextflow.file.FileSystemPathFactory']) Closure<T> criteria) {
final factories = factories0()
for( int i=0; i<factories.size(); i++ ) {
final FileSystemPathFactory it = factories[i]
final result = criteria.call(it)
if( result!=null )
return result
}
return null
}
}

View File

@@ -0,0 +1,37 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.file
import java.nio.file.CopyOption
import java.nio.file.Path
/**
* @author : jorge <jorge.aguilera@seqera.io>
*
*/
interface FileSystemTransferAware {
boolean canUpload(Path source, Path target)
boolean canDownload(Path source, Path target)
void download(Path source, Path target, CopyOption... options) throws IOException
void upload(Path source, Path target, CopyOption... options) throws IOException
}

View File

@@ -0,0 +1,43 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.file
import java.lang.reflect.Method
import groovy.transform.CompileStatic
/**
* Helper class to access {@link sun.nio.fs.Globs#toRegexPattern(java.lang.String, boolean)} method
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class Globs {
final private static Method toRegexPattern
static {
def clazz = sun.nio.fs.Globs
toRegexPattern = clazz.getDeclaredMethod('toRegexPattern', String, boolean)
toRegexPattern.setAccessible(true)
}
static String toUnixRegexPattern(String globPattern) {
toRegexPattern.invoke(null, globPattern, false)
}
}

View File

@@ -0,0 +1,64 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.file
import java.nio.file.LinkOption
import java.nio.file.Path
import java.nio.file.Paths
import org.codehaus.groovy.runtime.InvokerHelper
/**
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Deprecated
class StagePath {
final Path target
StagePath( Path source ) {
this.target = source
}
StagePath( String fileName ) {
this.target = Paths.get(fileName)
}
Object getProperty( String name ) {
InvokerHelper.getProperty(target,name)
}
void setProperty(String property, Object newValue) {
InvokerHelper.setProperty(target, property, newValue)
}
Object invokeMethod(String name, Object args) {
InvokerHelper.invokeMethod(target, name, args)
}
Path getFileName() {
target.getFileName()
}
Path toRealPath(LinkOption... options) {
return target
}
String toString() {
target.getFileName().toString()
}
}

View File

@@ -0,0 +1,31 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.file
/**
* Defines the interface for annotate a file with one or more tags
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
interface TagAwareFile {
void setTags(Map<String,String> tags)
void setContentType(String type)
void setStorageClass(String storageClass)
}

View File

@@ -0,0 +1,99 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.io
import java.nio.file.Path
import groovy.transform.CompileStatic
import groovy.transform.EqualsAndHashCode
import groovy.transform.ToString
import nextflow.util.StringUtils
/**
* Parse a cloud bucket uri and decompose in scheme, bucket and path
* components eg. s3://foo/some/file.txt
* - scheme: s3
* - bucket: foo
* - path : /some/file.txt
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
@ToString(includeNames = true, includePackage = false, ignoreNulls=false)
@EqualsAndHashCode(includeFields = true)
@CompileStatic
class BucketParser {
private String scheme
private String bucket
private Path path
String getScheme() { scheme }
String getBucket() { bucket }
Path getPath() { path }
protected BucketParser(String s, String b, String p) {
this.scheme = s
this.bucket = b
this.path = p
? (p.startsWith('/') || !bucket ? Path.of(p) : Path.of('/'+p) )
: Path.of('/')
}
protected BucketParser() {
this.path = Path.of('/')
}
static BucketParser from(String uri) {
new BucketParser().parse(uri)
}
BucketParser parse(String uri) {
final m = StringUtils.URL_PROTOCOL.matcher(uri)
if( !m.matches() ) {
path = Path.of(uri)
return this
}
this.scheme = m.group(1)
final location = m.group(2)
final p = location.indexOf('/')
if( p==-1 ) {
bucket = location
path = Path.of('/')
}
else {
bucket = location.substring(0,p)
path = Path.of(location.substring(p))
}
if( bucket.startsWith('/') || bucket.endsWith('/') )
throw new IllegalArgumentException("Invalid bucket URI path: $uri")
return this
}
/**
* @return The object key, essentially the same as {@link #path} stripping the leading slash character
*/
String getKey() {
final result = path.toString()
return result.substring(1)
}
}

View File

@@ -0,0 +1,54 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.io
import java.nio.ByteBuffer
import groovy.transform.CompileStatic
/**
* An {@code InputStream} adaptor which reads data from a {@code ByteBuffer}
*
* See http://stackoverflow.com/a/6603018/395921
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class ByteBufferBackedInputStream extends InputStream {
ByteBuffer buf;
ByteBufferBackedInputStream(ByteBuffer buf) {
this.buf = buf;
}
int read() throws IOException {
if (!buf.hasRemaining()) {
return -1;
}
return buf.get() & 0xFF;
}
int read(byte[] bytes, int off, int len) throws IOException {
if (!buf.hasRemaining()) {
return -1;
}
len = Math.min(len, buf.remaining());
buf.get(bytes, off, len);
return len;
}
}

View File

@@ -0,0 +1,127 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.io
import java.util.concurrent.CountDownLatch
import java.util.concurrent.TimeUnit
import groovy.util.logging.Slf4j
/**
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
class ByteDumper extends Thread {
InputStream fInputStream
boolean fTerminated
CountDownLatch barrier = new CountDownLatch(1)
Closure fCallback
File fInputFile
ByteDumper(InputStream input0, Closure callback0 ) {
assert input0
this.fInputStream = new BufferedInputStream(input0)
this.fCallback = callback0
setDaemon(true)
}
ByteDumper( File file0, Closure callback0 ) {
assert file0
this.fInputFile = file0
this.fCallback = callback0
setDaemon(true)
}
/**
* Interrupt the dumper thread
*/
def void terminate() { fTerminated = true }
/**
* Await that the thread finished to read the process stdout
*
* @param millis Maximum time (in millis) to await
*/
def void await(long millis=0) {
if( millis ) {
barrier.await(millis, TimeUnit.MILLISECONDS)
}
else {
barrier.await()
}
}
@Override
def void run() {
try {
consume()
}
finally{
if( fInputStream ) fInputStream.closeQuietly()
barrier.countDown()
}
}
def void consume() {
// -- if a file reference has been provided instead of stream
// wait for the file to exist
while( fInputStream == null && fInputFile !=null ) {
if( fTerminated ) {
log.trace "consume '${getName()}' -- terminated"
return
}
if( fInputFile.exists() ) {
log.trace "consume '${getName()}' -- ${fInputFile} exists: true"
fInputStream = new BufferedInputStream(new FileInputStream(fInputFile))
}
else {
sleep 200
}
}
if ( !fCallback ) {
log.trace "consume '${getName()}' -- no callback"
return
}
byte[] buf = new byte[8192]
int next
try {
while ((next = fInputStream.read(buf)) != -1 && !fTerminated) {
log.trace "consume '${getName()}' -- reading "
fCallback.call(buf, next)
}
log.trace "consume '${getName()}' -- exit -- terminated: ${fTerminated}"
} catch (IOException e) {
throw new RuntimeException("exception while dumping process stream", e)
}
}
}

View File

@@ -0,0 +1,46 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.io
import groovy.transform.CompileStatic
/**
* Implements a {@link InputStream} object backed on a {@link DataInput} instance
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class DataInputStreamAdapter extends InputStream {
final private DataInput target
DataInputStreamAdapter( DataInput input ) { this.target = input }
@Override
int read() throws IOException {
try {
return target.readUnsignedByte()
}
catch( EOFException | IndexOutOfBoundsException e ) {
return -1
}
}
long skip(long n) {
target.skipBytes(n as int)
}
}

View File

@@ -0,0 +1,36 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.io
import groovy.transform.CompileStatic
/**
* Implements a {@link OutputStream} object backed on a {@link DataOutput} instance
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class DataOutputStreamAdapter extends OutputStream {
final private DataOutput target
DataOutputStreamAdapter( DataOutput out ) { target = out }
@Override
void write(int b) throws IOException { target.write(b) }
}

View File

@@ -0,0 +1,187 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.io;
/*
* This source is a derivation of the
* https://commons.apache.org/proper/commons-exec/apidocs/src-html/org/apache/commons/exec/LogOutputStream.html
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
/**
* Base class to connect a logging system to the output and/or
* error stream of then external process. The implementation
* parses the incoming data to construct a line and passes
* the complete line to an user-defined implementation.
*
* @version $Id: LogOutputStream.java 1636056 2014-11-01 21:12:52Z ggregory $
*/
public abstract class LogOutputStream
extends OutputStream {
/** Initial buffer size. */
private static final int INTIAL_SIZE = 132;
/** Carriage return */
private static final int CR = 0x0d;
/** Linefeed */
private static final int LF = 0x0a;
/** the internal buffer */
private final ByteArrayOutputStream buffer = new ByteArrayOutputStream(
INTIAL_SIZE);
private boolean skip = false;
private final int level;
/**
* Creates a new instance of this class.
* Uses the default level of 999.
*/
public LogOutputStream() {
this(999);
}
/**
* Creates a new instance of this class.
*
* @param level loglevel used to log data written to this stream.
*/
public LogOutputStream(final int level) {
this.level = level;
}
/**
* Write the data to the buffer and flush the buffer, if a line separator is
* detected.
*
* @param cc data to log (byte).
* @see java.io.OutputStream#write(int)
*/
@Override
public void write(final int cc) throws IOException {
final byte c = (byte) cc;
if (c == '\n' || c == '\r') {
if (!skip) {
processBuffer();
}
} else {
buffer.write(cc);
}
skip = c == '\r';
}
/**
* Flush this log stream.
*
* @see java.io.OutputStream#flush()
*/
@Override
public void flush() {
if (buffer.size() > 0) {
processBuffer();
}
}
/**
* Writes all remaining data from the buffer.
*
* @see java.io.OutputStream#close()
*/
@Override
public void close() throws IOException {
if (buffer.size() > 0) {
processBuffer();
}
super.close();
}
/**
* @return the trace level of the log system
*/
public int getMessageLevel() {
return level;
}
/**
* Write a block of characters to the output stream
*
* @param b the array containing the data
* @param off the offset into the array where data starts
* @param len the length of block
* @throws java.io.IOException if the data cannot be written into the stream.
* @see java.io.OutputStream#write(byte[], int, int)
*/
@Override
public void write(final byte[] b, final int off, final int len)
throws IOException {
// find the line breaks and pass other chars through in blocks
int offset = off;
int blockStartOffset = offset;
int remaining = len;
while (remaining > 0) {
while (remaining > 0 && b[offset] != LF && b[offset] != CR) {
offset++;
remaining--;
}
// either end of buffer or a line separator char
final int blockLength = offset - blockStartOffset;
if (blockLength > 0) {
buffer.write(b, blockStartOffset, blockLength);
}
while (remaining > 0 && (b[offset] == LF || b[offset] == CR)) {
write(b[offset]);
offset++;
remaining--;
}
blockStartOffset = offset;
}
}
/**
* Converts the buffer to a string and sends it to {@code processLine}.
*/
protected void processBuffer() {
processLine(buffer.toString());
buffer.reset();
}
/**
* Logs a line to the log system of the user.
*
* @param line
* the line to log.
*/
protected void processLine(final String line) {
processLine(line, level);
}
/**
* Logs a line to the log system of the user.
*
* @param line the line to log.
* @param logLevel the log level to use
*/
protected abstract void processLine(final String line, final int logLevel);
}

View File

@@ -0,0 +1,293 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.io;
import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.Charset;
import java.nio.charset.CharsetEncoder;
import java.nio.charset.CoderResult;
import java.nio.charset.CodingErrorAction;
/**
* {@link InputStream} implementation that reads a character stream from a {@link Reader}
* and transforms it to a byte stream using a specified charset encoding. The stream
* is transformed using a {@link CharsetEncoder} object, guaranteeing that all charset
* encodings supported by the JRE are handled correctly. In particular for charsets such as
* UTF-16, the implementation ensures that one and only one byte order marker
* is produced.
* <p>
* Since in general it is not possible to predict the number of characters to be read from the
* {@link Reader} to satisfy a read request on the {@link ReaderInputStream}, all reads from
* the {@link Reader} are buffered. There is therefore no well defined correlation
* between the current position of the {@link Reader} and that of the {@link ReaderInputStream}.
* This also implies that in general there is no need to wrap the underlying {@link Reader}
* in a {@link java.io.BufferedReader}.
* <p>
* {@link ReaderInputStream} implements the inverse transformation of {@link java.io.InputStreamReader};
* in the following example, reading from <tt>in2</tt> would return the same byte
* sequence as reading from <tt>in</tt> (provided that the initial byte sequence is legal
* with respect to the charset encoding):
* <pre>
* InputStream in = ...
* Charset cs = ...
* InputStreamReader reader = new InputStreamReader(in, cs);
* ReaderInputStream in2 = new ReaderInputStream(reader, cs);</pre>
* {@link ReaderInputStream} implements the same transformation as {@link java.io.OutputStreamWriter},
* except that the control flow is reversed: both classes transform a character stream
* into a byte stream, but {@link java.io.OutputStreamWriter} pushes data to the underlying stream,
* while {@link ReaderInputStream} pulls it from the underlying stream.
* <p>
* Note that while there are use cases where there is no alternative to using
* this class, very often the need to use this class is an indication of a flaw
* in the design of the code. This class is typically used in situations where an existing
* API only accepts an {@link InputStream}, but where the most natural way to produce the data
* is as a character stream, i.e. by providing a {@link Reader} instance. An example of a situation
* where this problem may appear is when implementing the {@link javax.activation.DataSource}
* interface from the Java Activation Framework.
* <p>
* Given the fact that the {@link Reader} class doesn't provide any way to predict whether the next
* read operation will block or not, it is not possible to provide a meaningful
* implementation of the {@link InputStream#available()} method. A call to this method
* will always return 0. Also, this class doesn't support {@link InputStream#mark(int)}.
* <p>
* Instances of {@link ReaderInputStream} are not thread safe.
*
* @see WriterOutputStream
*
* @since 2.0
*/
public class ReaderInputStream extends InputStream {
private static final int DEFAULT_BUFFER_SIZE = 1024;
private final Reader reader;
private final CharsetEncoder encoder;
/**
* CharBuffer used as input for the decoder. It should be reasonably
* large as we read data from the underlying Reader into this buffer.
*/
private final CharBuffer encoderIn;
/**
* ByteBuffer used as output for the decoder. This buffer can be small
* as it is only used to transfer data from the decoder to the
* buffer provided by the caller.
*/
private final ByteBuffer encoderOut;
private CoderResult lastCoderResult;
private boolean endOfInput;
/**
* Construct a new {@link ReaderInputStream}.
*
* @param reader the target {@link Reader}
* @param encoder the charset encoder
* @since 2.1
*/
public ReaderInputStream(Reader reader, CharsetEncoder encoder) {
this(reader, encoder, DEFAULT_BUFFER_SIZE);
}
/**
* Construct a new {@link ReaderInputStream}.
*
* @param reader the target {@link Reader}
* @param encoder the charset encoder
* @param bufferSize the size of the input buffer in number of characters
* @since 2.1
*/
public ReaderInputStream(Reader reader, CharsetEncoder encoder, int bufferSize) {
this.reader = reader;
this.encoder = encoder;
this.encoderIn = CharBuffer.allocate(bufferSize);
this.encoderIn.flip();
this.encoderOut = ByteBuffer.allocate(128);
this.encoderOut.flip();
}
/**
* Construct a new {@link ReaderInputStream}.
*
* @param reader the target {@link Reader}
* @param charset the charset encoding
* @param bufferSize the size of the input buffer in number of characters
*/
public ReaderInputStream(Reader reader, Charset charset, int bufferSize) {
this(reader,
charset.newEncoder()
.onMalformedInput(CodingErrorAction.REPLACE)
.onUnmappableCharacter(CodingErrorAction.REPLACE),
bufferSize);
}
/**
* Construct a new {@link ReaderInputStream} with a default input buffer size of
* 1024 characters.
*
* @param reader the target {@link Reader}
* @param charset the charset encoding
*/
public ReaderInputStream(Reader reader, Charset charset) {
this(reader, charset, DEFAULT_BUFFER_SIZE);
}
/**
* Construct a new {@link ReaderInputStream}.
*
* @param reader the target {@link Reader}
* @param charsetName the name of the charset encoding
* @param bufferSize the size of the input buffer in number of characters
*/
public ReaderInputStream(Reader reader, String charsetName, int bufferSize) {
this(reader, Charset.forName(charsetName), bufferSize);
}
/**
* Construct a new {@link ReaderInputStream} with a default input buffer size of
* 1024 characters.
*
* @param reader the target {@link Reader}
* @param charsetName the name of the charset encoding
*/
public ReaderInputStream(Reader reader, String charsetName) {
this(reader, charsetName, DEFAULT_BUFFER_SIZE);
}
/**
* Construct a new {@link ReaderInputStream} that uses the default character encoding
* with a default input buffer size of 1024 characters.
*
* @param reader the target {@link Reader}
*/
public ReaderInputStream(Reader reader) {
this(reader, Charset.defaultCharset());
}
/**
* Fills the internal char buffer from the reader.
*
* @throws IOException
* If an I/O error occurs
*/
private void fillBuffer() throws IOException {
if (!endOfInput && (lastCoderResult == null || lastCoderResult.isUnderflow())) {
encoderIn.compact();
int position = encoderIn.position();
// We don't use Reader#read(CharBuffer) here because it is more efficient
// to write directly to the underlying char array (the default implementation
// copies data to a temporary char array).
int c = reader.read(encoderIn.array(), position, encoderIn.remaining());
if (c == -1) {
endOfInput = true;
} else {
encoderIn.position(position+c);
}
encoderIn.flip();
}
encoderOut.compact();
lastCoderResult = encoder.encode(encoderIn, encoderOut, endOfInput);
encoderOut.flip();
}
/**
* Read the specified number of bytes into an array.
*
* @param b the byte array to read into
* @param off the offset to start reading bytes into
* @param len the number of bytes to read
* @return the number of bytes read or <code>-1</code>
* if the end of the stream has been reached
* @throws IOException if an I/O error occurs
*/
@Override
public int read(byte[] b, int off, int len) throws IOException {
if (b == null) {
throw new NullPointerException("Byte array must not be null");
}
if (len < 0 || off < 0 || (off + len) > b.length) {
throw new IndexOutOfBoundsException("Array Size=" + b.length +
", offset=" + off + ", length=" + len);
}
int read = 0;
if (len == 0) {
return 0; // Always return 0 if len == 0
}
while (len > 0) {
if (encoderOut.hasRemaining()) {
int c = Math.min(encoderOut.remaining(), len);
encoderOut.get(b, off, c);
off += c;
len -= c;
read += c;
} else {
fillBuffer();
if (endOfInput && !encoderOut.hasRemaining()) {
break;
}
}
}
return read == 0 && endOfInput ? -1 : read;
}
/**
* Read the specified number of bytes into an array.
*
* @param b the byte array to read into
* @return the number of bytes read or <code>-1</code>
* if the end of the stream has been reached
* @throws IOException if an I/O error occurs
*/
@Override
public int read(byte[] b) throws IOException {
return read(b, 0, b.length);
}
/**
* Read a single byte.
*
* @return either the byte read or <code>-1</code> if the end of the stream
* has been reached
* @throws IOException if an I/O error occurs
*/
@Override
public int read() throws IOException {
for (;;) {
if (encoderOut.hasRemaining()) {
return encoderOut.get() & 0xFF;
} else {
fillBuffer();
if (endOfInput && !encoderOut.hasRemaining()) {
return -1;
}
}
}
}
/**
* Close the stream. This method will cause the underlying {@link Reader}
* to be closed.
* @throws IOException if an I/O error occurs
*/
@Override
public void close() throws IOException {
reader.close();
}
}

View File

@@ -0,0 +1,26 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.io
/**
* Marker interface for generic serializable value object.
* This is not me
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
interface SerializableMarker extends Serializable {
}

View File

@@ -0,0 +1,57 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.io
import java.lang.annotation.ElementType
import java.lang.annotation.Retention
import java.lang.annotation.RetentionPolicy
import java.lang.annotation.Target
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.codehaus.groovy.ast.ASTNode
import org.codehaus.groovy.ast.ClassNode
import org.codehaus.groovy.control.CompilePhase
import org.codehaus.groovy.control.SourceUnit
import org.codehaus.groovy.transform.ASTTransformation
import org.codehaus.groovy.transform.GroovyASTTransformation
import org.codehaus.groovy.transform.GroovyASTTransformationClass
/**
* Declares an AST xform to automatically add the {@link SerializableMarker} interface
* for a class marked as @SerializableObject
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Retention(RetentionPolicy.RUNTIME)
@Target([ElementType.TYPE])
@GroovyASTTransformationClass(classes = [ValueObjectImpl])
@interface SerializableObject {
//--- === implementation === ---
@Slf4j
@CompileStatic
@GroovyASTTransformation(phase = CompilePhase.SEMANTIC_ANALYSIS)
class ValueObjectImpl implements ASTTransformation {
@Override
void visit(ASTNode[] nodes, SourceUnit source) {
final target = (ClassNode)nodes[1]
target.addInterface( new ClassNode(SerializableMarker.class) )
}
}
}

View File

@@ -0,0 +1,123 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.io;
import java.io.IOException;
import java.io.InputStream;
/**
* An input stream that skips the first `n` lines in a text stream
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
public class SkipLinesInputStream extends InputStream {
private static final int LF = '\n';
private static final int CR = '\r';
private InputStream target;
private int skip;
private int buffer=-1;
private int count;
private StringBuilder header;
/**
* Creates a <code>FilterInputStream</code>
* by assigning the argument <code>in</code>
* to the field <code>this.in</code> so as
* to remember it for later use.
*
* @param inputStream the underlying input stream, or <code>null</code> if
* this instance is to be created without an underlying stream.
*/
public SkipLinesInputStream(InputStream inputStream, int skip) {
this.target = inputStream;
this.skip = skip;
}
@Override
public void close() throws IOException {
target.close();
}
public long skip(long n) throws IOException {
throw new UnsupportedOperationException("Skip operation is not supported by " + this.getClass().getName());
}
@Override
public int read() throws IOException {
while( count < skip ) {
int ch = read0();
if( ch == -1 )
return -1;
}
return read0();
}
/**
* A line is considered to be terminated by any one
* of a line feed ('\n'), a carriage return ('\r'), or a carriage return
* followed immediately by a linefeed.
*
*/
private int read0() throws IOException {
if( buffer != -1 ) {
int result = buffer;
buffer=-1;
return result;
}
int ch = target.read();
if( header!=null && count<skip && ch!=-1 )
header.append((char)ch);
if( ch == LF ) {
count++;
}
else if( ch == CR ) {
count++;
int next = target.read();
if( next == LF ) {
if( header!=null )
header.append((char)next);
}
else {
buffer = next;
if( header!=null && count<skip && next!=-1 )
header.append((char)next);
}
}
return ch;
}
public String consumeHeader() throws IOException {
header = new StringBuilder();
while( count<skip && read0()!=-1 ) ;
return header.toString();
}
public String getHeader() {
return header != null ? header.toString() : null;
}
}

View File

@@ -0,0 +1,42 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.io
import java.lang.annotation.ElementType
import java.lang.annotation.Retention
import java.lang.annotation.RetentionPolicy
import java.lang.annotation.Target
import groovy.transform.AnnotationCollector
import groovy.transform.AnnotationCollectorMode
import groovy.transform.AutoClone
import groovy.transform.Immutable
/**
* Declares an AST xform to automatically add the {@link SerializableMarker} interface
* for a class marked as @SerializableObject
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@AutoClone
@Immutable(copyWith=true)
@SerializableObject
@AnnotationCollector(mode = AnnotationCollectorMode.PREFER_EXPLICIT_MERGED)
@Retention(RetentionPolicy.RUNTIME)
@Target([ElementType.TYPE])
@interface ValueObject {
}

View File

@@ -0,0 +1,305 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.io;
import java.io.IOException;
import java.io.OutputStream;
import java.io.Writer;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CoderResult;
import java.nio.charset.CodingErrorAction;
/**
* {@link OutputStream} implementation that transforms a byte stream to a
* character stream using a specified charset encoding and writes the resulting
* stream to a {@link Writer}. The stream is transformed using a
* {@link CharsetDecoder} object, guaranteeing that all charset
* encodings supported by the JRE are handled correctly.
* <p>
* The output of the {@link CharsetDecoder} is buffered using a fixed size buffer.
* This implies that the data is written to the underlying {@link Writer} in chunks
* that are no larger than the size of this buffer. By default, the buffer is
* flushed only when it overflows or when {@link #flush()} or {@link #close()}
* is called. In general there is therefore no need to wrap the underlying {@link Writer}
* in a {@link java.io.BufferedWriter}. {@link WriterOutputStream} can also
* be instructed to flush the buffer after each write operation. In this case, all
* available data is written immediately to the underlying {@link Writer}, implying that
* the current position of the {@link Writer} is correlated to the current position
* of the {@link WriterOutputStream}.
* <p>
* {@link WriterOutputStream} implements the inverse transformation of {@link java.io.OutputStreamWriter};
* in the following example, writing to <tt>out2</tt> would have the same result as writing to
* <tt>out</tt> directly (provided that the byte sequence is legal with respect to the
* charset encoding):
* <pre>
* OutputStream out = ...
* Charset cs = ...
* OutputStreamWriter writer = new OutputStreamWriter(out, cs);
* WriterOutputStream out2 = new WriterOutputStream(writer, cs);</pre>
* {@link WriterOutputStream} implements the same transformation as {@link java.io.InputStreamReader},
* except that the control flow is reversed: both classes transform a byte stream
* into a character stream, but {@link java.io.InputStreamReader} pulls data from the underlying stream,
* while {@link WriterOutputStream} pushes it to the underlying stream.
* <p>
* Note that while there are use cases where there is no alternative to using
* this class, very often the need to use this class is an indication of a flaw
* in the design of the code. This class is typically used in situations where an existing
* API only accepts an {@link OutputStream} object, but where the stream is known to represent
* character data that must be decoded for further use.
* <p>
* Instances of {@link WriterOutputStream} are not thread safe.
*
* @see ReaderInputStream
*
* @since 2.0
*/
public class WriterOutputStream extends OutputStream {
private static final int DEFAULT_BUFFER_SIZE = 1024;
private final Writer writer;
private final CharsetDecoder decoder;
private final boolean writeImmediately;
/**
* ByteBuffer used as input for the decoder. This buffer can be small
* as it is used only to transfer the received data to the
* decoder.
*/
private final ByteBuffer decoderIn = ByteBuffer.allocate(128);
/**
* CharBuffer used as output for the decoder. It should be
* somewhat larger as we write from this buffer to the
* underlying Writer.
*/
private final CharBuffer decoderOut;
/**
* Constructs a new {@link WriterOutputStream} with a default output buffer size of
* 1024 characters. The output buffer will only be flushed when it overflows or when
* {@link #flush()} or {@link #close()} is called.
*
* @param writer the target {@link Writer}
* @param decoder the charset decoder
* @since 2.1
*/
public WriterOutputStream(Writer writer, CharsetDecoder decoder) {
this(writer, decoder, DEFAULT_BUFFER_SIZE, false);
}
/**
* Constructs a new {@link WriterOutputStream}.
*
* @param writer the target {@link Writer}
* @param decoder the charset decoder
* @param bufferSize the size of the output buffer in number of characters
* @param writeImmediately If <tt>true</tt> the output buffer will be flushed after each
* write operation, i.e. all available data will be written to the
* underlying {@link Writer} immediately. If <tt>false</tt>, the
* output buffer will only be flushed when it overflows or when
* {@link #flush()} or {@link #close()} is called.
* @since 2.1
*/
public WriterOutputStream(Writer writer, CharsetDecoder decoder, int bufferSize, boolean writeImmediately) {
this.writer = writer;
this.decoder = decoder;
this.writeImmediately = writeImmediately;
decoderOut = CharBuffer.allocate(bufferSize);
}
/**
* Constructs a new {@link WriterOutputStream}.
*
* @param writer the target {@link Writer}
* @param charset the charset encoding
* @param bufferSize the size of the output buffer in number of characters
* @param writeImmediately If <tt>true</tt> the output buffer will be flushed after each
* write operation, i.e. all available data will be written to the
* underlying {@link Writer} immediately. If <tt>false</tt>, the
* output buffer will only be flushed when it overflows or when
* {@link #flush()} or {@link #close()} is called.
*/
public WriterOutputStream(Writer writer, Charset charset, int bufferSize, boolean writeImmediately) {
this(writer,
charset.newDecoder()
.onMalformedInput(CodingErrorAction.REPLACE)
.onUnmappableCharacter(CodingErrorAction.REPLACE)
.replaceWith("?"),
bufferSize,
writeImmediately);
}
/**
* Constructs a new {@link WriterOutputStream} with a default output buffer size of
* 1024 characters. The output buffer will only be flushed when it overflows or when
* {@link #flush()} or {@link #close()} is called.
*
* @param writer the target {@link Writer}
* @param charset the charset encoding
*/
public WriterOutputStream(Writer writer, Charset charset) {
this(writer, charset, DEFAULT_BUFFER_SIZE, false);
}
/**
* Constructs a new {@link WriterOutputStream}.
*
* @param writer the target {@link Writer}
* @param charsetName the name of the charset encoding
* @param bufferSize the size of the output buffer in number of characters
* @param writeImmediately If <tt>true</tt> the output buffer will be flushed after each
* write operation, i.e. all available data will be written to the
* underlying {@link Writer} immediately. If <tt>false</tt>, the
* output buffer will only be flushed when it overflows or when
* {@link #flush()} or {@link #close()} is called.
*/
public WriterOutputStream(Writer writer, String charsetName, int bufferSize, boolean writeImmediately) {
this(writer, Charset.forName(charsetName), bufferSize, writeImmediately);
}
/**
* Constructs a new {@link WriterOutputStream} with a default output buffer size of
* 1024 characters. The output buffer will only be flushed when it overflows or when
* {@link #flush()} or {@link #close()} is called.
*
* @param writer the target {@link Writer}
* @param charsetName the name of the charset encoding
*/
public WriterOutputStream(Writer writer, String charsetName) {
this(writer, charsetName, DEFAULT_BUFFER_SIZE, false);
}
/**
* Constructs a new {@link WriterOutputStream} that uses the default character encoding
* and with a default output buffer size of 1024 characters. The output buffer will only
* be flushed when it overflows or when {@link #flush()} or {@link #close()} is called.
*
* @param writer the target {@link Writer}
*/
public WriterOutputStream(Writer writer) {
this(writer, Charset.defaultCharset(), DEFAULT_BUFFER_SIZE, false);
}
/**
* Write bytes from the specified byte array to the stream.
*
* @param b the byte array containing the bytes to write
* @param off the start offset in the byte array
* @param len the number of bytes to write
* @throws IOException if an I/O error occurs
*/
@Override
public void write(byte[] b, int off, int len) throws IOException {
while (len > 0) {
int c = Math.min(len, decoderIn.remaining());
decoderIn.put(b, off, c);
processInput(false);
len -= c;
off += c;
}
if (writeImmediately) {
flushOutput();
}
}
/**
* Write bytes from the specified byte array to the stream.
*
* @param b the byte array containing the bytes to write
* @throws IOException if an I/O error occurs
*/
@Override
public void write(byte[] b) throws IOException {
write(b, 0, b.length);
}
/**
* Write a single byte to the stream.
*
* @param b the byte to write
* @throws IOException if an I/O error occurs
*/
@Override
public void write(int b) throws IOException {
write(new byte[] { (byte)b }, 0, 1);
}
/**
* Flush the stream. Any remaining content accumulated in the output buffer
* will be written to the underlying {@link Writer}. After that
* {@link Writer#flush()} will be called.
* @throws IOException if an I/O error occurs
*/
@Override
public void flush() throws IOException {
flushOutput();
writer.flush();
}
/**
* Close the stream. Any remaining content accumulated in the output buffer
* will be written to the underlying {@link Writer}. After that
* {@link Writer#close()} will be called.
* @throws IOException if an I/O error occurs
*/
@Override
public void close() throws IOException {
processInput(true);
flushOutput();
writer.close();
}
/**
* Decode the contents of the input ByteBuffer into a CharBuffer.
*
* @param endOfInput indicates end of input
* @throws IOException if an I/O error occurs
*/
private void processInput(boolean endOfInput) throws IOException {
// Prepare decoderIn for reading
decoderIn.flip();
CoderResult coderResult;
while (true) {
coderResult = decoder.decode(decoderIn, decoderOut, endOfInput);
if (coderResult.isOverflow()) {
flushOutput();
} else if (coderResult.isUnderflow()) {
break;
} else {
// The decoder is configured to replace malformed input and unmappable characters,
// so we should not get here.
throw new IOException("Unexpected coder result");
}
}
// Discard the bytes that have been read
decoderIn.compact();
}
/**
* Flush the output.
*
* @throws IOException if an I/O error occurs
*/
private void flushOutput() throws IOException {
if (decoderOut.position() > 0) {
writer.write(decoderOut.array(), 0, decoderOut.position());
decoderOut.rewind();
}
}
}

View File

@@ -0,0 +1,62 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.plugin
import groovy.transform.CompileStatic
import groovy.transform.PackageScope
import nextflow.BuildInfo
import nextflow.exception.AbortOperationException
import org.pf4j.Plugin
import org.pf4j.PluginWrapper
import org.slf4j.Logger
import org.slf4j.LoggerFactory
/**
* Base class for NF plugins
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
abstract class BasePlugin extends Plugin {
private static Logger log = LoggerFactory.getLogger(BasePlugin)
BasePlugin(PluginWrapper wrapper) {
super(wrapper)
}
@PackageScope boolean verMatches(String requires, String current=BuildInfo.version) {
return getWrapper()
.getPluginManager()
.getVersionManager()
.checkVersionConstraint(current, requires)
}
@Override
void start() {
final desc = getWrapper().getDescriptor()
final name = "${desc.pluginId}@${desc.version}"
if( desc.requires && !verMatches(desc.requires)) {
throw new AbortOperationException("Failed requirement - Plugin $name requires Nextflow version $desc.requires (current $BuildInfo.version)")
}
log.debug "Plugin started $name"
}
@Override
void stop() {
log.debug "Plugin stopped ${wrapper.descriptor.pluginId}"
}
}

View File

@@ -0,0 +1,67 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.plugin
import java.nio.file.Path
import groovy.transform.CompileStatic
import nextflow.BuildInfo
import org.pf4j.DefaultPluginManager
import org.pf4j.ExtensionFactory
import org.pf4j.PluginWrapper
import org.pf4j.SingletonExtensionFactory
import org.pf4j.VersionManager
/**
* Custom plugin manager to that allow accessing to {@code loadPluginFromPath} and
* {@code resolvePlugins} method
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class CustomPluginManager extends DefaultPluginManager {
CustomPluginManager() {}
CustomPluginManager(Path root) {
super(root)
}
@Override
PluginWrapper loadPluginFromPath(Path pluginPath) {
super.loadPluginFromPath(pluginPath)
}
@Override
void resolvePlugins() {
super.resolvePlugins()
}
@Override
protected VersionManager createVersionManager() {
return new CustomVersionManager()
}
@Override
String getSystemVersion() {
return BuildInfo.version
}
@Override
protected ExtensionFactory createExtensionFactory() {
return new SingletonExtensionFactory(this)
}
}

View File

@@ -0,0 +1,79 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.plugin
import java.util.regex.Pattern
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.util.VersionNumber
import org.pf4j.DefaultVersionManager
/**
* Extends default version manager adding the ability to support
* Calendar versioning
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
class CustomVersionManager extends DefaultVersionManager {
public static Pattern CAL_VER = ~/2\d\.\d\d\.\S+/
@Override
boolean checkVersionConstraint(String version, String constraint) {
if( !version || !constraint || constraint=='*' || version==constraint )
return true
try {
return safeCheck0(version, constraint)
}
catch (Throwable e) {
log.debug "Failed to check version constraint - version: $version; constraint: $constraint"
return false
}
}
private boolean safeCheck0(String version, String constraint) {
if( version =~ CAL_VER ) {
if( constraint.startsWith('nextflow@') ) {
constraint = constraint.substring('nextflow@'.size())
}
if( version.endsWith('-SNAPSHOT') ) {
version = stripSuffix(version)
constraint = stripSuffix(constraint)
}
return new VersionNumber(version).matches(constraint)
}
else {
return super.checkVersionConstraint(version, constraint)
}
}
private String stripSuffix(String str) {
int p = str.lastIndexOf('-')
return p==-1 ? str : str.substring(0,p)
}
@Override
int compareVersions(String v1, String v2) {
return v1=~CAL_VER || v2=~CAL_VER
? new VersionNumber(v1) <=> new VersionNumber(v2)
: super.compareVersions(v1, v2)
}
}

View File

@@ -0,0 +1,75 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.plugin
import groovy.transform.CompileStatic
/**
* Model the collection of default plugins used if no plugins are provided by the user
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class DefaultPlugins {
public static final DefaultPlugins INSTANCE = new DefaultPlugins()
private Map<String,PluginRef> plugins = new HashMap<>(20)
protected DefaultPlugins() {
final meta = this.class.getResourceAsStream('/META-INF/plugins-info.txt')?.text
plugins = parseMeta(meta)
}
protected Map<String,PluginRef> parseMeta(String meta) {
if( !meta )
return Collections.emptyMap()
final result = new HashMap(20)
for( String line : meta.readLines() ) {
final spec = PluginRef.parse(line)
result[spec.id] = spec
}
return result
}
PluginRef getPlugin(String pluginId) throws IllegalArgumentException {
if( !pluginId )
throw new IllegalArgumentException("Missing pluginId argument")
final result = plugins.get(pluginId)
if( !result )
throw new IllegalArgumentException("Unknown Nextflow plugin '$pluginId'")
return result
}
boolean hasPlugin(String pluginId) {
return plugins.containsKey(pluginId)
}
List<PluginRef> getPlugins() {
return new ArrayList(plugins.values())
}
String toSortedString(String divisor=',') {
getPlugins().sort().join(divisor)
}
@Override
String toString() {
return "DefaultPlugins${getPlugins()}"
}
}

View File

@@ -0,0 +1,60 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.plugin
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j;
import org.pf4j.PluginClasspath;
/**
* Customise classpath loader for Groovy based
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
class DevPluginClasspath extends PluginClasspath {
private boolean logged
DevPluginClasspath() {
// the path where classes are resources should be found in the dev environment
// for each plugin project directory
addClassesDirectories("build/classes/groovy/main", "build/resources/main", 'build/classes/main')
// note: this path is not created automatically by Gradle, it should be created by a custom task
// see `copyPluginLibs` task in the base plugins `build.gradle`
addJarsDirectories('build/target/libs')
}
@Override
Set<String> getClassesDirectories() {
if( !logged ) {
log.debug "Groovy DEV plugin classpath: classes-dirs=${super.getClassesDirectories()}; jars-dirs=${super.getJarsDirectories()}"
logged = true
}
return super.getClassesDirectories()
}
@Override
Set<String> getJarsDirectories() {
if( !logged ) {
log.debug "Groovy DEV plugin classpath: classes-dirs=${super.getClassesDirectories()}; jars-dirs=${super.getJarsDirectories()}"
logged = true
}
return super.getJarsDirectories()
}
}

View File

@@ -0,0 +1,38 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.plugin
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.pf4j.BasePluginLoader
import org.pf4j.PluginManager
/**
* Plugin loader for Groovy based development environment
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
class DevPluginLoader extends BasePluginLoader {
DevPluginLoader(PluginManager pluginManager) {
super(pluginManager, new DevPluginClasspath())
}
}

View File

@@ -0,0 +1,77 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.plugin
import java.nio.file.Path
import java.nio.file.Paths
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.pf4j.CompoundPluginRepository
import org.pf4j.DevelopmentPluginRepository
import org.pf4j.ManifestPluginDescriptorFinder
import org.pf4j.PluginDescriptorFinder
import org.pf4j.PluginLoader
import org.pf4j.PluginRepository
/**
* Custom plugin manager that allow loading plugins from Groovy/Gradle/Intellij build environment
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
class DevPluginManager extends CustomPluginManager {
DevPluginManager(Path root) {
super(root)
}
private List<Path> getExtensionRoots() {
final ext = System .getenv('NXF_PLUGINS_DEV')
if( !ext )
return Collections.emptyList()
return ext
.tokenize(',')
.collect { Paths.get(it).toAbsolutePath() }
}
@Override
protected PluginDescriptorFinder createPluginDescriptorFinder() {
return new ManifestPluginDescriptorFinder()
}
@Override
protected PluginLoader createPluginLoader() {
return new DevPluginLoader(this)
}
@Override
protected PluginRepository createPluginRepository() {
def repos = new CompoundPluginRepository()
// main dev repo
log.debug "Add plugin root repository: ${getPluginsRoot()}"
repos.add( new DevelopmentPluginRepository(getPluginsRoot()) )
// extension repos
for( Path it : extensionRoots ) {
log.debug "Add plugin dev repository: $it"
repos.add( new DevelopmentPluginRepository(it) )
}
return repos
}
}

View File

@@ -0,0 +1,43 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.plugin
import groovy.transform.CompileStatic
/**
* Updater disabling plugin installation and update when
* running the plugin manager in dev mode
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class DevPluginUpdater extends PluginUpdater {
DevPluginUpdater(CustomPluginManager pluginManager) {
super(pluginManager)
}
@Override
boolean installPlugin(String id, String version) {
throw new UnsupportedOperationException("Install is not supported on dev mode - Missing plugin $id ${version?:''}")
}
@Override
boolean updatePlugin(String id, String version) {
throw new UnsupportedOperationException("Update is not supported on dev mode - Missing plugin $id ${version?:''}")
}
}

View File

@@ -0,0 +1,55 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.plugin
import java.nio.file.Path
import groovy.transform.CompileStatic
import org.pf4j.DefaultPluginLoader
import org.pf4j.DefaultPluginRepository
import org.pf4j.ManifestPluginDescriptorFinder
import org.pf4j.PluginDescriptorFinder
import org.pf4j.PluginLoader
import org.pf4j.PluginRepository
/**
* Implements a plugin manager used when running in embedded mode
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class EmbeddedPluginManager extends CustomPluginManager {
EmbeddedPluginManager(Path repository) {
super(repository)
}
@Override
protected PluginDescriptorFinder createPluginDescriptorFinder() {
return new ManifestPluginDescriptorFinder()
}
@Override
protected PluginLoader createPluginLoader() {
return new DefaultPluginLoader(this)
}
@Override
protected PluginRepository createPluginRepository() {
return new DefaultPluginRepository(getPluginsRoot())
}
}

View File

@@ -0,0 +1,221 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.plugin
import java.net.http.HttpRequest
import java.net.http.HttpResponse
import nextflow.serde.gson.GsonEncoder
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import io.seqera.http.HxClient
import io.seqera.npr.api.schema.v1.ListDependenciesResponse
import io.seqera.npr.api.schema.v1.PluginDependency
import nextflow.BuildInfo
import nextflow.util.RetryConfig
import org.pf4j.PluginRuntimeException
import org.pf4j.update.FileDownloader
import org.pf4j.update.FileVerifier
import org.pf4j.update.PluginInfo
import org.pf4j.update.PluginInfo.PluginRelease
import org.pf4j.update.SimpleFileDownloader
import org.pf4j.update.verifier.CompoundVerifier
/**
* Represents an update repository served via an HTTP api.
*
* It implements PrefetchUpdateRepository so that all relevant
* plugin metadata can be loaded with a single HTTP request, rather
* than a request-per-plugin.
*
* Metadata is prefetched into memory when Nextflow starts and expires
* upon termination (or when 'refresh()' is called).
*/
@Slf4j
@CompileStatic
class HttpPluginRepository implements PrefetchUpdateRepository {
private final String id
private final URI url
private final HxClient httpClient
private Map<String, PluginInfo> plugins
HttpPluginRepository(String id, URI url) {
this.id = id
// ensure url ends with a slash
this.url = !url.toString().endsWith("/")
? URI.create(url.toString() + "/")
: url
this.httpClient = HxClient.newBuilder()
.retryConfig(RetryConfig.config())
.build()
}
// NOTE ON PREFETCHING
//
// The prefetch mechanism is used to work around a limitation in the
// UpdateRepository interface from pf4j.
//
// Specifically, p4fj expects that getPlugins() returns a Map<> of all
// metadata about all plugins. To implement this for an HTTP repository
// would require either downloading the entire contents of the remote
// repository or implementing a lazy map and making an HTTP request for
// each required plugin.
//
// Instead we can use the list of configured plugins to load all relevant
// metadata in a single HTTP request at startup, and use this to populate
// the map. Once the prefetch is complete, this repository will behave
// like any other implementation of UpdateRepository.
@Override
void prefetch(List<PluginRef> plugins) {
if (plugins && !plugins.isEmpty()) {
this.plugins = fetchMetadata(plugins)
}
}
@Override
String getId() {
return id
}
@Override
URL getUrl() {
return url.toURL()
}
@Override
Map<String, PluginInfo> getPlugins() {
if (plugins==null) {
log.warn "getPlugins() called before prefetch() - plugins map will be empty"
return Map.of()
}
return Collections.unmodifiableMap(plugins)
}
@Override
PluginInfo getPlugin(String id) {
return plugins.computeIfAbsent(id) { key -> fetchMetadataByIds([key]).get(key) }
}
@Override
void refresh() {
plugins = fetchMetadataByIds(plugins.keySet())
}
@Override
FileDownloader getFileDownloader() {
return new SimpleFileDownloader()
}
@Override
FileVerifier getFileVerifier() {
return new CompoundVerifier()
}
// ----------------------------------------------------------------------------
// http handling
private Map<String, PluginInfo> fetchMetadataByIds(Collection<String> ids) {
def specs = ids.collect(id -> new PluginRef(id, null))
return fetchMetadata(specs)
}
private Map<String, PluginInfo> fetchMetadata(Collection<PluginRef> specs) {
final ordered = specs.sort(false)
return fetchMetadata0(ordered)
}
private Map<String, PluginInfo> fetchMetadata0(List<PluginRef> specs) {
def pluginsParam = specs.collect { "${it.id}${it.version ? '@' + it.version : ''}" }.join(',')
def uri = url.resolve("v1/plugins/dependencies?plugins=${URLEncoder.encode(pluginsParam, 'UTF-8')}&nextflowVersion=${URLEncoder.encode(BuildInfo.version, 'UTF-8')}")
def req = HttpRequest.newBuilder()
.uri(uri)
.GET()
.build()
try {
return sendAndParse(req)
}
catch (PluginRuntimeException e) {
throw e
}
catch (Exception e) {
throw new PluginRuntimeException(e, "Unable to connect to %s - cause: %s", uri, e.message)
}
}
private Map<String, PluginInfo> sendAndParse(HttpRequest req) {
final encoder = new GsonEncoder<ListDependenciesResponse>() {}
final resp = httpClient.send(req, HttpResponse.BodyHandlers.ofString())
final body = resp.body()
log.debug "Registry request: ${resp.uri()}\n- code: ${resp.statusCode()}\n- body: ${body}"
if( resp.statusCode() != 200 ) {
final msg = "Invalid response while fetching plugin metadata from: ${req.uri()}\n- http status: ${resp.statusCode()}\n- response : ${body}"
throw new PluginRuntimeException(msg)
}
try {
final ListDependenciesResponse decoded = encoder.decode(body)
if( decoded.plugins == null ) {
throw new PluginRuntimeException("Failed to download plugin metadata: Failed to parse response body")
}
final result = new HashMap<String, PluginInfo>()
for( PluginDependency plugin : decoded.plugins ) {
if( plugin.releases ) {
final pluginInfo = mapToPluginInfo(plugin)
result.put(plugin.id, pluginInfo)
}
else
log.debug "Registry ${resp.uri().host} has no releases for plugin: ${plugin}"
}
return result
}
catch( Exception e ) {
final msg = "Unexpected error while fetching plugin metadata from: ${req.uri()}\n- message : ${e.message}\n- response: ${body}"
throw new PluginRuntimeException(msg)
}
}
/**
* Maps a Plugin object from the repository API to a PluginInfo object for pf4j compatibility.
* Handles conversion of OffsetDateTime to Date and ensures the releases collection is never null.
*
* @param plugin The Plugin object from the repository API response
* @return A PluginInfo object compatible with pf4j's update repository interface
*/
static protected PluginInfo mapToPluginInfo(PluginDependency plugin) {
assert plugin.releases, "Plugin releases cannot be empty"
final pluginInfo = new PluginInfo()
pluginInfo.id = plugin.id
pluginInfo.projectUrl = plugin.projectUrl
pluginInfo.provider = plugin.provider
// Map releases to PluginInfo.PluginRelease
pluginInfo.releases = new ArrayList<>()
for (def release : plugin.releases) {
final pluginRelease = new PluginRelease()
pluginRelease.version = release.version
pluginRelease.date = release.date ? Date.from(release.date.toInstant()) : null
pluginRelease.url = release.url
pluginRelease.sha512sum = release.sha512sum
pluginRelease.requires = release.requires
pluginInfo.releases.add(pluginRelease)
}
return pluginInfo
}
}

View File

@@ -0,0 +1,136 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.plugin
import java.nio.file.FileAlreadyExistsException
import java.nio.file.Files
import java.nio.file.Path
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.Const
import nextflow.extension.FilesEx
import org.pf4j.DefaultPluginLoader
import org.pf4j.DefaultPluginManager
import org.pf4j.ManifestPluginDescriptorFinder
import org.pf4j.PluginDescriptorFinder
import org.pf4j.PluginLoader
import org.pf4j.PluginRepository
import org.pf4j.PluginWrapper
/**
* Custom plugin manager creating tracking plugins with symlinks to
* the parent repository
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
class LocalPluginManager extends CustomPluginManager {
static private Path PLUGINS_LOCAL_ROOT = Const.appCacheDir.resolve('plr')
private Path repository
LocalPluginManager(Path repository) {
super(makeLocalRoot())
if( !localRoot ) throw new IllegalArgumentException("Missing Local plugins root directory")
if( !repository ) throw new IllegalArgumentException("Missing plugins repository directory")
this.repository = repository
}
static protected Path makeLocalRoot() {
final result = PLUGINS_LOCAL_ROOT.resolve(UUID.randomUUID().toString())
if( !FilesEx.mkdirs(result) )
throw new IOException("Unable to create plugins local directory: $result -- Make sure you have write permissions in this directory path")
Runtime.addShutdownHook { FilesEx.deleteDir(result) }
return result
}
protected Path getLocalRoot() { getPluginsRoot() }
@Override
protected PluginDescriptorFinder createPluginDescriptorFinder() {
return new ManifestPluginDescriptorFinder()
}
@Override
protected PluginLoader createPluginLoader() {
return new DefaultPluginLoader(this)
}
@Override
protected PluginRepository createPluginRepository() {
return new LocalPluginRepository(localRoot)
}
/**
* Override parent {@link DefaultPluginManager#loadPlugin(java.nio.file.Path)} method
* creating a symlink the plugin path directory in the current plugin root
*
* @param pluginPath
* The file path where the plugin is stored unzipped) in the
* central plugins directory
*/
@Override
String loadPlugin(Path pluginPath) {
final symlink = createLinkFromPath(pluginPath)
super.loadPlugin(symlink)
}
@Override
PluginWrapper loadPluginFromPath(Path pluginPath) {
final symlink = createLinkFromPath(pluginPath)
return super.loadPluginFromPath(symlink)
}
private Path createLinkFromPath(Path pluginPath) {
if( pluginPath.startsWith(localRoot))
return pluginPath
if( !pluginPath )
throw new IllegalArgumentException("Plugin path cannot be null")
if( !Files.isDirectory(pluginPath) )
throw new IllegalArgumentException("Plugin path must be a directory: $pluginPath")
// create a symlink relative to the current root
final symlink = localRoot.resolve(pluginPath.getFileName())
createLink0(symlink, pluginPath)
return symlink
}
private void createLink0(Path symlink, Path pluginPath) {
try {
log.trace "Creating local plugins root link: $symlink → $pluginPath"
Files.createSymbolicLink(symlink, pluginPath)
}
catch (FileAlreadyExistsException e) {
log.debug "Deleting existing local plugins root link: $symlink"
if( !sameTarget(symlink, pluginPath) ) {
Files.delete(symlink)
Files.createSymbolicLink(symlink, pluginPath)
}
}
}
private boolean sameTarget(Path link, Path target) {
try {
return Files.readSymbolicLink(link) == target
}
catch (IOException e) {
return false
}
}
}

View File

@@ -0,0 +1,61 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.plugin
import java.nio.file.Files
import java.nio.file.Path
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.pf4j.DefaultPluginRepository
/**
* Extends the default plugin repository to avoid the deletion
* of the plugin directory and delete instead the local symbolic link
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
class LocalPluginRepository extends DefaultPluginRepository {
LocalPluginRepository(Path pluginsRoot) {
super(pluginsRoot)
}
/**
* Override {@link DefaultPluginRepository#deletePluginPath(java.nio.file.Path)}
* to prevent deleting the real plugin directory
*
* @param pluginPath
* @return
*/
@Override
boolean deletePluginPath(Path pluginPath) {
if(Files.isSymbolicLink(pluginPath))
try {
Files.delete(pluginPath)
return true
}
catch (Exception e) {
log.debug "Unable to delete plugin path: $pluginPath"
return false
}
else
return super.deletePluginPath(pluginPath)
}
}

View File

@@ -0,0 +1,124 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.plugin
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.extension.FilesEx
import org.pf4j.ManifestPluginDescriptorFinder
import org.pf4j.PluginDescriptor
import org.pf4j.update.FileDownloader
import org.pf4j.update.FileVerifier
import org.pf4j.update.PluginInfo
import org.pf4j.update.UpdateRepository
import org.pf4j.update.verifier.CompoundVerifier
import java.nio.file.Path
/**
* Implementation of UpdateRepository which looks in a local directory of already-downloaded
* plugins to find available versions.
*/
@CompileStatic
@Slf4j
class LocalUpdateRepository implements UpdateRepository {
private final String id
private final Path dir
private Map<String, PluginInfo> plugins
LocalUpdateRepository(String id, Path dir) {
this.id = id
this.dir = dir
}
@Override
String getId() {
return id
}
@Override
URL getUrl() {
return dir.toUri().toURL()
}
@Override
Map<String, PluginInfo> getPlugins() {
if( !plugins )
plugins = loadPlugins(dir)
return plugins
}
@Override
PluginInfo getPlugin(String id) {
return getPlugins().get(id)
}
@Override
void refresh() {
this.plugins = null
}
@Override
FileDownloader getFileDownloader() {
// plugins in this repo are already downloaded, so treat any download url as a file path
return (URL url) -> Path.of(url.toURI())
}
@Override
FileVerifier getFileVerifier() {
return new CompoundVerifier()
}
private static Map<String, PluginInfo> loadPlugins(Path dir) {
// each plugin is stored in a dir called $id-$version; grab the descriptor from each
final manifestReader = new ManifestPluginDescriptorFinder()
final descriptors = FilesEx.listFiles(dir)
.collect { plugin -> new LocalPlugin(plugin, manifestReader.find(plugin)) }
// now group the descriptors by id, to create a PluginInfo with list of versions
return descriptors
.groupBy { d -> d.getPluginId() }
.collectEntries { id, versions -> Map.entry(id, toPluginInfo(id, versions)) }
}
private static PluginInfo toPluginInfo(String id, List<LocalPlugin> versions) {
final info = new PluginInfo()
info.id = id
info.releases = versions.collect { v ->
final release = new PluginInfo.PluginRelease()
release.version = v.version
release.requires = v.requires
release.url = v.path.toUri().toURL()
if( !info.provider && v.provider )
info.provider = v.provider
return release
}
return info
}
private static class LocalPlugin {
Path path
@Delegate
PluginDescriptor descriptor
LocalPlugin(Path path, PluginDescriptor descriptor) {
this.path = path
this.descriptor = descriptor
}
}
}

View File

@@ -0,0 +1,75 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.plugin
import com.google.common.hash.Hasher
import groovy.transform.Canonical
import nextflow.util.CacheFunnel
import nextflow.util.CacheHelper
/**
* Model a plugin Id and version
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Canonical
class PluginRef implements CacheFunnel, Comparable<PluginRef> {
/**
* Plugin unique ID
*/
String id
/**
* The plugin version
*/
String version
/**
* Parse a plugin fully-qualified ID eg. nf-amazon@1.2.0
*
* @param fqid The fully qualified plugin id
* @return A {@link PluginRef} representing the plugin
*/
static PluginRef parse(String fqid, DefaultPlugins defaultPlugins=null) {
final tokens = fqid.tokenize('@') as List<String>
final id = tokens[0]
final ver = tokens[1]
if( ver || defaultPlugins==null )
return new PluginRef(id, ver)
if( defaultPlugins.hasPlugin(id) )
return defaultPlugins.getPlugin(id)
return new PluginRef(id)
}
@Override
Hasher funnel(Hasher hasher, CacheHelper.HashMode mode) {
hasher.putUnencodedChars(id)
if( version )
hasher.putUnencodedChars(version)
return hasher
}
@Override
String toString() {
version ? "${id}@${version}" : id
}
@Override
int compareTo(PluginRef that) {
return this.toString() <=> that.toString()
}
}

View File

@@ -0,0 +1,505 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.plugin
import static java.nio.file.StandardCopyOption.*
import java.nio.file.Files
import java.nio.file.Path
import java.util.function.Predicate
import java.util.regex.Pattern
import com.github.zafarkhaja.semver.Version
import dev.failsafe.Failsafe
import dev.failsafe.RetryPolicy
import dev.failsafe.event.ExecutionAttemptedEvent
import dev.failsafe.function.CheckedSupplier
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.BuildInfo
import nextflow.SysEnv
import nextflow.extension.FilesEx
import nextflow.file.FileHelper
import nextflow.file.FileMutex
import org.pf4j.InvalidPluginDescriptorException
import org.pf4j.PluginDependency
import org.pf4j.PluginRuntimeException
import org.pf4j.PluginState
import org.pf4j.PluginWrapper
import org.pf4j.RuntimeMode
import org.pf4j.update.DefaultUpdateRepository
import org.pf4j.update.PluginInfo
import org.pf4j.update.UpdateManager
import org.pf4j.update.UpdateRepository
import org.pf4j.util.FileUtils
/**
* Implements the download/install/update for nextflow plugins
*
* Plugins are downloaded and stored uncompressed in the
* directory defined by the variable `NXF_PLUGINS_DIR`, $NXF_HOME/nextflow
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
class PluginUpdater extends UpdateManager {
static final public Pattern META_REGEX = ~/(.+)-(\d+\.\d+\.\d+\S*)-meta\.json/
private CustomPluginManager pluginManager
private Path pluginsStore
private boolean pullOnly
private boolean offline
private DefaultPlugins defaultPlugins = DefaultPlugins.INSTANCE
protected PluginUpdater(CustomPluginManager pluginManager) {
super(pluginManager)
this.pluginManager = pluginManager
}
PluginUpdater(CustomPluginManager pluginManager, Path pluginsRoot, URL repo, boolean offline) {
super(pluginManager, wrap(repo, pluginsRoot, offline))
this.offline = offline
this.pluginsStore = pluginsRoot
this.pluginManager = pluginManager
}
static private List<UpdateRepository> wrap(URL remote, Path local, boolean offline) {
List<UpdateRepository> result = new ArrayList<>(1)
if( offline ) {
log.debug "Using local update repository: ${local}"
result.add(new LocalUpdateRepository('downloaded', local))
}
else {
def remoteRepo = remote.path.endsWith('.json')
? new DefaultUpdateRepository('nextflow.io', remote)
: new HttpPluginRepository('registry', remote.toURI())
log.debug "Using plugin repository: ${remoteRepo.getClass().getSimpleName()} [${remoteRepo.id}]; url=${remote}"
result.add(remoteRepo)
result.addAll(customRepos())
}
return result
}
static private List<DefaultUpdateRepository> customRepos() {
final repos = SysEnv.get('NXF_PLUGINS_TEST_REPOSITORY')
if( !repos )
return List.of()
// warn the user that a custom
final msg = """\
=======================================================================
= WARNING =
= You are running this script using a un-official plugin repository. =
= =
= ${repos}
= =
= This is only meant to be used for plugin testing purposes. =
=============================================================================
""".stripIndent(true)
log.warn(msg)
final result = new ArrayList<DefaultUpdateRepository>(10)
// the repos string can contain one or more plugin repository uri separated by comma
for( String it : repos.tokenize(',') )
result.add(customRepo(it))
return result
}
static private DefaultUpdateRepository customRepo(String uri) {
// Check if it's a plugin meta file. The name must match the pattern `<plugin id>-X.Y.Z-meta.json`
final matcher = META_REGEX.matcher(uri.tokenize('/')[-1])
if( matcher.matches() ) {
try {
final pluginId = matcher.group(1)
final temp = File.createTempFile('nxf-','json')
temp.deleteOnExit()
temp.text = /[{"id":"${pluginId}", "releases":[ ${new URL(uri).text} ]}]/
uri = 'file://' + temp.absolutePath
}
catch (FileNotFoundException e) {
throw new IllegalArgumentException("Provided repository URL does not exist or cannot be accessed: $uri")
}
}
// create the update repository instance
final fileName = uri.tokenize('/')[-1]
return new DefaultUpdateRepository('uri', new URL(uri), fileName)
}
/**
* Prefetch metadata for plugins. This gives an opportunity for certain
* repository types to perform some data-loading optimisations.
*/
void prefetchMetadata(List<PluginRef> plugins) {
// use direct field access to avoid the refresh() call in getRepositories()
// which could fail anything which hasn't had a chance to prefetch yet
for( def repo : this.@repositories ) {
if( repo instanceof PrefetchUpdateRepository ) {
log.trace "Prefetching plugin metadata from repository: ${repo.getClass().getSimpleName()} [${repo.id}]; plugins=${plugins}"
repo.prefetch(plugins)
}
}
}
/**
* Resolve a plugin installing or updating the dependencies if necessary
* and start the plugin
*
* @param pluginId The plugin Id
* @param version The required version, can be null for the latest version
*/
void prepareAndStart(String pluginId, String version) {
final PluginWrapper current = pluginManager.getPlugin(pluginId)
if( !current ) {
log.debug "Installing plugin ${pluginId} version: ${version ?: 'latest'}"
// install & start the plugin
installPlugin(pluginId, version)
}
else if( shouldUpdate(pluginId, version, current) ) {
log.debug "Updating plugin ${pluginId} version: ${version ?: 'latest'} [current version: $current.descriptor.version]"
// update & start the plugin
updatePlugin(pluginId, version)
}
else {
if( !version ) version = current.descriptor.version
log.debug "Starting plugin ${pluginId} version: ${version}"
pluginManager.startPlugin(pluginId)
}
}
void pullPlugins(List<String> plugins) {
pullOnly=true
try {
final specs = plugins.collect(it -> PluginRef.parse(it,defaultPlugins))
prefetchMetadata(specs)
for( PluginRef spec : specs ) {
pullPlugin0(spec.id, spec.version)
}
}
finally {
pullOnly=false
}
}
void pullPlugin0(String pluginId, String version) {
final PluginWrapper current = pluginManager.getPlugin(pluginId)
if( !current ) {
log.debug "Installing plugin ${pluginId} version: ${version ?: 'latest'}"
// install & start the plugin
installPlugin(pluginId, version)
}
else if( shouldUpdate(pluginId, version, current) ) {
log.debug "Updating plugin ${pluginId} version: ${version ?: 'latest'} [current version: $current.descriptor.version]"
// update & start the plugin
updatePlugin(pluginId, version)
}
}
/**
* Install a new plugin downloading the artifact from the remote source if needed
*
* @param id The plugin Id
* @param version The plugin version
* @return {@code true} when the plugin is correctly download, installed and started, {@code false} otherwise
*/
@Override
boolean installPlugin(String id, String version) {
if( !pluginsStore )
throw new IllegalStateException("Missing pluginStore attribute")
return load0(id, version)
}
private Path download0(String id, String version) {
// 0. check if version is specified
if( !version )
throw new InvalidPluginDescriptorException("Missing version for plugin $id")
log.info "Downloading plugin ${id}@${version}"
// 1. check if already exists
final pluginPath = pluginsStore.resolve("$id-$version")
if( FilesEx.exists(pluginPath) ) {
return pluginPath
}
// 2. download to temporary location
Path downloaded = safeDownloadPlugin(id, version);
// 3. unzip the content and delete downloaded file
Path dir = FileUtils.expandIfZip(downloaded)
FileHelper.deletePath(downloaded)
// 4. move the final destination the plugin directory
assert pluginPath.getFileName() == dir.getFileName()
try {
safeMove(dir, pluginPath)
}
catch (IOException e) {
throw new PluginRuntimeException(e, "Failed to write file '$pluginPath' to plugins folder")
}
return pluginPath
}
protected Path safeDownloadPlugin(String id, String version) {
final CheckedSupplier<Path> supplier = () -> downloadPlugin(id, version)
final policy = retryPolicy(id,version)
return Failsafe.with(policy).get(supplier)
}
protected <T> RetryPolicy<T> retryPolicy(String id, String version) {
final listener = new dev.failsafe.event.EventListener<ExecutionAttemptedEvent<T>>() {
@Override
void accept(ExecutionAttemptedEvent<T> event) throws Throwable {
log.debug("Failed to download plugin: $id; version: $version - attempt: ${event.attemptCount}", event.lastFailure)
}
}
final condition = new Predicate<Throwable>() {
@Override
boolean test(Throwable error) {
return error?.cause instanceof ConnectException
}
}
return RetryPolicy.<T>builder()
.handleIf(condition)
.withMaxAttempts(3)
.onRetry(listener)
.build()
}
protected void safeMove(Path source, Path target) {
try {
Files.move(source, target, ATOMIC_MOVE, REPLACE_EXISTING)
}
catch (IOException e) {
log.debug "Failed atomic move for plugin $source -> $target - Reason: ${e.message ?: e} - Fallback on safe move"
safeMove0(source, target)
}
}
protected void safeMove0(Path source, Path target) {
// make sure to the target path does not exist
FileHelper.deletePath(target)
// copy the source to the more
FileHelper.copyPath(source, target)
// finally remove the source
try {
FileHelper.deletePath(source)
}
catch (IOException e) {
log.warn("Unable to delete plugin directory: $source", e)
}
}
/**
* Race condition safe plugin download. Multiple instances are synchronised
* using a file system lock created in the tmp directory
*
* @param id The plugin Id
* @param version The plugin version string
* @return The uncompressed plugin directory path
*/
private Path safeDownload(String id, String version) {
final sentinel = lockFile(id,version)
final mutex = new FileMutex(target: sentinel, timeout: '10 min')
try {
return mutex.lock { download0(id, version) }
}
finally {
sentinel.delete()
}
}
/**
* Get the file to be used a lock to synchronise concurrent downloaded from multiple Nextflow launches
*
* @param id The plugin Id
* @param version The plugin version
* @return The lock the path
*/
private File lockFile(String id, String version) {
def tmp = System.getProperty('java.io.tmpdir')
new File(tmp, "nextflow-plugin-${id}-${version}.lock")
}
private boolean load0(String id, String requestedVersion) {
assert id, "Missing plugin Id"
if( offline && !requestedVersion ) {
throw new IllegalStateException("Cannot find version for $id plugin -- plugin versions MUST be specified in offline mode")
}
def version = requestedVersion
if( !version ) {
version = getLastPluginRelease(id)?.version
}
else if( !Version.isValid(version) ) {
// a version is 'valid' if it's an exact semver version "major.minor.patch" so
// if it's not that, treat it as a version constraint and look for matches
version = findNewestMatchingRelease(id, version)?.version
}
if( !version ) {
final msg = requestedVersion
? "Cannot find version of $id plugin matching $requestedVersion"
: "Cannot find latest version of $id plugin"
throw new IllegalStateException(msg)
}
if( version != requestedVersion ) {
log.debug "Plugin $id version $requestedVersion resolved to: $version"
}
def pluginPath = pluginsStore.resolve("$id-$version")
if( !FilesEx.exists(pluginPath) ) {
pluginPath = safeDownload(id, version)
}
// verify the plugin install path contains the expected manifest path
if( !FilesEx.exists(pluginPath.resolve('classes/META-INF/MANIFEST.MF')) ) {
log.warn("Plugin '${pluginPath.getFileName()}' installation looks corrupted - Delete the following directory and run nextflow again: $pluginPath")
}
// load the plugin from the file system
PluginWrapper wrapper = pluginManager.loadPluginFromPath(pluginPath)
// pull all required required deps
final deps = wrapper.descriptor.dependencies ?: Collections.<PluginDependency>emptyList()
for( PluginDependency it : deps ) {
// 1. check for installed version satisfying req
final installed = checkInstalled(it.pluginId, it.pluginVersionSupport)
if( installed ) {
log.debug "Plugin $id requires $it.pluginId supported version: $it.pluginVersionSupport - found version: $installed"
continue
}
// 2. find latest satisfying req
// -- if it's a core nextflow plugin use the version expected by it
def depVersion = defaultPlugins.getPlugin(it.pluginId)?.version
// -- otherwise try to find the newest matching release
if( !depVersion )
depVersion = findNewestMatchingRelease(it.pluginId, it.pluginVersionSupport)?.version
log.debug "Plugin $id requires $it.pluginId supported version: $it.pluginVersionSupport - available version: $depVersion"
if( pullOnly )
pullPlugin0(it.pluginId, depVersion)
else
prepareAndStart(it.pluginId, depVersion)
}
if( pullOnly )
return false
// resolve the plugins
pluginManager.resolvePlugins()
// finally start it
PluginState state = pluginManager.startPlugin(id)
return PluginState.STARTED == state
}
/**
* Update an existing plugin, unloading the current version and installing
* the requested one downloading the artifact if needed
*
* @param id The plugin Id
* @param version The plugin version
* @return {@code true} when the plugin is updated and started or {@code false} otherwise
*/
@Override
boolean updatePlugin(String id, String version) {
if (pluginManager.getPlugin(id) == null) {
throw new PluginRuntimeException("Plugin $id cannot be updated since it is not installed")
}
PluginInfo pluginInfo = getPluginsMap().get(id)
if (pluginInfo == null) {
throw new PluginRuntimeException("Plugin $id does not exist in any repository")
}
if (!pluginManager.deletePlugin(id)) {
return false
}
load0(id, version)
}
protected boolean shouldUpdate(String pluginId, String version, PluginWrapper current) {
if( pluginManager.runtimeMode == RuntimeMode.DEVELOPMENT ) {
log.debug "Update not supported during development mode"
return false
}
if( offline ) {
log.debug "Update not supported in offline mode"
return false
}
if( !version )
version = getLastPluginRelease(pluginId)?.version
if( !version ) {
log.warn "Cannot find latest version for plugin $pluginId [skip update]"
return false
}
return pluginManager
.getVersionManager()
.compareVersions(current.descriptor.version, version) < 0
}
protected String checkInstalled(String id, String verConstraint) {
final versionManager = pluginManager.getVersionManager()
// check if the installed version satisfies the requirement
final current = pluginManager.getPlugin(id)
if( !current )
return null
final found = versionManager.checkVersionConstraint(current.descriptor.version, verConstraint)
return found ? current.descriptor.version : null
}
/**
* Find newest release version matching the requested plugin constraint and nextflow runtime
*
* @param id The plugin id
* @param verConstraint The version constraint
* @return The plugin release satisfying the requested criteria or null it none is found
*/
protected PluginInfo.PluginRelease findNewestMatchingRelease(String id, String verConstraint) {
assert verConstraint
final versionManager = pluginManager.getVersionManager()
PluginInfo pluginInfo = getPluginsMap().get(id)
if( !pluginInfo )
throw new IllegalArgumentException("Unknown plugin id: $id")
// note: order releases list by descending version numbers ie. latest version comes first
def releases = pluginInfo.releases.sort(false) { a,b -> Version.parse(b.version) <=> Version.parse(a.version) }
for (PluginInfo.PluginRelease rel : releases ) {
if( !versionManager.checkVersionConstraint(rel.version, verConstraint) || !rel.url )
continue
if( versionManager.checkVersionConstraint(BuildInfo.version, rel.requires) )
return rel
}
return null
}
}

View File

@@ -0,0 +1,94 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.plugin
import java.nio.file.Path
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.pf4j.PluginManager
/**
* Plugin manager specialized for Nextflow build environment
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
class Plugins {
private final static PluginsFacade INSTANCE = new PluginsFacade()
static PluginManager getManager() { INSTANCE.manager }
static synchronized void init(boolean embeddedMode=false) {
INSTANCE.init(embeddedMode)
}
static synchronized void init(Path root, String mode, CustomPluginManager pluginManager) {
INSTANCE.init(root, mode, pluginManager)
}
/**
* @param config
*/
static void load(Map config) {
INSTANCE.load(config)
}
static void start(String pluginId) {
INSTANCE.start(pluginId)
}
static synchronized void stop() {
INSTANCE.stop()
}
static <T> List<T> getExtensions(Class<T> type) {
INSTANCE.getExtensions(type)
}
static <T> List<T> getPriorityExtensions(Class<T> type, String group=null) {
INSTANCE.getPriorityExtensions(type,group)
}
static <T> T getExtension(Class<T> type) {
final allExtensions = INSTANCE.getExtensions(type)
return allExtensions ? allExtensions.first() : null
}
static <T> List<T> getExtensionsInPluginId(Class<T> type, String pluginId) {
final allExtensions = INSTANCE.getExtensions(type, pluginId)
return allExtensions
}
static void pull(List<String> ids) {
INSTANCE.pullPlugins(ids)
}
static boolean startIfMissing(String pluginId) {
if( INSTANCE ) {
return INSTANCE.startIfMissing(pluginId)
} else {
log.debug "Plugins subsystem not available - Ignoring installIfMissing('$pluginId')"
return false
}
}
static boolean isStarted(String pluginId) {
INSTANCE.isStarted(pluginId)
}
}

View File

@@ -0,0 +1,608 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.plugin
import java.nio.file.Path
import java.nio.file.Paths
import groovy.transform.CompileStatic
import groovy.transform.Memoized
import groovy.transform.PackageScope
import groovy.util.logging.Slf4j
import nextflow.SysEnv
import nextflow.exception.AbortOperationException
import nextflow.extension.Bolts
import nextflow.extension.FilesEx
import org.pf4j.DefaultPluginManager
import org.pf4j.PluginManager
import org.pf4j.PluginState
import org.pf4j.PluginStateEvent
import org.pf4j.PluginStateListener
/**
* Manage plugins installation and configuration
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
class PluginsFacade implements PluginStateListener {
@PackageScope
final static String LEGACY_PLUGINS_REPO = 'https://raw.githubusercontent.com/nextflow-io/plugins/main/plugins.json'
final static String NEXTFLOW_PLUGINS_REPO = 'https://registry.nextflow.io/api'
private static final String DEV_MODE = 'dev'
private static final String PROD_MODE = 'prod'
private Map<String,String> env = SysEnv.get()
private String mode
private Path root
private boolean offline
private PluginUpdater updater
private CustomPluginManager manager
private DefaultPlugins defaultPlugins = DefaultPlugins.INSTANCE
private String indexUrl
private boolean embedded
PluginsFacade() {
mode = getPluginsMode()
root = getPluginsDir()
indexUrl = getPluginsRegistryUrl()
offline = env.get('NXF_OFFLINE') == 'true'
if( mode==DEV_MODE && root.toString()=='plugins' ) {
// In dev mode with default 'plugins' path, try to detect the actual plugins directory
// by looking for the nextflow project root. This is needed when running tests via Gradle
// where classes are loaded from JARs but the plugins are in the project directory.
final detected = detectPluginsDevRoot0()
if( detected )
root = detected
}
System.setProperty('pf4j.mode', mode)
}
PluginsFacade(Path root, String mode=PROD_MODE, boolean offline=false,
String indexUrl=LEGACY_PLUGINS_REPO) {
this.mode = mode
this.root = root
this.offline = offline
this.indexUrl = indexUrl
System.setProperty('pf4j.mode', mode)
}
/**
* Determine if it's running from a JAR archive
* @return {@code true} if the code is running from a JAR artifact, {@code false} otherwise
*/
protected String isRunningFromDistArchive() {
final className = this.class.name.replace('.', '/');
final classJar = this.class.getResource("/" + className + ".class").toString();
return classJar.startsWith("jar:")
}
protected Path getPluginsDir() {
final dir = env.get('NXF_PLUGINS_DIR')
if( dir ) {
log.trace "Detected NXF_PLUGINS_DIR=$dir"
return Paths.get(dir)
}
else if( env.containsKey('NXF_HOME') ) {
log.trace "Detected NXF_HOME - Using ${env.NXF_HOME}/plugins"
return Paths.get(env.NXF_HOME, 'plugins')
}
else {
log.trace "Using local plugins directory"
return Paths.get('plugins')
}
}
static protected boolean isSupportedIndex(String url) {
if( !url ) {
throw new IllegalArgumentException("Missing plugins registry URL")
}
if( !url.startsWith('https://') && !url.startsWith('http://') ) {
throw new IllegalArgumentException("Plugins registry URL must start with 'http://' or 'https://': $url")
}
if( url == LEGACY_PLUGINS_REPO ) {
return true
}
final hostname = URI.create(url).authority
return hostname.endsWith('.nextflow.io')
|| hostname.endsWith('.nextflow.com')
|| hostname.endsWith('.seqera.io')
}
protected String getPluginsRegistryUrl() {
final url = env.get('NXF_PLUGINS_REGISTRY_URL')
if( !url ) {
log.trace "Using default plugins url: ${NEXTFLOW_PLUGINS_REPO}"
return NEXTFLOW_PLUGINS_REPO
}
log.debug "Detected NXF_PLUGINS_REGISTRY_URL=$url"
if( !isSupportedIndex(url) ) {
// warn that this is experimental behaviour
log.warn """\
=======================================================================
= WARNING =
= This workflow run us using an unofficial plugins registry. =
= =
= ${url}
= =
= Its usage is unsupported and not recommended for production workloads. =
=============================================================================
""".stripIndent(true)
}
return url
}
private boolean isNextflowDevRoot(File file) {
file.name=='nextflow' && file.isDirectory() && new File(file, 'settings.gradle').isFile()
}
private Path pluginsDevRoot(File path) {
// main project root
if( isNextflowDevRoot(path) )
return path.toPath().resolve('plugins')
// when nextflow is included into another build, check the sibling directory
if( new File(path,'settings.gradle').exists() && isNextflowDevRoot(path=new File(path,'../nextflow')) )
return path.toPath().resolve('plugins')
else
return null
}
/**
* Try to detect the development plugin root without throwing an exception.
*
* @return The nextflow plugins project path, or null if not found
*/
protected Path detectPluginsDevRoot0() {
def file = new File('.').canonicalFile
while( file!=null ) {
final root = pluginsDevRoot(file)
if( root ) {
log.debug "Detected dev plugins root: $root"
return root
}
file = file.parentFile
}
return null
}
/**
* Determine the development plugin root. This is required to
* allow running unit tests for plugin projects importing the
* nextflow core runtime.
*
* The nextflow main project is expected to be cloned into
* a sibling directory respect to the plugin project
*
* @return The nextflow plugins project path in the local file system
* @throws IllegalStateException if the plugins root cannot be detected
*/
protected Path detectPluginsDevRoot() {
final root = detectPluginsDevRoot0()
if( root )
return root
throw new IllegalStateException("Unable to detect local plugins root")
}
protected String getPluginsMode() {
final mode = env.get('NXF_PLUGINS_MODE')
if( mode ) {
log.trace "Detected NXF_PLUGINS_MODE=$mode"
return mode
}
else if( env.containsKey('NXF_HOME') ) {
log.trace "Detected NXF_HOME - Using plugins mode=prod"
return PROD_MODE
}
else {
log.debug "Using dev plugins mode"
return DEV_MODE
}
}
protected boolean getPluginsDefault() {
if( env.containsKey('NXF_PLUGINS_DEFAULT')) {
log.trace "Detected NXF_PLUGINS_DEFAULT=$env.NXF_PLUGINS_DEFAULT"
return env.NXF_PLUGINS_DEFAULT!='false'
}
else if( env.containsKey('NXF_HOME') ) {
log.trace "Detected NXF_HOME - Using plugin defaults"
return true
}
else {
log.trace "Disabling plugin defaults"
return false
}
}
private CustomPluginManager newPluginManager(Path root, boolean embedded) {
if( mode==DEV_MODE ) {
// plugin manage for dev purposes
return new DevPluginManager(root)
}
if( embedded ) {
// use the custom plugin manager to by-pass the creation of a local plugin repository
return new EmbeddedPluginManager(root)
}
return new LocalPluginManager(root)
}
protected CustomPluginManager createManager(Path root, boolean embedded) {
final result = newPluginManager(root, embedded)
result.addPluginStateListener(this)
return result
}
protected PluginUpdater createUpdater(Path root, CustomPluginManager manager) {
return ( mode!=DEV_MODE
? new PluginUpdater(manager, root, new URL(indexUrl), offline)
: new DevPluginUpdater(manager) )
}
@Override
void pluginStateChanged(PluginStateEvent ev) {
final err = ev.plugin.failedException
final dsc = ev.plugin.descriptor
if( err ) {
throw new IllegalStateException("Unable to start plugin id=${dsc.pluginId} version=${dsc.version} -- cause: ${err.message ?: err}", err)
}
}
PluginManager getManager() { manager }
void init(boolean embedded=false) {
if( manager )
throw new IllegalArgumentException("Plugin system already setup")
log.debug "Setting up plugin manager > mode=${mode}; embedded=$embedded; plugins-dir=$root; core-plugins: ${defaultPlugins.toSortedString()}"
// make sure plugins dir exists
if( mode!=DEV_MODE && !FilesEx.exists(root) && !FilesEx.mkdirs(root) )
throw new IOException("Unable to create plugins dir: $root")
this.manager = createManager(root, embedded)
this.updater = createUpdater(root, manager)
manager.loadPlugins()
if( embedded ) {
manager.startPlugins()
this.embedded = embedded
}
}
void init(Path root, String mode, CustomPluginManager pluginManager) {
if( manager )
throw new IllegalArgumentException("Plugin system already setup")
this.root = root
this.mode = mode
// setup plugin manager
this.manager = pluginManager
this.manager.addPluginStateListener(this)
// setup the updater
this.updater = createUpdater(root, manager)
// load plugins
manager.loadPlugins()
if( embedded ) {
manager.startPlugins()
this.embedded = embedded
}
}
void load(Map config) {
if( !manager )
throw new IllegalArgumentException("Plugin system has not been initialized")
start(pluginsRequirement(config))
}
synchronized void stop() {
if( manager ) {
manager.stopPlugins()
manager = null
}
}
/**
* Return a list of extension matching the requested interface type
*
* @param type
* The request extension interface
* @return
* The list of extensions matching the requested interface.
*/
def <T> List<T> getExtensions(Class<T> type) {
if( manager ) {
return manager.getExtensions(type)
}
else {
// this should only be used to load system extensions
// i.e. included in the app class path not provided by
// a plugin extension
log.debug "Using Default plugin manager"
return defaultManager().getExtensions(type)
}
}
/**
* Return a list of extension matching the requested interface type in a plugin
*
* @param type
* The request extension interface
* @return
* The list of extensions matching the requested interface.
*/
def <T> List<T> getExtensions(Class<T> type, String pluginId) {
if( manager ) {
return manager.getExtensions(type, pluginId)
}
else {
return List.of()
}
}
/**
* Return a list of extension matching the requested type
* ordered by a priority value. The element at the beginning
* of the list (index 0) has higher priority
*
* @param type
* The request extension interface
* @return
* The list of extensions matching the requested interface.
* The extension with higher priority appears first (lower index)
*/
def <T> List<T> getPriorityExtensions(Class<T> type,String group=null) {
def extensions = getExtensions(type)
if( group )
extensions = extensions.findAll(it -> group0(it)==group )
final result = extensions.sort( it -> priority0(it) )
if( log.isTraceEnabled() )
log.trace "Discovered extensions for type ${type.getName()}: ${extensions.join(',')}"
return result
}
protected int priority0(Object it) {
final annot = it.getClass().getAnnotation(Priority)
return annot ? annot.value() : 0
}
protected String group0(Object it) {
final annot = it.getClass().getAnnotation(Priority)
return annot && annot.group() ? annot.group() : null
}
@Memoized
private PluginManager defaultManager() {
new DefaultPluginManager()
}
void start(String pluginId) {
start(List.of(PluginRef.parse(pluginId, defaultPlugins)))
}
void start(List<PluginRef> specs) {
// check if the plugins are allowed to start
final disallow = specs.find(it-> !isAllowed(it))
if( disallow ) {
throw new AbortOperationException("Refuse to use plugin '${disallow.id}' - allowed plugins are: ${allowedPluginsString()}")
}
// when running in embedded mode, default plugins should not be started
// the following split partition the "specs" collection in to list
// - the first holding the plugins for which "start" is not required
// - the second all remaining plugins that requires a start invocation
final split = specs.split(plugin -> isEmbedded() && defaultPlugins.hasPlugin(plugin.id))
final skippable = split[0]
final startable = split[1]
// just report a debug line for the skipped ones
if( skippable ) {
final skippedIds = skippable.collect{ plugin -> plugin.id }
log.debug "Plugin 'start' is not required in embedded mode -- ignoring for plugins: $skippedIds"
}
// prefetch the plugins meta
updater.prefetchMetadata(startable)
// finally start the plugins
for( PluginRef plugin : startable ) {
updater.prepareAndStart(plugin.id, plugin.version)
}
}
boolean isStarted(String pluginId) {
manager.getPlugin(pluginId)?.pluginState == PluginState.STARTED
}
/**
* @return {@code true} when running in embedded mode ie. the nextflow distribution
* include also plugin libraries. When running is this mode, plugins should not be started
* and cannot be updated.
*/
protected boolean isEmbedded() {
return embedded
}
protected List<PluginRef> pluginsRequirement(Map config) {
def specs = parseConf(config)
if( isEmbedded() && specs ) {
// custom plugins are not allowed for nextflow self-contained package
log.warn "Nextflow embedded mode only core plugins -- User config plugins will be ignored: ${specs.join(',')}"
return Collections.emptyList()
}
if( specs ) {
log.debug "Plugins declared=$specs"
}
if( getPluginsDefault() ){
final defSpecs = defaultPluginsConf(config)
specs = mergePluginSpecs(specs, defSpecs)
log.debug "Plugins default=$defSpecs"
}
// add tower plugin when config contains tower options
if( (Bolts.navigate(config,'tower.enabled') || Bolts.navigate(config,'fusion.enabled') || env.TOWER_ACCESS_TOKEN ) && !specs.find {it.id == 'nf-tower' } ) {
specs << defaultPlugins.getPlugin('nf-tower')
}
if( (Bolts.navigate(config,'wave.enabled') || Bolts.navigate(config,'fusion.enabled')) && !specs.find {it.id == 'nf-wave' } ) {
specs << defaultPlugins.getPlugin('nf-wave')
}
if( Bolts.navigate(config,'process.executor')=='seqera') {
specs << defaultPlugins.getPlugin('nf-seqera')
}
// add cloudcache plugin when cloudcache is enabled in the config
if( Bolts.navigate(config, 'cloudcache.enabled')==true ) {
specs << defaultPlugins.getPlugin('nf-cloudcache')
}
log.debug "Plugins resolved requirement=$specs"
return specs
}
protected List<PluginRef> defaultPluginsConf(Map config) {
// retrieve the list from the env var
final commaSepList = env.get('NXF_PLUGINS_DEFAULT')
if( commaSepList && commaSepList !in ['true','false'] ) {
// if the plugin id in the list does *not* contain the @version suffix, it picks the version
// specified in the defaults list. Otherwise parse the provider id@version string to the corresponding spec
return commaSepList
.tokenize(',')
.collect( it-> defaultPlugins.hasPlugin(it) ? defaultPlugins.getPlugin(it) : PluginRef.parse(it) )
}
// infer from app config
final plugins = new ArrayList<PluginRef>()
final workDir = config.workDir as String
final bucketDir = config.bucketDir as String
final executor = Bolts.navigate(config, 'process.executor')
if( executor == 'awsbatch' || workDir?.startsWith('s3://') || bucketDir?.startsWith('s3://') || env.containsKey('NXF_ENABLE_AWS_SES') )
plugins << defaultPlugins.getPlugin('nf-amazon')
if( executor == 'google-lifesciences' || executor == 'google-batch' || workDir?.startsWith('gs://') || bucketDir?.startsWith('gs://') )
plugins << defaultPlugins.getPlugin('nf-google')
if( executor == 'azurebatch' || workDir?.startsWith('az://') || bucketDir?.startsWith('az://') )
plugins << defaultPlugins.getPlugin('nf-azure')
if( executor == 'k8s' )
plugins << defaultPlugins.getPlugin('nf-k8s')
if( Bolts.navigate(config, 'weblog.enabled'))
plugins << new PluginRef('nf-weblog')
return plugins
}
/**
* Lookup for plugins declared in the nextflow.config using the `plugins` scope
*
* @param config The nextflow config as a Map object
* @return The list of declared plugins
*/
protected List<PluginRef> parseConf(Map config) {
final pluginsConf = config.plugins as List<String>
final result = new ArrayList( pluginsConf?.size() ?: 0 )
if(pluginsConf) for( String it : pluginsConf ) {
result.add( PluginRef.parse(it, defaultPlugins) )
}
return result
}
synchronized void pullPlugins(List<String> ids) {
updater.pullPlugins(ids)
}
boolean startIfMissing(String pluginId) {
if( env.NXF_PLUGINS_DEFAULT == 'false' )
return false
if( isEmbedded() && defaultPlugins.hasPlugin(pluginId) )
return false
if( isStarted(pluginId) )
return false
synchronized (this) {
if( isStarted(pluginId) )
return false
start(pluginId)
return true
}
}
/**
* Merge two lists of plugin requirements
*
* @param configPlugins
* The list of plugins specified via the configuration file. This has higher priority
* @param defaultPlugins
* The list of plugins specified via the environment
* @return
* The list of plugins resulting from merging the two lists
*/
protected List<PluginRef> mergePluginSpecs(List<PluginRef> configPlugins, List<PluginRef> defaultPlugins) {
final map = new LinkedHashMap<String,PluginRef>(10)
// add all plugins in the 'configPlugins' argument
for( PluginRef plugin : configPlugins ) {
map.put(plugin.id, plugin)
}
// add the plugin in the 'defaultPlugins' argument
// if the map already contains the plugin,
// override it only if it does not specify a version
for( PluginRef plugin : defaultPlugins ) {
if( !map[plugin.id] || !map[plugin.id].version ) {
map.put(plugin.id, plugin)
}
}
return new ArrayList<PluginRef>(map.values())
}
protected List<PluginRef> parseAllowedPlugins(Map<String,String> env) {
final list = env.get('NXF_PLUGINS_ALLOWED')
// note: empty string means no plugins is allowed
return list!=null
? list.tokenize(',').collect(it-> PluginRef.parse(it))
: null
}
/**
* The list of allowed plugins to be used defined by the variable NXF_PLUGINS_ALLOWED
*
* @return
* The list of allowed plugins. {@code null} mean all. Empty list means none
*/
@Memoized
protected List<PluginRef> getAllowedPlugins() {
return parseAllowedPlugins(env)
}
protected boolean isAllowed(String pluginId) {
final list = getAllowedPlugins()
return list != null
? list.any(it->it.id==pluginId)
: true
}
protected isAllowed(PluginRef plugin) {
return isAllowed(plugin.id)
}
private String allowedPluginsString() {
final list = getAllowedPlugins()
if( list == null )
return '(all)'
if( list.isEmpty() )
return '(none)'
return list.collect(it->it.id).join('')
}
}

View File

@@ -0,0 +1,36 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.plugin
import groovy.transform.CompileStatic
import org.pf4j.update.UpdateRepository
/**
* Extension to pf4j's UpdateRepository which supports pre-fetching
* metadata for a specified set of plugins.
*
* This gives the ability to avoid downloading metadata for unused
* plugins.
*/
@CompileStatic
interface PrefetchUpdateRepository extends UpdateRepository {
/**
* This will be called when Nextflow starts, before
* initialising the plugins.
*/
void prefetch(List<PluginRef> plugins)
}

View File

@@ -0,0 +1,35 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.plugin
import java.lang.annotation.ElementType
import java.lang.annotation.Retention
import java.lang.annotation.RetentionPolicy
import java.lang.annotation.Target
/**
* Allow the definition of plugin priority order, smaller value is an higher priority
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.TYPE)
@interface Priority {
int value()
String group() default ''
}

View File

@@ -0,0 +1,36 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.plugin
import java.lang.annotation.ElementType
import java.lang.annotation.Retention
import java.lang.annotation.RetentionPolicy
import java.lang.annotation.Target
/**
* Allow the definition of plugin priority order, smaller value is an higher priority
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Deprecated
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.TYPE)
@interface Scoped {
String value()
int priority() default 0
}

View File

@@ -0,0 +1,218 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.plugin.util
import java.util.regex.Matcher
import java.util.regex.Pattern
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.exception.AbortOperationException
import nextflow.extension.FilesEx
@Slf4j
@CompileStatic
class PluginRefactor {
private String pluginName
private String providerName
private String packageName
private File pluginDir
private String pluginClassPrefix
private File gradleSettingsFile
private File gradleBuildFile
private Map<String,String> tokenMapping = new HashMap<>()
String getPluginName() {
return pluginName
}
String getPackageName() {
return packageName
}
String getProviderName() {
return providerName
}
File getPluginDir() {
return pluginDir
}
PluginRefactor withPluginDir(File directory) {
this.pluginDir = directory.absoluteFile.canonicalFile
return this
}
PluginRefactor withPluginName(String name) {
this.pluginName = normalizeToKebabCase(name)
this.pluginClassPrefix = normalizeToClassName(name)
if( pluginName.toLowerCase()=='plugin' )
throw new IllegalStateException("Invalid plugin name: '$name'")
if( !pluginClassPrefix )
throw new IllegalStateException("Invalid plugin name: '$name'")
return this
}
PluginRefactor withProviderName(String name) {
this.providerName = name?.trim()
if( !providerName )
throw new AbortOperationException("Provider name cannot be empty or blank")
this.packageName = normalizeToPackageNameSegment(name)
if( !packageName )
throw new AbortOperationException("Invalid provider name: $name")
return this
}
protected void init() {
if( !pluginName )
throw new IllegalStateException("Missing plugin name")
if( !providerName )
throw new IllegalStateException("Missing provider name")
// initial
this.gradleBuildFile = new File(pluginDir, 'build.gradle')
this.gradleSettingsFile = new File(pluginDir, 'settings.gradle')
if( !gradleBuildFile.exists() )
throw new AbortOperationException("Plugin file does not exist: $gradleBuildFile")
if( !gradleSettingsFile.exists() )
throw new AbortOperationException("Plugin file does not exist: $gradleSettingsFile")
if( !providerName )
throw new AbortOperationException("Plugin provider name is missing")
if( !packageName )
throw new AbortOperationException("Plugin package name is missing")
// packages to be updates
tokenMapping.put('acme', packageName)
tokenMapping.put('provider-name', providerName)
tokenMapping.put('nf-plugin-template', pluginName)
}
void apply() {
init()
replacePrefixInFiles(pluginDir, pluginClassPrefix)
renameDirectory(new File(pluginDir, "src/main/groovy/acme"), new File(pluginDir, "src/main/groovy/${packageName}"))
renameDirectory(new File(pluginDir, "src/test/groovy/acme"), new File(pluginDir, "src/test/groovy/${packageName}"))
updateClassNamesAndSymbols(pluginDir)
}
protected void replacePrefixInFiles(File rootDir, String newPrefix) {
if (!rootDir.exists() || !rootDir.isDirectory()) {
throw new IllegalStateException("Invalid directory: $rootDir")
}
rootDir.eachFileRecurse { file ->
if (file.isFile() && file.name.startsWith('My') && FilesEx.getExtension(file) in ['groovy']) {
final newName = file.name.replaceFirst(/^My/, newPrefix)
final renamedFile = new File(file.parentFile, newName)
if (file.renameTo(renamedFile)) {
log.debug "Renamed: ${file.name} -> ${renamedFile.name}"
final source = FilesEx.getBaseName(file)
final target = FilesEx.getBaseName(renamedFile)
tokenMapping.put(source, target)
}
else {
throw new IllegalStateException("Failed to rename: ${file.name}")
}
}
}
}
protected void updateClassNamesAndSymbols(File rootDir) {
rootDir.eachFileRecurse { file ->
if (file.isFile() && FilesEx.getExtension(file) in ['groovy','gradle','md']) {
replaceTokensInFile(file, tokenMapping)
}
}
}
protected void replaceTokensInFile(File inputFile, Map<String, String> replacements, File outputFile = inputFile) {
def content = inputFile.text
// Replace each key with its corresponding value
for( Map.Entry<String,String> entry : replacements ) {
content = content.replaceAll(Pattern.quote(entry.key), Matcher.quoteReplacement(entry.value))
}
outputFile.text = content
log.debug "Replacements done in: ${outputFile.path}"
}
protected void renameDirectory(File oldDir, File newDir) {
if (!oldDir.exists() || !oldDir.isDirectory()) {
throw new AbortOperationException("Plugin template directory to rename does not exist: $oldDir")
}
if( oldDir==newDir ) {
log.debug "Unneeded path rename: $oldDir -> $newDir"
}
if (newDir.exists()) {
throw new AbortOperationException("Plugin target directory already exists: $newDir")
}
if (oldDir.renameTo(newDir)) {
log.debug "Successfully renamed: $oldDir -> $newDir"
}
else {
throw new AbortOperationException("Unable to replace plugin template path: $oldDir -> $newDir")
}
}
static String normalizeToClassName(String input) {
// Replace non-alphanumeric characters with spaces (except underscores)
final cleaned = input.replaceAll(/[^a-zA-Z0-9_]/, ' ')
.replaceAll(/_/, ' ')
.trim()
// Split by whitespace, capitalize each word, join them
final parts = cleaned.split(/\s+/).collect { it.capitalize() }
final result = parts.join('').replace('Plugin','')
// Remove "Nf" prefix only
return result.startsWith('Nf') ? result.substring(2) : result
}
static String normalizeToKebabCase(String input) {
// Insert spaces before capital letters (handles CamelCase)
def spaced = input.replaceAll(/([a-z])([A-Z])/, '$1 $2')
.replaceAll(/([A-Z]+)([A-Z][a-z])/, '$1 $2')
// Replace non-alphanumeric characters and underscores with spaces
def cleaned = spaced.replaceAll(/[^a-zA-Z0-9]/, ' ')
.trim()
// Split, lowercase, and join with hyphens
def parts = cleaned.split(/\s+/).collect { it.toLowerCase() }
return parts.join('-')
}
static String normalizeToPackageNameSegment(String input) {
// Replace non-alphanumeric characters with spaces
def cleaned = input.replaceAll(/[^a-zA-Z0-9]/, ' ')
.trim()
// Split into lowercase words and join
def parts = cleaned.split(/\s+/).collect { it.toLowerCase() }
def name = parts.join('')
// Strip leading digits
name = name.replaceFirst(/^\d+/, '')
return name ?: null
}
}

View File

@@ -0,0 +1,45 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.serde
/**
* An interface for encoding and decoding objects between two types.
*
* @param <T> the type of the original object to be encoded.
* @param <S> the type of the encoded representation.
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
interface Encoder<T,S> {
/**
* Encodes an object of type {@code T} into its corresponding encoded representation of type {@code S}.
*
* @param object the object to encode
* @return the encoded representation of the object
*/
S encode(T object)
/**
* Decodes an encoded representation of type {@code S} back into its original form of type {@code T}.
*
* @param encoded the encoded representation to decode
* @return the decoded object
*/
T decode(S encoded)
}

View File

@@ -0,0 +1,26 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.serde
/**
* Implements a marker interface for Json serialization objects.
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
interface JsonSerializable {
}

View File

@@ -0,0 +1,39 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.serde.gson
import java.lang.reflect.Type
import com.google.gson.JsonElement
import com.google.gson.JsonPrimitive
import com.google.gson.JsonSerializationContext
import com.google.gson.JsonSerializer
import groovy.transform.CompileStatic
/**
* Implements a Gson serializer for Groovy GString
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class GStringSerializer implements JsonSerializer<GString> {
@Override
JsonElement serialize(GString src, Type typeOfSrc, JsonSerializationContext context) {
// Convert GString to plain String
return new JsonPrimitive(src.toString())
}
}

View File

@@ -0,0 +1,104 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.serde.gson
import java.lang.reflect.Type
import java.nio.file.Path
import java.time.Instant
import java.time.OffsetDateTime
import com.google.gson.Gson
import com.google.gson.GsonBuilder
import com.google.gson.TypeAdapterFactory
import groovy.transform.CompileStatic
import nextflow.serde.Encoder
import nextflow.util.TypeHelper
import org.codehaus.groovy.runtime.GStringImpl
/**
* Implement a JSON encoder based on Google Gson
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
abstract class GsonEncoder<T> implements Encoder<T, String> {
private Type type
private TypeAdapterFactory factory
private boolean prettyPrint
private boolean serializeNulls
private volatile Gson gson
protected GsonEncoder() {
this.type = TypeHelper.getGenericType(this, 0)
}
GsonEncoder<T> withTypeAdapterFactory(TypeAdapterFactory factory) {
this.factory = factory
return this
}
GsonEncoder<T> withPrettyPrint(boolean value) {
this.prettyPrint = value
return this
}
GsonEncoder<T> withSerializeNulls(boolean value) {
this.serializeNulls = value
return this
}
private Gson gson0() {
if( gson )
return gson
synchronized (this) {
if( gson )
return gson
return gson = create0()
}
}
private Gson create0() {
final builder = new GsonBuilder()
builder.registerTypeAdapter(Instant.class, new InstantAdapter())
builder.registerTypeAdapter(OffsetDateTime.class, new OffsetDateTimeAdapter())
builder.registerTypeAdapter(GStringImpl.class, new GStringSerializer())
builder.registerTypeHierarchyAdapter(Path.class, new PathAdapter())
if( factory )
builder.registerTypeAdapterFactory(factory)
if( prettyPrint )
builder.setPrettyPrinting()
if( serializeNulls )
builder.serializeNulls()
return builder.create()
}
@Override
String encode(T object) {
return gson0().toJson(object, type)
}
@Override
T decode(String json) {
gson0().fromJson(json, type)
}
}

View File

@@ -0,0 +1,47 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.serde.gson
import java.time.Instant
import com.google.gson.TypeAdapter
import com.google.gson.stream.JsonReader
import com.google.gson.stream.JsonToken
import com.google.gson.stream.JsonWriter
import groovy.transform.CompileStatic
/**
* Implements a Gson adapter for {@link Instant}
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class InstantAdapter extends TypeAdapter<Instant> {
@Override
void write(JsonWriter writer, Instant value) throws IOException {
writer.value(value?.toString())
}
@Override
Instant read(JsonReader reader) throws IOException {
if( reader.peek() == JsonToken.NULL ) {
reader.nextNull()
return null
}
return Instant.parse(reader.nextString())
}
}

View File

@@ -0,0 +1,48 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.serde.gson
import java.time.Instant
import java.time.OffsetDateTime
import com.google.gson.TypeAdapter
import com.google.gson.stream.JsonReader
import com.google.gson.stream.JsonToken
import com.google.gson.stream.JsonWriter
import groovy.transform.CompileStatic
/**
* Implements a Gson adapter for {@link Instant}
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class OffsetDateTimeAdapter extends TypeAdapter<OffsetDateTime> {
@Override
void write(JsonWriter writer, OffsetDateTime value) throws IOException {
writer.value(value?.toString())
}
@Override
OffsetDateTime read(JsonReader reader) throws IOException {
if (reader.peek() == JsonToken.NULL) {
reader.nextNull()
return null
}
return OffsetDateTime.parse(reader.nextString())
}
}

View File

@@ -0,0 +1,40 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.serde.gson
import com.google.gson.TypeAdapter
import com.google.gson.stream.JsonReader
import com.google.gson.stream.JsonWriter
import nextflow.file.FileHelper
import java.nio.file.Path
class PathAdapter extends TypeAdapter<Path> {
@Override
void write(JsonWriter writer, Path value) throws IOException {
writer.value(value?.toUriString())
}
@Override
Path read(JsonReader reader) throws IOException {
if (reader.peek() == JsonToken.NULL) {
reader.nextNull()
return null
}
return FileHelper.asPath(reader.nextString())
}
}

View File

@@ -0,0 +1,354 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.serde.gson;
import java.io.IOException;
import java.util.LinkedHashMap;
import java.util.Map;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
import com.google.gson.Gson;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParseException;
import com.google.gson.JsonPrimitive;
import com.google.gson.TypeAdapter;
import com.google.gson.TypeAdapterFactory;
import com.google.gson.reflect.TypeToken;
import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonWriter;
/*
* NOTE: this class is copied from Gson extra module which is not included in the default
* library distribution.
*
* See
* https://github.com/google/gson/blob/main/extras/src/main/java/com/google/gson/typeadapters/RuntimeTypeAdapterFactory.java
*
*/
/**
* Adapts values whose runtime type may differ from their declaration type. This is necessary when a
* field's type is not the same type that GSON should create when deserializing that field. For
* example, consider these types:
*
* <pre>{@code
* abstract class Shape {
* int x;
* int y;
* }
* class Circle extends Shape {
* int radius;
* }
* class Rectangle extends Shape {
* int width;
* int height;
* }
* class Diamond extends Shape {
* int width;
* int height;
* }
* class Drawing {
* Shape bottomShape;
* Shape topShape;
* }
* }</pre>
*
* <p>Without additional type information, the serialized JSON is ambiguous. Is the bottom shape in
* this drawing a rectangle or a diamond?
*
* <pre>{@code
* {
* "bottomShape": {
* "width": 10,
* "height": 5,
* "x": 0,
* "y": 0
* },
* "topShape": {
* "radius": 2,
* "x": 4,
* "y": 1
* }
* }
* }</pre>
*
* This class addresses this problem by adding type information to the serialized JSON and honoring
* that type information when the JSON is deserialized:
*
* <pre>{@code
* {
* "bottomShape": {
* "type": "Diamond",
* "width": 10,
* "height": 5,
* "x": 0,
* "y": 0
* },
* "topShape": {
* "type": "Circle",
* "radius": 2,
* "x": 4,
* "y": 1
* }
* }
* }</pre>
*
* Both the type field name ({@code "type"}) and the type labels ({@code "Rectangle"}) are
* configurable.
*
* <h2>Registering Types</h2>
*
* Create a {@code RuntimeTypeAdapterFactory} by passing the base type and type field name to the
* {@link #of} factory method. If you don't supply an explicit type field name, {@code "type"} will
* be used.
*
* <pre>{@code
* RuntimeTypeAdapterFactory<Shape> shapeAdapterFactory
* = RuntimeTypeAdapterFactory.of(Shape.class, "type");
* }</pre>
*
* Next register all of your subtypes. Every subtype must be explicitly registered. This protects
* your application from injection attacks. If you don't supply an explicit type label, the type's
* simple name will be used.
*
* <pre>{@code
* shapeAdapterFactory.registerSubtype(Rectangle.class, "Rectangle");
* shapeAdapterFactory.registerSubtype(Circle.class, "Circle");
* shapeAdapterFactory.registerSubtype(Diamond.class, "Diamond");
* }</pre>
*
* Finally, register the type adapter factory in your application's GSON builder:
*
* <pre>{@code
* Gson gson = new GsonBuilder()
* .registerTypeAdapterFactory(shapeAdapterFactory)
* .create();
* }</pre>
*
* Like {@code GsonBuilder}, this API supports chaining:
*
* <pre>{@code
* RuntimeTypeAdapterFactory<Shape> shapeAdapterFactory = RuntimeTypeAdapterFactory.of(Shape.class)
* .registerSubtype(Rectangle.class)
* .registerSubtype(Circle.class)
* .registerSubtype(Diamond.class);
* }</pre>
*
* <h2>Serialization and deserialization</h2>
*
* In order to serialize and deserialize a polymorphic object, you must specify the base type
* explicitly.
*
* <pre>{@code
* Diamond diamond = new Diamond();
* String json = gson.toJson(diamond, Shape.class);
* }</pre>
*
* And then:
*
* <pre>{@code
* Shape shape = gson.fromJson(json, Shape.class);
* }</pre>
*/
public class RuntimeTypeAdapterFactory<T> implements TypeAdapterFactory {
private final Class<?> baseType;
private final String typeFieldName;
private final Map<String, Class<?>> labelToSubtype = new LinkedHashMap<>();
private final Map<Class<?>, String> subtypeToLabel = new LinkedHashMap<>();
private final boolean maintainType;
private boolean recognizeSubtypes;
protected RuntimeTypeAdapterFactory(Class<?> baseType, String typeFieldName, boolean maintainType) {
if (typeFieldName == null || baseType == null) {
throw new NullPointerException();
}
this.baseType = baseType;
this.typeFieldName = typeFieldName;
this.maintainType = maintainType;
}
/**
* Creates a new runtime type adapter for {@code baseType} using {@code typeFieldName} as the type
* field name. Type field names are case sensitive.
*
* @param maintainType true if the type field should be included in deserialized objects
*/
public static <T> RuntimeTypeAdapterFactory<T> of(
Class<T> baseType, String typeFieldName, boolean maintainType) {
return new RuntimeTypeAdapterFactory<>(baseType, typeFieldName, maintainType);
}
/**
* Creates a new runtime type adapter for {@code baseType} using {@code typeFieldName} as the type
* field name. Type field names are case sensitive.
*/
public static <T> RuntimeTypeAdapterFactory<T> of(Class<T> baseType, String typeFieldName) {
return new RuntimeTypeAdapterFactory<>(baseType, typeFieldName, false);
}
/**
* Creates a new runtime type adapter for {@code baseType} using {@code "type"} as the type field
* name.
*/
public static <T> RuntimeTypeAdapterFactory<T> of(Class<T> baseType) {
return new RuntimeTypeAdapterFactory<>(baseType, "type", false);
}
/**
* Ensures that this factory will handle not just the given {@code baseType}, but any subtype of
* that type.
*/
@CanIgnoreReturnValue
public RuntimeTypeAdapterFactory<T> recognizeSubtypes() {
this.recognizeSubtypes = true;
return this;
}
/**
* Registers {@code type} identified by {@code label}. Labels are case sensitive.
*
* @throws IllegalArgumentException if either {@code type} or {@code label} have already been
* registered on this type adapter.
*/
@CanIgnoreReturnValue
public RuntimeTypeAdapterFactory<T> registerSubtype(Class<? extends T> type, String label) {
if (type == null || label == null) {
throw new NullPointerException();
}
if (subtypeToLabel.containsKey(type) || labelToSubtype.containsKey(label)) {
throw new IllegalArgumentException("types and labels must be unique");
}
labelToSubtype.put(label, type);
subtypeToLabel.put(type, label);
return this;
}
/**
* Registers {@code type} identified by its {@link Class#getSimpleName simple name}. Labels are
* case sensitive.
*
* @throws IllegalArgumentException if either {@code type} or its simple name have already been
* registered on this type adapter.
*/
@CanIgnoreReturnValue
public RuntimeTypeAdapterFactory<T> registerSubtype(Class<? extends T> type) {
return registerSubtype(type, type.getSimpleName());
}
protected Class<?> getSubTypeFromLabel(String label){
return labelToSubtype.get(label);
}
protected String getLabelFromSubtype(Class<?> subType){
return subtypeToLabel.get(subType);
}
protected String getTypeFieldName(){
return typeFieldName;
}
@Override
public <R> TypeAdapter<R> create(Gson gson, TypeToken<R> type) {
if (type == null) {
return null;
}
Class<?> rawType = type.getRawType();
boolean handle =
recognizeSubtypes ? baseType.isAssignableFrom(rawType) : baseType.equals(rawType);
if (!handle) {
return null;
}
TypeAdapter<JsonElement> jsonElementAdapter = gson.getAdapter(JsonElement.class);
Map<String, TypeAdapter<?>> labelToDelegate = new LinkedHashMap<>();
Map<Class<?>, TypeAdapter<?>> subtypeToDelegate = new LinkedHashMap<>();
for (Map.Entry<String, Class<?>> entry : labelToSubtype.entrySet()) {
TypeAdapter<?> delegate = gson.getDelegateAdapter(this, TypeToken.get(entry.getValue()));
labelToDelegate.put(entry.getKey(), delegate);
subtypeToDelegate.put(entry.getValue(), delegate);
}
return new TypeAdapter<R>() {
@Override
public R read(JsonReader in) throws IOException {
JsonElement jsonElement = jsonElementAdapter.read(in);
JsonElement labelJsonElement;
if (maintainType) {
labelJsonElement = jsonElement.getAsJsonObject().get(typeFieldName);
} else {
labelJsonElement = jsonElement.getAsJsonObject().remove(typeFieldName);
}
if (labelJsonElement == null) {
throw new JsonParseException(
"cannot deserialize "
+ baseType
+ " because it does not define a field named "
+ typeFieldName);
}
String label = labelJsonElement.getAsString();
@SuppressWarnings("unchecked") // registration requires that subtype extends T
TypeAdapter<R> delegate = (TypeAdapter<R>) labelToDelegate.get(label);
if (delegate == null) {
throw new JsonParseException(
"cannot deserialize "
+ baseType
+ " subtype named "
+ label
+ "; did you forget to register a subtype?");
}
return delegate.fromJsonTree(jsonElement);
}
@Override
public void write(JsonWriter out, R value) throws IOException {
Class<?> srcType = value.getClass();
String label = subtypeToLabel.get(srcType);
@SuppressWarnings("unchecked") // registration requires that subtype extends T
TypeAdapter<R> delegate = (TypeAdapter<R>) subtypeToDelegate.get(srcType);
if (delegate == null) {
throw new JsonParseException(
"cannot serialize " + srcType.getName() + "; did you forget to register a subtype?");
}
JsonObject jsonObject = delegate.toJsonTree(value).getAsJsonObject();
if (maintainType) {
jsonElementAdapter.write(out, jsonObject);
return;
}
JsonObject clone = new JsonObject();
if (jsonObject.has(typeFieldName)) {
throw new JsonParseException(
"cannot serialize "
+ srcType.getName()
+ " because it already defines a field named "
+ typeFieldName);
}
clone.add(typeFieldName, new JsonPrimitive(label));
for (Map.Entry<String, JsonElement> e : jsonObject.entrySet()) {
clone.add(e.getKey(), e.getValue());
}
jsonElementAdapter.write(out, clone);
}
}.nullSafe();
}
}

View File

@@ -0,0 +1,27 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.ui
/**
* A generic decorator for {@code TextLabel} class
*/
interface LabelDecorator {
String apply( TextLabel label, String value )
}

View File

@@ -0,0 +1,269 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.ui
/**
* Used to render text based table for UI purpose. Example:
*
* <pre>
* def table = new TableBuilder().head('col1').head('col2').head('col3', 10)
* table << x << y << z << table.closeRow()
* table << p << q << w << table.closeRow()
*
* :
* println table.toString()
* </pre>
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
class TableBuilder {
/**
* The list defining the table header
*/
List<TextLabel> headers = []
List<TextLabel.Align> colsAlign = []
/**
* All the rows
*/
List<List<TextLabel>> allRows = []
List<Integer> dim = []
List<Integer> maxColsWidth = []
String cellSeparator = ' '
String rowSeparator = '\n'
List<TextLabel> currentRow = []
/**
* Defines a single column header, for example:
* <pre>
* def table = new TableBuilder().head('col1').head('col2').head('col3', 10)
* table << x << y << z
* </pre>
*
* @param name The string value to be used as column header
* @param maxWidth The column max width
* @return The table itself, to enable method chaining
*/
TableBuilder head( String name, int maxWidth = 0 ) {
def label = new TextLabel(name)
headers << label
maxColsWidth << maxWidth
colsAlign << null
def widths = headers .collect { it?.toString()?.size() }
trackWidths(widths)
return this
}
TableBuilder head( String name, TextLabel.Align align ) {
assert align
head(name)
colsAlign[-1] = align
return this
}
/**
* Use the specific string array as the table header definition
* @param cols
* @return
*/
TableBuilder setHeaders( String... cols ) {
assert cols != null
setHeaders( cols.collect { new TextLabel(it) } as TextLabel[] )
}
TableBuilder setHeaders( TextLabel... cols ) {
assert cols != null
// copy the header
this.headers = new ArrayList<>(cols as List<TextLabel>)
// keep track of the columns width
def widths = cols .collect { it?.toString()?.size() }
trackWidths(widths)
//return the object itself
return this
}
TableBuilder setMaxColsWidth( int...colsWidth ) {
assert colsWidth != null
maxColsWidth = new ArrayList<>(colsWidth as List<Integer>)
return this
}
TableBuilder append( Object... values ) {
append( values as List )
}
/**
* Append the specified list of values as the next row in the table, the columns width
* are adapted accordingly
*
* @param values
* @return
*/
TableBuilder append( List values ) {
assert values != null
def row = new ArrayList<TextLabel>(values.size())
def len = new ArrayList<Integer>(values.size())
values.each{ it ->
row << ( it instanceof TextLabel ? it : new TextLabel(it) )
len << it?.toString()?.size()
}
trackWidths(len)
allRows << row
this
}
/**
* Defines the left-shift operator useful to build the table using the following syntax
* <pre>
* def table = new TableBuilder()
* table << col1 << col2 << col3 << table.closeRow()
* :
* </pre>
*
*
* @param value The value to be added in the table at the current row
* @return The table instance itself
*/
TableBuilder leftShift( def value ) {
if( value == this ) {
return this
}
if( value instanceof TextLabel ) {
currentRow << value
}
else {
currentRow << new TextLabel(value)
}
return this
}
/**
* Close a row in the
* @return
*/
TableBuilder closeRow() {
append(currentRow)
currentRow = []
return this
}
protected void trackWidths( List<Integer> newDim ) {
def size = Math.min(dim.size(), newDim.size())
for( int i=0; i<size; i++ ) {
if ( dim[i] < newDim[i]) {
dim[i] = newDim[i]
}
}
if( newDim.size() > size ) {
for( int i=size; i<newDim.size(); i++ ) {
dim << newDim[i]
}
}
}
/**
* @return Render the final table and return the string
*/
String toString() {
// check if there's some rows no closed
if( currentRow ) {
closeRow()
}
StringBuilder result = new StringBuilder()
def count=0
/*
* render the header
*/
if( headers ) {
count++
headers.eachWithIndex { TextLabel cell, int index ->
renderCell( result, cell, index )
}
}
/*
* render the table
*/
allRows.each{ List<TextLabel> row ->
// render the 'rowSeparator' (only after the first row
if( count++ && rowSeparator!=null ) { result.append(rowSeparator) }
// render the row
row.eachWithIndex { TextLabel cell, int index ->
renderCell( result, cell, index )
}
}
result.toString()
}
/**
* Render a cell in the table
*
* @param result The {@code StringBuilder} collecting the result table text
* @param cell The cell to the rendered
* @param index The current index in the row of the cell to be rendered
*/
private void renderCell( StringBuilder result, TextLabel cell, int index ) {
// the 'cellSeparator' only after the first col
if( index && cellSeparator != null ) result.append(cellSeparator)
// set the max col width
if( maxColsWidth && index<maxColsWidth.size() && maxColsWidth[index] ) {
cell.max( maxColsWidth[index] )
}
// set the max
cell.width( dim[index] )
if( colsAlign[index] ) {
cell.setAlign( colsAlign[index] )
}
// render the cell
result.append( cell.toString() )
}
}

View File

@@ -0,0 +1,161 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.ui
/**
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
import groovy.transform.EqualsAndHashCode
@EqualsAndHashCode(includes='value')
class TextLabel {
enum Align { RIGHT, LEFT }
int width
Align align = Align.LEFT
boolean active
def value
def List<LabelDecorator> decorators = []
def Integer max
/**
* Create a label with the provided value
*/
TextLabel( def value ) {
this.value = value
this.align = ( value instanceof Number || value?.toString()?.isNumber() ) ? Align.RIGHT : Align.LEFT
}
def TextLabel width( int num ) { this.width = num; this }
def TextLabel left() { this.align = Align.LEFT; this }
def TextLabel right() { this.align = Align.RIGHT; this }
def TextLabel number() { this.align = Align.RIGHT; this }
def TextLabel max( int value ) { this.max = value; this }
/**
* Switch OFF the decorators rendering
*/
def TextLabel switchOff() { active = false; this }
/**
* Turn ON the decorators rendering
*/
def TextLabel switchOn() {
if ( !decorators ) {
decorators << AnsiStyle.style().negative()
}
active = true;
return this
}
def TextLabel leftShift( LabelDecorator deco ) {
this.decorators.add(deco)
return this
}
def TextLabel add( LabelDecorator deco ) {
this.decorators.add(deco)
return this
}
/**
* @return Renders the string applying the provided decorators
*/
def String toString() {
String result = value != null ? value.toString() : '-'
if( width && align == Align.LEFT ) {
result = result.padRight(width)
}
else if ( width && align == Align.RIGHT ) {
result = result.padLeft(width)
}
else {
result
}
// apply the max rule
if ( max != null ) {
result = applyMax(result)
}
// if not active return as it is
if( !active ) {
return result
}
decorators.each {
result = it.apply(this,result)
}
result
}
String applyMax( String str ) {
assert str
if( str.length()<=max) {
return str
}
def cut
if( align == Align.LEFT ) {
cut = str.substring(max)
str = str.substring(0,max)
}
else {
int p = str.size()-max
cut = str.substring( 0, p )
str = str.substring( p, str.size() )
}
if( str.size()>3 && !cut.isAllWhitespace() ) {
if ( align == Align.LEFT ) {
str = str[0..-3] + '..'
}
else if( align == Align.RIGHT ) {
str = '..' + str[2..-1]
}
}
return str
}
/**
* Create a new {@code TextLabel} object with the specified value
*/
static TextLabel of( def value ) {
new TextLabel( value )
}
}

View File

@@ -0,0 +1,30 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.ui.console
import org.pf4j.ExtensionPoint
/**
* Define the extension interface for console app
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
interface ConsoleExtension extends ExtensionPoint {
void run(String...args)
}

View File

@@ -0,0 +1,126 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.ListIterator;
import nextflow.script.types.Tuple;
/**
* Provides a basic tuple implementation extending an {@link ArrayList}
* and not allowing any content change operation
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
public class ArrayTuple<E> extends ArrayList<E> implements Tuple {
private static final long serialVersionUID = - 4765828600345948947L;
public ArrayTuple() {}
public ArrayTuple(Collection<E> other) {
super(other);
}
public ArrayTuple(int initialCapacity) {
super(initialCapacity);
}
public boolean add(E e) {
throw new UnsupportedOperationException();
}
public boolean remove(Object o) {
throw new UnsupportedOperationException();
}
public boolean addAll(Collection<? extends E> coll) {
throw new UnsupportedOperationException();
}
public boolean removeAll(Collection<?> coll) {
throw new UnsupportedOperationException();
}
public boolean retainAll(Collection<?> coll) {
throw new UnsupportedOperationException();
}
public void clear() {
throw new UnsupportedOperationException();
}
public E set(int index, E element) {
throw new UnsupportedOperationException();
}
public void add(int index, E element) {
throw new UnsupportedOperationException();
}
public E remove(int index) {
throw new UnsupportedOperationException();
}
public boolean addAll(int index, Collection<? extends E> c) {
throw new UnsupportedOperationException();
}
public Iterator<E> iterator() {
return new Iterator<E>() {
private final Iterator<? extends E> i = ArrayTuple.super.iterator();
public boolean hasNext() {return i.hasNext(); }
public E next() {return i.next(); }
public void remove() {
throw new UnsupportedOperationException();
}
};
}
public ListIterator<E> listIterator(final int index) {
return new ListIterator<E>() {
private final ListIterator<? extends E> i = ArrayTuple.super.listIterator(index);
public boolean hasNext() {return i.hasNext();}
public E next() {return i.next();}
public boolean hasPrevious() {return i.hasPrevious();}
public E previous() {return i.previous();}
public int nextIndex() {return i.nextIndex();}
public int previousIndex() {return i.previousIndex();}
public void remove() {
throw new UnsupportedOperationException();
}
public void set(E e) {
throw new UnsupportedOperationException();
}
public void add(E e) {
throw new UnsupportedOperationException();
}
};
}
public List<E> subList(int fromIndex, int toIndex) {
return new ArrayTuple<E>(super.subList(fromIndex, toIndex));
}
}

View File

@@ -0,0 +1,99 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util;
import java.math.BigInteger;
import java.util.regex.Pattern;
import groovy.transform.CompileStatic;
import static java.util.Objects.requireNonNull;
/**
* Base62 encoder/decoder.
* <p>
* This is free and unencumbered public domain software
* <p>
* Source: https://github.com/opencoinage/opencoinage/blob/master/src/java/org/opencoinage/util/Base62.java
*/
@CompileStatic
public class Base62 {
private static final BigInteger BASE = BigInteger.valueOf(62);
private static final String DIGITS = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
private Base62() { }
/**
* Encodes a number using Base62 encoding.
*
* @param number a positive integer
* @return a Base62 string
*
* @throws IllegalArgumentException if <code>number</code> is a negative integer
*/
static String encode(BigInteger number) {
if (number.compareTo(BigInteger.ZERO) < 0) {
throwIllegalArgumentException("number must not be negative");
}
StringBuilder result = new StringBuilder();
while (number.compareTo(BigInteger.ZERO) > 0) {
BigInteger[] divmod = number.divideAndRemainder(BASE);
number = divmod[0];
int digit = divmod[1].intValue();
result.insert(0, DIGITS.charAt(digit));
}
return (result.length() == 0) ? DIGITS.substring(0, 1) : result.toString();
}
private static BigInteger throwIllegalArgumentException(String format, Object... args) {
throw new IllegalArgumentException(String.format(format, args));
}
/**
* Decodes a string using Base62 encoding.
*
* @param string a Base62 string
* @return a positive integer
*
* @throws IllegalArgumentException if <code>string</code> is empty
*/
static BigInteger decode(final String string) {
return decode(string, 128);
}
static BigInteger decode(final String string, int bitLimit) {
requireNonNull(string, "Decoded string must not be null");
if (string.length() == 0) {
return throwIllegalArgumentException("String '%s' must not be empty", string);
}
if (!Pattern.matches("[" + DIGITS + "]*", string)) {
throwIllegalArgumentException("String '%s' contains illegal characters, only '%s' are allowed", string, DIGITS);
}
BigInteger result = BigInteger.ZERO;
int digits = string.length();
for (int index = 0; index < digits; index++) {
int digit = DIGITS.indexOf(string.charAt(digits - index - 1));
result = result.add(BigInteger.valueOf(digit).multiply(BASE.pow(index)));
if (bitLimit > 0 && result.bitLength() > bitLimit) {
throwIllegalArgumentException("String contains '%s' more than 128-bit information (%sbit)", string, result.bitLength());
}
}
return result;
}
}

View File

@@ -0,0 +1,32 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util
import com.google.common.hash.Hasher
import groovy.transform.CompileStatic
/**
* Interface to delegate cache hashing to
* a the implementing object
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
interface CacheFunnel {
Hasher funnel(Hasher hasher, CacheHelper.HashMode mode)
}

View File

@@ -0,0 +1,81 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hasher;
import org.slf4j.LoggerFactory;
/**
* Provide helper method to handle caching
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
public class CacheHelper {
public enum HashMode {
STANDARD, DEEP, LENIENT, SHA256;
private static HashMode defaultValue;
static {
if( System.getenv().containsKey("NXF_CACHE_MODE") )
defaultValue = valueOf(System.getenv().get("NXF_CACHE_MODE"));
}
public static HashMode DEFAULT() {
return defaultValue != null ? defaultValue : STANDARD;
}
public static HashMode of( Object obj ) {
if( obj==null || obj instanceof Boolean )
return null;
if( obj instanceof CharSequence ) {
if( "true".equals(obj) || "false".equals(obj) )
return null;
if( "standard".equals(obj) )
return STANDARD;
if( "lenient".equals(obj) )
return LENIENT;
if( "deep".equals(obj) )
return DEEP;
if( "sha256".equals(obj) )
return SHA256;
}
LoggerFactory.getLogger(HashMode.class).warn("Unknown cache mode: {}", obj.toString());
return null;
}
}
public static Hasher hasher( Object value ) {
return hasher(value, HashMode.STANDARD);
}
public static Hasher hasher( Object value, HashMode mode ) {
return hasher( HashBuilder.defaultHasher(), value, mode );
}
public static Hasher hasher( HashFunction function, Object value, HashMode mode ) {
return hasher( function.newHasher(), value, mode );
}
public static Hasher hasher( Hasher hasher, Object value, HashMode mode ) {
return HashBuilder.hasher(hasher, value, mode);
}
}

View File

@@ -0,0 +1,51 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util
import java.nio.charset.Charset
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
/**
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
class CharsetHelper {
static Charset getCharset( object ) {
if( object instanceof Map ) {
if( object.containsKey('charset') )
object = (object as Map).charset
else
return Charset.defaultCharset()
}
if( object instanceof Charset )
return (Charset)object
if( object instanceof String && Charset.isSupported(object) )
return Charset.forName(object)
if( object != null )
log.warn "Invalid charset object: $object -- using default: ${Charset.defaultCharset()}"
Charset.defaultCharset()
}
}

View File

@@ -0,0 +1,121 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util
import java.util.regex.Pattern
import groovy.transform.CompileStatic
import nextflow.extension.Bolts
/**
* Validation helper
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class CheckHelper {
/**
* Valid a method named parameters map
*
* @param name The name of the method, only in the error reported message
* @param params The actual parameters map
* @param valid A map providing for each parameter name the valid values
* @throws IllegalArgumentException when the parameter include an unexpected parameter name or value
*/
static void checkParams( String name, Map<String,?> params, Map<String,?> valid ) {
if( !params ) return
def allKeys = valid.keySet()
for( String key : params.keySet() ) {
if( !allKeys.contains(key) )
throw new IllegalArgumentException("Unknown argument '${key}' for operator '$name' -- Possible arguments: ${allKeys.join(', ')}")
final value = params.get(key)
final accepted = valid.get(key)
if( accepted instanceof Collection ) {
boolean ok = false
final itr = accepted.iterator()
while( !ok && itr.hasNext() )
ok |= isValid(value, itr.next())
if( !ok )
throw new IllegalArgumentException("Value '${value}' cannot be used in in parameter '${key}' for operator '$name' -- Possible values: ${(accepted as Collection).join(', ')}")
}
else if( !isValid(value, accepted) )
throw new IllegalArgumentException("Value '${value}' cannot be used in in parameter '${key}' for operator '$name' -- Value don't match: ${accepted}")
}
}
/**
* Check if the provide value is included in the specified range
*
* @param value A value to verify
* @param range The range it may be a {@link Class} a {@link Collection} of values, a regexp {@link Pattern} or a specific value
* @return {@code true} is the match is satisfied or {@code false} otherwise
*/
static boolean isValid( value, range ) {
if( range instanceof Class && value != null )
return range.isAssignableFrom(value.class)
if( range instanceof Collection )
return range.contains(value)
if( value != null && range instanceof Pattern )
return range.matcher(value.toString()).matches()
value == range
}
/**
* Verify that all method named parameters are included in the provided list
*
* @param name The method name used in the reported error message
* @param params The list of accepted named parameters
* @param valid The list of accepted parameter names
* @throws IllegalArgumentException a parameter is not included the in valid names list
*/
static void checkParams( String name, Map<String,?> params, List<String> valid ) {
if( !params ) return
for( String key : params.keySet() ) {
if( !valid.contains(key) ) {
def matches = Bolts.closest(valid,key) ?: valid
def message = "Unknown argument '${key}' for operator '$name'. Did you mean one of these?\n" + matches.collect { " $it"}.join('\n')
throw new IllegalArgumentException(message)
}
}
}
/**
* Verify that all method named parameters are included in the provided list
*
* @param name The method name used in the reported error message
* @param params The list of accepted named parameters
* @param valid The list of accepted parameter names
* @throws IllegalArgumentException a parameter is not included the in valid names list
*/
static void checkParams( String name, Map<String,?> params, String... valid ) {
checkParams(name, params, Arrays.asList(valid))
}
}

View File

@@ -0,0 +1,151 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util
import groovy.transform.CompileStatic
import java.util.regex.Pattern
/**
* Implement command line parsing helpers
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class CmdLineHelper {
static private Pattern CLI_OPT = ~/--([a-zA-Z_-]+)(?:\W.*)?$|-([a-zA-Z])(?:\W.*)?$/
private List<String> args
CmdLineHelper( String cmdLineToBeParsed ) {
args = splitter(cmdLineToBeParsed ?: '')
}
private boolean contains(String argument) {
return args.indexOf(argument) != -1
}
private getArg( String argument ) {
int pos = args.indexOf(argument)
if( pos == -1 ) return null
List<String> result = []
for( int i=pos+1; i<args.size(); i++ ) {
if( args[i].startsWith('-') ) {
break
}
result.add(args[i])
}
if( result.size()==0 ) {
return true
}
else if( result.size()==1 ) {
return result[0]
}
else {
return result
}
}
/**
* Given a string the splitter method separate it by blank returning a list of string.
* Tokens wrapped by a single quote or double quotes are considered as a contiguous string
* and is added as a single element in the returned list.
* <p>
* For example the string: {@code "alpha beta 'delta gamma'"} will return the following result
* {@code ["alpha", "beta", "delta gamma"]}
*
* @param cmdline The string to be splitted in single elements
* @return A list of string on which each entry represent a command line argument, or an
* empty list if the {@code cmdline} parameter is empty
*/
static List<String> splitter( String cmdline ) {
List<String> result = []
if( cmdline ) {
QuoteStringTokenizer tokenizer = new QuoteStringTokenizer(cmdline);
while( tokenizer.hasNext() ) {
result.add(tokenizer.next())
}
}
result
}
static String toLine( String... args ) {
toLine( args as List)
}
static String toLine( List<String> args ) {
def result = new ArrayList(args.size())
args.each { result << (it.contains(' ') ? "'${it}'" : it) }
return result.join(' ')
}
/**
* Parse command line and returns the options and their values as a map object.
*
* @param cmdline
* The command line as single string
* @return
* A map object holding the option key-value(s) associations
*/
static CmdLineOptionMap parseGnuArgs(String cmdline) {
final BLANK = ' ' as char
final result = new CmdLineOptionMap()
if( !cmdline )
return result
final tokenizer = new QuoteStringTokenizer(cmdline, BLANK);
String opt = null
String last = null
while( tokenizer.hasNext() ) {
final String token = tokenizer.next()
if( !token || token=='--')
continue
final matcher = CLI_OPT.matcher(token)
if( matcher.matches() ) {
if( opt ) {
result.addOption(opt,'true')
}
opt = matcher.group(1) ?: matcher.group(2)
}
else {
if( !opt ) {
if( !last ) continue
result.addOption(last, token)
}
else {
result.addOption(opt, token)
last = opt
opt = null
}
}
}
if( opt )
result.addOption(opt, 'true')
return result
}
}

View File

@@ -0,0 +1,98 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util
import com.google.common.hash.Hasher
import groovy.transform.CompileStatic
import groovy.transform.EqualsAndHashCode
import groovy.transform.ToString
/**
* Holder for parsed command line options.
*
* @author Manuele Simi <manuele.simi@gmail.com>
*/
@CompileStatic
@ToString(includes = 'options', includeFields = true)
@EqualsAndHashCode(includes = 'options', includeFields = true)
class CmdLineOptionMap implements CacheFunnel {
final private Map<String, List<String>> options = new LinkedHashMap<String, List<String>>()
final private static CmdLineOptionMap EMPTY = new CmdLineOptionMap()
protected CmdLineOptionMap addOption(String key, String value) {
if ( !options.containsKey(key) )
options[key] = new ArrayList<String>(10)
options[key].add(value)
return this
}
boolean hasMultipleValues(String key) {
options.containsKey(key) ? options[key].size() > 1 : false
}
boolean hasOptions() {
options.size()
}
List<String> getValues(String key) {
return options.containsKey(key) ? options[key] : new ArrayList<String>(10)
}
def getFirstValue(String key) {
getFirstValueOrDefault(key, null)
}
boolean asBoolean() {
return options.size()>0
}
boolean exists(String key) {
options.containsKey(key)
}
def getFirstValueOrDefault(String key, String alternative) {
options.containsKey(key) && options[key].get(0) ? options[key].get(0) : alternative
}
static CmdLineOptionMap fromMap(final Map map) {
def optionMap = new CmdLineOptionMap()
map.each {
optionMap.addOption(it.key as String, it.value as String)
}
return optionMap
}
static CmdLineOptionMap emptyOption() {
return EMPTY
}
@Override
String toString() {
def serialized = []
options.each {
serialized << "option{${it.key}: ${it.value.each {it}}}"
}
return "[${serialized.join(', ')}]"
}
@Override
Hasher funnel(Hasher hasher, CacheHelper.HashMode mode) {
return CacheHelper.hasher(hasher, options, mode)
}
}

View File

@@ -0,0 +1,57 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util
import groovy.transform.CompileStatic
/**
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class CollectionHelper {
static List flatten( Collection collection ) {
def result = []
flatten(collection, { result << it })
return result
}
static void flatten( Collection list, Closure row ) {
def maxLen = 1
list.each { if( it instanceof Collection ) { maxLen = Math.max(maxLen,((Collection)it).size()) } }
for( int i=0; i<maxLen; i++ ) {
def set = new ArrayList(list.size())
for( int j=0; j<list.size(); j++ ) {
def val = list[j]
if( val instanceof Collection )
set[j] = i < ((Collection)val).size() ? ((Collection)val)[i] : null
else
set[j] = val
}
row.call(set)
}
}
}

View File

@@ -0,0 +1,135 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util
import groovy.transform.CompileStatic
import groovy.transform.PackageScope
import groovy.transform.ToString
import org.apache.commons.lang3.StringUtils
/**
* Simple comma-separated-values parser
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@ToString(includeFields = true, includeNames = true)
@CompileStatic
class CsvParser {
final static private char COMMA = ',' as char
private char quote
private char separator = COMMA
private String empty
private boolean strip
CsvParser setQuote(char ch) {
this.quote = ch
return this
}
CsvParser setQuote(String ch) {
this.quote = firstChar(ch)
return this
}
CsvParser setSeparator(char ch) {
this.separator = ch ?: COMMA
return this
}
CsvParser setSeparator(String ch) {
this.separator = firstChar(ch) ?: COMMA
return this
}
CsvParser setStrip(boolean value) {
this.strip = value
return this
}
static private char firstChar(String str) {
if( !str )
return 0 as char
if( str.size()>1 )
throw new IllegalArgumentException("Not a valid CSV character: $str")
str.charAt(0)
}
List<String> parse( String line ) {
List<String> result = []
while( line != null ) {
if( !line ) {
result.add(empty)
break
}
else if( quote && line.charAt(0)==quote ) {
line = readQuotedValue(line, result)
}
else {
line = readSimpleValue(line, result)
}
}
return result
}
private String readSimpleValue(String line, List<String> result) {
def p = line.indexOf( (int)separator )
if( p == -1 ) {
result.add(stripBlanks(line))
return null
}
else {
result.add(stripBlanks(line.substring(0,p)) ?: empty)
return line.substring(p+1)
}
}
private String readQuotedValue(String line, List<String> result) {
def value = line.substring(1)
def p = value.indexOf( (int)quote )
if( p == -1 )
throw new IllegalStateException("Missing double-quote termination in CSV value -- offending line: $line")
result.add( stripBlanks(value.substring(0,p)) )
def next = p+1
if( next<value.size() ) {
if( value.charAt(next) != separator )
throw new IllegalStateException("Invalid CSV value -- offending line: $line")
return value.substring(next+1)
}
else {
return null
}
}
@PackageScope String stripBlanks(String str) {
final value = strip ? StringUtils.strip(str) : str
if( !quote )
return value
if( value.length()>1 && value.charAt(0)==quote && value.charAt(value.size()-1)==quote )
return value.substring(1, value.length()-1)
else
return value
}
}

View File

@@ -0,0 +1,43 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util
/**
* Some utils for debugging
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
final class DebugUtils {
private static final String SEPARATOR = "\n";
private DebugUtils() {
}
static String formatStackTrace(StackTraceElement[] stackTrace) {
StringBuilder buffer = new StringBuilder();
for (StackTraceElement element : stackTrace) {
buffer.append(element).append(SEPARATOR);
}
return buffer.toString();
}
static String formatCurrentStacktrace() {
StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace();
return formatStackTrace(stackTrace);
}
}

View File

@@ -0,0 +1,434 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util
import java.time.temporal.ChronoUnit
import java.time.temporal.Temporal
import java.time.temporal.TemporalAmount
import java.time.temporal.TemporalUnit
import java.time.temporal.UnsupportedTemporalTypeException
import java.util.concurrent.TimeUnit
import groovy.transform.CompileStatic
import groovy.transform.EqualsAndHashCode
import groovy.util.logging.Slf4j
import nextflow.script.types.Duration as IDuration
import org.apache.commons.lang3.time.DurationFormatUtils
/**
* A simple time duration representation
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
@EqualsAndHashCode(includes = 'durationInMillis')
class Duration implements IDuration, TemporalAmount, Comparable<Duration>, Serializable, Cloneable {
static private final List<TemporalUnit> SUPPORTED_UNITS = List.<TemporalUnit>of(ChronoUnit.DAYS, ChronoUnit.HOURS, ChronoUnit.MINUTES, ChronoUnit.SECONDS, ChronoUnit.MILLIS)
static private final FORMAT = ~/^(\d+\.?\d*)\s*([a-zA-Z]+)/
static private final LEGACY = ~/^(\d{1,2}):(\d{1,2}):(\d{1,2})$/
static private final List<String> MILLIS = ['ms','milli','millis']
static private final List<String> SECONDS = ['s','sec','second','seconds']
static private final List<String> MINUTES = ['m','min','minute','minutes']
static private final List<String> HOURS = ['h','hour','hours']
static private final List<String> DAYS = ['d','day','days']
static public final List<String> UNITS
static {
UNITS = []
UNITS.addAll(MILLIS)
UNITS.addAll(SECONDS)
UNITS.addAll(MINUTES)
UNITS.addAll(HOURS)
UNITS.addAll(DAYS)
}
/**
* Duration in millis
*/
final long durationInMillis
/**
* Create e a duration object having the specified number of millis
*
* @param duration The duration as milliseconds
*/
Duration(long duration) {
assert duration>=0, "Duration unit cannot be a negative number"
this.durationInMillis = duration
}
/**
* Default constructor is required by Kryo serializer
* Do not removed or use it directly
*/
private Duration() { durationInMillis=0 }
/**
* Create the object using a string 'duration' format.
* Accepted prefix are:
* <li>{@code ms}, {@code milli}, {@code millis}: for milliseconds
* <li>{@code s}, {@code second}, {@code seconds}: for seconds
* <li>{@code m}, {@code minute}, {@code minutes}: for minutes
* <li>{@code h}, {@code hour}, {@code hours}: for hours
* <li>{@code d}, {@code day}, {@code days}: for days
*
*
* @param str
*/
Duration(String str) {
try {
try {
durationInMillis = parseSimple(str)
}
catch( IllegalArgumentException e ) {
durationInMillis = parseLegacy(str)
}
}
catch( IllegalArgumentException e ) {
throw e
}
catch( Exception e ) {
throw new IllegalArgumentException("Not a valid duration value: ${str}", e)
}
}
/**
* Parse a duration string in legacy format i.e. hh:mm:ss
*
* @param str The string to be parsed e.g. {@code 05:10:30} (5 hours, 10 min, 30 seconds)
* @return The duration in millisecond
*/
private long parseLegacy( String str ) {
def matcher = (str =~ LEGACY)
if( !matcher.matches() )
new IllegalArgumentException("Not a valid duration value: ${str}")
def groups = (List<String>)matcher[0]
def hh = groups[1].toInteger()
def mm = groups[2].toInteger()
def ss = groups[3].toInteger()
return TimeUnit.HOURS.toMillis(hh) + TimeUnit.MINUTES.toMillis(mm) + TimeUnit.SECONDS.toMillis(ss)
}
/**
* Parse a duration string
*
* @param str A duration string containing one or more component e.g. {@code 1d 3h 10mins}
* @return The duration in millisecond
*/
private long parseSimple( String str ) {
long result=0
for( int i=0; true; i++ ) {
def matcher = (str =~ FORMAT)
if( matcher.find() ) {
def groups = (List<String>)matcher[0]
def all = groups[0]
def digit = groups[1]
def unit = groups[2]
result += convert( digit.toDouble(), unit )
str = str.substring(all.length()).trim()
continue
}
if( i == 0 || str )
throw new IllegalArgumentException("Not a valid duration value: ${str}")
break
}
return result
}
/**
* Parse a single duration component
*
* @param digit
* @param unit A valid duration unit e.g. {@code d}, {@code d}, {@code h}, {@code hour}, etc
* @return The duration in millisecond
*/
private long convert( double digit, String unit ) {
if( unit in MILLIS ) {
return Math.round(digit)
}
if ( unit in SECONDS ) {
return Math.round(digit * 1_000)
}
if ( unit in MINUTES ) {
return Math.round(digit * 60 * 1_000)
}
if ( unit in HOURS ) {
return Math.round(digit * 60 * 60 * 1_000)
}
if ( unit in DAYS ) {
return Math.round(digit * 24 * 60 * 60 * 1_000)
}
throw new IllegalStateException()
}
Duration(long value, TimeUnit unit) {
assert value>=0, "Duration unit cannot be a negative number"
assert unit, "Time unit cannot be null"
this.durationInMillis = unit.toMillis(value)
}
static Duration of( long value ) {
new Duration(value)
}
static Duration of( String str ) {
new Duration(str)
}
static Duration of( String str, Duration fallback ) {
try {
return new Duration(str)
}
catch( IllegalArgumentException e ) {
log.debug "Not a valid duration value: $str -- Fallback on default value: $fallback"
return fallback
}
}
static Duration between( Temporal start, Temporal end ) {
new Duration(java.time.Duration.between(start, end).toMillis())
}
@Override
long toMillis() {
durationInMillis
}
long getMillis() {
durationInMillis
}
@Override
long toSeconds() {
TimeUnit.MILLISECONDS.toSeconds(durationInMillis)
}
long getSeconds() {
toSeconds()
}
@Override
long toMinutes() {
TimeUnit.MILLISECONDS.toMinutes(durationInMillis)
}
long getMinutes() {
toMinutes()
}
@Override
long toHours() {
TimeUnit.MILLISECONDS.toHours(durationInMillis)
}
long getHours() {
toHours()
}
@Override
long toDays() {
TimeUnit.MILLISECONDS.toDays(durationInMillis)
}
long getDays() {
toDays()
}
/**
* Duration formatting utilities and constants. The following table describes the tokens used in the pattern language for formatting.
* <p>
* <pre>
* character duration element
* y years
* d days
* H hours
* m minutes
* s seconds
* </pre>
*
* @param fmt
* @return
*/
String format( String fmt ) {
DurationFormatUtils.formatDuration(durationInMillis, fmt)
}
String toString() {
// just prints the milliseconds
if( durationInMillis < 1_000 ) {
return durationInMillis + 'ms'
}
// when less than 60 seconds round up to 100th of millis
if( durationInMillis < 60_000 ) {
return String.valueOf( Math.round(durationInMillis / 1_000 * 10 as double) / 10 ) + 's'
}
def secs
def mins
def hours
def days
def result = []
// round up to seconds
secs = Math.round( (double)(durationInMillis / 1_000) )
mins = secs.intdiv(60)
secs = secs % 60
if( secs )
result.add( secs+'s' )
hours = mins.intdiv(60)
mins = mins % 60
if( mins )
result.add(0, mins+'m' )
days = hours.intdiv(24)
hours = hours % 24
if( hours )
result.add(0, hours+'h' )
if( days )
result.add(0, days+'d')
return result.join(' ')
}
def plus( Duration value ) {
return new Duration( durationInMillis + value.durationInMillis )
}
def minus( Duration value ) {
return new Duration( durationInMillis - value.durationInMillis )
}
def multiply( Number value ) {
return new Duration( (long)(durationInMillis * value) )
}
def div( Number value ) {
return new Duration( Math.round((double)(durationInMillis / value)) )
}
boolean asBoolean() {
return durationInMillis != 0
}
@Override
int compareTo(Duration that) {
return this.durationInMillis <=> that.durationInMillis
}
static int compareTo(Duration left, Object right) {
assert left
if( right==null )
throw new IllegalArgumentException("Not a valid duration value: null")
if( right instanceof Duration )
return left <=> (Duration)right
if( right instanceof Number )
return left.durationInMillis <=> right.toLong()
if( right instanceof CharSequence )
return left <=> Duration.of(right.toString())
throw new IllegalArgumentException("Not a valid duration value: $right")
}
// TemporalAmount interface methods
/**
* Gets the value of the requested unit.
*
* @param unit the TemporalUnit for which to return the value
* @return the long value of the unit
*/
@Override
long get(TemporalUnit unit) {
if (unit == ChronoUnit.MILLIS) {
return durationInMillis
}
if (unit == ChronoUnit.SECONDS) {
return toSeconds()
}
if (unit == ChronoUnit.MINUTES) {
return toMinutes()
}
if (unit == ChronoUnit.HOURS) {
return toHours()
}
if (unit == ChronoUnit.DAYS) {
return toDays()
}
throw new UnsupportedTemporalTypeException("Unsupported unit: " + unit)
}
/**
* Returns a list of units uniquely defining the value of this TemporalAmount.
*
* @return a list of the supported ChronoUnits, not null
*/
@Override
List<TemporalUnit> getUnits() {
return SUPPORTED_UNITS
}
/**
* Adds this amount to the specified temporal object.
*
* @param temporal the temporal object to adjust, not null
* @return an object of the same type with the adjustment made, not null
*/
@Override
Temporal addTo(Temporal temporal) {
return temporal.plus(durationInMillis, ChronoUnit.MILLIS)
}
/**
* Subtracts this amount from the specified temporal object.
*
* @param temporal the temporal object to adjust, not null
* @return an object of the same type with the adjustment made, not null
*/
@Override
Temporal subtractFrom(Temporal temporal) {
return temporal.minus(durationInMillis, ChronoUnit.MILLIS)
}
}

View File

@@ -0,0 +1,112 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util
import java.nio.file.Path
import groovy.transform.CompileStatic
import nextflow.extension.FilesEx
/**
* Escape helper class
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class Escape {
final private static List<String> SPECIAL_CHARS = ["'", '"', ' ', '(', ')', '\\', '!', '&', '|', '<', '>', '`', ':', ';']
final private static List<String> VAR_CHARS = ['$']
final private static List<String> WILDCARDS = ["*", "?", "{", "}", "[", "]", "'", '"', ' ', '(', ')', '\\', '!', '&', '|', '<', '>', '`', ':']
private static String replace(List<String> special, String str, boolean doNotEscapeComplement=false) {
def copy = new StringBuilder(str.size() +10)
for( int i=0; i<str.size(); i++) {
def ch = str[i]
def p = special.indexOf(ch)
if( p != -1 ) {
// when ! is the first character after a `[` it should not be escaped
// see http://man7.org/linux/man-pages/man7/glob.7.html
final isComplement = doNotEscapeComplement && ch=='!' && ( i>0 && str[i-1]=='[' && (i==1 || str[i-2]!='\\') && str.substring(i).contains(']'))
if( !isComplement )
copy.append('\\')
}
copy.append(str[i])
}
return copy.toString()
}
static String wildcards(String str) {
replace(WILDCARDS, str)
}
static String path(String val) {
replace(SPECIAL_CHARS, val, true)
}
static String path(Path val) {
path(val.toString())
}
static String path(File val) {
path(val.toString())
}
static String path(GString val) {
path(val.toString())
}
static String cli(String[] args) {
args.collect { cli(it) }.join(' ')
}
static String cli(String arg) {
if( arg.contains("'") )
return wildcards(arg)
def x = wildcards(arg)
x == arg ? arg : "'" + arg + "'"
}
static String uriPath(Path file) {
final scheme = FilesEx.getScheme(file)
if( scheme == 'file' )
return path(file)
final uri = FilesEx.toUriString(file)
final prefix = "$scheme://"
assert uri.startsWith(prefix)
def loc = uri.substring(prefix.length())
while( loc.size()>1 && loc.endsWith('/') )
loc = loc.substring(0,loc.length()-1)
return prefix + path(loc)
}
static String blanks(String str) {
str
.replaceAll('\n',/\\n/)
.replaceAll('\t',/\\t/)
.replaceAll('\r',/\\r/)
.replaceAll('\f',/\\f/)
}
static String variable(String val) {
replace(VAR_CHARS, val, false)
}
}

View File

@@ -0,0 +1,243 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import nextflow.script.types.Bag;
/**
* Implementation of a bag based on a frequency map.
*
* @author Ben Sherman <bentshermann@gmail.com>
*/
public class HashBag<E> implements Bag<E> {
private Map<E, Integer> counts;
public HashBag() {
this(10);
}
public HashBag(int initialCapacity) {
counts = new HashMap<>(initialCapacity);
}
public HashBag(Collection<? extends E> c) {
this(c.size());
addAll(c);
}
/**
* Ensures that this collection contains the specified element.
*
* @param e
*/
@Override
public boolean add(E e) {
var v = counts.computeIfAbsent(e, (k) -> 0);
counts.put(e, v + 1);
return true;
}
/**
* Adds all of the elements in the specified collection to this collection.
*
* @param c
*/
@Override
public boolean addAll(Collection<? extends E> c) {
for( var e : c )
add(e);
return true;
}
/**
* Removes all of the elements from this collection.
*/
@Override
public void clear() {
counts.clear();
}
/**
* Returns true if this collection contains the specified element.
*
* @param o
*/
@Override
public boolean contains(Object o) {
return counts.containsKey(o);
}
/**
* Returns true if this collection contains all of the elements in the specified collection.
*
* @param c
*/
@Override
public boolean containsAll(Collection<?> c) {
for( var e : c ) {
if( !contains(e) )
return false;
}
return true;
}
/**
* Compares the specified object with this collection for equality.
*
* @param o
*/
@Override
public boolean equals(Object o) {
return o instanceof HashBag that && this.counts.equals(that.counts);
}
/**
* Returns the hash code value for this collection.
*/
@Override
public int hashCode() {
return counts.hashCode();
}
/**
* Returns true if this collection contains no elements.
*/
@Override
public boolean isEmpty() {
return counts.isEmpty();
}
/**
* Returns an iterator over the elements in this collection.
*/
@Override
public Iterator<E> iterator() {
return new Iterator<E>() {
private Iterator<Map.Entry<E,Integer>> itor = counts.entrySet().iterator();
private E currentElement;
private int currentCount;
@Override
public boolean hasNext() {
return currentCount > 0 || itor.hasNext();
}
@Override
public E next() {
if( currentCount == 0 ) {
var entry = itor.next();
currentElement = entry.getKey();
currentCount = entry.getValue();
}
currentCount--;
return currentElement;
}
};
}
/**
* Removes a single instance of the specified element from this collection, if it is present.
*
* @param o
*/
@Override
public boolean remove(Object o) {
var count = counts.get(o);
if( count == null )
return false;
if( count.intValue() == 1 )
counts.remove(o);
else
counts.put((E) o, count.intValue() - 1);
return true;
}
/**
* Removes all of this collection's elements that are also contained in the specified collection.
*
* @param c
*/
@Override
public boolean removeAll(Collection<?> c) {
var changed = false;
for( var e : c ) {
if( remove(e) )
changed = true;
}
return changed;
}
/**
* Retains only the elements in this collection that are contained in the specified collection.
*
* @param c
*/
@Override
public boolean retainAll(Collection<?> c) {
var changed = false;
for( var e : new HashSet<>(counts.keySet()) ) {
if( !c.contains(e) ) {
counts.remove(e);
changed = true;
}
}
return changed;
}
/**
* Returns the number of elements in this collection.
*/
@Override
public int size() {
var result = 0;
for( var count : counts.values() )
result += count;
return result;
}
@Override
public Object[] toArray() {
throw new UnsupportedOperationException();
}
@Override
public <T> T[] toArray(T[] a) {
throw new UnsupportedOperationException();
}
@Override
public String toString() {
var builder = new StringBuilder();
builder.append('[');
var itor = iterator();
while( itor.hasNext() ) {
builder.append(itor.next().toString());
if( itor.hasNext() )
builder.append(", ");
}
builder.append(']');
return builder.toString();
}
}

View File

@@ -0,0 +1,553 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.ProviderMismatchException;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.hash.Funnels;
import com.google.common.hash.HashCode;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hasher;
import com.google.common.hash.Hashing;
import com.google.common.io.ByteStreams;
import nextflow.Global;
import nextflow.ISession;
import nextflow.extension.Bolts;
import nextflow.extension.FilesEx;
import nextflow.io.SerializableMarker;
import nextflow.script.types.Bag;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static nextflow.Const.DEFAULT_ROOT;
import static nextflow.util.CacheHelper.HashMode;
/**
* Implements the hashing of objects
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
public class HashBuilder {
private static final Logger log = LoggerFactory.getLogger(HashBuilder.class);
private static final HashFunction DEFAULT_HASHING = Hashing.murmur3_128();
private static final int HASH_BITS = DEFAULT_HASHING.bits();
private static final int HASH_BYTES = HASH_BITS / 8;
private static final Map<String,Object> FIRST_ONLY;
static {
FIRST_ONLY = new HashMap<>(1);
FIRST_ONLY.put("firstOnly", Boolean.TRUE);
}
public static Hasher defaultHasher() {
return DEFAULT_HASHING.newHasher();
}
private Hasher hasher = defaultHasher();
private HashMode mode = HashMode.STANDARD;
private Path basePath;
public HashBuilder() {}
public HashBuilder withHasher(Hasher hasher) {
this.hasher = hasher;
return this;
}
public HashBuilder withMode(HashMode mode) {
this.mode = mode;
return this;
}
public HashBuilder withBasePath(Path basePath) {
this.basePath = basePath;
return this;
}
public HashBuilder with(Object value) {
if( value == null )
return this;
else if( value instanceof Boolean )
hasher.putBoolean((Boolean) value);
else if( value instanceof Short )
hasher.putShort((Short) value);
else if( value instanceof Integer)
hasher.putInt((Integer) value);
else if( value instanceof Long )
hasher.putLong((Long) value);
else if( value instanceof Float )
hasher.putFloat((Float) value);
else if( value instanceof Double )
hasher.putDouble( (Double)value );
else if( value instanceof Byte )
hasher.putByte( (Byte)value );
else if( value instanceof Number )
// reduce all other number types (BigInteger, BigDecimal, AtomicXxx, etc) to string equivalent
hasher.putUnencodedChars(value.toString());
else if( value instanceof Character )
hasher.putChar( (Character)value );
else if( value instanceof CharSequence )
hasher.putUnencodedChars( (CharSequence)value );
else if( value instanceof byte[] )
hasher.putBytes( (byte[])value );
else if( value instanceof Object[]) {
for( Object item : ((Object[])value) )
with(item);
}
else if( value instanceof CacheFunnel )
((CacheFunnel)value).funnel(hasher, mode);
else if( value instanceof Map )
hashUnorderedCollection(hasher, ((Map) value).entrySet(), mode);
else if( value instanceof Map.Entry ) {
Map.Entry entry = (Map.Entry)value;
with(entry.getKey());
with(entry.getValue());
}
else if( value instanceof Bag || value instanceof Set )
hashUnorderedCollection(hasher, (Collection) value, mode);
else if( value instanceof Collection)
for( Object item : ((Collection)value) )
with(item);
else if( value instanceof Path )
hashFile(hasher, (Path)value, mode, basePath);
else if( value instanceof java.io.File )
hashFile(hasher, (java.io.File)value, mode, basePath);
else if( value instanceof UUID ) {
UUID uuid = (UUID)value;
hasher.putLong(uuid.getMostSignificantBits()).putLong(uuid.getLeastSignificantBits());
}
else if( value instanceof VersionNumber )
hasher.putInt( value.hashCode() );
else if( value instanceof SerializableMarker)
hasher.putInt( value.hashCode() );
else if( value instanceof Enum )
hasher.putUnencodedChars( value.getClass().getName() + "." + value );
else {
Bolts.debug1(log, FIRST_ONLY, "[WARN] Unknown hashing type: " + value.getClass());
hasher.putInt( value.hashCode() );
}
return this;
}
public Hasher getHasher() {
return hasher;
}
public HashCode build() {
return hasher.hash();
}
public static Hasher hasher( Hasher hasher, Object value, HashMode mode ) {
return new HashBuilder()
.withHasher(hasher)
.withMode(mode)
.with(value)
.getHasher();
}
/**
* Hash a file using only the relative file name instead of
* the absolute file path.
*
* @param path
* @param basePath
* @param mode
*/
public static HashCode hashPath(Path path, Path basePath, HashMode mode) {
return new HashBuilder().withMode(mode).withBasePath(basePath).with(path).build();
}
/**
* Hashes the specified file
*
* @param hasher The current {@code Hasher} object
* @param file The {@code File} object to hash
* @param mode When {@code mode} is equals to the string {@code deep} is used the file content
* in order to create the hash key for this file, otherwise just the file metadata information
* (full name, size and last update timestamp)
* @return The updated {@code Hasher} object
*/
static private Hasher hashFile( Hasher hasher, java.io.File file, HashMode mode, Path basePath ) {
return hashFile(hasher, file.toPath(), mode, basePath);
}
/**
* Hashes the specified file
*
* @param hasher The current {@code Hasher} object
* @param path The {@code Path} object to hash
* @param mode When {@code mode} is equals to the string {@code deep} is used the file content
* in order to create the hash key for this file, otherwise just the file metadata information
* (full name, size and last update timestamp)
* @return The updated {@code Hasher} object
*/
static private Hasher hashFile( Hasher hasher, Path path, HashMode mode, Path basePath ) {
BasicFileAttributes attrs=null;
try {
attrs = Files.readAttributes(path, BasicFileAttributes.class);
}
catch(IOException e) {
log.debug("Unable to get file attributes file: {} -- Cause: {}", FilesEx.toUriString(path), e.toString());
}
catch(ProviderMismatchException e) {
// see https://github.com/nextflow-io/nextflow/pull/1382
log.warn("File system is unable to get file attributes file: {} -- Cause: {}", FilesEx.toUriString(path), e.toString());
}
catch(Exception e) {
log.warn("Unable to get file attributes file: {} -- Cause: {}", FilesEx.toUriString(path), e.toString());
}
if( (mode==HashMode.STANDARD || mode==HashMode.LENIENT) && isAssetFile(path, DEFAULT_ROOT) ) {
if( attrs==null ) {
// when file attributes are not avail, or it's a directory
// hash the file using the file name path and the repository
log.warn("Unable to fetch attribute for file: {} - Hash is inferred from Git repository commit Id", FilesEx.toUriString(path));
return hashFileAsset(hasher, path);
}
final Path base = Global.getSession().getBaseDir();
if( attrs.isDirectory() ) {
// hash all the directory content
return hashDirSha256(hasher, path, base);
}
else {
// hash the content being an asset file
// (i.e. included in the project repository) it's expected to small file
// which makes the content hashing doable
return hashFileSha256(hasher, path, base);
}
}
if( mode==HashMode.DEEP && attrs!=null && attrs.isRegularFile() )
return hashFileContent(hasher, path);
if( mode==HashMode.SHA256 && attrs!=null && attrs.isRegularFile() )
return hashFileSha256(hasher, path, null);
// default
return hashFileMetadata(hasher, path, attrs, mode, basePath);
}
static private LoadingCache<Path,String> sha256Cache = CacheBuilder
.newBuilder()
.maximumSize(10_000)
.build(new CacheLoader<Path, String>() {
@Override
public String load(Path key) throws Exception {
return hashFileSha256Impl0(key);
}
});
static protected Hasher hashFileSha256( Hasher hasher, Path path, Path base ) {
try {
log.trace("Hash sha-256 file content path={} - base={}", path, base);
// the file relative base
if( base!=null )
hasher.putUnencodedChars(base.relativize(path).toString());
// file content hash
String sha256 = sha256Cache.get(path);
hasher.putUnencodedChars(sha256);
}
catch (ExecutionException t) {
Throwable err = t.getCause()!=null ? t.getCause() : t;
String msg = err.getMessage()!=null ? err.getMessage() : err.toString();
log.warn("Unable to compute sha-256 hashing for file: {} - Cause: {}", FilesEx.toUriString(path), msg);
}
return hasher;
}
/**
* Compute an, order independent, hash of a directory path traversing recursively the directory content.
*
* @param hasher
* The {@link Hasher} object to which the resulting directory hash will be added.
* @param dir
* The target directory path to be hashed.
* @param base
* The "base" directory path against which resolve relative paths.
* @return
* The resulting {@link Hasher} object updated with the directory path.
*/
static protected Hasher hashDirSha256( Hasher hasher, Path dir, Path base ) {
if( base==null )
throw new IllegalArgumentException("Argument 'base' cannot be null");
// the byte array used as "accumulator" for
final byte[] resultBytes = new byte[HASH_BYTES];
try {
Files.walkFileTree(dir, new SimpleFileVisitor<Path>() {
public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IOException {
log.trace("Hash sha-256 dir content [FILE] path={} - base={}", path, base);
try {
// the file relative base
final String relPath = base.relativize(path).toString();
// the file content sha-256 checksum
final String sha256 = sha256Cache.get(path);
// compute the file path hash and sum to the result hash
// since the sum is commutative, the traverse order does not matter
// compute a hash of the (file path, file hash) pair.
// since the sum is commutative, the resulting hash in `resultBytes` is invariant to the file traversal order.
// however, the file path and file hash do need to be processed together,
// otherwise this introduces an edge case with directories with similar contents with have the same sha (see nextflow-io/nextflow#6198)
sumBytes(resultBytes, hashBytes(Map.entry(relPath, sha256), HashMode.STANDARD));
return FileVisitResult.CONTINUE;
}
catch (ExecutionException t) {
throw new IOException(t);
}
}
public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes attrs) {
log.trace("Hash sha-256 dir content [DIR] path={} - base={}", path, base);
// the file relative base
final String relPath = base.relativize(path).toString();
// compute the file path hash and sum to the result hash
// since the sum is commutative, the traverse order does not matter
sumBytes(resultBytes, hashBytes(relPath, HashMode.STANDARD));
return FileVisitResult.CONTINUE;
}
});
// finally put the result bytes in the hashing
hasher.putBytes(resultBytes);
}
catch (IOException t) {
Throwable err = t.getCause()!=null ? t.getCause() : t;
String msg = err.getMessage()!=null ? err.getMessage() : err.toString();
log.warn("Unable to compute sha-256 hashing for directory: {} - Cause: {}", FilesEx.toUriString(dir), msg);
}
return hasher;
}
static protected String hashFileSha256Impl0(Path path) throws IOException {
log.debug("Hash asset file sha-256: {}", path);
Hasher hasher = Hashing.sha256().newHasher();
ByteStreams.copy(Files.newInputStream(path), Funnels.asOutputStream(hasher));
return hasher.hash().toString();
}
static private Hasher hashFileAsset( Hasher hasher, Path path ) {
log.debug("Hash asset file: {}", path);
hasher.putUnencodedChars( Global.getSession().getCommitId() );
return hasher;
}
/**
* Hashes the file by using the metadata information: full path string, size and last update timestamp
*
* @param hasher The current {@code Hasher} object
* @param file file The {@code Path} object to hash
* @return The updated {@code Hasher} object
*/
static private Hasher hashFileMetadata( Hasher hasher, Path file, BasicFileAttributes attrs, HashMode mode, Path basePath ) {
String filename = basePath != null && file.startsWith(basePath)
? basePath.relativize(file).toString()
: file.toAbsolutePath().toString();
hasher = hasher.putUnencodedChars( filename );
if( attrs != null ) {
hasher = hasher.putLong(attrs.size());
if( attrs.lastModifiedTime() != null && mode != HashMode.LENIENT ) {
hasher = hasher.putLong( attrs.lastModifiedTime().toMillis() );
}
}
if( log.isTraceEnabled() ) {
log.trace("Hashing file meta: path={}; size={}, lastModified={}, mode={}",
file.toAbsolutePath().toString(),
attrs!=null ? attrs.size() : "--",
attrs!=null && attrs.lastModifiedTime() != null && mode != HashMode.LENIENT ? attrs.lastModifiedTime().toMillis() : "--",
mode
);
}
return hasher;
}
/**
* Hashes the file by reading file content
*
* @param hasher The current {@code Hasher} object
* @param path file The {@code Path} object to hash
* @return The updated {@code Hasher} object
*/
static private Hasher hashFileContent( Hasher hasher, Path path ) {
OutputStream output = Funnels.asOutputStream(hasher);
try {
Files.copy(path, output);
}
catch( IOException e ) {
throw new IllegalStateException("Unable to hash content: " + FilesEx.toUriString(path), e);
}
finally {
FilesEx.closeQuietly(output);
}
return hasher;
}
static HashCode hashContent( Path file ) {
return hashContent(file, null);
}
static HashCode hashContent( Path file, HashFunction function ) {
if( function == null )
function = DEFAULT_HASHING;
Hasher hasher = function.newHasher();
return hashFileContent(hasher, file).hash();
}
static private Hasher hashUnorderedCollection(Hasher hasher, Collection collection, HashMode mode) {
byte[] resultBytes = new byte[HASH_BYTES];
for (Object item : collection) {
// hash ghe collection item
byte[] nextBytes = hashBytes(item, mode);
// sum the hash bytes to the "resultBytes" accumulator
// since the sum is a commutative operation the order does not matter
sumBytes(resultBytes, nextBytes);
}
// add the result bytes and return the resulting object
return hasher.putBytes(resultBytes);
}
static private byte[] hashBytes(Object item, HashMode mode) {
return hasher(defaultHasher(), item, mode).hash().asBytes();
}
/**
* Sum two arras of bytes having the same length, required to compute hash of unordered collections.
*
* - For each byte position, add the corresponding byte from nextBytes into resultBytes
* - Order doesn't matter: addition is commutative (a + b = b + a), so the final result is
* the same no matter the order of items.
* - This is what makes it suitable for unordered collections
*
* @param resultBytes
* The first argument to be summed. This array is used as the accumulator array (i.e. the result)
* @param nextBytes
* The second argument to be summed.
* @return
* The array resulting adding the bytes in the second array to the first one. Note,
* the result array instance is the same object passed as first argument.
*
*/
static private byte[] sumBytes(byte[] resultBytes, byte[] nextBytes) {
if( nextBytes.length != resultBytes.length )
throw new IllegalStateException("All hash codes must have the same bit length");
for (int i = 0; i < nextBytes.length; i++) {
resultBytes[i] += nextBytes[i];
}
return resultBytes;
}
/**
* Check if the argument is an asset file i.e. a file that makes part of the
* pipeline Git repository.
*
* <p>Asset files are hashed using their content (SHA-256) rather than metadata
* to maintain cache validity across different clones where timestamps may differ
* on remote executors like batch processing systems.
*
* <p>This method checks two locations:
* <ol>
* <li>Files under {@code session.getBaseDir()} - the script's working directory</li>
* <li>Files under {@code assetRoot} - the repository root (typically ~/.nextflow/assets)</li>
* </ol>
*
* The distinction is important when executing workflows from subdirectories using
* the main-script parameter, as repository assets may exist outside the script's
* directory but still be part of the repository.
*
* @param path
* The item to check.
* @param assetRoot
* Location where assets are being stored (the repository root).
* @return
* {@code true} if the path is included in the pipeline Git repository,
* {@code false} otherwise.
*
* @see <a href="https://github.com/nextflow-io/nextflow/issues/6604">Issue #6604</a>
* @see <a href="https://github.com/nextflow-io/nextflow/pull/6605">PR #6605</a>
*/
static protected boolean isAssetFile(Path path, File assetRoot) {
final ISession session = Global.getSession();
if( session==null )
return false;
// if the commit ID is null the current run is not launched from a repo
if( session.getCommitId()==null )
return false;
// if the file belong to different file system, cannot be a file belonging to the repo
if( session.getBaseDir().getFileSystem()!=path.getFileSystem() )
return false;
// Check both the script's base directory and the repository root.
// This handles cases where a workflow is executed from a subdirectory
// (using the main-script parameter) but references assets elsewhere in the repo.
// The assetRoot check ensures these non-sibling assets are still recognized.
return path.startsWith(session.getBaseDir()) || path.startsWith(assetRoot.toPath());
}
}

View File

@@ -0,0 +1,165 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util
import java.nio.file.Files
import java.nio.file.Path
import java.util.regex.Matcher
import java.util.regex.Pattern
import groovy.transform.CompileStatic
/**
* Read a INI file
*
* See http://stackoverflow.com/a/15638381/395921
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class IniFile {
private Pattern fSection = Pattern.compile( "\\s*\\[([^]]*)\\]\\s*" );
private Pattern fKeyValue = Pattern.compile( "\\s*([^=]*)=(.*)" );
private Map<String, Map<String, String>> fEntries = new HashMap<>();
private Path fFile;
IniFile() {}
IniFile( Path path ) throws IOException {
load( path );
}
IniFile(String path) {
load(path)
}
IniFile load( String path ) {
assert path
load(path as Path)
}
IniFile load( File file ) {
assert file
load(file.toPath())
}
IniFile load( Path path, cs = null ) {
assert path
this.fFile = path
if( Files.exists(path) ) {
final charset = CharsetHelper.getCharset(cs)
final reader = Files.newBufferedReader(path, charset)
try {
load(reader)
}
finally{
reader.close()
}
}
return this
}
IniFile load( Reader br ) {
assert br
String line;
String section = null;
while(( line = br.readLine()) != null ) {
Matcher m = fSection.matcher( line );
if( m.matches()) {
section = m.group( 1 ).trim();
}
else if( section != null ) {
m = fKeyValue.matcher( line );
if( m.matches()) {
String key = m.group( 1 ).trim();
String value = m.group( 2 ).trim();
Map< String, String > kv = fEntries.get( section );
if( kv == null ) {
fEntries.put( section, kv = new HashMap<>());
}
kv.put( key, value );
}
}
}
return this
}
String getString( String section, String key, String defValue = null ) {
Map< String, String > kv = fEntries.get( section );
if( kv == null ) {
return defValue;
}
return kv.get(key) ?: defValue
}
int getInt( String section, String key, int defValue = 0) {
Map< String, String > kv = fEntries.get( section );
if( kv == null ) {
return defValue;
}
kv.containsKey(key) ? Integer.parseInt( kv.get( key )) : defValue;
}
float getFloat( String section, String key, float defValue = 0 ) {
Map< String, String > kv = fEntries.get( section );
if( kv == null ) {
return defValue;
}
kv.containsKey(key) ? Float.parseFloat( kv.get( key )) : defValue
}
double getDouble( String section, String key, double defValue = 0 ) {
Map< String, String > kv = fEntries.get( section );
if( kv == null ) {
return defValue;
}
kv.containsKey(key) ? Double.parseDouble( kv.get( key )) : defValue
}
boolean getBool( String section, String key, boolean defValue = false ) {
Map< String, String > kv = fEntries.get( section );
if( kv == null ) {
return defValue;
}
kv.containsKey(key) ? Boolean.parseBoolean( kv.get( key )) : defValue
}
Map<String,String> section(String section) {
def result = fEntries.get(section)
return result != null ? Collections.unmodifiableMap(result) : Collections.<String,String>emptyMap()
}
def propertyMissing(String name) {
if( fEntries.containsKey(name) )
return section(name)
throw new MissingPropertyException(name,IniFile)
}
def getFile() {
return fFile
}
}

View File

@@ -0,0 +1,65 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
/**
* A object deserializer that allows you to specify a custom class loader
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
class InputStreamDeserializer extends ObjectInputStream {
ClassLoader loader
InputStreamDeserializer( InputStream input, ClassLoader loader ) {
super(input)
this.loader = loader
}
protected Class<?> resolveClass(ObjectStreamClass desc)
throws IOException, ClassNotFoundException
{
try {
return super.resolveClass(desc)
}
catch( ClassNotFoundException e ) {
log.debug "Trying to load class: ${desc.getName()}"
return Class.forName( desc.getName(), false, loader )
}
}
/**
* Deserialize an object given its bytes representation and a custom class-loader
*
* @param bytes
* @param loader
* @return
*/
static <T> T deserialize( byte[] bytes, ClassLoader loader ) {
assert bytes
def buffer = new ByteArrayInputStream(bytes)
(T)new InputStreamDeserializer(buffer, loader).readObject()
}
}

View File

@@ -0,0 +1,49 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util
import groovy.transform.CompileStatic
/**
* A {@link Map} that handles keys in a case insensitive manner
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class InsensitiveMap<K,V> implements Map<K,V> {
@Delegate
private Map<K,V> target
private InsensitiveMap(Map<K,V> map) {
this.target = map
}
@Override
boolean containsKey(Object key) {
target.any( it -> key?.toString()?.toLowerCase() == it.key?.toString()?.toLowerCase())
}
@Override
V get(Object key) {
target.find(it -> key?.toString()?.toLowerCase() == it.key?.toString()?.toLowerCase())?.value
}
static <K,V> Map<K,V> of(Map<K,V> target) {
new InsensitiveMap(target)
}
}

View File

@@ -0,0 +1,41 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util
import groovy.transform.CompileStatic
/**
* Math helper functions
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class MathHelper {
static int ceilDiv(int x, int y) {
return -Math.floorDiv(-x,y)
}
static long ceilDiv(long x, long y){
return -Math.floorDiv(-x,y)
}
static long ceilDiv(long x, int y){
return -Math.floorDiv(-x,y)
}
}

View File

@@ -0,0 +1,196 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util
import java.text.DecimalFormat
import java.text.DecimalFormatSymbols
import java.util.regex.Pattern
import groovy.transform.CompileStatic
import groovy.transform.EqualsAndHashCode
import nextflow.script.types.MemoryUnit as IMemoryUnit
/**
* Represent a memory unit
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
@EqualsAndHashCode(includes = 'size', includeFields = true)
class MemoryUnit implements IMemoryUnit, Comparable<MemoryUnit>, Serializable, Cloneable {
final static public MemoryUnit ZERO = new MemoryUnit(0)
final static private Pattern FORMAT = ~/([0-9\.]+)\s*(\S)?B?/
final static public List UNITS = [ "B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB" ]
private long size
static final private DecimalFormatSymbols formatSymbols
static {
formatSymbols = new DecimalFormatSymbols()
formatSymbols.setDecimalSeparator('.' as char)
}
/**
* Default constructor is required by Kryo serializer
* Do not remove of use directly
*/
private MemoryUnit() { this.size=0 }
/**
* Create a memory unit instance
*
* @param value The number of bytes it represent
*/
MemoryUnit( long value ) {
assert value>=0, "Memory unit cannot be a negative number"
this.size = value
}
/**
* Create a memory unit instance with the given semantic string
*
* @param str A string using the following of of the following units: B, KB, MB, GB, TB, PB, EB, ZB
*/
MemoryUnit( String str ) {
def matcher = FORMAT.matcher(str)
if( !matcher.matches() ) {
throw new IllegalArgumentException("Not a valid FileSize value: '$str'")
}
final value = matcher.group(1)
final unit = matcher.group(2)?.toUpperCase()
if ( !unit || unit == "B" ) {
size = Long.parseLong(value)
}
else {
int p = UNITS.indexOf(unit)
if ( p == -1 ) {
// try adding a 'B' specified
p = UNITS.indexOf(unit+'B')
if( p == -1 ) {
throw new IllegalArgumentException("Not a valid file size unit: ${str}")
}
}
size = Math.round( Double.parseDouble(value) * Math.pow(1024, p) )
}
}
@Override
long toBytes() {
size
}
long getBytes() { size }
@Override
long toKilo() { size >> 10 }
long getKilo() { size >> 10 }
@Override
long toMega() { size >> 20 }
long getMega() { size >> 20 }
@Override
long toGiga() { size >> 30 }
long getGiga() { size >> 30 }
MemoryUnit plus( MemoryUnit value ) {
return value != null ? new MemoryUnit( size + value.size ) : this
}
MemoryUnit minus( MemoryUnit value ) {
return value != null ? new MemoryUnit( size - value.size ) : this
}
MemoryUnit multiply( Number value ) {
return new MemoryUnit( (long)(size * value) )
}
MemoryUnit div( Number value ) {
return new MemoryUnit( Math.round((double)(size / value)) )
}
String toString() {
if(size <= 0) {
return "0"
}
// see http://stackoverflow.com/questions/2510434/format-bytes-to-kilobytes-megabytes-gigabytes
int digitGroups = (int) (Math.log10(size) / Math.log10(1024))
final formatter = new DecimalFormat("0.#", formatSymbols)
formatter.setGroupingUsed(false)
formatter.format(size / Math.pow(1024, digitGroups)) + " " + UNITS[digitGroups]
}
@Override
int compareTo(MemoryUnit that) {
return this.size <=> that.size
}
static int compareTo(MemoryUnit left, Object right) {
assert left
if( right==null )
throw new IllegalArgumentException("Not a valid memory value: null")
if( right instanceof MemoryUnit )
return left <=> (MemoryUnit)right
if( right instanceof Number )
return left.size <=> right.toLong()
if( right instanceof CharSequence )
return left <=> MemoryUnit.of(right.toString())
throw new IllegalArgumentException("Not a valid memory value: $right")
}
static MemoryUnit of( String value ) {
new MemoryUnit(value)
}
static MemoryUnit of( long value ) {
new MemoryUnit(value)
}
boolean asBoolean() {
return size != 0
}
/**
* Function to parse/convert given memory unit
*
* @param unit String expressing memory unit in bytes, e.g. KB, MB, GB
*/
@Override
long toUnit(String unit){
int p = UNITS.indexOf(unit)
if (p==-1)
throw new IllegalArgumentException("Not a valid memory unit: $unit")
return size.intdiv(Math.round(Math.pow(1024,p)))
}
}

View File

@@ -0,0 +1,64 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util
import java.lang.management.ManagementFactory
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
/**
* Implements helper methods for {@link Process} objects
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
class ProcessHelper {
static long pid(Process process) {
try {
// with Java 9 and later pid can be access with `.toHandle().getPid()` method
final toHandle = Process.class.getMethod("toHandle")
final handle = toHandle.invoke(process)
return (Long) Class.forName("java.lang.ProcessHandle").getMethod("pid").invoke(handle)
}
catch (Exception e) {
// ignore it and fallback on next
}
// Fallback for Java6+ on unix. This is known not to work on Windows.
try {
final pidField = process.class.getDeclaredField("pid")
pidField.setAccessible(true)
return pidField.getLong(process)
}
catch(Exception e) {
throw new UnsupportedOperationException("Unable to access process pid for class: ${process.getClass().getName()}",e )
}
}
static long selfPid() {
try {
Long.parseLong(ManagementFactory.getRuntimeMXBean().getName().split("@")[0])
}
catch(Exception e) {
throw new UnsupportedOperationException("Unable to find current process pid",e)
}
}
}

View File

@@ -0,0 +1,126 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util
/**
* Split a string into literal tokens i.e. sequence of chars that does not contain a blank
* or wrapper by a quote or double quote
*
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*
*/
class QuoteStringTokenizer implements Iterator<String>, Iterable<String> {
private List<Character> chars = Arrays.<Character>asList(' ' as Character);
private List<String> tokens = new ArrayList<String>();
private Iterator<String> itr;
public QuoteStringTokenizer( String value ) {
this(value, ' ' as Character);
}
public QuoteStringTokenizer( String value, char... separators ) {
if( separators == null || separators.length==0 ) {
chars = new ArrayList<Character>(3);
chars .add(' ' as Character);
}
else {
chars = new ArrayList<Character>(3);
for( char ch : separators ) { chars.add(ch); }
}
chars.add("\"" as char);
chars.add("'" as char);
/* start parsing */
parseNext(value != null ? value.trim() : "");
}
void parseNext( String value ) {
for( int i=0; i<value.length(); i++ ) {
char ch=value.charAt(i);
if( chars.contains(ch) ) {
if( i>0 ) {
tokens.add(value.substring(0,i));
}
if( ch=='"' as char || ch=='\'' as char ) {
parseQuote(value.substring(i+1), ch);
}
else if( chars.contains(ch) ) {
parseNext(value.substring(i+1));
}
break;
}
// and of the string
else if( i+1 == value.length() ) {
tokens.add(value);
}
}
}
void parseQuote( String value, char delim ) {
int p = value.indexOf(delim as String);
if( p == -1 ) {
tokens.add(value);
return;
}
tokens.add( value.substring(0,p) );
parseNext(value.substring(p+1));
}
public boolean hasNext() {
return itr().hasNext();
}
public String next() {
return itr().next();
}
public void remove() {
throw new UnsupportedOperationException("Remove not supported");
}
public String toString() {
return tokens.toString();
}
public Iterator<String> iterator() {
return itr();
}
/*
* lazy iterator creator
*/
private Iterator<String> itr() {
if( itr == null ) {
itr = tokens.iterator();
}
return itr;
}
}

View File

@@ -0,0 +1,119 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util
import com.google.common.util.concurrent.RateLimiter
import groovy.transform.CompileStatic
import groovy.transform.EqualsAndHashCode
/**
* Model an execution rate limit measure unit
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
@EqualsAndHashCode(includeFields=true,includes='rate')
class RateUnit {
private static String RATE_FORMAT = ~/^(\d+\.?\d*)\s*([a-zA-Z]*)/
private double rate
static RateUnit of(String str) { new RateUnit(str) }
/**
* Create a rate unit by a double value representing the number of events per second
*
* @param rate The rate unit as a double value
*/
RateUnit(double rate) {
assert rate>0, "Rate unit must be greater than zero"
this.rate = rate
}
/**
* Create a rate unit using a string annotation as `10 sec` ie. 10 per second
* or `100 / 5 min` 100 events each 5 minutes
*
* @param str The rate string
*/
RateUnit(String str) {
this.rate = parse(str)
}
/**
* @return The actual rate unit as a double
*/
double getRate() { rate }
/**
* Create a {@link RateLimiter} object for rate unit
*
* @return An instance of {@link RateLimiter}
*/
RateLimiter getRateLimiter() { RateLimiter.create(rate) }
protected double parse(String limit) {
def tokens = limit.tokenize('/')
if( tokens.size() == 2 ) {
/*
* the rate limit is provide num of task over a duration
* - eg. 100 / 5 min
* - ie. max 100 task per 5 minutes
*/
final X = tokens[0].trim()
final Y = tokens[1].trim()
return parse0(X, Y, limit)
}
/*
* the rate limit is provide as a duration
* - eg. 200 min
* - ie. max 200 task per minutes
*/
final matcher = (limit =~ RATE_FORMAT)
if( !matcher.matches() )
throw new IllegalArgumentException("Invalid submit-rate-limit value: $limit -- It must be formatted as `num request sec|min|hour` eg. 10 sec ie. max 10 tasks per second")
final num = matcher.group(1) ?: '_'
final unit = matcher.group(2) ?: 'sec'
return parse0(num, "1 $unit", limit)
}
private double parse0(String X, String Y, String limit ) {
if( !X.isInteger() )
throw new IllegalArgumentException("Invalid submit-rate-limit value: $limit -- It must be formatted as `num request / duration` eg. 10/1s")
final num = Integer.parseInt(X)
final duration = Y.isInteger() ? Duration.of( Y+'sec' ) : ( Y[0].isInteger() ? Duration.of(Y) : Duration.of('1'+Y) )
long seconds = duration.toSeconds()
if( !seconds )
throw new IllegalArgumentException("Invalid submit-rate-limit value: $limit -- The interval must be at least 1 second")
num / seconds as double
}
String toString() {
String.format(Locale.ROOT, "%.2f", rate) + '/sec'
}
}

View File

@@ -0,0 +1,74 @@
/*
* Copyright 2013-2025, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.Map;
import nextflow.script.types.Record;
import org.codehaus.groovy.runtime.DefaultGroovyMethods;
/**
* Implements Record as an immutable map.
*
* @author Ben Sherman <bentshermann@gmail.com>
*/
public class RecordMap extends LinkedHashMap<String,Object> implements Record {
public RecordMap() {}
public RecordMap(Map<String,Object> props) {
super(props);
}
public RecordMap(int initialCapacity) {
super(initialCapacity);
}
@Override
public void clear() {
throw new UnsupportedOperationException();
}
@Override
public Object put(String key, Object value) {
throw new UnsupportedOperationException();
}
@Override
public void putAll(Map<? extends String, ? extends Object> m) {
throw new UnsupportedOperationException();
}
@Override
public Object remove(Object key) {
throw new UnsupportedOperationException();
}
public Record subMap(Collection<String> keys) {
return new RecordMap(DefaultGroovyMethods.subMap(this, keys));
}
public RecordMap plus(RecordMap other) {
var result = new LinkedHashMap<String,Object>();
result.putAll(this);
result.putAll(other);
return new RecordMap(result);
}
}

View File

@@ -0,0 +1,152 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util
import com.google.common.base.CaseFormat
import groovy.transform.CompileDynamic
import groovy.transform.CompileStatic
import groovy.transform.EqualsAndHashCode
import groovy.transform.Memoized
import groovy.transform.ToString
import groovy.util.logging.Slf4j
import io.seqera.util.retry.Retryable
import nextflow.Global
import nextflow.SysEnv
/**
* Models retry policy configuration
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@ToString(includePackage = false, includeNames = true)
@EqualsAndHashCode
@CompileStatic
class RetryConfig implements Retryable.Config {
public static final Duration DEFAULT_DELAY = Duration.of('350ms')
public static final Duration DEFAULT_MAX_DELAY = Duration.of('90s')
public static final Integer DEFAULT_MAX_ATTEMPTS = 5
public static final Double DEFAULT_JITTER = 0.25
public static final double DEFAULT_MULTIPLIER = 2.0
private final static String ENV_PREFIX = 'NXF_RETRY_POLICY_'
final private Duration delay
final private Duration maxDelay
final private int maxAttempts
final private double jitter
final private double multiplier
RetryConfig() {
this(Collections.emptyMap())
}
RetryConfig(Map config) {
delay =
valueOf(config, 'delay', ENV_PREFIX, DEFAULT_DELAY, Duration)
maxDelay =
valueOf(config, 'maxDelay', ENV_PREFIX, DEFAULT_MAX_DELAY, Duration)
maxAttempts =
valueOf(config, 'maxAttempts', ENV_PREFIX, DEFAULT_MAX_ATTEMPTS, Integer)
jitter =
valueOf(config, 'jitter', ENV_PREFIX, DEFAULT_JITTER, Double)
multiplier =
valueOf(config, 'multiplier', ENV_PREFIX, DEFAULT_MULTIPLIER, Double)
}
Duration getDelay() {
return delay
}
Duration getMaxDelay() {
return maxDelay
}
@Override
int getMaxAttempts() {
return maxAttempts
}
@Override
double getJitter() {
return jitter
}
@Override
double getMultiplier() {
return multiplier
}
static RetryConfig config() {
config(Global.config)
}
@Memoized
static RetryConfig config(Map config) {
if( config==null ) {
log.debug "Missing nextflow session - using default retry config"
config = Collections.emptyMap()
}
return new RetryConfig(getNestedConfig(config, 'nextflow', 'retryPolicy') ?: Collections.emptyMap())
}
private static Map getNestedConfig(Map config, String... keys) {
def current = config
for (String key : keys) {
if (current instanceof Map && current.containsKey(key)) {
current = current.get(key)
} else {
return null
}
}
return current instanceof Map ? current : null
}
static <T> T valueOf(Map config, String name, String prefix, T defValue, Class<T> type) {
assert name, "Argument 'name' cannot be null or empty"
assert type, "Argument 'type' cannot be null"
// try to get the value from the config map
final cfg = config?.get(name)
if( cfg != null ) {
return toType(cfg, type)
}
// try to fallback to the sys environment
if( !prefix.endsWith('_') )
prefix += '_'
final key = prefix.toUpperCase() + CaseFormat.LOWER_CAMEL.to(CaseFormat.UPPER_UNDERSCORE, name)
final env = SysEnv.get(key)
if( env != null ) {
return toType(env, type)
}
// return the default value
return defValue
}
@CompileDynamic
static protected <T> T toType(Object value, Class<T> type) {
if( value == null )
return null
if( type==Boolean.class ) {
return type.cast(Boolean.valueOf(value.toString()))
}
else {
return value.asType(type)
}
}
}

View File

@@ -0,0 +1,59 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util
import java.security.SecureRandom
/**
* Rnd key generator
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
class Rnd {
private static final BigInteger B = BigInteger.ONE.shiftLeft(80); // 2^64
private static final SecureRandom random = new SecureRandom();
private Rnd() {}
/**
* Generate random key base62 encoded ie. containing only [0-9a-zA-Z] characters
* guaranteed to be unique.
*
* Tested with 100 mln iteration -> 0 collision
*
* @return A random generated alphanumeric between 9-14 characters
*/
static String key() {
byte[] buffer = new byte[10]
random.nextBytes(buffer)
def big = new BigInteger(buffer)
if (big.signum() < 0) {
big = big.add(B)
}
return Base62.encode(big)
}
static String hex() {
byte[] buffer = new byte[10]
random.nextBytes(buffer)
return buffer.encodeHex()
}
}

View File

@@ -0,0 +1,42 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util
import groovy.transform.CompileStatic
import org.pf4j.ExtensionPoint
/**
* Register a serializer class in the Kryo serializers list
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
interface SerializerRegistrant extends ExtensionPoint {
/**
* Serializer should implement this method adding the
* serialized class and the serializer class to the
* map passed as argument
*
* @param
* serializers The serializer map, where the key element
* represent the class to be serialized and the value
* the serialization object
*/
void register(Map<Class,Object> serializers)
}

View File

@@ -0,0 +1,182 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util
import groovy.transform.CompileStatic
/**
* A service loader inspired to {@link ServiceLoader}
* that allows to load only service names or classes without instantiating
* the actual instances
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class ServiceDiscover<S> {
private static final String PREFIX = "META-INF/services/";
private Class<S> service
private ClassLoader classLoader
static <S> List<Class<S>> load(Class<S> service) {
new ServiceDiscover<S>(service).load()
}
ServiceDiscover(Class<S> service) {
assert service
this.service = service
this.classLoader = Thread.currentThread().getContextClassLoader()
}
ServiceDiscover(Class<S> service, ClassLoader classLoader) {
assert service
assert classLoader
this.service = service
this.classLoader = classLoader
}
private int parseLine(Class service, URL u, BufferedReader r, int lc, List<String> names)
throws IOException, ServiceConfigurationError
{
String ln = r.readLine();
if (ln == null) {
return -1
}
int ci = ln.indexOf('#')
if (ci >= 0)
ln = ln.substring(0, ci)
ln = ln.trim()
int n = ln.length()
if (n != 0) {
if ((ln.indexOf(' ') >= 0) || (ln.indexOf('\t') >= 0))
fail(service, u, lc, "Illegal configuration file syntax")
int cp = ln.codePointAt(0)
if (!Character.isJavaIdentifierStart(cp))
fail(service, u, lc, "Illegal provider-class name: " + ln)
for (int i = Character.charCount(cp); i < n; i += Character.charCount(cp)) {
cp = ln.codePointAt(i)
if (!Character.isJavaIdentifierPart(cp) && (cp != '.'))
fail(service, u, lc, "Illegal provider-class name: " + ln)
}
if ( !names.contains(ln) )
names.add(ln)
}
return lc + 1
}
private List<String> parse(Class service, URL u) throws ServiceConfigurationError
{
InputStream stream = null;
BufferedReader reader = null;
ArrayList<String> names = new ArrayList<>();
try {
stream = u.openStream();
reader = new BufferedReader(new InputStreamReader(stream, "utf-8"));
int lc = 1;
while ((lc = parseLine(service, u, reader, lc, names)) >= 0) { /*empty*/ }
}
catch (IOException x) {
fail(service, "Error reading configuration file", x);
}
finally {
try {
if (reader != null) reader.close();
if (stream != null) stream.close();
}
catch (IOException y) {
fail(service, "Error closing configuration file", y);
}
}
return names
}
private static void fail(Class service, String msg, Throwable cause)
throws ServiceConfigurationError
{
throw new ServiceConfigurationError(service.getName() + ": " + msg,
cause);
}
private static void fail(Class service, String msg)
throws ServiceConfigurationError
{
throw new ServiceConfigurationError(service.getName() + ": " + msg);
}
private static void fail(Class service, URL u, int line, String msg)
throws ServiceConfigurationError
{
fail(service, u.toString() + ":" + line + ": " + msg);
}
private Class<S> classForName(String clazzName) {
Class<?> result = null
try {
result = Class.forName(clazzName, false, classLoader);
}
catch (ClassNotFoundException x) {
fail(service, "Provider $clazzName not found");
}
if (!service.isAssignableFrom(result)) {
fail(service, "Provider $clazzName not a subtype");
}
try {
return (Class<S>)result;
}
catch (Throwable x) {
fail(service, "Provider $clazzName could not be instantiated: " + x, x);
}
throw new Error()
}
List<String> names() {
Enumeration<URL> configs = null
try {
configs = classLoader.getResources(PREFIX + service.getName())
} catch (IOException x) {
fail(service, "Error locating configuration files", x);
}
List<String> result = []
for( URL url : configs ) {
for( String name : parse(service, url) ) {
result << name
}
}
return result
}
List<Class<S>> load() {
names().collect{ classForName(it) }
}
}

View File

@@ -0,0 +1,147 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util
import java.util.regex.Matcher
import java.util.regex.Pattern
import com.google.common.net.InetAddresses
import groovy.transform.CompileStatic
/**
* String helper routines
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class StringUtils {
static final public Pattern URL_PROTOCOL = ~/^([a-zA-Z0-9]*):\\/\\/(.+)/
static final private Pattern URL_PASSWORD = ~/^[a-zA-Z][a-zA-Z0-9]*:\\/\\/(.+)@.+/
/**
* Deprecated. Use {@link nextflow.file.FileHelper#getUrlProtocol(java.lang.String)} instead
*/
@Deprecated
static String getUrlProtocol(String str) {
final m = URL_PROTOCOL.matcher(str)
return m.matches() ? m.group(1) : null
}
static final private Pattern BASE_URL = ~/(?i)((?:[a-z][a-zA-Z0-9]*)?:\/\/[^:|\/]+(?::\d*)?)(?:$|\/.*)/
/**
* Deprecated. Use {@link nextflow.file.FileHelper#baseUrl(java.lang.String)} instead
*/
@Deprecated
static String baseUrl(String url) {
if( !url )
return null
final m = BASE_URL.matcher(url)
return m.matches() ? m.group(1).toLowerCase() : null
}
static private Pattern multilinePattern = ~/["']?(password|token|secret|license)["']?\s?[:=]\s?["']?(\w+)["']?/
static String stripSecrets(String message) {
if (message == null) {
return message
}
StringBuilder sb = new StringBuilder(message)
Matcher matcher = multilinePattern.matcher(sb)
while (matcher.find()) {
for(int idx=0; idx<matcher.groupCount(); idx+=2){
int ini = matcher.start(idx+2)
int end = matcher.end(idx+2)
sb.delete(ini, end)
sb.insert(ini, '********')
}
matcher.reset()
}
return sb.toString();
}
static private boolean isSensitive(Object key) {
final str = key.toString().toLowerCase()
return str.contains('password') \
|| str.contains('token') \
|| str.contains('secret') \
|| str.contains('license') \
|| str.contains('auth')
}
static Map stripSecrets(Map map) {
if( map==null )
return null
final copy = new HashMap(map.size())
for( Map.Entry entry : map.entrySet() ) {
if( entry.value instanceof Map ) {
copy.put( entry.key, stripSecrets((Map)entry.value))
}
else if( isSensitive(entry.key) ) {
copy.put(entry.key, redact(entry.value))
}
else {
copy.put(entry.key, redactUrlPassword(entry.value))
}
}
return copy
}
static String redact(Object value) {
if( value==null )
return '(null)'
if( !value )
return '(empty)'
final str = value.toString()
return str.length()>=10 ? str[0..2] + '****' : '****'
}
static String redactUrlPassword(value) {
final str = value.toString()
final m = URL_PASSWORD.matcher(str)
if( m.matches() ) {
return replaceGroup(m, str, 1, redact(m.group(1)))
}
return str
}
static String replaceGroup(Matcher matcher, String source, int groupToReplace, String replacement) {
return new StringBuilder(source)
.replace(matcher.start(groupToReplace), matcher.end(groupToReplace), replacement)
.toString()
}
static boolean isIpV6String(String address) {
if( !address || !address.contains(':') )
return false
try {
InetAddresses.forString(address).getAddress().length==16
}
catch (IllegalArgumentException e) {
return false
}
}
static String formatHostName(String host, String port) {
if( !port || !host )
return host
final ipv6 = isIpV6String(host)
return ipv6 ? "[$host]:$port" : "$host:$port"
}
}

View File

@@ -0,0 +1,226 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util
import java.lang.management.ManagementFactory
import java.text.SimpleDateFormat
import java.time.OffsetDateTime
import com.sun.management.OperatingSystemMXBean
import groovy.transform.CompileStatic
import groovy.transform.Memoized
import groovy.transform.PackageScope
import groovy.util.logging.Slf4j
import nextflow.SysEnv
import nextflow.file.FileHelper
/**
* System helper methods
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
class SysHelper {
public static final String DEFAULT_DOCKER_PLATFORM = 'linux/amd64'
private static final String DATE_FORMAT = 'dd-MMM-yyyy HH:mm:ss'
/**
* Given a timestamp as epoch time convert to a string representation
* using the {@link #DATE_FORMAT}
*
* @param dateInMillis
* The date as number of milliseconds from 1 Jan 1970
* @param tz
* The {@link TimeZone} to use to format the date, or {@code null} to use the default time-zone
* @return
* The formatted date string
*/
static String fmtDate(long dateInMillis, TimeZone tz=null) {
fmtDate(new Date(dateInMillis), tz)
}
/**
* Given a {@link Date} object convert to a string representation
* according the {@link #DATE_FORMAT}
*
* @param date
* The date to render as a string
* @param tz
* The {@link TimeZone} to use to format the date, or {@code null} to use the default time-zone
* @return
* The formatted date string
*/
static String fmtDate(Date date, TimeZone tz=null) {
final formatter=new SimpleDateFormat(fmtEnv(), Locale.ENGLISH)
if(tz) formatter.setTimeZone(tz)
formatter.format(date)
}
/**
* Given a {@link java.time.OffsetDateTime} object convert to a string representation
* according the {@link #DATE_FORMAT}
*
* @param dateTime
* The OffsetDateTime to render as a string
* @return
* The formatted date string
*/
static String fmtDate(OffsetDateTime dateTime) {
// Convert to Date while preserving the original timezone
final date = Date.from(dateTime.toInstant())
final timezone = TimeZone.getTimeZone(dateTime.getOffset())
fmtDate(date, timezone)
}
static private String fmtEnv() {
final result = SysEnv.get('NXF_DATE_FORMAT', DATE_FORMAT)
return result.toLowerCase() == 'iso'
? "yyyy-MM-dd'T'HH:mm:ssXXX"
: result
}
/**
* Read the system uptime as returned by the {@code uptime} Linux tool
*
* @return The {@code uptime} stdout text
* @throws IllegalStateException If the {@code uptime} command return a non-zero exit status
*/
static String getUptimeText() throws IllegalStateException {
def proc = new ProcessBuilder('uptime').start()
def status = proc.waitFor()
if( status == 0 ) {
return proc.text.trim()
}
throw new IllegalStateException("Unable to run system `uptime` command: exit=$status")
}
/**
* The system uptime
*
* @return A {@link Duration} object representing the uptime of the system
*/
static Duration getUptimeDuration() {
def text = getUptimeText()
def result = parseUptimeText(text)
log.trace "Uptime $result -- parsed text: $text"
return result
}
static long getBootTimeMillis() {
System.currentTimeMillis() - getUptimeDuration().toMillis()
}
static String getBootTimeString() {
fmtDate(new Date(getBootTimeMillis()))
}
/**
* Parse `uptime` command line tool output
*
* @param str The text stdout produced by the {@code uptime} system tool
* @return
* @throws IllegalArgumentException
*/
@PackageScope
static Duration parseUptimeText( String str ) throws IllegalArgumentException {
try {
def p = str.indexOf(' up ')
def items = str.substring(p+4).tokenize(',')
def uptime = items[0].trim().replace('mins','min').replace('hrs','hour')
if( uptime.contains(':') ) {
uptime = reformatHourAndMin(uptime)
}
else if( items.size()>1 && items[1].contains(":")) {
uptime += " ${reformatHourAndMin(items[1])}"
}
return Duration.of(uptime)
}
catch( Exception e ) {
throw new IllegalArgumentException("Not a valid uptime text: `$str`", e)
}
}
private static String reformatHourAndMin(String text) {
def items = text.tokenize(':')
"${items[0]}h ${items[1]}m"
}
/**
* @return The actual free space in the node local storage
*/
static MemoryUnit getAvailDisk() {
final free = FileHelper.getLocalTempPath().toFile().getFreeSpace()
new MemoryUnit(free)
}
@Memoized
static private OperatingSystemMXBean getSystemMXBean() {
(OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean()
}
/**
* @return The number of CPUs available
*/
static int getAvailCpus() {
final int result = getSystemMXBean().getAvailableProcessors()
return result
}
/**
* @return The total system memory available
*/
static MemoryUnit getAvailMemory() {
new MemoryUnit(getSystemMXBean().getTotalPhysicalMemorySize())
}
/**
* @return The system host name
*/
static String getHostName() {
System.getenv('HOSTNAME') ?: InetAddress.getLocalHost().getHostName()
}
static String getArch() {
final os = (java.lang.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean()
return os.getArch()
}
/**
* Dump th stack trace of current running threads
* @return
*/
static String dumpThreads() {
def buffer = new StringBuffer()
Map<Thread, StackTraceElement[]> m = Thread.getAllStackTraces();
for(Map.Entry<Thread, StackTraceElement[]> e : m.entrySet()) {
buffer.append('\n').append(e.getKey().toString()).append('\n')
for (StackTraceElement s : e.getValue()) {
buffer.append(" " + s).append('\n')
}
}
return buffer.toString()
}
}

View File

@@ -0,0 +1,35 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util
import static java.lang.annotation.ElementType.*
import java.lang.annotation.Retention
import java.lang.annotation.RetentionPolicy
import java.lang.annotation.Target
/**
* Marker interface to annotate types and methods that are meant to be used only
* for testing purpose.
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Retention(RetentionPolicy.SOURCE)
@Target(value=[CONSTRUCTOR, FIELD, LOCAL_VARIABLE, METHOD, PACKAGE, PARAMETER, TYPE])
@interface TestOnly {
}

View File

@@ -0,0 +1,48 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util
import groovy.transform.CompileStatic
import nextflow.SysEnv
/**
* Helper class for threads handling
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class Threads {
static boolean useVirtual() {
SysEnv.get('NXF_ENABLE_VIRTUAL_THREADS')=='true'
}
static Thread start(Closure action) {
return useVirtual()
? Thread.startVirtualThread(action)
: Thread.startDaemon(action)
}
static Thread start(String name, Closure action) {
if( !useVirtual() )
return Thread.startDaemon(name, action)
// create a new virtual thread and change the name
final result = Thread.startVirtualThread(action)
result.setName(name)
return result
}
}

View File

@@ -0,0 +1,230 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util
import java.util.concurrent.ConcurrentHashMap
import groovy.transform.CompileStatic
import groovy.transform.EqualsAndHashCode
/**
* Limit the execution of a code block on a specified time basis
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class Throttle {
static final Map<Integer,ThrottleObj> throttleMap = new ConcurrentHashMap<>()
@EqualsAndHashCode
static class ThrottleObj {
Object result
long timestamp
ThrottleObj() {}
ThrottleObj( value, long timestamp ) {
this.result = value
this.timestamp = timestamp
}
}
/**
* Executes the specified code block and returns the resulting value. All following
* invocations within the specified time {@code period} are skipped and the cached
* result value is returned instead.
*
* @param period
* The time period in milliseconds, in which the throttler will allow at most to invoke
* the specified closure
* @param closure
* The code block to execute
* @return
* The value returned by the closure execution
*/
static Object every( long period, Closure closure ) {
throttle0( period, null, closure)
}
/**
* Executes the specified code block and returns the resulting value. All following
* invocations within the specified time {@code period} are skipped and the cached
* result value is returned instead.
*
* @param period
* The time duration, in which the throttler will allow at most to invoke
* the specified closure
* @param closure
* The code block to execute
* @return
* The value returned by the closure execution
*/
static Object every( Duration period, Closure closure ) {
throttle0( period.millis, null, closure)
}
/**
* Executes the specified code block and returns the resulting value. All following
* invocations within the specified time {@code period} are skipped and the cached
* result value is returned instead.
*
* @param period
* The duration string, in which the throttler will allow at most to invoke
* the specified closure
* @param closure
* The code block to execute
* @return
* The value returned by the closure execution
*/
static Object every( String period, Closure closure ) {
throttle0( Duration.of(period).millis, null, closure)
}
/**
* Allows at most one execution after the specified time {@code delay} returning
* {@code null} before the first execution, and the cached result value the
* following times
*
* @param delay
* @param closure
* @return
*/
static Object after( long delay, Closure closure ) {
final obj = new ThrottleObj( null, System.currentTimeMillis() )
throttle0( delay, obj, closure)
}
/**
* Allows at most one execution after the specified time {@code delay} returning
* {@code null} before the first execution, and the cached result value the
* following times
*
* @param delay
* @param closure
* @return
*/
static Object after( Duration delay, Closure closure ) {
def obj = new ThrottleObj( null, System.currentTimeMillis() )
throttle0( delay.millis, obj, closure)
}
/**
* Allows at most one execution after the specified time {@code delay} returning
* {@code null} before the first execution, and the cached result value the
* following times
*
* @param delay
* @param closure
* @return
*/
static Object after( String delay, Closure closure ) {
def obj = new ThrottleObj( null, System.currentTimeMillis() )
throttle0( Duration.of(delay).millis, obj, closure)
}
/**
* Allows at most one execution after the specified time {@code delay} returning
* {@code initialValue} before the first execution, and the cached result value the
* following times
*
* @param delay
* @param initialValue
* @param closure
* @return
*/
static Object after( long delay, initialValue, Closure closure ) {
final obj = new ThrottleObj( initialValue, System.currentTimeMillis() )
throttle0( delay, obj, closure)
}
/**
* Allows at most one execution after the specified time {@code delay} returning
* {@code initialValue} before the first execution, and the cached result value the
* following times
*
* @param delay
* @param initialValue
* @param closure
* @return
*/
static Object after( Duration delay, initialValue, Closure closure ) {
def obj = new ThrottleObj( initialValue, System.currentTimeMillis() )
throttle0( delay.millis, obj, closure)
}
/**
* Allows at most one execution after the specified time {@code delay} returning
* {@code initialValue} before the first execution, and the cached result value the
* following times
*
* @param delay
* @param initialValue
* @param closure
* @return
*/
static Object after( String delay, initialValue, Closure closure ) {
def obj = new ThrottleObj( initialValue, System.currentTimeMillis() )
throttle0( Duration.of(delay).millis, obj, closure)
}
private static throttle0( long timeout, ThrottleObj initialValue, Closure closure ) {
assert closure != null
def key = 17
key = 31 * key + closure.class.name.hashCode()
key = 31 * key + closure.owner.hashCode()
key = 31 * key + closure.delegate?.hashCode() ?: 0
ThrottleObj obj = throttleMap.get(key)
if( obj == null ) {
obj = initialValue ?: new ThrottleObj()
throttleMap.put(key,obj)
}
if( System.currentTimeMillis() - obj.timestamp > timeout ) {
obj.timestamp = System.currentTimeMillis()
obj.result = closure.call()
}
obj.result
}
static <V> V cache( Object key, Duration eviction, Closure<V> action ) {
cache(key, eviction.toMillis(), action)
}
static <V> V cache( Object key, long eviction, Closure<V> closure ) {
final int hash = key?.hashCode() ?: 0
ThrottleObj obj = throttleMap.get(hash)
if( obj == null ) {
obj = new ThrottleObj()
obj.result = closure.call()
obj.timestamp = System.currentTimeMillis()
throttleMap.put(hash,obj)
}
else if( System.currentTimeMillis() - obj.timestamp > eviction ) {
obj.timestamp = System.currentTimeMillis()
obj.result = closure.call()
}
(V)obj.result
}
}

View File

@@ -0,0 +1,195 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.util
import java.lang.reflect.Field
import java.lang.reflect.ParameterizedType
import java.lang.reflect.Type
import java.nio.file.Files
import java.nio.file.Path
import groovy.transform.CompileStatic
import groovy.transform.Memoized
import nextflow.exception.AbortOperationException
import nextflow.file.FileHelper
import nextflow.script.dsl.Nullable
import nextflow.script.types.Bag
import nextflow.script.types.Record
import nextflow.util.HashBag
import nextflow.util.RecordMap
import org.codehaus.groovy.runtime.typehandling.DefaultTypeTransformation
import org.codehaus.groovy.runtime.typehandling.GroovyCastException
/**
* Utility functions for working with types at runtime.
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
* @author Ben Sherman <bentshermann@gmail.com>
*/
@CompileStatic
class TypeHelper {
/**
* Returns the given type argument from the given value's superclass.
*
* This method assumes that the value's class extends a parameterized type.
*
* @param value
* @param index
*
* @example
* <pre>
* class ExampleClass extends GenericBase&lt;String, Integer&gt; {}
*
* Type type = TypeHelper.getGenericType(new ExampleClass(), 1);
* System.out.println(type); // Output: class java.lang.Integer
* </pre>
*/
static Type getGenericType(Object value, int index) {
final params = (ParameterizedType) value.getClass().getGenericSuperclass()
return params.getActualTypeArguments()[index]
}
/**
* Get the concrete Java class for a type.
*
* @param type
*/
static Class getRawType(Type type) {
return \
type instanceof Class ? type :
type instanceof ParameterizedType ? type.getRawType() :
Object
}
/**
* Determine whether a type is a collection type (Bag, List, Set).
*
* @param type
*/
static boolean isCollectionType(Type type) {
return Collection.class.isAssignableFrom(getRawType(type))
}
/**
* Determine whether a type is a record type.
*
* @param type
*/
static boolean isRecordType(Type type) {
return type instanceof Class && Record.class.isAssignableFrom(type)
}
/**
* Convert a value to the given type.
*
* @param value
* @param type
*/
static Object asType(Object value, Type type) {
if( value == null )
return null
if( isCollectionType(type) )
return asCollectionType(value as Collection, type)
if( isRecordType(type) )
return asRecordType(value as Map, (Class) type)
if( type == Path )
return TypeHelper.asPathType(value.toString())
return DefaultTypeTransformation.castToType(value, getRawType(type))
}
/**
* Convert a collection to the given collection type.
*
* If the type specifies an element type, each element in the
* collection is converted to that type.
*
* @param collection
* @param type
*/
static Collection asCollectionType(Collection collection, Type type) {
if( type instanceof ParameterizedType ) {
final elementType = type.getActualTypeArguments()[0]
collection = collection.collect { el -> asType(el, elementType) }
}
return switch( getRawType(type) ) {
case Bag.class -> new HashBag<>(collection)
case List.class -> collection as List
case Set.class -> collection as Set
default -> collection
}
}
/**
* Convert a string representing a file path to a Path.
* Report an error if the path does not exist.
*
* @param str
*/
static Path asPathType(String str) {
final result = FileHelper.asPath(str)
if( !Files.exists(result) )
throw new AbortOperationException("Input file '${str}' does not exist")
return result
}
/**
* Convert a map to a record, validating it against the given
* record type.
*
* @param map
* @param type
*/
static Record asRecordType(Map<String,?> map, Class type) {
final fields = recordFields(type)
for( final field : fields.values() ) {
if( field.isAnnotationPresent(Nullable.class) )
continue
if( map.get(field.getName()) == null )
throw new AbortOperationException("Input record ${map} is missing field '${field.getName()}' required by record type '${type.getSimpleName()}'")
}
final result = new HashMap<String,Object>(map.size())
for( final entry : map.entrySet() ) {
final name = entry.key
final value = fields.containsKey(name)
? asType(entry.value, fields[name].getGenericType())
: entry.value
result.put(name, value)
}
return new RecordMap(result)
}
@Memoized
private static Map<String,Field> recordFields(Class type) {
final fields = type.getDeclaredFields()
final result = new HashMap<String,Field>(fields.size())
for( final field : fields ) {
if( field.isSynthetic() )
continue
result.put(field.getName(), field)
}
return result
}
}

Some files were not shown because too many files have changed in this diff Show More