add nextflow d30e48d
This commit is contained in:
50
nextflow/modules/nf-commons/build.gradle
Normal file
50
nextflow/modules/nf-commons/build.gradle
Normal file
@@ -0,0 +1,50 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
apply plugin: 'groovy'
|
||||
|
||||
sourceSets {
|
||||
main.java.srcDirs = []
|
||||
main.groovy.srcDirs = ['src/main']
|
||||
main.resources.srcDirs = ['src/resources']
|
||||
test.groovy.srcDirs = ['src/test']
|
||||
test.java.srcDirs = []
|
||||
test.resources.srcDirs = ['src/testResources']
|
||||
}
|
||||
|
||||
dependencies {
|
||||
api(project(':nf-lang'))
|
||||
api "ch.qos.logback:logback-classic:1.5.26"
|
||||
api "org.apache.groovy:groovy:4.0.31"
|
||||
api "org.apache.groovy:groovy-nio:4.0.31"
|
||||
api "org.apache.commons:commons-lang3:3.18.0"
|
||||
api 'com.google.guava:guava:33.0.0-jre'
|
||||
api 'org.pf4j:pf4j:3.14.1'
|
||||
api 'org.pf4j:pf4j-update:2.3.0'
|
||||
api 'dev.failsafe:failsafe:3.1.0'
|
||||
api 'io.seqera:lib-httpx:2.1.0'
|
||||
api 'io.seqera:lib-retry:2.0.0'
|
||||
// patch gson dependency required by pf4j
|
||||
api 'com.google.code.gson:gson:2.13.1'
|
||||
api 'io.seqera:npr-api:0.22.0'
|
||||
|
||||
/* testImplementation inherited from top gradle build file */
|
||||
testImplementation(testFixtures(project(":nextflow")))
|
||||
testFixturesImplementation(project(":nextflow"))
|
||||
|
||||
testImplementation "org.apache.groovy:groovy-json:4.0.31" // needed by wiremock
|
||||
testImplementation ('org.wiremock:wiremock:3.13.1') { exclude module: 'groovy-all' }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow
|
||||
|
||||
import static nextflow.extension.Bolts.*
|
||||
|
||||
import java.text.SimpleDateFormat
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
/**
|
||||
* Model app build information
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class BuildInfo {
|
||||
|
||||
private static Properties properties
|
||||
|
||||
static {
|
||||
final BUILD_INFO = '/META-INF/build-info.properties'
|
||||
properties = new Properties()
|
||||
try {
|
||||
properties.load( BuildInfo.getResourceAsStream(BUILD_INFO) )
|
||||
}
|
||||
catch( Exception e ) {
|
||||
log.warn "Unable to parse $BUILD_INFO - Cause ${e.message ?: e}"
|
||||
}
|
||||
}
|
||||
|
||||
static Properties getProperties() { properties }
|
||||
|
||||
static String getVersion() { properties.getProperty('version') }
|
||||
|
||||
static String getCommitId() { properties.getProperty('commitId')}
|
||||
|
||||
static String getBuildNum() { properties.getProperty('build') }
|
||||
|
||||
static long getTimestampMillis() { properties.getProperty('timestamp') as long }
|
||||
|
||||
static String getTimestampUTC() {
|
||||
def tz = TimeZone.getTimeZone('UTC')
|
||||
def fmt = new SimpleDateFormat(DATETIME_FORMAT)
|
||||
fmt.setTimeZone(tz)
|
||||
fmt.format(new Date(getTimestampMillis())) + ' ' + tz.getDisplayName( true, TimeZone.SHORT )
|
||||
}
|
||||
|
||||
static String getTimestampLocal() {
|
||||
def tz = TimeZone.getDefault()
|
||||
def fmt = new SimpleDateFormat(DATETIME_FORMAT)
|
||||
fmt.setTimeZone(tz)
|
||||
fmt.format(new Date(getTimestampMillis())) + ' ' + tz.getDisplayName( true, TimeZone.SHORT )
|
||||
}
|
||||
|
||||
static String getTimestampDelta() {
|
||||
if( getTimestampUTC() == getTimestampLocal() ) {
|
||||
return ''
|
||||
}
|
||||
|
||||
final utc = getTimestampUTC().tokenize(' ')
|
||||
final loc = getTimestampLocal().tokenize(' ')
|
||||
|
||||
final result = utc[0] == loc[0] ? loc[1,-1].join(' ') : loc.join(' ')
|
||||
return "($result)"
|
||||
}
|
||||
|
||||
static String getFullVersion() {
|
||||
"${version}_${commitId}"
|
||||
}
|
||||
|
||||
}
|
||||
91
nextflow/modules/nf-commons/src/main/nextflow/Const.groovy
Normal file
91
nextflow/modules/nf-commons/src/main/nextflow/Const.groovy
Normal file
@@ -0,0 +1,91 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow
|
||||
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
/**
|
||||
* Application main constants
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class Const {
|
||||
|
||||
static final public String ISO_8601_DATETIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ss'Z'"
|
||||
|
||||
static final public transient BOOL_YES = ['true','yes','on']
|
||||
|
||||
static final public transient BOOL_NO = ['false','no','off']
|
||||
|
||||
/**
|
||||
* The application main package name
|
||||
*/
|
||||
static public final String MAIN_PACKAGE = Const.name.split('\\.')[0]
|
||||
|
||||
/**
|
||||
* The application main name
|
||||
*/
|
||||
static public final String APP_NAME = MAIN_PACKAGE
|
||||
|
||||
/**
|
||||
* The application home folder
|
||||
*/
|
||||
static public final Path APP_HOME_DIR = getHomeDir(APP_NAME)
|
||||
|
||||
static Path sysHome() {
|
||||
def home = System.getProperty("user.home")
|
||||
if( !home || home=='?' )
|
||||
home = SysEnv.get('HOME')
|
||||
if( !home )
|
||||
throw new IllegalStateException("Unable to detect system home path - Make sure the variable HOME or NXF_HOME is defined in your environment")
|
||||
return Path.of(home)
|
||||
}
|
||||
|
||||
private static Path getHomeDir(String appname) {
|
||||
final home = SysEnv.get('NXF_HOME')
|
||||
final result = home ? Paths.get(home) : sysHome().resolve(".$appname")
|
||||
|
||||
if( !result.exists() && !result.mkdir() ) {
|
||||
throw new IllegalStateException("Cannot create path '${result}' -- check file system access permission")
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
static final Path getAppCacheDir() {
|
||||
return Path.of(SysEnv.get('NXF_CACHE_DIR', '.nextflow'))
|
||||
}
|
||||
|
||||
static public final String ROLE_WORKER = 'worker'
|
||||
|
||||
static public final String ROLE_MASTER = 'master'
|
||||
|
||||
static public final String MANIFEST_FILE_NAME = 'nextflow.config'
|
||||
|
||||
static public final String DEFAULT_MAIN_FILE_NAME = 'main.nf'
|
||||
|
||||
static public final String DEFAULT_ORGANIZATION = SysEnv.get('NXF_ORG') ?: 'nextflow-io'
|
||||
|
||||
static public final String DEFAULT_HUB = SysEnv.get('NXF_HUB') ?: 'github'
|
||||
|
||||
static public final File DEFAULT_ROOT = SysEnv.get('NXF_ASSETS') ? new File(SysEnv.get('NXF_ASSETS')) : APP_HOME_DIR.resolve('assets').toFile()
|
||||
|
||||
static public final String DEFAULT_BRANCH = 'master'
|
||||
|
||||
static public final String SCOPE_SEP = ':'
|
||||
|
||||
}
|
||||
120
nextflow/modules/nf-commons/src/main/nextflow/Global.groovy
Normal file
120
nextflow/modules/nf-commons/src/main/nextflow/Global.groovy
Normal file
@@ -0,0 +1,120 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow
|
||||
|
||||
|
||||
import java.util.function.Consumer
|
||||
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.util.Duration
|
||||
import nextflow.util.IniFile
|
||||
import nextflow.util.MemoryUnit
|
||||
import nextflow.util.TestOnly
|
||||
import org.apache.commons.lang3.StringUtils
|
||||
import org.apache.commons.lang3.exception.ExceptionUtils
|
||||
/**
|
||||
* Hold global variables
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
class Global {
|
||||
|
||||
/**
|
||||
* When {@code true}, Fusion trace metrics replace the bash command-trace wrapper
|
||||
*/
|
||||
static boolean isFusionTraceEnabled() {
|
||||
return SysEnv.get('NXF_FUSION_TRACE', 'false') == 'true'
|
||||
}
|
||||
|
||||
/**
|
||||
* The pipeline session instance
|
||||
*/
|
||||
static private ISession session
|
||||
|
||||
/**
|
||||
* Allow to load session in a lazy manner
|
||||
*/
|
||||
static private Closure<ISession> loader
|
||||
|
||||
/**
|
||||
* The main configuration object
|
||||
*/
|
||||
static Map config
|
||||
|
||||
/**
|
||||
* @return The object instance representing the current session
|
||||
*/
|
||||
static ISession getSession() {
|
||||
if( session != null )
|
||||
return session
|
||||
if( loader != null )
|
||||
session = loader.call()
|
||||
return session
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the application session object
|
||||
*
|
||||
* @param value An object instance representing the current session
|
||||
*/
|
||||
static void setSession( ISession value ) {
|
||||
session = value
|
||||
}
|
||||
|
||||
/**
|
||||
* Set a session lazy loader
|
||||
*
|
||||
* @param loader
|
||||
*/
|
||||
static void setLazySession( Closure<ISession> loader ) {
|
||||
Global.loader = loader
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the specified closure on application termination
|
||||
*
|
||||
* @param callback A closure to be executed on application shutdown
|
||||
*/
|
||||
static void onCleanup(Consumer<ISession> callback) {
|
||||
if( callback==null ) {
|
||||
log.warn "Cleanup consumer cannot be null\n${ExceptionUtils.getStackTrace(new Exception())}"
|
||||
return
|
||||
}
|
||||
hooks.add(callback)
|
||||
}
|
||||
|
||||
static final private List<Consumer<ISession>> hooks = []
|
||||
|
||||
static synchronized cleanUp() {
|
||||
for( Consumer<ISession> c : hooks ) {
|
||||
try {
|
||||
c.accept(session)
|
||||
}
|
||||
catch( Exception e ) {
|
||||
log.debug("Error during on cleanup", e )
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@TestOnly
|
||||
static void reset() {
|
||||
session = null
|
||||
config = null
|
||||
hooks.clear()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
/**
|
||||
* Nextflow session interface
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
interface ISession {
|
||||
|
||||
/**
|
||||
* The folder where tasks temporary files are stored
|
||||
*/
|
||||
Path getWorkDir()
|
||||
|
||||
/**
|
||||
* The folder where the main script is contained
|
||||
*/
|
||||
Path getBaseDir()
|
||||
|
||||
/**
|
||||
* Holds the configuration object
|
||||
*/
|
||||
Map getConfig()
|
||||
|
||||
/**
|
||||
* The pipeline script name (without parent path)
|
||||
*/
|
||||
String getScriptName()
|
||||
|
||||
/**
|
||||
* @return List of path added to application classpath at runtime
|
||||
*/
|
||||
List<Path> getLibDir()
|
||||
|
||||
/**
|
||||
* The unique identifier of this session
|
||||
*/
|
||||
UUID getUniqueId()
|
||||
|
||||
/**
|
||||
* @return The global script binding object
|
||||
*/
|
||||
Binding getBinding()
|
||||
|
||||
boolean isCacheable()
|
||||
|
||||
boolean isResumeMode()
|
||||
|
||||
String getCommitId()
|
||||
|
||||
}
|
||||
99
nextflow/modules/nf-commons/src/main/nextflow/SysEnv.groovy
Normal file
99
nextflow/modules/nf-commons/src/main/nextflow/SysEnv.groovy
Normal file
@@ -0,0 +1,99 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.PackageScope
|
||||
|
||||
/**
|
||||
* Helper class that holds a reference system environment and
|
||||
* allow to swap to a different one for testing purposes
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class SysEnv {
|
||||
|
||||
private static Holder holder = new Holder(System.getenv())
|
||||
|
||||
private static final List<Map<String,String>> history = new ArrayList<Map<String,String>>()
|
||||
|
||||
static boolean containsKey(String key) {
|
||||
return holder.containsKey(key)
|
||||
}
|
||||
|
||||
static Map<String,String> get() {
|
||||
return holder
|
||||
}
|
||||
|
||||
static String get(String name) {
|
||||
return holder.get(name)
|
||||
}
|
||||
|
||||
static String get(String name, String defValue) {
|
||||
return holder.containsKey(name) ? holder.get(name) : defValue
|
||||
}
|
||||
|
||||
static boolean getBool(String name, boolean defValue) {
|
||||
final result = get(name,String.valueOf(defValue))
|
||||
return Boolean.parseBoolean(result) || result == '1'
|
||||
}
|
||||
|
||||
static Integer getInteger(String name, Integer defValue) {
|
||||
final result = get(name, defValue!=null ? String.valueOf(defValue) : null)
|
||||
return result!=null ? Integer.valueOf(result) : null
|
||||
}
|
||||
|
||||
static Long getLong(String name, Long defValue) {
|
||||
final result = get(name, defValue!=null ? String.valueOf(defValue) : null)
|
||||
return result!=null ? Long.valueOf(result) : null
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if agent output mode is enabled via environment variables.
|
||||
* When enabled, Nextflow replaces interactive ANSI logging with minimal,
|
||||
* structured output optimized for AI agents.
|
||||
*
|
||||
* Supported variables (any truthy value activates the mode):
|
||||
* {@code NXF_AGENT_MODE}, {@code AGENT}, {@code CLAUDECODE}.
|
||||
*
|
||||
* @return {@code true} if agent mode is enabled
|
||||
*/
|
||||
static boolean isAgentMode() {
|
||||
return getBool('NXF_AGENT_MODE', false) ||
|
||||
getBool('AGENT', false) ||
|
||||
getBool('CLAUDECODE', false)
|
||||
}
|
||||
|
||||
static void push(Map<String,String> env) {
|
||||
history.push(holder.getTarget())
|
||||
holder.setTarget(env)
|
||||
}
|
||||
|
||||
static void pop() {
|
||||
holder.setTarget(history.pop())
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@PackageScope
|
||||
class Holder implements Map<String,String> {
|
||||
@Delegate Map<String,String> target
|
||||
Holder(Map<String,String> target) { this.target = target }
|
||||
void setTarget(Map<String,String> target) { this.target=target }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.config
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.SysEnv
|
||||
import nextflow.config.spec.ConfigOption
|
||||
import nextflow.config.spec.ConfigScope
|
||||
import nextflow.config.spec.ScopeName
|
||||
import nextflow.script.dsl.Description
|
||||
|
||||
/**
|
||||
* Configuration scope for module registry settings
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
@ScopeName("registry")
|
||||
@Description("""
|
||||
The `registry` scope provides configuration for the Nextflow module registry.
|
||||
This includes the registry URL(s) and API key for authentication.
|
||||
""")
|
||||
@CompileStatic
|
||||
class RegistryConfig implements ConfigScope {
|
||||
|
||||
public static final String DEFAULT_REGISTRY_URL = 'https://registry.nextflow.io/api'
|
||||
|
||||
@ConfigOption
|
||||
@Description("Registry URL or list of registry URLs in priority order (primary URL first)")
|
||||
private final Collection<String> url
|
||||
|
||||
@ConfigOption
|
||||
@Description("API key for authenticating with the primary registry")
|
||||
private final String apiKey
|
||||
|
||||
/* required by extension point -- do not remove */
|
||||
RegistryConfig() {
|
||||
this.url = [DEFAULT_REGISTRY_URL]
|
||||
this.apiKey = null
|
||||
}
|
||||
|
||||
RegistryConfig(Map opts) {
|
||||
final urlObject = opts.url ?: [DEFAULT_REGISTRY_URL]
|
||||
if (urlObject instanceof Collection<String>)
|
||||
this.url = (urlObject as Collection<String>).collect { it.replaceAll(/\/+$/, '') }
|
||||
else
|
||||
this.url = [urlObject.toString().replaceAll(/\/+$/, '')]
|
||||
this.apiKey = opts.apiKey as String
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the primary (first) registry URL
|
||||
*
|
||||
* @return The primary registry URL
|
||||
*/
|
||||
String getUrl() {
|
||||
return this.url ? url[0] as String : DEFAULT_REGISTRY_URL
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all registry URLs (primary first, fallbacks after)
|
||||
*
|
||||
* @return Collection of registry URLs
|
||||
*/
|
||||
Collection<String> getAllUrls() {
|
||||
return this.url ?: [DEFAULT_REGISTRY_URL]
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the API key for the primary registry.
|
||||
* Authentication is only supported for the primary registry.
|
||||
*
|
||||
* @return The API key, or 'NXF_REGISTRY_TOKEN' env value if not configured
|
||||
*/
|
||||
String getApiKey() {
|
||||
return apiKey ?: SysEnv.get('NXF_REGISTRY_TOKEN')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.exception
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.InheritConstructors
|
||||
|
||||
/**
|
||||
* Abort the current operation execution
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@InheritConstructors
|
||||
@CompileStatic
|
||||
class AbortOperationException extends RuntimeException { }
|
||||
@@ -0,0 +1,981 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.extension
|
||||
|
||||
import java.nio.file.Path
|
||||
import java.text.DateFormat
|
||||
import java.text.SimpleDateFormat
|
||||
import java.time.OffsetDateTime
|
||||
import java.time.format.DateTimeFormatter
|
||||
import java.util.concurrent.locks.Lock
|
||||
import java.util.regex.Pattern
|
||||
|
||||
import groovy.transform.CompileDynamic
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.Memoized
|
||||
import groovy.transform.stc.ClosureParams
|
||||
import groovy.transform.stc.FirstParam
|
||||
import nextflow.file.FileHelper
|
||||
import nextflow.file.FileMutex
|
||||
import nextflow.util.CheckHelper
|
||||
import nextflow.util.Duration
|
||||
import nextflow.util.MemoryUnit
|
||||
import nextflow.util.RateUnit
|
||||
import org.apache.commons.lang3.StringUtils
|
||||
import org.codehaus.groovy.runtime.DefaultGroovyMethods
|
||||
import org.codehaus.groovy.runtime.GStringImpl
|
||||
import org.codehaus.groovy.runtime.ResourceGroovyMethods
|
||||
import org.codehaus.groovy.runtime.StringGroovyMethods
|
||||
import org.slf4j.Logger
|
||||
/**
|
||||
* Generic extensions
|
||||
*
|
||||
* See more about extension methods
|
||||
* http://docs.codehaus.org/display/GROOVY/Creating+an+extension+module
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class Bolts {
|
||||
|
||||
static public final String DATETIME_FORMAT = 'dd-MM-yyyy HH:mm'
|
||||
|
||||
static private Pattern PATTERN_RIGHT_TRIM = ~/\s+$/
|
||||
|
||||
static private Pattern PATTERN_LEFT_TRIM = ~/^\s+/
|
||||
|
||||
@Memoized
|
||||
static private ThreadLocal<DateFormat> getLocalDateFormat(String fmt, TimeZone tz) {
|
||||
|
||||
return new ThreadLocal<DateFormat>() {
|
||||
@Override
|
||||
protected DateFormat initialValue() {
|
||||
def result = new SimpleDateFormat(fmt, Locale.ENGLISH)
|
||||
if(tz) result.setTimeZone(tz)
|
||||
return result
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Format a {@link Date} object
|
||||
*
|
||||
* @param self The {@link Date} object to format
|
||||
* @param format The date format to use eg. {@code dd-MM-yyyy HH:mm}.
|
||||
* @param tz The timezone to be used eg. {@code UTC}. If {@code null} the current timezone is used.
|
||||
* @return The date-time formatted as a string
|
||||
*/
|
||||
static String format(Date self, String format=null, String tz=null) {
|
||||
TimeZone zone = tz ? TimeZone.getTimeZone(tz) : null
|
||||
getLocalDateFormat(format ?: DATETIME_FORMAT, zone).get().format(self)
|
||||
}
|
||||
|
||||
static String format(OffsetDateTime self, String format) {
|
||||
return self.format(DateTimeFormatter.ofPattern(format).withLocale(Locale.ENGLISH))
|
||||
}
|
||||
|
||||
/**
|
||||
* Format a {@link Date} object
|
||||
*
|
||||
* @param self The {@link Date} object to format
|
||||
* @param format The date format to use eg. {@code dd-MM-yyyy HH:mm}
|
||||
* @param tz The timezone to be used. If {@code null} the current timezone is used.
|
||||
* @return The date-time formatted as a string
|
||||
*/
|
||||
static String format(Date self, String format, TimeZone tz) {
|
||||
getLocalDateFormat(format ?: DATETIME_FORMAT, tz).get().format(self)
|
||||
}
|
||||
|
||||
static List pairs(Map self, Map opts=null) {
|
||||
def flat = opts?.flat == true
|
||||
def result = []
|
||||
for( Map.Entry entry : self.entrySet() ) {
|
||||
if( flat && entry.value instanceof Collection )
|
||||
entry.value.iterator().each { result << [entry.key, it] }
|
||||
else
|
||||
result << [entry.key, entry.value]
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove the left side after a dot (including it) e.g.
|
||||
* <pre>
|
||||
* 0.10 => 0
|
||||
* 10000.00 => 10000
|
||||
* </pre>
|
||||
*
|
||||
* @param self
|
||||
* @return
|
||||
*/
|
||||
static String trimDotZero(String self) {
|
||||
int p = self?.indexOf('.')
|
||||
p!=-1 ? self.substring(0,p) : self
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove blank chars at the end of the string
|
||||
*
|
||||
* @param self The string itself
|
||||
* @return The string with blanks removed
|
||||
*/
|
||||
|
||||
static String rightTrim(String self) {
|
||||
self.replaceAll(PATTERN_RIGHT_TRIM,'')
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove blank chars at string beginning
|
||||
*
|
||||
* @param self The string itself
|
||||
* @return The string with blanks removed
|
||||
*/
|
||||
static String leftTrim( String self ) {
|
||||
self.replaceAll(PATTERN_LEFT_TRIM,'')
|
||||
}
|
||||
|
||||
/**
|
||||
* <p>Strips any of a set of characters from the start and end of a String.
|
||||
* This is similar to {@link String#trim()} but allows the characters
|
||||
* to be stripped to be controlled.</p>
|
||||
*
|
||||
* <p>A <code>null</code> input String returns <code>null</code>.
|
||||
* An empty string ("") input returns the empty string.</p>
|
||||
*
|
||||
* <p>If the stripChars String is <code>null</code>, whitespace is
|
||||
* stripped as defined by {@link Character#isWhitespace(char)}.
|
||||
* Alternatively use {@link #strip(String)}.</p>
|
||||
*
|
||||
* <pre>
|
||||
* StringUtils.strip(null, *) = null
|
||||
* StringUtils.strip("", *) = ""
|
||||
* StringUtils.strip("abc", null) = "abc"
|
||||
* StringUtils.strip(" abc", null) = "abc"
|
||||
* StringUtils.strip("abc ", null) = "abc"
|
||||
* StringUtils.strip(" abc ", null) = "abc"
|
||||
* StringUtils.strip(" abcyx", "xyz") = " abc"
|
||||
* </pre>
|
||||
*
|
||||
* @param str the String to remove characters from, may be null
|
||||
* @param stripChars the characters to remove, null treated as whitespace
|
||||
* @return the stripped String, <code>null</code> if null String input
|
||||
*/
|
||||
static String strip( String self, String stripChars = null ) {
|
||||
StringUtils.strip(self, stripChars)
|
||||
}
|
||||
|
||||
/**
|
||||
* <p>Strips any of a set of characters from the start of a String.</p>
|
||||
*
|
||||
* <p>A <code>null</code> input String returns <code>null</code>.
|
||||
* An empty string ("") input returns the empty string.</p>
|
||||
*
|
||||
* <p>If the stripChars String is <code>null</code>, whitespace is
|
||||
* stripped as defined by {@link Character#isWhitespace(char)}.</p>
|
||||
*
|
||||
* <pre>
|
||||
* StringUtils.stripStart(null, *) = null
|
||||
* StringUtils.stripStart("", *) = ""
|
||||
* StringUtils.stripStart("abc", "") = "abc"
|
||||
* StringUtils.stripStart("abc", null) = "abc"
|
||||
* StringUtils.stripStart(" abc", null) = "abc"
|
||||
* StringUtils.stripStart("abc ", null) = "abc "
|
||||
* StringUtils.stripStart(" abc ", null) = "abc "
|
||||
* StringUtils.stripStart("yxabc ", "xyz") = "abc "
|
||||
* </pre>
|
||||
*
|
||||
* @param str the String to remove characters from, may be null
|
||||
* @param stripChars the characters to remove, null treated as whitespace
|
||||
* @return the stripped String, <code>null</code> if null String input
|
||||
*/
|
||||
static String stripStart( String self, String stripChars = null ) {
|
||||
StringUtils.stripStart(self, stripChars)
|
||||
}
|
||||
|
||||
/**
|
||||
* <p>Strips any of a set of characters from the end of a String.</p>
|
||||
*
|
||||
* <p>A <code>null</code> input String returns <code>null</code>.
|
||||
* An empty string ("") input returns the empty string.</p>
|
||||
*
|
||||
* <p>If the stripChars String is <code>null</code>, whitespace is
|
||||
* stripped as defined by {@link Character#isWhitespace(char)}.</p>
|
||||
*
|
||||
* <pre>
|
||||
* StringUtils.stripEnd(null, *) = null
|
||||
* StringUtils.stripEnd("", *) = ""
|
||||
* StringUtils.stripEnd("abc", "") = "abc"
|
||||
* StringUtils.stripEnd("abc", null) = "abc"
|
||||
* StringUtils.stripEnd(" abc", null) = " abc"
|
||||
* StringUtils.stripEnd("abc ", null) = "abc"
|
||||
* StringUtils.stripEnd(" abc ", null) = " abc"
|
||||
* StringUtils.stripEnd(" abcyx", "xyz") = " abc"
|
||||
* StringUtils.stripEnd("120.00", ".0") = "12"
|
||||
* </pre>
|
||||
*
|
||||
* @param str the String to remove characters from, may be null
|
||||
* @param stripChars the set of characters to remove, null treated as whitespace
|
||||
* @return the stripped String, <code>null</code> if null String input
|
||||
*/
|
||||
static String stripEnd( String self, String stripChars = null ) {
|
||||
StringUtils.stripEnd(self, stripChars)
|
||||
}
|
||||
|
||||
/**
|
||||
* <p>Capitalizes a String changing the first letter to title case as
|
||||
* per {@link Character#toTitleCase(char)}. No other letters are changed.</p>
|
||||
*
|
||||
* <p>For a word based algorithm, see {@link org.apache.commons.lang.WordUtils#capitalize(String)}.
|
||||
* A <code>null</code> input String returns <code>null</code>.</p>
|
||||
*
|
||||
* <pre>
|
||||
* StringUtils.capitalize(null) = null
|
||||
* StringUtils.capitalize("") = ""
|
||||
* StringUtils.capitalize("cat") = "Cat"
|
||||
* StringUtils.capitalize("cAt") = "CAt"
|
||||
* </pre>
|
||||
*
|
||||
*/
|
||||
static String capitalize(String self) {
|
||||
StringUtils.capitalize(self)
|
||||
}
|
||||
|
||||
/**
|
||||
* <p>Uncapitalizes a String changing the first letter to title case as
|
||||
* per {@link Character#toLowerCase(char)}. No other letters are changed.</p>
|
||||
*
|
||||
* <p>For a word based algorithm, see {@link org.apache.commons.lang.WordUtils#uncapitalize(String)}.
|
||||
* A <code>null</code> input String returns <code>null</code>.</p>
|
||||
*
|
||||
* <pre>
|
||||
* StringUtils.uncapitalize(null) = null
|
||||
* StringUtils.uncapitalize("") = ""
|
||||
* StringUtils.uncapitalize("Cat") = "cat"
|
||||
* StringUtils.uncapitalize("CAT") = "cAT"
|
||||
* </pre>
|
||||
*
|
||||
*/
|
||||
static String uncapitalize(String self) {
|
||||
StringUtils.uncapitalize(self)
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a alphabetic characters in a string are lowercase. Non alphabetic characters are ignored
|
||||
* @param self The string to check
|
||||
* @return {@true} if the string contains no uppercase characters, {@code false} otherwise
|
||||
*/
|
||||
static boolean isLowerCase(String self) {
|
||||
if( self ) for( int i=0; i<self.size(); i++ ) {
|
||||
if( Character.isUpperCase(self.charAt(i)))
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a alphabetic characters in a string are uppercase. Non alphabetic characters are ignored
|
||||
* @param self The string to check
|
||||
* @return {@true} if the string contains no lowercase characters, {@code false} otherwise
|
||||
*/
|
||||
static boolean isUpperCase(String self) {
|
||||
if( self ) for( int i=0; i<self.size(); i++ ) {
|
||||
if( Character.isLowerCase(self.charAt(i)))
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if ALL characters in a string are lowercase.
|
||||
* @param self The string to check
|
||||
* @return {@true} when all characters are uppercase, {@code false} otherwise
|
||||
*/
|
||||
static boolean isAllLowerCase(String self) {
|
||||
StringUtils.isAllLowerCase(self)
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if ALL characters in a string are uppercase.
|
||||
* @param self The string to check
|
||||
* @return {@true} when all characters are uppercase, {@code false} otherwise
|
||||
*/
|
||||
static boolean isAllUpperCase(String self) {
|
||||
StringUtils.isAllUpperCase(self)
|
||||
}
|
||||
|
||||
/**
|
||||
* Invokes the specify closure including it with a lock/unlock calls pair
|
||||
*
|
||||
* @param self
|
||||
* @param interruptible
|
||||
* @param closure
|
||||
* @return the closure result
|
||||
*/
|
||||
static <T> T withLock( Lock self, boolean interruptible = false, Closure<T> closure ) {
|
||||
// acquire the lock
|
||||
if( interruptible )
|
||||
self.lockInterruptibly()
|
||||
else
|
||||
self.lock()
|
||||
|
||||
try {
|
||||
return closure.call()
|
||||
}
|
||||
finally {
|
||||
self.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Invokes the specify closure only if it is able to acquire a lock
|
||||
*
|
||||
* @param self
|
||||
* @param interruptible
|
||||
* @param closure
|
||||
* @return the closure result
|
||||
*/
|
||||
static boolean tryLock( Lock self, Closure closure ) {
|
||||
if( !self.tryLock() )
|
||||
return false
|
||||
|
||||
try {
|
||||
closure.call()
|
||||
}
|
||||
finally {
|
||||
self.unlock()
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a file system wide lock that prevent two or more JVM instances/process
|
||||
* to work on the same file
|
||||
*
|
||||
* Note: this does not protected against multiple-thread accessing the file in a
|
||||
* concurrent manner.
|
||||
*
|
||||
* @param
|
||||
* self The file over which define the lock
|
||||
* @param
|
||||
* timeout An option timeout elapsed which the a {@link InterruptedException} is thrown
|
||||
* @param
|
||||
* closure The action to apply during the lock file spawn
|
||||
* @return
|
||||
* The user provided {@code closure} result
|
||||
*
|
||||
* @throws
|
||||
* InterruptedException if the lock cannot be acquired within the specified {@code timeout}
|
||||
*/
|
||||
static withLock(File self, Duration timeout = null, Closure closure) {
|
||||
def locker = new FileMutex(self)
|
||||
if( timeout )
|
||||
locker.setTimeout(timeout)
|
||||
locker.lock(closure)
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Creates a file system wide lock that prevent two or more JVM instances/process
|
||||
* to work on the same file
|
||||
*
|
||||
* Note: this does not protected against multiple-thread accessing the file in a
|
||||
* concurrent manner.
|
||||
*
|
||||
* @param
|
||||
* self The file over which define the lock
|
||||
* @param
|
||||
* timeout An option timeout elapsed which the a {@link InterruptedException} is thrown
|
||||
* @param
|
||||
* closure The action to apply during the lock file spawn
|
||||
* @return
|
||||
* The user provided {@code closure} result
|
||||
*
|
||||
* @throws
|
||||
* InterruptedException if the lock cannot be acquired within the specified {@code timeout}
|
||||
*/
|
||||
static withLock( Path self, Duration timeout, Closure closure ) {
|
||||
def locker = new FileMutex(self.toFile())
|
||||
if( timeout )
|
||||
locker.setTimeout(timeout)
|
||||
locker.lock(closure)
|
||||
}
|
||||
|
||||
static Duration toDuration(Number number) {
|
||||
new Duration(number.toLong())
|
||||
}
|
||||
|
||||
static MemoryUnit toMemory(Number number) {
|
||||
new MemoryUnit(number.toLong())
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a {@code String} to a {@code Duration} object
|
||||
*
|
||||
* @param self
|
||||
* @param type
|
||||
* @return
|
||||
*/
|
||||
static Object asType( String self, Class<Object> type ) {
|
||||
if( type == Duration ) {
|
||||
return new Duration(self)
|
||||
}
|
||||
else if( Path.isAssignableFrom(type) ) {
|
||||
return FileHelper.asPath(self)
|
||||
}
|
||||
else if( type == MemoryUnit ) {
|
||||
return new MemoryUnit(self)
|
||||
}
|
||||
else if( type == RateUnit ) {
|
||||
return new RateUnit(self)
|
||||
}
|
||||
else if ( type == URL ) {
|
||||
return new URL(self)
|
||||
}
|
||||
else if ( type == URI ) {
|
||||
return URI.create(self)
|
||||
}
|
||||
|
||||
StringGroovyMethods.asType(self, type);
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a {@code GString} to a {@code Duration} object
|
||||
*
|
||||
* @param self
|
||||
* @param type
|
||||
* @return
|
||||
*/
|
||||
static Object asType( GString self, Class<Object> type ) {
|
||||
if( type == Duration ) {
|
||||
return new Duration(self.toString())
|
||||
}
|
||||
else if( Path.isAssignableFrom(type) ) {
|
||||
return FileHelper.asPath(self)
|
||||
}
|
||||
else if( type == MemoryUnit ) {
|
||||
return new MemoryUnit(self.toString())
|
||||
}
|
||||
else if ( type == URL ) {
|
||||
return new URL(self.toString())
|
||||
}
|
||||
else if ( type == URI ) {
|
||||
return URI.create(self.toString())
|
||||
}
|
||||
|
||||
StringGroovyMethods.asType(self, type);
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a {@code Number} to a {@code Duration} object
|
||||
*
|
||||
* @param self
|
||||
* @param type
|
||||
* @return
|
||||
*/
|
||||
static Object asType( Number self, Class type ) {
|
||||
if( type == Duration ) {
|
||||
return new Duration(self.longValue())
|
||||
}
|
||||
if( type == MemoryUnit ) {
|
||||
return new MemoryUnit(self.longValue())
|
||||
}
|
||||
DefaultGroovyMethods.asType(self, type);
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a {@code File} to a {@code Path} object
|
||||
*
|
||||
* @param self
|
||||
* @param type
|
||||
* @return
|
||||
*/
|
||||
static Object asType( File self, Class<Object> type ) {
|
||||
if( Path.isAssignableFrom(type) ) {
|
||||
return self.toPath()
|
||||
}
|
||||
|
||||
ResourceGroovyMethods.asType(self, type);
|
||||
}
|
||||
|
||||
static <K,V> V getOrCreate( Map<K,V> self, K key, @ClosureParams(FirstParam.FirstGenericType) Closure <V> value ) {
|
||||
getOrCreate0(self,key,value)
|
||||
}
|
||||
|
||||
static <K,V> V getOrCreate( Map<K,V> self, K key, V value ) {
|
||||
getOrCreate0(self,key,value)
|
||||
}
|
||||
|
||||
static private <K,V> V getOrCreate0(Map<K,V> self, K key, value) {
|
||||
if( self.containsKey(key) )
|
||||
return self.get(key)
|
||||
|
||||
synchronized (self) {
|
||||
if( self.containsKey(key) )
|
||||
return self.get(key)
|
||||
|
||||
final result = (V)(value instanceof Closure ? value.call(key) : value)
|
||||
self.put(key,result)
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate a map of maps traversing multiple attribute using dot notation. For example:
|
||||
* {@code x.y.z }
|
||||
*
|
||||
* @param self The root map object
|
||||
* @param key A dot separated list of keys
|
||||
* @param closure An optional closure to be applied. Only if all keys exists
|
||||
* @return The value associated to the specified key(s) or null on first missing entry
|
||||
*/
|
||||
static def navigate( Map self, String key, Closure closure = null ) {
|
||||
assert key
|
||||
def items = key.split(/\./)
|
||||
def current = self.get(items[0])
|
||||
|
||||
for( int i=1; i<items.length; i++ ) {
|
||||
if( current instanceof Map ) {
|
||||
if( current.containsKey(items[i]))
|
||||
current = current.get(items[i])
|
||||
else
|
||||
return null
|
||||
}
|
||||
else if( !current ) {
|
||||
return null
|
||||
}
|
||||
else {
|
||||
throw new IllegalArgumentException("Cannot navigate map attribute: '$key' -- Content: $self")
|
||||
}
|
||||
}
|
||||
|
||||
return closure ? closure(current) : current
|
||||
}
|
||||
|
||||
static def navigate(Map self, String key, defValue) {
|
||||
def result = navigate(self,key)
|
||||
return result!=null ? result : defValue
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts {@code ConfigObject}s to a plain {@code Map}
|
||||
*
|
||||
* @param config
|
||||
* @return A normalized config object
|
||||
*/
|
||||
static Map toMap( ConfigObject config ) {
|
||||
assert config != null
|
||||
(Map)normalize0((Map)config)
|
||||
}
|
||||
|
||||
static ConfigObject toConfigObject(Map self) {
|
||||
|
||||
def result = new ConfigObject()
|
||||
self.each { key, value ->
|
||||
if( value instanceof Map ) {
|
||||
result.put( key, toConfigObject((Map)value) )
|
||||
}
|
||||
else {
|
||||
result.put( key, value )
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
static private normalize0( config ) {
|
||||
|
||||
if( config instanceof Map ) {
|
||||
Map result = new LinkedHashMap(config.size())
|
||||
config.keySet().each { name ->
|
||||
def value = (config as Map).get(name)
|
||||
result.put(name, normalize0(value))
|
||||
}
|
||||
return result
|
||||
}
|
||||
else if( config instanceof Collection ) {
|
||||
List result = new ArrayList(config.size())
|
||||
for( entry in config ) {
|
||||
result << normalize0(entry)
|
||||
}
|
||||
return result
|
||||
}
|
||||
else if( config instanceof GString ) {
|
||||
return config.toString()
|
||||
}
|
||||
else {
|
||||
return config
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Indent each line in the given test by a specified prefix
|
||||
*
|
||||
* @param text
|
||||
* @param prefix
|
||||
* @return The string indented
|
||||
*/
|
||||
static String indent( String text, String prefix = ' ' ) {
|
||||
def result = new StringBuilder()
|
||||
def lines = text ? text.readLines() : Collections.emptyList()
|
||||
for( int i=0; i<lines.size(); i++ ) {
|
||||
result << prefix
|
||||
result << lines.get(i)
|
||||
result << '\n'
|
||||
}
|
||||
return result.toString()
|
||||
}
|
||||
|
||||
/**
|
||||
* Find all the best matches for the given example string in a list of values
|
||||
*
|
||||
* @param options A list of string
|
||||
* @param sample The example string -- cannot be empty
|
||||
* @return The list of options that best matches to the specified example -- return an empty list if none match
|
||||
*/
|
||||
static List<String> bestMatches( Collection<String> options, String sample ) {
|
||||
assert sample
|
||||
assert options
|
||||
|
||||
// Otherwise look for the most similar
|
||||
Map<String,Integer> diffs = [:]
|
||||
options.each {
|
||||
diffs[it] = StringUtils.getLevenshteinDistance(sample, it)
|
||||
}
|
||||
|
||||
// sort the Levenshtein Distance and get the fist entry
|
||||
def sorted = diffs.sort { Map.Entry<String,Integer> it -> it.value }
|
||||
def nearest = (Map.Entry<String,Integer>)sorted.find()
|
||||
int min = nearest.value
|
||||
int len = sample.length()
|
||||
|
||||
int threshold = len<=3 ? 1 : ( len > 10 ? 5 : Math.floorDiv(len,2))
|
||||
|
||||
List<String> result
|
||||
if( min <= threshold ) {
|
||||
result = (List<String>)sorted.findAll { it.value==min } .collect { it.key }
|
||||
}
|
||||
else {
|
||||
result = []
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
}
|
||||
|
||||
static boolean isCamelCase(String str) {
|
||||
if( !str ) return false
|
||||
for( int i=0; i<str.size()-1; i++ )
|
||||
if( Character.getType(str.charAt(i)) == Character.LOWERCASE_LETTER && Character.getType(str.charAt(i+1)) == Character.UPPERCASE_LETTER)
|
||||
return true
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
/**
|
||||
* Clone a {@link Closure} object and set the specified map as delegate object
|
||||
* in the resulting closure
|
||||
*
|
||||
* @param self The {@link Closure} object to be cloned
|
||||
* @param binding The delegate object to set in the new closure
|
||||
* @return The cloned {@link Closure} object
|
||||
*/
|
||||
static <T extends Closure> T cloneWith( T self, binding ) {
|
||||
|
||||
def copy = (Closure)self.clone()
|
||||
if( binding != null ) {
|
||||
copy.setDelegate(binding)
|
||||
copy.setResolveStrategy( Closure.DELEGATE_FIRST )
|
||||
}
|
||||
|
||||
return (T)copy
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a copy of a {@link GString} object cloning all values that are instance of {@link Closure}
|
||||
*
|
||||
* @param self The {@link GString} itself
|
||||
* @param binding A {@link Map} object that is set as delegate object in the cloned closure.
|
||||
* @return The cloned {@link GString} instance
|
||||
*/
|
||||
static GString cloneAsLazy(GString self, binding) {
|
||||
|
||||
def values = new Object[ self.valueCount ]
|
||||
|
||||
// clone the GString setting the delegate for each closure argument
|
||||
for( int i=0; i<self.valueCount; i++ ) {
|
||||
values[i] = ( self.values[i] instanceof Closure
|
||||
? cloneWith(self.values[i] as Closure, binding)
|
||||
: self.values[i] )
|
||||
}
|
||||
|
||||
new GStringImpl(values, self.strings)
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Find all the best matches for the given example string in a list of values
|
||||
*
|
||||
* @param sample The example string -- cannot be empty
|
||||
* @param options A list of string
|
||||
* @return The list of options that best matches to the specified example -- return an empty list if none match
|
||||
*/
|
||||
@CompileDynamic
|
||||
static List<String> closest(Collection<String> options, String sample ) {
|
||||
assert sample
|
||||
|
||||
if( !options )
|
||||
return Collections.emptyList()
|
||||
|
||||
// Otherwise look for the most similar
|
||||
def diffs = [:]
|
||||
options.each {
|
||||
diffs[it] = StringUtils.getLevenshteinDistance(sample, it)
|
||||
}
|
||||
|
||||
// sort the Levenshtein Distance and get the fist entry
|
||||
def sorted = diffs.sort { it.value }
|
||||
def nearest = sorted.find()
|
||||
def min = nearest.value
|
||||
def len = sample.length()
|
||||
|
||||
def threshold = len<=3 ? 1 : ( len > 10 ? 5 : Math.floor(len/2))
|
||||
|
||||
def result
|
||||
if( min <= threshold ) {
|
||||
result = sorted.findAll { it.value==min } .collect { it.key }
|
||||
}
|
||||
else {
|
||||
result = []
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
private static HashMap<Object,Long> LOGGER_CACHE = new LinkedHashMap<Object,Long>() {
|
||||
protected boolean removeEldestEntry(Map.Entry<Object, Long> eldest) {
|
||||
return size() > 10_000
|
||||
}
|
||||
}
|
||||
|
||||
private static final Duration LOG_DFLT_THROTTLE = Duration.of('1min')
|
||||
|
||||
static synchronized private checkLogCache( Object msg, Map params, Closure action ) {
|
||||
|
||||
// -- check if this message has already been printed
|
||||
final String str = msg.toString()
|
||||
final Throwable error = params?.causedBy as Throwable
|
||||
final Duration throttle = params?.throttle as Duration ?: LOG_DFLT_THROTTLE
|
||||
final firstOnly = params?.firstOnly == true
|
||||
final key = params?.cacheKey ?: str
|
||||
|
||||
long now = System.currentTimeMillis()
|
||||
Long ts = LOGGER_CACHE.get(key)
|
||||
if( ts && (now - ts <= throttle.toMillis() || firstOnly) ) {
|
||||
return
|
||||
}
|
||||
LOGGER_CACHE.put(key, now)
|
||||
|
||||
action.call(str, error)
|
||||
}
|
||||
|
||||
private static Map<String,?> LOGGER_PARAMS = [ cacheKey: Object, causedBy: Throwable, throttle: [String, Number, Duration], firstOnly: Boolean ]
|
||||
|
||||
|
||||
/**
|
||||
* Append a `trace` level entry in the application log.
|
||||
*
|
||||
* @param log
|
||||
* The {@link Logger} object
|
||||
* @param params
|
||||
* Optional named parameters
|
||||
* - `causedBy`: A {@link Throwable} object that raised the error
|
||||
* - `throttle`: When specified suppress identical logs within the specified time {@link Duration}
|
||||
* @param msg
|
||||
* The message to print
|
||||
*
|
||||
*/
|
||||
static void trace1(Logger log, Map params=null, Object msg ) {
|
||||
CheckHelper.checkParams('trace1', params, LOGGER_PARAMS)
|
||||
if( !log.isTraceEnabled() || msg==null ) return
|
||||
checkLogCache(msg,params) { String str, Throwable t -> t ? log.trace(str,t) : log.trace(str) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Append a `debug` level entry in the application log.
|
||||
*
|
||||
* @param log
|
||||
* The {@link Logger} object
|
||||
* @param params
|
||||
* Optional named parameters
|
||||
* - `causedBy`: A {@link Throwable} object that raised the error
|
||||
* - `throttle`: When specified suppress identical logs within the specified time {@link Duration}
|
||||
* @param msg
|
||||
* The message to print
|
||||
*
|
||||
*/
|
||||
static void debug1(Logger log, Map params=null, Object msg ) {
|
||||
CheckHelper.checkParams('debug1', params, LOGGER_PARAMS)
|
||||
if( !log.isDebugEnabled() || msg==null ) return
|
||||
checkLogCache(msg,params) { String str, Throwable t -> t ? log.debug(str,t) : log.debug(str) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Append a `info` level entry in the application log.
|
||||
*
|
||||
* @param log
|
||||
* The {@link Logger} object
|
||||
* @param params
|
||||
* Optional named parameters
|
||||
* - `causedBy`: A {@link Throwable} object that raised the error
|
||||
* - `throttle`: When specified suppress identical logs within the specified time {@link Duration}
|
||||
* @param msg
|
||||
* The message to print
|
||||
*
|
||||
*/
|
||||
static void info1(Logger log, Map params=null, Object msg ) {
|
||||
CheckHelper.checkParams('info1', params, LOGGER_PARAMS)
|
||||
if( !log.isInfoEnabled() || msg==null ) return
|
||||
checkLogCache(msg,params) { String str, Throwable t -> t ? log.info(str,t) : log.info(str) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Append a `warn` level entry in the application log.
|
||||
*
|
||||
* @param log
|
||||
* The {@link Logger} object
|
||||
* @param params
|
||||
* Optional named parameters
|
||||
* - `causedBy`: A {@link Throwable} object that raised the error
|
||||
* - `throttle`: When specified suppress identical logs within the specified time {@link Duration}
|
||||
* @param msg
|
||||
* The message to print
|
||||
*
|
||||
*/
|
||||
static void warn1(Logger log, Map params=null, Object msg ) {
|
||||
CheckHelper.checkParams('warn1', params, LOGGER_PARAMS)
|
||||
if( !log.isWarnEnabled() || msg==null ) return
|
||||
checkLogCache(msg,params) { String str, Throwable t -> t ? log.warn(str,t) : log.warn(str) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Append a `error` level entry in the application log.
|
||||
*
|
||||
* @param log
|
||||
* The {@link Logger} object
|
||||
* @param params
|
||||
* Optional named parameters
|
||||
* - `causedBy`: A {@link Throwable} object that raised the error
|
||||
* - `throttle`: When specified suppress identical logs within the specified time {@link Duration}
|
||||
* @param msg
|
||||
* The message to print
|
||||
*
|
||||
*/
|
||||
static void error1(Logger log, Map params=null, Object msg ) {
|
||||
CheckHelper.checkParams('error1', params, LOGGER_PARAMS)
|
||||
if( !log.isErrorEnabled() || msg==null ) return
|
||||
checkLogCache(msg,params) { String str, Throwable t -> t ? log.error(str,t) : log.error(str) }
|
||||
}
|
||||
|
||||
static void trace(Logger log, Object msg) {
|
||||
if( log.isTraceEnabled() ) {
|
||||
log.trace(msg.toString())
|
||||
}
|
||||
}
|
||||
|
||||
static void trace(Logger log, Object msg, Throwable e) {
|
||||
if( log.isTraceEnabled() ) {
|
||||
log.trace(msg.toString(),e)
|
||||
}
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
static redact(String self, int max=5, String suffix='...') {
|
||||
if( !self )
|
||||
return self
|
||||
if( self.size()<max )
|
||||
return suffix
|
||||
else
|
||||
return self.substring(0,Math.min(self.size()-max, max)) + suffix
|
||||
}
|
||||
|
||||
protected static <T extends Serializable> T deepClone0(T obj) {
|
||||
final buffer = new ByteArrayOutputStream()
|
||||
final oos = new ObjectOutputStream(buffer)
|
||||
oos.writeObject(obj)
|
||||
oos.flush()
|
||||
|
||||
final inputStream = new ByteArrayInputStream(buffer.toByteArray())
|
||||
return (T) new ObjectInputStream(inputStream).readObject()
|
||||
}
|
||||
|
||||
static <T extends Map> T deepClone(T map) {
|
||||
if( map == null)
|
||||
return null
|
||||
final result = map instanceof LinkedHashMap ? new LinkedHashMap<>(map) : new HashMap<>(map)
|
||||
for( def key : map.keySet() ) {
|
||||
def value = map.get(key)
|
||||
if( value instanceof Map ) {
|
||||
result.put(key, deepClone(value))
|
||||
}
|
||||
}
|
||||
return (T)result
|
||||
}
|
||||
|
||||
static Map deepMerge(Map target, Map source) {
|
||||
final result = cloneMap0(target)
|
||||
for (Object name : source.keySet()) {
|
||||
// to prevent side-effects with ConfigObject object (which creates a value on-fly
|
||||
// when getting it, always use `containsKey` before
|
||||
if( result.containsKey(name) && result.get(name) instanceof Map && source.containsKey(name) && source.get(name) instanceof Map ) {
|
||||
result.put(name, deepMerge( (Map)result.get(name), (Map)source.get(name)))
|
||||
}
|
||||
else {
|
||||
result.put(name, source.get(name))
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
static private Map cloneMap0(Map map) {
|
||||
if( map instanceof ConfigObject )
|
||||
return ((ConfigObject)map).clone()
|
||||
if( map instanceof LinkedHashMap )
|
||||
return new LinkedHashMap<>(map)
|
||||
else
|
||||
return new HashMap<>(map)
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a lazy expression (e.g. closure, gstring) against
|
||||
* a delegate (i.e. binding).
|
||||
*
|
||||
* @param binding
|
||||
* @param value
|
||||
*/
|
||||
static Object resolveLazy(Object binding, Object value) {
|
||||
if( value instanceof Closure )
|
||||
return cloneWith(value, binding).call()
|
||||
|
||||
if( value instanceof GString )
|
||||
return cloneAsLazy(value, binding).toString()
|
||||
|
||||
return value
|
||||
}
|
||||
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,194 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.file;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.AtomicMoveNotSupportedException;
|
||||
import java.nio.file.CopyOption;
|
||||
import java.nio.file.FileAlreadyExistsException;
|
||||
import java.nio.file.FileVisitOption;
|
||||
import java.nio.file.FileVisitResult;
|
||||
import java.nio.file.FileVisitor;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.LinkOption;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.SimpleFileVisitor;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.nio.file.attribute.BasicFileAttributes;
|
||||
import java.util.EnumSet;
|
||||
|
||||
import nextflow.extension.FilesEx;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
|
||||
/**
|
||||
* Helper class to handle copy/move files and directories
|
||||
*/
|
||||
public class CopyMoveHelper {
|
||||
/**
|
||||
* True if currently performing a copy of a foreign file.
|
||||
*/
|
||||
public static final ThreadLocal<Boolean> IN_FOREIGN_COPY = new ThreadLocal<>();
|
||||
|
||||
private static Logger log = LoggerFactory.getLogger(CopyMoveHelper.class);
|
||||
|
||||
private CopyMoveHelper() { }
|
||||
|
||||
|
||||
/**
|
||||
* Converts the given array of options for moving a file to options suitable
|
||||
* for copying the file when a move is implemented as copy + delete.
|
||||
*/
|
||||
private static CopyOption[] convertMoveToCopyOptions(CopyOption... options)
|
||||
throws AtomicMoveNotSupportedException
|
||||
{
|
||||
int len = options.length;
|
||||
CopyOption[] newOptions = new CopyOption[len+1];
|
||||
for (int i=0; i<len; i++) {
|
||||
CopyOption option = options[i];
|
||||
if (option == StandardCopyOption.ATOMIC_MOVE) {
|
||||
throw new AtomicMoveNotSupportedException(null, null,
|
||||
"Atomic move between providers is not supported");
|
||||
}
|
||||
newOptions[i] = option;
|
||||
}
|
||||
newOptions[len] = LinkOption.NOFOLLOW_LINKS;
|
||||
return newOptions;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Copy a file to a local or foreign file system
|
||||
*
|
||||
* @param source The source file path
|
||||
* @param target The target file path
|
||||
* @param foreign When {@code true} create a copy on the remote file system
|
||||
* @param options Copy options
|
||||
* @throws IOException
|
||||
*/
|
||||
private static void copyFile(Path source, Path target, boolean foreign, CopyOption... options)
|
||||
throws IOException
|
||||
{
|
||||
if( !foreign ) {
|
||||
source.getFileSystem().provider().copy(source, target, options);
|
||||
return;
|
||||
}
|
||||
|
||||
IN_FOREIGN_COPY.set(true);
|
||||
try (InputStream in = Files.newInputStream(source)) {
|
||||
Files.copy(in, target);
|
||||
} finally {
|
||||
IN_FOREIGN_COPY.set(false);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a directory content copy
|
||||
*
|
||||
* @param source The source directory to copy
|
||||
* @param target The target path where directory will be copied
|
||||
* @param options Copy options
|
||||
* @throws IOException
|
||||
*/
|
||||
static void copyDirectory( final Path source, final Path target, final CopyOption... options )
|
||||
throws IOException
|
||||
{
|
||||
|
||||
final boolean foreign = source.getFileSystem().provider() != target.getFileSystem().provider();
|
||||
|
||||
FileVisitor<Path> visitor = new SimpleFileVisitor<Path>() {
|
||||
|
||||
public FileVisitResult preVisitDirectory(Path current, BasicFileAttributes attr)
|
||||
throws IOException
|
||||
{
|
||||
// get the *delta* path against the source path
|
||||
Path rel = source.relativize(current);
|
||||
String delta = rel != null ? rel.toString() : null;
|
||||
Path newFolder = delta != null ? target.resolve(delta) : target;
|
||||
if(log.isTraceEnabled())
|
||||
log.trace("Copy DIR: " + current + " -> " + newFolder);
|
||||
// this `copy` creates the new folder, but does not copy the contained files
|
||||
Files.createDirectory(newFolder);
|
||||
return FileVisitResult.CONTINUE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileVisitResult visitFile(Path current, BasicFileAttributes attr)
|
||||
throws IOException
|
||||
{
|
||||
// get the *delta* path against the source path
|
||||
Path rel = source.relativize(current);
|
||||
String delta = rel != null ? rel.toString() : null;
|
||||
Path newFile = delta != null ? target.resolve(delta) : target;
|
||||
if( log.isTraceEnabled())
|
||||
log.trace("Copy file: " + current + " -> "+ FilesEx.toUriString(newFile));
|
||||
copyFile(current, newFile, foreign, options);
|
||||
return FileVisitResult.CONTINUE;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
Files.walkFileTree(source, EnumSet.of(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, visitor);
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Simple copy for use when source and target are associated with different
|
||||
* providers
|
||||
*/
|
||||
static void copyToForeignTarget(Path source, Path target, CopyOption... options)
|
||||
throws IOException
|
||||
{
|
||||
CopyOptions opts = CopyOptions.parse(options);
|
||||
LinkOption[] linkOptions = (opts.followLinks()) ? new LinkOption[0] : new LinkOption[] { LinkOption.NOFOLLOW_LINKS };
|
||||
|
||||
// attributes of source file
|
||||
BasicFileAttributes attrs = Files.readAttributes(source, BasicFileAttributes.class, linkOptions);
|
||||
if (attrs.isSymbolicLink())
|
||||
throw new IOException("Copying of symbolic links not supported");
|
||||
|
||||
// delete target if it exists and REPLACE_EXISTING is specified
|
||||
if (opts.replaceExisting()) {
|
||||
FileHelper.deletePath(target);
|
||||
}
|
||||
else if (Files.exists(target))
|
||||
throw new FileAlreadyExistsException(target.toString());
|
||||
|
||||
// create directory or copy file
|
||||
if (attrs.isDirectory()) {
|
||||
copyDirectory(source, target);
|
||||
}
|
||||
else {
|
||||
copyFile(source, target, true);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Simple move implements as copy+delete for use when source and target are
|
||||
* associated with different providers
|
||||
*/
|
||||
static void moveToForeignTarget(Path source, Path target, CopyOption... options)
|
||||
throws IOException
|
||||
{
|
||||
copyToForeignTarget(source, target, convertMoveToCopyOptions(options));
|
||||
FileHelper.deletePath(source);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.file;
|
||||
|
||||
import java.nio.file.CopyOption;
|
||||
import java.nio.file.LinkOption;
|
||||
import java.nio.file.StandardCopyOption
|
||||
|
||||
import groovy.transform.Canonical
|
||||
import groovy.transform.CompileStatic;
|
||||
|
||||
/**
|
||||
* Parses the arguments for a file copy operation.
|
||||
*/
|
||||
@CompileStatic
|
||||
@Canonical
|
||||
class CopyOptions {
|
||||
private boolean replaceExisting = false;
|
||||
private boolean copyAttributes = false;
|
||||
private boolean followLinks = true;
|
||||
|
||||
boolean replaceExisting() { replaceExisting }
|
||||
boolean copyAttributes() { copyAttributes }
|
||||
boolean followLinks() { followLinks }
|
||||
|
||||
private CopyOptions() { }
|
||||
|
||||
static public CopyOptions parse(CopyOption... options) {
|
||||
CopyOptions result = new CopyOptions();
|
||||
for (CopyOption option: options) {
|
||||
if (option == StandardCopyOption.REPLACE_EXISTING) {
|
||||
result.replaceExisting = true;
|
||||
continue;
|
||||
}
|
||||
if (option == LinkOption.NOFOLLOW_LINKS) {
|
||||
result.followLinks = false;
|
||||
continue;
|
||||
}
|
||||
if (option == StandardCopyOption.COPY_ATTRIBUTES) {
|
||||
result.copyAttributes = true;
|
||||
continue;
|
||||
}
|
||||
if (option == null)
|
||||
throw new NullPointerException();
|
||||
throw new UnsupportedOperationException("'" + option +
|
||||
"' is not a recognized copy option");
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
1199
nextflow/modules/nf-commons/src/main/nextflow/file/FileHelper.groovy
Normal file
1199
nextflow/modules/nf-commons/src/main/nextflow/file/FileHelper.groovy
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,113 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.file
|
||||
|
||||
|
||||
import java.nio.file.FileSystems
|
||||
import java.nio.file.Path
|
||||
|
||||
import com.google.common.hash.Hasher
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import groovy.transform.PackageScope
|
||||
import groovy.transform.ToString
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.util.CacheFunnel
|
||||
import nextflow.util.CacheHelper
|
||||
import nextflow.util.CacheHelper.HashMode
|
||||
/**
|
||||
* Implements a special {@code Path} used to stage files in the work area
|
||||
*/
|
||||
@Slf4j
|
||||
@ToString(includePackage = false, includeNames = true)
|
||||
@EqualsAndHashCode
|
||||
@CompileStatic
|
||||
class FileHolder implements CacheFunnel {
|
||||
|
||||
final def sourceObj
|
||||
|
||||
final Path storePath
|
||||
|
||||
final String stageName
|
||||
|
||||
FileHolder( Path path ) {
|
||||
this.sourceObj = path
|
||||
this.storePath = real(path)
|
||||
this.stageName = norm(path.getFileName())
|
||||
}
|
||||
|
||||
FileHolder( def origin, Path path ) {
|
||||
this.sourceObj = origin
|
||||
this.storePath = real(path)
|
||||
this.stageName = norm(path.getFileName())
|
||||
}
|
||||
|
||||
protected FileHolder( def source, Path store, def stageName ) {
|
||||
this.sourceObj = source
|
||||
this.storePath = store
|
||||
this.stageName = norm(stageName)
|
||||
}
|
||||
|
||||
FileHolder withName( def stageName ) {
|
||||
new FileHolder( this.sourceObj, this.storePath, stageName )
|
||||
}
|
||||
|
||||
Path getSourcePath() {
|
||||
sourceObj instanceof Path ? sourceObj : null
|
||||
}
|
||||
|
||||
Path getStorePath() { storePath }
|
||||
|
||||
String getStageName() { stageName }
|
||||
|
||||
@Override
|
||||
Hasher funnel(Hasher hasher, HashMode mode) {
|
||||
return CacheHelper.hasher(hasher, sourceObj, mode)
|
||||
}
|
||||
|
||||
@PackageScope
|
||||
static FileHolder get( def path, def name = null ) {
|
||||
Path storePath = path as Path
|
||||
def target = name ? name : storePath.getFileName()
|
||||
new FileHolder( path, storePath, target )
|
||||
}
|
||||
|
||||
/**
|
||||
* Make sure the stage name does not start with a slash character
|
||||
*
|
||||
* @param path
|
||||
* @return The normalised path
|
||||
*/
|
||||
static private String norm(path) {
|
||||
def result = path.toString()
|
||||
return result.startsWith('/') ? result.substring(1) : result
|
||||
}
|
||||
|
||||
static private Path real( Path path ) {
|
||||
try {
|
||||
// main reason for this is to resolve symlinks to real file location
|
||||
// hence apply only for default file system
|
||||
// note: also for Google Cloud storage path it may convert to relative path
|
||||
// it may return invalid (relative) paths therefore do not apply it
|
||||
return path.getFileSystem() == FileSystems.default ? path.toRealPath() : path
|
||||
}
|
||||
catch( Exception e ) {
|
||||
log.trace "Unable to get real path for: $path"
|
||||
return path
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.file
|
||||
|
||||
import java.nio.channels.FileLock
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.util.Duration
|
||||
|
||||
/**
|
||||
* Implements a mutual exclusive file lock strategy to prevent
|
||||
* two or more JVMs/process to access the same file or resource
|
||||
*
|
||||
* NOTE: it should NOT be used to synchronise concurrent from different
|
||||
* threads in the same JVM
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class FileMutex {
|
||||
|
||||
String waitMessage
|
||||
|
||||
String errorMessage
|
||||
|
||||
File target
|
||||
|
||||
long delayMillis = 100
|
||||
|
||||
private long timeoutMillis
|
||||
|
||||
FileMutex() { }
|
||||
|
||||
FileMutex(File target) {
|
||||
this.target = target
|
||||
}
|
||||
|
||||
FileMutex setTimeout( value ) {
|
||||
if( value instanceof Duration )
|
||||
timeoutMillis = value.millis
|
||||
else if( value instanceof CharSequence )
|
||||
timeoutMillis = Duration.of(value.toString()).millis
|
||||
else if( value instanceof Number )
|
||||
timeoutMillis = value.longValue()
|
||||
else if( value != null )
|
||||
throw new IllegalArgumentException("Not a valid Duration value: $value [${value.class.name}]")
|
||||
|
||||
return this
|
||||
}
|
||||
|
||||
|
||||
FileMutex setWaitMessage( String str ) {
|
||||
this.waitMessage = str
|
||||
return this
|
||||
}
|
||||
|
||||
FileMutex setErrorMessage( String str ) {
|
||||
this.errorMessage = str
|
||||
return this
|
||||
}
|
||||
|
||||
|
||||
def <T> T lock(Closure<T> closure) {
|
||||
assert target
|
||||
final file = new RandomAccessFile(target, "rw")
|
||||
try {
|
||||
/*
|
||||
* wait to acquire a lock
|
||||
*/
|
||||
boolean showed=0
|
||||
long max = timeoutMillis ? timeoutMillis : Long.MAX_VALUE
|
||||
long begin = System.currentTimeMillis()
|
||||
FileLock lock
|
||||
while( !(lock=file.getChannel().tryLock()) ) {
|
||||
if( waitMessage && !showed ) {
|
||||
log.info waitMessage
|
||||
showed = true
|
||||
}
|
||||
|
||||
if( System.currentTimeMillis()-begin > max )
|
||||
throw new InterruptedException("Cannot acquire FileLock on $target")
|
||||
|
||||
sleep delayMillis
|
||||
}
|
||||
|
||||
/*
|
||||
* now it can do the job
|
||||
*/
|
||||
try {
|
||||
return closure.call()
|
||||
}
|
||||
finally {
|
||||
lock.close()
|
||||
}
|
||||
}
|
||||
finally {
|
||||
file.close()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,255 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.file
|
||||
|
||||
import java.nio.file.Path
|
||||
import java.util.regex.Pattern
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
/**
|
||||
* Parse a file path to isolate the parent, file-name and whenever it contains a glob|regex pattern
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class FilePatternSplitter {
|
||||
|
||||
static final public FilePatternSplitter GLOB = glob()
|
||||
|
||||
static enum Syntax { GLOB, REGEX }
|
||||
|
||||
static final public Pattern QUESTION_MARK_REGEX = ~/(?<!\\)\?/
|
||||
|
||||
static final public Pattern STAR_CHAR_REGEX = ~/(?<!\\)\*/
|
||||
|
||||
static final public Pattern GLOB_CURLY_BRACKETS = Pattern.compile(/(.*)(?<!\\)(\{.*,.*\})(.*)/)
|
||||
|
||||
static final public Pattern GLOB_SQUARE_BRACKETS = Pattern.compile(/(.*)(?<!\\)(\[.+\])(.*)/)
|
||||
|
||||
static final private char BACK_SLASH = '\\' as char
|
||||
|
||||
static final private String GLOB_CHARS = '*?[]{}'
|
||||
|
||||
static final private String REGEX_CHARS = '.^$+{}[]|()'
|
||||
|
||||
private boolean pattern
|
||||
|
||||
private final Syntax syntax
|
||||
|
||||
private String scheme
|
||||
|
||||
private String parent
|
||||
|
||||
private String fileName
|
||||
|
||||
boolean isPattern() { pattern }
|
||||
|
||||
String getFileName() { fileName }
|
||||
|
||||
String getParent() { parent }
|
||||
|
||||
String getScheme() { scheme }
|
||||
|
||||
static FilePatternSplitter glob() { new FilePatternSplitter(Syntax.GLOB) }
|
||||
|
||||
static FilePatternSplitter regex() { new FilePatternSplitter(Syntax.REGEX) }
|
||||
|
||||
|
||||
static boolean isMatchingPattern(pattern) {
|
||||
if( !pattern )
|
||||
return false
|
||||
if( pattern instanceof Pattern )
|
||||
return true
|
||||
def str = pattern.toString()
|
||||
if( STAR_CHAR_REGEX.matcher(str).find() )
|
||||
return true
|
||||
if( QUESTION_MARK_REGEX.matcher(str).find() )
|
||||
return true
|
||||
if( GLOB_CURLY_BRACKETS.matcher(str).find() )
|
||||
return true
|
||||
if( GLOB_SQUARE_BRACKETS.matcher(str).find() )
|
||||
return true
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
|
||||
FilePatternSplitter( Syntax syntax ) {
|
||||
this.syntax = syntax
|
||||
}
|
||||
|
||||
private String metaChars() {
|
||||
syntax == Syntax.GLOB ? GLOB_CHARS : REGEX_CHARS
|
||||
}
|
||||
|
||||
private boolean containsMetaChars(String str) {
|
||||
final meta = metaChars()
|
||||
|
||||
for( int i=0; i<str.length(); i++ ) {
|
||||
if( meta.contains(str[i]) )
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a file path detecting the schema, parent folder, file name and pattern components
|
||||
*
|
||||
* @param filePath The file path string to parse
|
||||
* @return
|
||||
*/
|
||||
FilePatternSplitter parse(String filePath) {
|
||||
// -- detected the file scheme if any
|
||||
int p = filePath.indexOf('://')
|
||||
if( p != -1 ) {
|
||||
scheme = filePath.substring(0, p)
|
||||
filePath = filePath.substring(p+3)
|
||||
}
|
||||
else {
|
||||
scheme = null
|
||||
}
|
||||
|
||||
//
|
||||
// split the path in two components
|
||||
// - folder: the part not containing meta characters
|
||||
// - pattern: the part containing a meta character
|
||||
|
||||
boolean found
|
||||
String norm = replaceMetaChars(filePath)
|
||||
p = firstMetaIndex(norm)
|
||||
|
||||
if( p == -1 ) {
|
||||
// find the last SLASH
|
||||
p = filePath.lastIndexOf('/')
|
||||
found = false
|
||||
}
|
||||
else {
|
||||
found = true
|
||||
// walk back to the first SLASH char
|
||||
int i = p
|
||||
p = -1
|
||||
while( --i >= 0 ) {
|
||||
if( filePath[i] == '/' ) {
|
||||
p = i
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if( p == -1 ) {
|
||||
parent = './'
|
||||
fileName = filePath
|
||||
pattern = found && pairedBrackets(norm)
|
||||
}
|
||||
else {
|
||||
parent = strip(filePath.substring(0,p+1))
|
||||
fileName = filePath.substring(p+1)
|
||||
pattern = found && pairedBrackets(norm)
|
||||
}
|
||||
|
||||
return this
|
||||
}
|
||||
|
||||
|
||||
private boolean pairedBrackets(String str) {
|
||||
if( syntax == Syntax.REGEX )
|
||||
return true
|
||||
|
||||
if( str.contains('{') || str.contains('}') )
|
||||
return GLOB_CURLY_BRACKETS.matcher(str).matches()
|
||||
|
||||
if( str.contains('[') || str.contains(']') )
|
||||
return GLOB_SQUARE_BRACKETS.matcher(str).matches()
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
protected String replaceMetaChars( String str, char marker = 0x0 ) {
|
||||
// create a version replacing escape meta chars with an 0x0
|
||||
final meta = metaChars()
|
||||
final result = new StringBuilder()
|
||||
int i=0;
|
||||
while( i<str.length() ) {
|
||||
def ch = str.charAt(i++)
|
||||
if( ch == BACK_SLASH && i<str.length() && meta.contains(str[i])) {
|
||||
result.append(ch).append(marker)
|
||||
i++
|
||||
}
|
||||
else {
|
||||
result.append(ch)
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
protected int firstMetaIndex(String str) {
|
||||
|
||||
// find the index of the first meta chars
|
||||
final meta = metaChars()
|
||||
def min = Integer.MAX_VALUE
|
||||
for( int i=0; i<meta.length(); i++ ) {
|
||||
def p = str.indexOf(meta[i])
|
||||
if( p!=-1 && p<min )
|
||||
min = p
|
||||
}
|
||||
|
||||
return min != Integer.MAX_VALUE ? min : -1
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Strips backslash characters from a path
|
||||
*/
|
||||
String strip( String str ) {
|
||||
int p = str.indexOf('\\')
|
||||
if( p == -1 )
|
||||
return str
|
||||
|
||||
final meta = metaChars()
|
||||
final result = new StringBuilder()
|
||||
int i=0;
|
||||
while( i<str.length() ) {
|
||||
def ch = str.charAt(i++)
|
||||
if( ch != BACK_SLASH || i==str.length() || !meta.contains(str[i])) {
|
||||
result.append(ch)
|
||||
}
|
||||
}
|
||||
|
||||
return result.toString()
|
||||
}
|
||||
|
||||
/**
|
||||
* Escape pattern meta-characters with a backslash
|
||||
*/
|
||||
String escape( Path path ) {
|
||||
escape(path.toString())
|
||||
}
|
||||
|
||||
/**
|
||||
* Escape pattern meta-characters with a backslash
|
||||
*/
|
||||
String escape( String str ) {
|
||||
final meta = metaChars()
|
||||
def result = str
|
||||
for( int i=0; i<meta.length(); i++ ) {
|
||||
result = result.replace( meta[i], '\\' + meta[i] )
|
||||
}
|
||||
return result
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.file
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.stc.ClosureParams
|
||||
import groovy.transform.stc.SimpleType
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.plugin.Plugins
|
||||
import org.pf4j.ExtensionPoint
|
||||
|
||||
/**
|
||||
* Define extension methods for supporting pluggable file remote file systems e.g. AWS S3 or Google Storage
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
abstract class FileSystemPathFactory implements ExtensionPoint {
|
||||
|
||||
/**
|
||||
* Converts path uri string to the corresponding {@link Path} object
|
||||
*
|
||||
* @param uri
|
||||
* A fully qualified path uri including the protocol scheme
|
||||
* @return
|
||||
* A {@link Path} for the given path or {@code null} if protocol is unknown
|
||||
*/
|
||||
abstract protected Path parseUri(String uri)
|
||||
|
||||
/**
|
||||
* Converts a {@link Path} object to a fully qualified uri string
|
||||
*
|
||||
* @param path
|
||||
* The {@link Path} object to be converted
|
||||
* @return
|
||||
* The uri string corresponding to the specified path or {@code null}
|
||||
* if the no provider is found
|
||||
*/
|
||||
abstract protected String toUriString(Path path)
|
||||
|
||||
/**
|
||||
* Bash helper library that implements the support for third-party storage
|
||||
* to be included in the job command wrapper script
|
||||
*
|
||||
* @return The Bash snippet implementing the support for third-party such as AWS S3
|
||||
* or {@code null} if not supported
|
||||
*/
|
||||
abstract protected String getBashLib(Path target)
|
||||
|
||||
/**
|
||||
* The name of a Bash helper function to upload a file to a remote file storage
|
||||
*
|
||||
* @return The name of the upload function or {@code null} if not supported
|
||||
*/
|
||||
abstract protected String getUploadCmd(String source, Path target)
|
||||
|
||||
|
||||
static Path parse(String uri) {
|
||||
lookup0 { it.parseUri(uri) }
|
||||
}
|
||||
|
||||
static String getUriString(Path path) {
|
||||
lookup0 { it.toUriString(path) }
|
||||
}
|
||||
|
||||
static String bashLib(Path target) {
|
||||
lookup0 { it.getBashLib(target) }
|
||||
}
|
||||
|
||||
static String uploadCmd(String source, Path target) {
|
||||
lookup0 { it.getUploadCmd(source, target) }
|
||||
}
|
||||
|
||||
private static List<FileSystemPathFactory> factories0() {
|
||||
final factories = new ArrayList(10)
|
||||
final itr = Plugins.getPriorityExtensions(FileSystemPathFactory).iterator()
|
||||
while( itr.hasNext() )
|
||||
factories.add(itr.next())
|
||||
log.trace "File system path factories: ${factories}"
|
||||
return factories
|
||||
}
|
||||
|
||||
private static <T> T lookup0( @ClosureParams(value = SimpleType, options = ['nextflow.file.FileSystemPathFactory']) Closure<T> criteria) {
|
||||
final factories = factories0()
|
||||
for( int i=0; i<factories.size(); i++ ) {
|
||||
final FileSystemPathFactory it = factories[i]
|
||||
final result = criteria.call(it)
|
||||
if( result!=null )
|
||||
return result
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.file
|
||||
|
||||
import java.nio.file.CopyOption
|
||||
import java.nio.file.Path
|
||||
|
||||
|
||||
/**
|
||||
* @author : jorge <jorge.aguilera@seqera.io>
|
||||
*
|
||||
*/
|
||||
interface FileSystemTransferAware {
|
||||
|
||||
boolean canUpload(Path source, Path target)
|
||||
|
||||
boolean canDownload(Path source, Path target)
|
||||
|
||||
void download(Path source, Path target, CopyOption... options) throws IOException
|
||||
|
||||
void upload(Path source, Path target, CopyOption... options) throws IOException
|
||||
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.file
|
||||
|
||||
import java.lang.reflect.Method
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
|
||||
/**
|
||||
* Helper class to access {@link sun.nio.fs.Globs#toRegexPattern(java.lang.String, boolean)} method
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class Globs {
|
||||
|
||||
final private static Method toRegexPattern
|
||||
|
||||
static {
|
||||
def clazz = sun.nio.fs.Globs
|
||||
toRegexPattern = clazz.getDeclaredMethod('toRegexPattern', String, boolean)
|
||||
toRegexPattern.setAccessible(true)
|
||||
}
|
||||
|
||||
static String toUnixRegexPattern(String globPattern) {
|
||||
toRegexPattern.invoke(null, globPattern, false)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.file
|
||||
|
||||
import java.nio.file.LinkOption
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
|
||||
import org.codehaus.groovy.runtime.InvokerHelper
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Deprecated
|
||||
class StagePath {
|
||||
|
||||
final Path target
|
||||
|
||||
StagePath( Path source ) {
|
||||
this.target = source
|
||||
}
|
||||
|
||||
StagePath( String fileName ) {
|
||||
this.target = Paths.get(fileName)
|
||||
}
|
||||
|
||||
Object getProperty( String name ) {
|
||||
InvokerHelper.getProperty(target,name)
|
||||
}
|
||||
|
||||
void setProperty(String property, Object newValue) {
|
||||
InvokerHelper.setProperty(target, property, newValue)
|
||||
}
|
||||
|
||||
Object invokeMethod(String name, Object args) {
|
||||
InvokerHelper.invokeMethod(target, name, args)
|
||||
}
|
||||
|
||||
Path getFileName() {
|
||||
target.getFileName()
|
||||
}
|
||||
|
||||
Path toRealPath(LinkOption... options) {
|
||||
return target
|
||||
}
|
||||
|
||||
String toString() {
|
||||
target.getFileName().toString()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.file
|
||||
|
||||
/**
|
||||
* Defines the interface for annotate a file with one or more tags
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
interface TagAwareFile {
|
||||
|
||||
void setTags(Map<String,String> tags)
|
||||
|
||||
void setContentType(String type)
|
||||
|
||||
void setStorageClass(String storageClass)
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.io
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import groovy.transform.ToString
|
||||
import nextflow.util.StringUtils
|
||||
/**
|
||||
* Parse a cloud bucket uri and decompose in scheme, bucket and path
|
||||
* components eg. s3://foo/some/file.txt
|
||||
* - scheme: s3
|
||||
* - bucket: foo
|
||||
* - path : /some/file.txt
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
@ToString(includeNames = true, includePackage = false, ignoreNulls=false)
|
||||
@EqualsAndHashCode(includeFields = true)
|
||||
@CompileStatic
|
||||
class BucketParser {
|
||||
|
||||
private String scheme
|
||||
private String bucket
|
||||
private Path path
|
||||
|
||||
String getScheme() { scheme }
|
||||
String getBucket() { bucket }
|
||||
Path getPath() { path }
|
||||
|
||||
protected BucketParser(String s, String b, String p) {
|
||||
this.scheme = s
|
||||
this.bucket = b
|
||||
this.path = p
|
||||
? (p.startsWith('/') || !bucket ? Path.of(p) : Path.of('/'+p) )
|
||||
: Path.of('/')
|
||||
}
|
||||
|
||||
protected BucketParser() {
|
||||
this.path = Path.of('/')
|
||||
}
|
||||
|
||||
static BucketParser from(String uri) {
|
||||
new BucketParser().parse(uri)
|
||||
}
|
||||
|
||||
BucketParser parse(String uri) {
|
||||
final m = StringUtils.URL_PROTOCOL.matcher(uri)
|
||||
if( !m.matches() ) {
|
||||
path = Path.of(uri)
|
||||
return this
|
||||
}
|
||||
|
||||
this.scheme = m.group(1)
|
||||
final location = m.group(2)
|
||||
|
||||
final p = location.indexOf('/')
|
||||
if( p==-1 ) {
|
||||
bucket = location
|
||||
path = Path.of('/')
|
||||
}
|
||||
else {
|
||||
bucket = location.substring(0,p)
|
||||
path = Path.of(location.substring(p))
|
||||
}
|
||||
|
||||
if( bucket.startsWith('/') || bucket.endsWith('/') )
|
||||
throw new IllegalArgumentException("Invalid bucket URI path: $uri")
|
||||
|
||||
return this
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @return The object key, essentially the same as {@link #path} stripping the leading slash character
|
||||
*/
|
||||
String getKey() {
|
||||
final result = path.toString()
|
||||
return result.substring(1)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package nextflow.io
|
||||
|
||||
import java.nio.ByteBuffer
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
|
||||
/**
|
||||
* An {@code InputStream} adaptor which reads data from a {@code ByteBuffer}
|
||||
*
|
||||
* See http://stackoverflow.com/a/6603018/395921
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class ByteBufferBackedInputStream extends InputStream {
|
||||
|
||||
ByteBuffer buf;
|
||||
|
||||
ByteBufferBackedInputStream(ByteBuffer buf) {
|
||||
this.buf = buf;
|
||||
}
|
||||
|
||||
int read() throws IOException {
|
||||
if (!buf.hasRemaining()) {
|
||||
return -1;
|
||||
}
|
||||
return buf.get() & 0xFF;
|
||||
}
|
||||
|
||||
int read(byte[] bytes, int off, int len) throws IOException {
|
||||
if (!buf.hasRemaining()) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
len = Math.min(len, buf.remaining());
|
||||
buf.get(bytes, off, len);
|
||||
return len;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.io
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
import groovy.util.logging.Slf4j
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
class ByteDumper extends Thread {
|
||||
|
||||
InputStream fInputStream
|
||||
boolean fTerminated
|
||||
CountDownLatch barrier = new CountDownLatch(1)
|
||||
Closure fCallback
|
||||
File fInputFile
|
||||
|
||||
ByteDumper(InputStream input0, Closure callback0 ) {
|
||||
assert input0
|
||||
|
||||
this.fInputStream = new BufferedInputStream(input0)
|
||||
this.fCallback = callback0
|
||||
setDaemon(true)
|
||||
}
|
||||
|
||||
ByteDumper( File file0, Closure callback0 ) {
|
||||
assert file0
|
||||
|
||||
this.fInputFile = file0
|
||||
this.fCallback = callback0
|
||||
setDaemon(true)
|
||||
}
|
||||
|
||||
/**
|
||||
* Interrupt the dumper thread
|
||||
*/
|
||||
def void terminate() { fTerminated = true }
|
||||
|
||||
/**
|
||||
* Await that the thread finished to read the process stdout
|
||||
*
|
||||
* @param millis Maximum time (in millis) to await
|
||||
*/
|
||||
def void await(long millis=0) {
|
||||
if( millis ) {
|
||||
barrier.await(millis, TimeUnit.MILLISECONDS)
|
||||
}
|
||||
else {
|
||||
barrier.await()
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
def void run() {
|
||||
|
||||
try {
|
||||
consume()
|
||||
}
|
||||
finally{
|
||||
if( fInputStream ) fInputStream.closeQuietly()
|
||||
barrier.countDown()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def void consume() {
|
||||
|
||||
// -- if a file reference has been provided instead of stream
|
||||
// wait for the file to exist
|
||||
while( fInputStream == null && fInputFile !=null ) {
|
||||
if( fTerminated ) {
|
||||
log.trace "consume '${getName()}' -- terminated"
|
||||
return
|
||||
}
|
||||
|
||||
if( fInputFile.exists() ) {
|
||||
log.trace "consume '${getName()}' -- ${fInputFile} exists: true"
|
||||
fInputStream = new BufferedInputStream(new FileInputStream(fInputFile))
|
||||
}
|
||||
else {
|
||||
sleep 200
|
||||
}
|
||||
}
|
||||
|
||||
if ( !fCallback ) {
|
||||
log.trace "consume '${getName()}' -- no callback"
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
byte[] buf = new byte[8192]
|
||||
int next
|
||||
try {
|
||||
while ((next = fInputStream.read(buf)) != -1 && !fTerminated) {
|
||||
log.trace "consume '${getName()}' -- reading "
|
||||
fCallback.call(buf, next)
|
||||
}
|
||||
|
||||
log.trace "consume '${getName()}' -- exit -- terminated: ${fTerminated}"
|
||||
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("exception while dumping process stream", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.io
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
|
||||
/**
|
||||
* Implements a {@link InputStream} object backed on a {@link DataInput} instance
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class DataInputStreamAdapter extends InputStream {
|
||||
|
||||
final private DataInput target
|
||||
|
||||
DataInputStreamAdapter( DataInput input ) { this.target = input }
|
||||
|
||||
@Override
|
||||
int read() throws IOException {
|
||||
try {
|
||||
return target.readUnsignedByte()
|
||||
}
|
||||
catch( EOFException | IndexOutOfBoundsException e ) {
|
||||
return -1
|
||||
}
|
||||
}
|
||||
|
||||
long skip(long n) {
|
||||
target.skipBytes(n as int)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.io
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
|
||||
/**
|
||||
* Implements a {@link OutputStream} object backed on a {@link DataOutput} instance
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
|
||||
@CompileStatic
|
||||
class DataOutputStreamAdapter extends OutputStream {
|
||||
|
||||
final private DataOutput target
|
||||
|
||||
DataOutputStreamAdapter( DataOutput out ) { target = out }
|
||||
|
||||
@Override
|
||||
void write(int b) throws IOException { target.write(b) }
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.io;
|
||||
|
||||
/*
|
||||
* This source is a derivation of the
|
||||
* https://commons.apache.org/proper/commons-exec/apidocs/src-html/org/apache/commons/exec/LogOutputStream.html
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
|
||||
/**
|
||||
* Base class to connect a logging system to the output and/or
|
||||
* error stream of then external process. The implementation
|
||||
* parses the incoming data to construct a line and passes
|
||||
* the complete line to an user-defined implementation.
|
||||
*
|
||||
* @version $Id: LogOutputStream.java 1636056 2014-11-01 21:12:52Z ggregory $
|
||||
*/
|
||||
public abstract class LogOutputStream
|
||||
extends OutputStream {
|
||||
|
||||
/** Initial buffer size. */
|
||||
private static final int INTIAL_SIZE = 132;
|
||||
|
||||
/** Carriage return */
|
||||
private static final int CR = 0x0d;
|
||||
|
||||
/** Linefeed */
|
||||
private static final int LF = 0x0a;
|
||||
|
||||
/** the internal buffer */
|
||||
private final ByteArrayOutputStream buffer = new ByteArrayOutputStream(
|
||||
INTIAL_SIZE);
|
||||
|
||||
private boolean skip = false;
|
||||
|
||||
private final int level;
|
||||
|
||||
/**
|
||||
* Creates a new instance of this class.
|
||||
* Uses the default level of 999.
|
||||
*/
|
||||
public LogOutputStream() {
|
||||
this(999);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new instance of this class.
|
||||
*
|
||||
* @param level loglevel used to log data written to this stream.
|
||||
*/
|
||||
public LogOutputStream(final int level) {
|
||||
this.level = level;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write the data to the buffer and flush the buffer, if a line separator is
|
||||
* detected.
|
||||
*
|
||||
* @param cc data to log (byte).
|
||||
* @see java.io.OutputStream#write(int)
|
||||
*/
|
||||
@Override
|
||||
public void write(final int cc) throws IOException {
|
||||
final byte c = (byte) cc;
|
||||
if (c == '\n' || c == '\r') {
|
||||
if (!skip) {
|
||||
processBuffer();
|
||||
}
|
||||
} else {
|
||||
buffer.write(cc);
|
||||
}
|
||||
skip = c == '\r';
|
||||
}
|
||||
|
||||
/**
|
||||
* Flush this log stream.
|
||||
*
|
||||
* @see java.io.OutputStream#flush()
|
||||
*/
|
||||
@Override
|
||||
public void flush() {
|
||||
if (buffer.size() > 0) {
|
||||
processBuffer();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes all remaining data from the buffer.
|
||||
*
|
||||
* @see java.io.OutputStream#close()
|
||||
*/
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
if (buffer.size() > 0) {
|
||||
processBuffer();
|
||||
}
|
||||
super.close();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the trace level of the log system
|
||||
*/
|
||||
public int getMessageLevel() {
|
||||
return level;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a block of characters to the output stream
|
||||
*
|
||||
* @param b the array containing the data
|
||||
* @param off the offset into the array where data starts
|
||||
* @param len the length of block
|
||||
* @throws java.io.IOException if the data cannot be written into the stream.
|
||||
* @see java.io.OutputStream#write(byte[], int, int)
|
||||
*/
|
||||
@Override
|
||||
public void write(final byte[] b, final int off, final int len)
|
||||
throws IOException {
|
||||
// find the line breaks and pass other chars through in blocks
|
||||
int offset = off;
|
||||
int blockStartOffset = offset;
|
||||
int remaining = len;
|
||||
while (remaining > 0) {
|
||||
while (remaining > 0 && b[offset] != LF && b[offset] != CR) {
|
||||
offset++;
|
||||
remaining--;
|
||||
}
|
||||
// either end of buffer or a line separator char
|
||||
final int blockLength = offset - blockStartOffset;
|
||||
if (blockLength > 0) {
|
||||
buffer.write(b, blockStartOffset, blockLength);
|
||||
}
|
||||
while (remaining > 0 && (b[offset] == LF || b[offset] == CR)) {
|
||||
write(b[offset]);
|
||||
offset++;
|
||||
remaining--;
|
||||
}
|
||||
blockStartOffset = offset;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts the buffer to a string and sends it to {@code processLine}.
|
||||
*/
|
||||
protected void processBuffer() {
|
||||
processLine(buffer.toString());
|
||||
buffer.reset();
|
||||
}
|
||||
|
||||
/**
|
||||
* Logs a line to the log system of the user.
|
||||
*
|
||||
* @param line
|
||||
* the line to log.
|
||||
*/
|
||||
protected void processLine(final String line) {
|
||||
processLine(line, level);
|
||||
}
|
||||
|
||||
/**
|
||||
* Logs a line to the log system of the user.
|
||||
*
|
||||
* @param line the line to log.
|
||||
* @param logLevel the log level to use
|
||||
*/
|
||||
protected abstract void processLine(final String line, final int logLevel);
|
||||
}
|
||||
@@ -0,0 +1,293 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package nextflow.io;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.Reader;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.CharBuffer;
|
||||
import java.nio.charset.Charset;
|
||||
import java.nio.charset.CharsetEncoder;
|
||||
import java.nio.charset.CoderResult;
|
||||
import java.nio.charset.CodingErrorAction;
|
||||
|
||||
/**
|
||||
* {@link InputStream} implementation that reads a character stream from a {@link Reader}
|
||||
* and transforms it to a byte stream using a specified charset encoding. The stream
|
||||
* is transformed using a {@link CharsetEncoder} object, guaranteeing that all charset
|
||||
* encodings supported by the JRE are handled correctly. In particular for charsets such as
|
||||
* UTF-16, the implementation ensures that one and only one byte order marker
|
||||
* is produced.
|
||||
* <p>
|
||||
* Since in general it is not possible to predict the number of characters to be read from the
|
||||
* {@link Reader} to satisfy a read request on the {@link ReaderInputStream}, all reads from
|
||||
* the {@link Reader} are buffered. There is therefore no well defined correlation
|
||||
* between the current position of the {@link Reader} and that of the {@link ReaderInputStream}.
|
||||
* This also implies that in general there is no need to wrap the underlying {@link Reader}
|
||||
* in a {@link java.io.BufferedReader}.
|
||||
* <p>
|
||||
* {@link ReaderInputStream} implements the inverse transformation of {@link java.io.InputStreamReader};
|
||||
* in the following example, reading from <tt>in2</tt> would return the same byte
|
||||
* sequence as reading from <tt>in</tt> (provided that the initial byte sequence is legal
|
||||
* with respect to the charset encoding):
|
||||
* <pre>
|
||||
* InputStream in = ...
|
||||
* Charset cs = ...
|
||||
* InputStreamReader reader = new InputStreamReader(in, cs);
|
||||
* ReaderInputStream in2 = new ReaderInputStream(reader, cs);</pre>
|
||||
* {@link ReaderInputStream} implements the same transformation as {@link java.io.OutputStreamWriter},
|
||||
* except that the control flow is reversed: both classes transform a character stream
|
||||
* into a byte stream, but {@link java.io.OutputStreamWriter} pushes data to the underlying stream,
|
||||
* while {@link ReaderInputStream} pulls it from the underlying stream.
|
||||
* <p>
|
||||
* Note that while there are use cases where there is no alternative to using
|
||||
* this class, very often the need to use this class is an indication of a flaw
|
||||
* in the design of the code. This class is typically used in situations where an existing
|
||||
* API only accepts an {@link InputStream}, but where the most natural way to produce the data
|
||||
* is as a character stream, i.e. by providing a {@link Reader} instance. An example of a situation
|
||||
* where this problem may appear is when implementing the {@link javax.activation.DataSource}
|
||||
* interface from the Java Activation Framework.
|
||||
* <p>
|
||||
* Given the fact that the {@link Reader} class doesn't provide any way to predict whether the next
|
||||
* read operation will block or not, it is not possible to provide a meaningful
|
||||
* implementation of the {@link InputStream#available()} method. A call to this method
|
||||
* will always return 0. Also, this class doesn't support {@link InputStream#mark(int)}.
|
||||
* <p>
|
||||
* Instances of {@link ReaderInputStream} are not thread safe.
|
||||
*
|
||||
* @see WriterOutputStream
|
||||
*
|
||||
* @since 2.0
|
||||
*/
|
||||
public class ReaderInputStream extends InputStream {
|
||||
private static final int DEFAULT_BUFFER_SIZE = 1024;
|
||||
|
||||
private final Reader reader;
|
||||
private final CharsetEncoder encoder;
|
||||
|
||||
/**
|
||||
* CharBuffer used as input for the decoder. It should be reasonably
|
||||
* large as we read data from the underlying Reader into this buffer.
|
||||
*/
|
||||
private final CharBuffer encoderIn;
|
||||
|
||||
/**
|
||||
* ByteBuffer used as output for the decoder. This buffer can be small
|
||||
* as it is only used to transfer data from the decoder to the
|
||||
* buffer provided by the caller.
|
||||
*/
|
||||
private final ByteBuffer encoderOut;
|
||||
|
||||
private CoderResult lastCoderResult;
|
||||
private boolean endOfInput;
|
||||
|
||||
/**
|
||||
* Construct a new {@link ReaderInputStream}.
|
||||
*
|
||||
* @param reader the target {@link Reader}
|
||||
* @param encoder the charset encoder
|
||||
* @since 2.1
|
||||
*/
|
||||
public ReaderInputStream(Reader reader, CharsetEncoder encoder) {
|
||||
this(reader, encoder, DEFAULT_BUFFER_SIZE);
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct a new {@link ReaderInputStream}.
|
||||
*
|
||||
* @param reader the target {@link Reader}
|
||||
* @param encoder the charset encoder
|
||||
* @param bufferSize the size of the input buffer in number of characters
|
||||
* @since 2.1
|
||||
*/
|
||||
public ReaderInputStream(Reader reader, CharsetEncoder encoder, int bufferSize) {
|
||||
this.reader = reader;
|
||||
this.encoder = encoder;
|
||||
this.encoderIn = CharBuffer.allocate(bufferSize);
|
||||
this.encoderIn.flip();
|
||||
this.encoderOut = ByteBuffer.allocate(128);
|
||||
this.encoderOut.flip();
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct a new {@link ReaderInputStream}.
|
||||
*
|
||||
* @param reader the target {@link Reader}
|
||||
* @param charset the charset encoding
|
||||
* @param bufferSize the size of the input buffer in number of characters
|
||||
*/
|
||||
public ReaderInputStream(Reader reader, Charset charset, int bufferSize) {
|
||||
this(reader,
|
||||
charset.newEncoder()
|
||||
.onMalformedInput(CodingErrorAction.REPLACE)
|
||||
.onUnmappableCharacter(CodingErrorAction.REPLACE),
|
||||
bufferSize);
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct a new {@link ReaderInputStream} with a default input buffer size of
|
||||
* 1024 characters.
|
||||
*
|
||||
* @param reader the target {@link Reader}
|
||||
* @param charset the charset encoding
|
||||
*/
|
||||
public ReaderInputStream(Reader reader, Charset charset) {
|
||||
this(reader, charset, DEFAULT_BUFFER_SIZE);
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct a new {@link ReaderInputStream}.
|
||||
*
|
||||
* @param reader the target {@link Reader}
|
||||
* @param charsetName the name of the charset encoding
|
||||
* @param bufferSize the size of the input buffer in number of characters
|
||||
*/
|
||||
public ReaderInputStream(Reader reader, String charsetName, int bufferSize) {
|
||||
this(reader, Charset.forName(charsetName), bufferSize);
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct a new {@link ReaderInputStream} with a default input buffer size of
|
||||
* 1024 characters.
|
||||
*
|
||||
* @param reader the target {@link Reader}
|
||||
* @param charsetName the name of the charset encoding
|
||||
*/
|
||||
public ReaderInputStream(Reader reader, String charsetName) {
|
||||
this(reader, charsetName, DEFAULT_BUFFER_SIZE);
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct a new {@link ReaderInputStream} that uses the default character encoding
|
||||
* with a default input buffer size of 1024 characters.
|
||||
*
|
||||
* @param reader the target {@link Reader}
|
||||
*/
|
||||
public ReaderInputStream(Reader reader) {
|
||||
this(reader, Charset.defaultCharset());
|
||||
}
|
||||
|
||||
/**
|
||||
* Fills the internal char buffer from the reader.
|
||||
*
|
||||
* @throws IOException
|
||||
* If an I/O error occurs
|
||||
*/
|
||||
private void fillBuffer() throws IOException {
|
||||
if (!endOfInput && (lastCoderResult == null || lastCoderResult.isUnderflow())) {
|
||||
encoderIn.compact();
|
||||
int position = encoderIn.position();
|
||||
// We don't use Reader#read(CharBuffer) here because it is more efficient
|
||||
// to write directly to the underlying char array (the default implementation
|
||||
// copies data to a temporary char array).
|
||||
int c = reader.read(encoderIn.array(), position, encoderIn.remaining());
|
||||
if (c == -1) {
|
||||
endOfInput = true;
|
||||
} else {
|
||||
encoderIn.position(position+c);
|
||||
}
|
||||
encoderIn.flip();
|
||||
}
|
||||
encoderOut.compact();
|
||||
lastCoderResult = encoder.encode(encoderIn, encoderOut, endOfInput);
|
||||
encoderOut.flip();
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the specified number of bytes into an array.
|
||||
*
|
||||
* @param b the byte array to read into
|
||||
* @param off the offset to start reading bytes into
|
||||
* @param len the number of bytes to read
|
||||
* @return the number of bytes read or <code>-1</code>
|
||||
* if the end of the stream has been reached
|
||||
* @throws IOException if an I/O error occurs
|
||||
*/
|
||||
@Override
|
||||
public int read(byte[] b, int off, int len) throws IOException {
|
||||
if (b == null) {
|
||||
throw new NullPointerException("Byte array must not be null");
|
||||
}
|
||||
if (len < 0 || off < 0 || (off + len) > b.length) {
|
||||
throw new IndexOutOfBoundsException("Array Size=" + b.length +
|
||||
", offset=" + off + ", length=" + len);
|
||||
}
|
||||
int read = 0;
|
||||
if (len == 0) {
|
||||
return 0; // Always return 0 if len == 0
|
||||
}
|
||||
while (len > 0) {
|
||||
if (encoderOut.hasRemaining()) {
|
||||
int c = Math.min(encoderOut.remaining(), len);
|
||||
encoderOut.get(b, off, c);
|
||||
off += c;
|
||||
len -= c;
|
||||
read += c;
|
||||
} else {
|
||||
fillBuffer();
|
||||
if (endOfInput && !encoderOut.hasRemaining()) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return read == 0 && endOfInput ? -1 : read;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the specified number of bytes into an array.
|
||||
*
|
||||
* @param b the byte array to read into
|
||||
* @return the number of bytes read or <code>-1</code>
|
||||
* if the end of the stream has been reached
|
||||
* @throws IOException if an I/O error occurs
|
||||
*/
|
||||
@Override
|
||||
public int read(byte[] b) throws IOException {
|
||||
return read(b, 0, b.length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a single byte.
|
||||
*
|
||||
* @return either the byte read or <code>-1</code> if the end of the stream
|
||||
* has been reached
|
||||
* @throws IOException if an I/O error occurs
|
||||
*/
|
||||
@Override
|
||||
public int read() throws IOException {
|
||||
for (;;) {
|
||||
if (encoderOut.hasRemaining()) {
|
||||
return encoderOut.get() & 0xFF;
|
||||
} else {
|
||||
fillBuffer();
|
||||
if (endOfInput && !encoderOut.hasRemaining()) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the stream. This method will cause the underlying {@link Reader}
|
||||
* to be closed.
|
||||
* @throws IOException if an I/O error occurs
|
||||
*/
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
reader.close();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.io
|
||||
|
||||
/**
|
||||
* Marker interface for generic serializable value object.
|
||||
* This is not me
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
interface SerializableMarker extends Serializable {
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.io
|
||||
|
||||
import java.lang.annotation.ElementType
|
||||
import java.lang.annotation.Retention
|
||||
import java.lang.annotation.RetentionPolicy
|
||||
import java.lang.annotation.Target
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import org.codehaus.groovy.ast.ASTNode
|
||||
import org.codehaus.groovy.ast.ClassNode
|
||||
import org.codehaus.groovy.control.CompilePhase
|
||||
import org.codehaus.groovy.control.SourceUnit
|
||||
import org.codehaus.groovy.transform.ASTTransformation
|
||||
import org.codehaus.groovy.transform.GroovyASTTransformation
|
||||
import org.codehaus.groovy.transform.GroovyASTTransformationClass
|
||||
|
||||
/**
|
||||
* Declares an AST xform to automatically add the {@link SerializableMarker} interface
|
||||
* for a class marked as @SerializableObject
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
@Target([ElementType.TYPE])
|
||||
@GroovyASTTransformationClass(classes = [ValueObjectImpl])
|
||||
@interface SerializableObject {
|
||||
|
||||
//--- === implementation === ---
|
||||
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
@GroovyASTTransformation(phase = CompilePhase.SEMANTIC_ANALYSIS)
|
||||
class ValueObjectImpl implements ASTTransformation {
|
||||
@Override
|
||||
void visit(ASTNode[] nodes, SourceUnit source) {
|
||||
final target = (ClassNode)nodes[1]
|
||||
target.addInterface( new ClassNode(SerializableMarker.class) )
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package nextflow.io;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
||||
/**
|
||||
* An input stream that skips the first `n` lines in a text stream
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
public class SkipLinesInputStream extends InputStream {
|
||||
|
||||
private static final int LF = '\n';
|
||||
|
||||
private static final int CR = '\r';
|
||||
|
||||
private InputStream target;
|
||||
|
||||
private int skip;
|
||||
|
||||
private int buffer=-1;
|
||||
|
||||
private int count;
|
||||
|
||||
private StringBuilder header;
|
||||
|
||||
/**
|
||||
* Creates a <code>FilterInputStream</code>
|
||||
* by assigning the argument <code>in</code>
|
||||
* to the field <code>this.in</code> so as
|
||||
* to remember it for later use.
|
||||
*
|
||||
* @param inputStream the underlying input stream, or <code>null</code> if
|
||||
* this instance is to be created without an underlying stream.
|
||||
*/
|
||||
public SkipLinesInputStream(InputStream inputStream, int skip) {
|
||||
this.target = inputStream;
|
||||
this.skip = skip;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
target.close();
|
||||
}
|
||||
|
||||
public long skip(long n) throws IOException {
|
||||
throw new UnsupportedOperationException("Skip operation is not supported by " + this.getClass().getName());
|
||||
}
|
||||
|
||||
@Override
|
||||
public int read() throws IOException {
|
||||
|
||||
while( count < skip ) {
|
||||
int ch = read0();
|
||||
if( ch == -1 )
|
||||
return -1;
|
||||
}
|
||||
|
||||
return read0();
|
||||
}
|
||||
|
||||
/**
|
||||
* A line is considered to be terminated by any one
|
||||
* of a line feed ('\n'), a carriage return ('\r'), or a carriage return
|
||||
* followed immediately by a linefeed.
|
||||
*
|
||||
*/
|
||||
private int read0() throws IOException {
|
||||
if( buffer != -1 ) {
|
||||
int result = buffer;
|
||||
buffer=-1;
|
||||
return result;
|
||||
}
|
||||
|
||||
int ch = target.read();
|
||||
if( header!=null && count<skip && ch!=-1 )
|
||||
header.append((char)ch);
|
||||
|
||||
if( ch == LF ) {
|
||||
count++;
|
||||
}
|
||||
else if( ch == CR ) {
|
||||
count++;
|
||||
int next = target.read();
|
||||
if( next == LF ) {
|
||||
if( header!=null )
|
||||
header.append((char)next);
|
||||
}
|
||||
else {
|
||||
buffer = next;
|
||||
if( header!=null && count<skip && next!=-1 )
|
||||
header.append((char)next);
|
||||
}
|
||||
|
||||
}
|
||||
return ch;
|
||||
}
|
||||
|
||||
public String consumeHeader() throws IOException {
|
||||
header = new StringBuilder();
|
||||
while( count<skip && read0()!=-1 ) ;
|
||||
return header.toString();
|
||||
}
|
||||
|
||||
public String getHeader() {
|
||||
return header != null ? header.toString() : null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.io
|
||||
|
||||
import java.lang.annotation.ElementType
|
||||
import java.lang.annotation.Retention
|
||||
import java.lang.annotation.RetentionPolicy
|
||||
import java.lang.annotation.Target
|
||||
|
||||
import groovy.transform.AnnotationCollector
|
||||
import groovy.transform.AnnotationCollectorMode
|
||||
import groovy.transform.AutoClone
|
||||
import groovy.transform.Immutable
|
||||
/**
|
||||
* Declares an AST xform to automatically add the {@link SerializableMarker} interface
|
||||
* for a class marked as @SerializableObject
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@AutoClone
|
||||
@Immutable(copyWith=true)
|
||||
@SerializableObject
|
||||
@AnnotationCollector(mode = AnnotationCollectorMode.PREFER_EXPLICIT_MERGED)
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
@Target([ElementType.TYPE])
|
||||
@interface ValueObject {
|
||||
|
||||
}
|
||||
@@ -0,0 +1,305 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package nextflow.io;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.io.Writer;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.CharBuffer;
|
||||
import java.nio.charset.Charset;
|
||||
import java.nio.charset.CharsetDecoder;
|
||||
import java.nio.charset.CoderResult;
|
||||
import java.nio.charset.CodingErrorAction;
|
||||
|
||||
/**
|
||||
* {@link OutputStream} implementation that transforms a byte stream to a
|
||||
* character stream using a specified charset encoding and writes the resulting
|
||||
* stream to a {@link Writer}. The stream is transformed using a
|
||||
* {@link CharsetDecoder} object, guaranteeing that all charset
|
||||
* encodings supported by the JRE are handled correctly.
|
||||
* <p>
|
||||
* The output of the {@link CharsetDecoder} is buffered using a fixed size buffer.
|
||||
* This implies that the data is written to the underlying {@link Writer} in chunks
|
||||
* that are no larger than the size of this buffer. By default, the buffer is
|
||||
* flushed only when it overflows or when {@link #flush()} or {@link #close()}
|
||||
* is called. In general there is therefore no need to wrap the underlying {@link Writer}
|
||||
* in a {@link java.io.BufferedWriter}. {@link WriterOutputStream} can also
|
||||
* be instructed to flush the buffer after each write operation. In this case, all
|
||||
* available data is written immediately to the underlying {@link Writer}, implying that
|
||||
* the current position of the {@link Writer} is correlated to the current position
|
||||
* of the {@link WriterOutputStream}.
|
||||
* <p>
|
||||
* {@link WriterOutputStream} implements the inverse transformation of {@link java.io.OutputStreamWriter};
|
||||
* in the following example, writing to <tt>out2</tt> would have the same result as writing to
|
||||
* <tt>out</tt> directly (provided that the byte sequence is legal with respect to the
|
||||
* charset encoding):
|
||||
* <pre>
|
||||
* OutputStream out = ...
|
||||
* Charset cs = ...
|
||||
* OutputStreamWriter writer = new OutputStreamWriter(out, cs);
|
||||
* WriterOutputStream out2 = new WriterOutputStream(writer, cs);</pre>
|
||||
* {@link WriterOutputStream} implements the same transformation as {@link java.io.InputStreamReader},
|
||||
* except that the control flow is reversed: both classes transform a byte stream
|
||||
* into a character stream, but {@link java.io.InputStreamReader} pulls data from the underlying stream,
|
||||
* while {@link WriterOutputStream} pushes it to the underlying stream.
|
||||
* <p>
|
||||
* Note that while there are use cases where there is no alternative to using
|
||||
* this class, very often the need to use this class is an indication of a flaw
|
||||
* in the design of the code. This class is typically used in situations where an existing
|
||||
* API only accepts an {@link OutputStream} object, but where the stream is known to represent
|
||||
* character data that must be decoded for further use.
|
||||
* <p>
|
||||
* Instances of {@link WriterOutputStream} are not thread safe.
|
||||
*
|
||||
* @see ReaderInputStream
|
||||
*
|
||||
* @since 2.0
|
||||
*/
|
||||
public class WriterOutputStream extends OutputStream {
|
||||
private static final int DEFAULT_BUFFER_SIZE = 1024;
|
||||
|
||||
private final Writer writer;
|
||||
private final CharsetDecoder decoder;
|
||||
private final boolean writeImmediately;
|
||||
|
||||
/**
|
||||
* ByteBuffer used as input for the decoder. This buffer can be small
|
||||
* as it is used only to transfer the received data to the
|
||||
* decoder.
|
||||
*/
|
||||
private final ByteBuffer decoderIn = ByteBuffer.allocate(128);
|
||||
|
||||
/**
|
||||
* CharBuffer used as output for the decoder. It should be
|
||||
* somewhat larger as we write from this buffer to the
|
||||
* underlying Writer.
|
||||
*/
|
||||
private final CharBuffer decoderOut;
|
||||
|
||||
/**
|
||||
* Constructs a new {@link WriterOutputStream} with a default output buffer size of
|
||||
* 1024 characters. The output buffer will only be flushed when it overflows or when
|
||||
* {@link #flush()} or {@link #close()} is called.
|
||||
*
|
||||
* @param writer the target {@link Writer}
|
||||
* @param decoder the charset decoder
|
||||
* @since 2.1
|
||||
*/
|
||||
public WriterOutputStream(Writer writer, CharsetDecoder decoder) {
|
||||
this(writer, decoder, DEFAULT_BUFFER_SIZE, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructs a new {@link WriterOutputStream}.
|
||||
*
|
||||
* @param writer the target {@link Writer}
|
||||
* @param decoder the charset decoder
|
||||
* @param bufferSize the size of the output buffer in number of characters
|
||||
* @param writeImmediately If <tt>true</tt> the output buffer will be flushed after each
|
||||
* write operation, i.e. all available data will be written to the
|
||||
* underlying {@link Writer} immediately. If <tt>false</tt>, the
|
||||
* output buffer will only be flushed when it overflows or when
|
||||
* {@link #flush()} or {@link #close()} is called.
|
||||
* @since 2.1
|
||||
*/
|
||||
public WriterOutputStream(Writer writer, CharsetDecoder decoder, int bufferSize, boolean writeImmediately) {
|
||||
this.writer = writer;
|
||||
this.decoder = decoder;
|
||||
this.writeImmediately = writeImmediately;
|
||||
decoderOut = CharBuffer.allocate(bufferSize);
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructs a new {@link WriterOutputStream}.
|
||||
*
|
||||
* @param writer the target {@link Writer}
|
||||
* @param charset the charset encoding
|
||||
* @param bufferSize the size of the output buffer in number of characters
|
||||
* @param writeImmediately If <tt>true</tt> the output buffer will be flushed after each
|
||||
* write operation, i.e. all available data will be written to the
|
||||
* underlying {@link Writer} immediately. If <tt>false</tt>, the
|
||||
* output buffer will only be flushed when it overflows or when
|
||||
* {@link #flush()} or {@link #close()} is called.
|
||||
*/
|
||||
public WriterOutputStream(Writer writer, Charset charset, int bufferSize, boolean writeImmediately) {
|
||||
this(writer,
|
||||
charset.newDecoder()
|
||||
.onMalformedInput(CodingErrorAction.REPLACE)
|
||||
.onUnmappableCharacter(CodingErrorAction.REPLACE)
|
||||
.replaceWith("?"),
|
||||
bufferSize,
|
||||
writeImmediately);
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructs a new {@link WriterOutputStream} with a default output buffer size of
|
||||
* 1024 characters. The output buffer will only be flushed when it overflows or when
|
||||
* {@link #flush()} or {@link #close()} is called.
|
||||
*
|
||||
* @param writer the target {@link Writer}
|
||||
* @param charset the charset encoding
|
||||
*/
|
||||
public WriterOutputStream(Writer writer, Charset charset) {
|
||||
this(writer, charset, DEFAULT_BUFFER_SIZE, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructs a new {@link WriterOutputStream}.
|
||||
*
|
||||
* @param writer the target {@link Writer}
|
||||
* @param charsetName the name of the charset encoding
|
||||
* @param bufferSize the size of the output buffer in number of characters
|
||||
* @param writeImmediately If <tt>true</tt> the output buffer will be flushed after each
|
||||
* write operation, i.e. all available data will be written to the
|
||||
* underlying {@link Writer} immediately. If <tt>false</tt>, the
|
||||
* output buffer will only be flushed when it overflows or when
|
||||
* {@link #flush()} or {@link #close()} is called.
|
||||
*/
|
||||
public WriterOutputStream(Writer writer, String charsetName, int bufferSize, boolean writeImmediately) {
|
||||
this(writer, Charset.forName(charsetName), bufferSize, writeImmediately);
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructs a new {@link WriterOutputStream} with a default output buffer size of
|
||||
* 1024 characters. The output buffer will only be flushed when it overflows or when
|
||||
* {@link #flush()} or {@link #close()} is called.
|
||||
*
|
||||
* @param writer the target {@link Writer}
|
||||
* @param charsetName the name of the charset encoding
|
||||
*/
|
||||
public WriterOutputStream(Writer writer, String charsetName) {
|
||||
this(writer, charsetName, DEFAULT_BUFFER_SIZE, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructs a new {@link WriterOutputStream} that uses the default character encoding
|
||||
* and with a default output buffer size of 1024 characters. The output buffer will only
|
||||
* be flushed when it overflows or when {@link #flush()} or {@link #close()} is called.
|
||||
*
|
||||
* @param writer the target {@link Writer}
|
||||
*/
|
||||
public WriterOutputStream(Writer writer) {
|
||||
this(writer, Charset.defaultCharset(), DEFAULT_BUFFER_SIZE, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Write bytes from the specified byte array to the stream.
|
||||
*
|
||||
* @param b the byte array containing the bytes to write
|
||||
* @param off the start offset in the byte array
|
||||
* @param len the number of bytes to write
|
||||
* @throws IOException if an I/O error occurs
|
||||
*/
|
||||
@Override
|
||||
public void write(byte[] b, int off, int len) throws IOException {
|
||||
while (len > 0) {
|
||||
int c = Math.min(len, decoderIn.remaining());
|
||||
decoderIn.put(b, off, c);
|
||||
processInput(false);
|
||||
len -= c;
|
||||
off += c;
|
||||
}
|
||||
if (writeImmediately) {
|
||||
flushOutput();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Write bytes from the specified byte array to the stream.
|
||||
*
|
||||
* @param b the byte array containing the bytes to write
|
||||
* @throws IOException if an I/O error occurs
|
||||
*/
|
||||
@Override
|
||||
public void write(byte[] b) throws IOException {
|
||||
write(b, 0, b.length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a single byte to the stream.
|
||||
*
|
||||
* @param b the byte to write
|
||||
* @throws IOException if an I/O error occurs
|
||||
*/
|
||||
@Override
|
||||
public void write(int b) throws IOException {
|
||||
write(new byte[] { (byte)b }, 0, 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Flush the stream. Any remaining content accumulated in the output buffer
|
||||
* will be written to the underlying {@link Writer}. After that
|
||||
* {@link Writer#flush()} will be called.
|
||||
* @throws IOException if an I/O error occurs
|
||||
*/
|
||||
@Override
|
||||
public void flush() throws IOException {
|
||||
flushOutput();
|
||||
writer.flush();
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the stream. Any remaining content accumulated in the output buffer
|
||||
* will be written to the underlying {@link Writer}. After that
|
||||
* {@link Writer#close()} will be called.
|
||||
* @throws IOException if an I/O error occurs
|
||||
*/
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
processInput(true);
|
||||
flushOutput();
|
||||
writer.close();
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode the contents of the input ByteBuffer into a CharBuffer.
|
||||
*
|
||||
* @param endOfInput indicates end of input
|
||||
* @throws IOException if an I/O error occurs
|
||||
*/
|
||||
private void processInput(boolean endOfInput) throws IOException {
|
||||
// Prepare decoderIn for reading
|
||||
decoderIn.flip();
|
||||
CoderResult coderResult;
|
||||
while (true) {
|
||||
coderResult = decoder.decode(decoderIn, decoderOut, endOfInput);
|
||||
if (coderResult.isOverflow()) {
|
||||
flushOutput();
|
||||
} else if (coderResult.isUnderflow()) {
|
||||
break;
|
||||
} else {
|
||||
// The decoder is configured to replace malformed input and unmappable characters,
|
||||
// so we should not get here.
|
||||
throw new IOException("Unexpected coder result");
|
||||
}
|
||||
}
|
||||
// Discard the bytes that have been read
|
||||
decoderIn.compact();
|
||||
}
|
||||
|
||||
/**
|
||||
* Flush the output.
|
||||
*
|
||||
* @throws IOException if an I/O error occurs
|
||||
*/
|
||||
private void flushOutput() throws IOException {
|
||||
if (decoderOut.position() > 0) {
|
||||
writer.write(decoderOut.array(), 0, decoderOut.position());
|
||||
decoderOut.rewind();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.plugin
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.PackageScope
|
||||
import nextflow.BuildInfo
|
||||
import nextflow.exception.AbortOperationException
|
||||
import org.pf4j.Plugin
|
||||
import org.pf4j.PluginWrapper
|
||||
import org.slf4j.Logger
|
||||
import org.slf4j.LoggerFactory
|
||||
/**
|
||||
* Base class for NF plugins
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
abstract class BasePlugin extends Plugin {
|
||||
|
||||
private static Logger log = LoggerFactory.getLogger(BasePlugin)
|
||||
|
||||
BasePlugin(PluginWrapper wrapper) {
|
||||
super(wrapper)
|
||||
}
|
||||
|
||||
@PackageScope boolean verMatches(String requires, String current=BuildInfo.version) {
|
||||
return getWrapper()
|
||||
.getPluginManager()
|
||||
.getVersionManager()
|
||||
.checkVersionConstraint(current, requires)
|
||||
}
|
||||
|
||||
@Override
|
||||
void start() {
|
||||
final desc = getWrapper().getDescriptor()
|
||||
final name = "${desc.pluginId}@${desc.version}"
|
||||
if( desc.requires && !verMatches(desc.requires)) {
|
||||
throw new AbortOperationException("Failed requirement - Plugin $name requires Nextflow version $desc.requires (current $BuildInfo.version)")
|
||||
}
|
||||
log.debug "Plugin started $name"
|
||||
}
|
||||
|
||||
@Override
|
||||
void stop() {
|
||||
log.debug "Plugin stopped ${wrapper.descriptor.pluginId}"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.plugin
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.BuildInfo
|
||||
import org.pf4j.DefaultPluginManager
|
||||
import org.pf4j.ExtensionFactory
|
||||
import org.pf4j.PluginWrapper
|
||||
import org.pf4j.SingletonExtensionFactory
|
||||
import org.pf4j.VersionManager
|
||||
/**
|
||||
* Custom plugin manager to that allow accessing to {@code loadPluginFromPath} and
|
||||
* {@code resolvePlugins} method
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class CustomPluginManager extends DefaultPluginManager {
|
||||
|
||||
CustomPluginManager() {}
|
||||
|
||||
CustomPluginManager(Path root) {
|
||||
super(root)
|
||||
}
|
||||
|
||||
@Override
|
||||
PluginWrapper loadPluginFromPath(Path pluginPath) {
|
||||
super.loadPluginFromPath(pluginPath)
|
||||
}
|
||||
|
||||
@Override
|
||||
void resolvePlugins() {
|
||||
super.resolvePlugins()
|
||||
}
|
||||
|
||||
@Override
|
||||
protected VersionManager createVersionManager() {
|
||||
return new CustomVersionManager()
|
||||
}
|
||||
|
||||
@Override
|
||||
String getSystemVersion() {
|
||||
return BuildInfo.version
|
||||
}
|
||||
|
||||
@Override
|
||||
protected ExtensionFactory createExtensionFactory() {
|
||||
return new SingletonExtensionFactory(this)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.plugin
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.util.VersionNumber
|
||||
import org.pf4j.DefaultVersionManager
|
||||
/**
|
||||
* Extends default version manager adding the ability to support
|
||||
* Calendar versioning
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class CustomVersionManager extends DefaultVersionManager {
|
||||
|
||||
public static Pattern CAL_VER = ~/2\d\.\d\d\.\S+/
|
||||
|
||||
@Override
|
||||
boolean checkVersionConstraint(String version, String constraint) {
|
||||
if( !version || !constraint || constraint=='*' || version==constraint )
|
||||
return true
|
||||
|
||||
try {
|
||||
return safeCheck0(version, constraint)
|
||||
}
|
||||
catch (Throwable e) {
|
||||
log.debug "Failed to check version constraint - version: $version; constraint: $constraint"
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
private boolean safeCheck0(String version, String constraint) {
|
||||
if( version =~ CAL_VER ) {
|
||||
if( constraint.startsWith('nextflow@') ) {
|
||||
constraint = constraint.substring('nextflow@'.size())
|
||||
}
|
||||
if( version.endsWith('-SNAPSHOT') ) {
|
||||
version = stripSuffix(version)
|
||||
constraint = stripSuffix(constraint)
|
||||
}
|
||||
return new VersionNumber(version).matches(constraint)
|
||||
}
|
||||
else {
|
||||
return super.checkVersionConstraint(version, constraint)
|
||||
}
|
||||
}
|
||||
|
||||
private String stripSuffix(String str) {
|
||||
int p = str.lastIndexOf('-')
|
||||
return p==-1 ? str : str.substring(0,p)
|
||||
}
|
||||
|
||||
@Override
|
||||
int compareVersions(String v1, String v2) {
|
||||
return v1=~CAL_VER || v2=~CAL_VER
|
||||
? new VersionNumber(v1) <=> new VersionNumber(v2)
|
||||
: super.compareVersions(v1, v2)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.plugin
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
|
||||
/**
|
||||
* Model the collection of default plugins used if no plugins are provided by the user
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class DefaultPlugins {
|
||||
|
||||
public static final DefaultPlugins INSTANCE = new DefaultPlugins()
|
||||
|
||||
private Map<String,PluginRef> plugins = new HashMap<>(20)
|
||||
|
||||
protected DefaultPlugins() {
|
||||
final meta = this.class.getResourceAsStream('/META-INF/plugins-info.txt')?.text
|
||||
plugins = parseMeta(meta)
|
||||
}
|
||||
|
||||
protected Map<String,PluginRef> parseMeta(String meta) {
|
||||
if( !meta )
|
||||
return Collections.emptyMap()
|
||||
|
||||
final result = new HashMap(20)
|
||||
for( String line : meta.readLines() ) {
|
||||
final spec = PluginRef.parse(line)
|
||||
result[spec.id] = spec
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
PluginRef getPlugin(String pluginId) throws IllegalArgumentException {
|
||||
if( !pluginId )
|
||||
throw new IllegalArgumentException("Missing pluginId argument")
|
||||
final result = plugins.get(pluginId)
|
||||
if( !result )
|
||||
throw new IllegalArgumentException("Unknown Nextflow plugin '$pluginId'")
|
||||
return result
|
||||
}
|
||||
|
||||
boolean hasPlugin(String pluginId) {
|
||||
return plugins.containsKey(pluginId)
|
||||
}
|
||||
|
||||
List<PluginRef> getPlugins() {
|
||||
return new ArrayList(plugins.values())
|
||||
}
|
||||
|
||||
String toSortedString(String divisor=',') {
|
||||
getPlugins().sort().join(divisor)
|
||||
}
|
||||
|
||||
@Override
|
||||
String toString() {
|
||||
return "DefaultPlugins${getPlugins()}"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.plugin
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j;
|
||||
import org.pf4j.PluginClasspath;
|
||||
|
||||
/**
|
||||
* Customise classpath loader for Groovy based
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class DevPluginClasspath extends PluginClasspath {
|
||||
|
||||
private boolean logged
|
||||
|
||||
DevPluginClasspath() {
|
||||
// the path where classes are resources should be found in the dev environment
|
||||
// for each plugin project directory
|
||||
addClassesDirectories("build/classes/groovy/main", "build/resources/main", 'build/classes/main')
|
||||
// note: this path is not created automatically by Gradle, it should be created by a custom task
|
||||
// see `copyPluginLibs` task in the base plugins `build.gradle`
|
||||
addJarsDirectories('build/target/libs')
|
||||
}
|
||||
|
||||
@Override
|
||||
Set<String> getClassesDirectories() {
|
||||
if( !logged ) {
|
||||
log.debug "Groovy DEV plugin classpath: classes-dirs=${super.getClassesDirectories()}; jars-dirs=${super.getJarsDirectories()}"
|
||||
logged = true
|
||||
}
|
||||
return super.getClassesDirectories()
|
||||
}
|
||||
|
||||
@Override
|
||||
Set<String> getJarsDirectories() {
|
||||
if( !logged ) {
|
||||
log.debug "Groovy DEV plugin classpath: classes-dirs=${super.getClassesDirectories()}; jars-dirs=${super.getJarsDirectories()}"
|
||||
logged = true
|
||||
}
|
||||
return super.getJarsDirectories()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.plugin
|
||||
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import org.pf4j.BasePluginLoader
|
||||
import org.pf4j.PluginManager
|
||||
|
||||
/**
|
||||
* Plugin loader for Groovy based development environment
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class DevPluginLoader extends BasePluginLoader {
|
||||
|
||||
DevPluginLoader(PluginManager pluginManager) {
|
||||
super(pluginManager, new DevPluginClasspath())
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.plugin
|
||||
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import org.pf4j.CompoundPluginRepository
|
||||
import org.pf4j.DevelopmentPluginRepository
|
||||
import org.pf4j.ManifestPluginDescriptorFinder
|
||||
import org.pf4j.PluginDescriptorFinder
|
||||
import org.pf4j.PluginLoader
|
||||
import org.pf4j.PluginRepository
|
||||
/**
|
||||
* Custom plugin manager that allow loading plugins from Groovy/Gradle/Intellij build environment
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class DevPluginManager extends CustomPluginManager {
|
||||
|
||||
DevPluginManager(Path root) {
|
||||
super(root)
|
||||
}
|
||||
|
||||
private List<Path> getExtensionRoots() {
|
||||
final ext = System .getenv('NXF_PLUGINS_DEV')
|
||||
if( !ext )
|
||||
return Collections.emptyList()
|
||||
|
||||
return ext
|
||||
.tokenize(',')
|
||||
.collect { Paths.get(it).toAbsolutePath() }
|
||||
}
|
||||
|
||||
@Override
|
||||
protected PluginDescriptorFinder createPluginDescriptorFinder() {
|
||||
return new ManifestPluginDescriptorFinder()
|
||||
}
|
||||
|
||||
@Override
|
||||
protected PluginLoader createPluginLoader() {
|
||||
return new DevPluginLoader(this)
|
||||
}
|
||||
|
||||
@Override
|
||||
protected PluginRepository createPluginRepository() {
|
||||
def repos = new CompoundPluginRepository()
|
||||
// main dev repo
|
||||
log.debug "Add plugin root repository: ${getPluginsRoot()}"
|
||||
repos.add( new DevelopmentPluginRepository(getPluginsRoot()) )
|
||||
// extension repos
|
||||
for( Path it : extensionRoots ) {
|
||||
log.debug "Add plugin dev repository: $it"
|
||||
repos.add( new DevelopmentPluginRepository(it) )
|
||||
}
|
||||
return repos
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.plugin
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
|
||||
/**
|
||||
* Updater disabling plugin installation and update when
|
||||
* running the plugin manager in dev mode
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class DevPluginUpdater extends PluginUpdater {
|
||||
|
||||
DevPluginUpdater(CustomPluginManager pluginManager) {
|
||||
super(pluginManager)
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean installPlugin(String id, String version) {
|
||||
throw new UnsupportedOperationException("Install is not supported on dev mode - Missing plugin $id ${version?:''}")
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean updatePlugin(String id, String version) {
|
||||
throw new UnsupportedOperationException("Update is not supported on dev mode - Missing plugin $id ${version?:''}")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.plugin
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import org.pf4j.DefaultPluginLoader
|
||||
import org.pf4j.DefaultPluginRepository
|
||||
import org.pf4j.ManifestPluginDescriptorFinder
|
||||
import org.pf4j.PluginDescriptorFinder
|
||||
import org.pf4j.PluginLoader
|
||||
import org.pf4j.PluginRepository
|
||||
|
||||
/**
|
||||
* Implements a plugin manager used when running in embedded mode
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class EmbeddedPluginManager extends CustomPluginManager {
|
||||
|
||||
EmbeddedPluginManager(Path repository) {
|
||||
super(repository)
|
||||
}
|
||||
|
||||
@Override
|
||||
protected PluginDescriptorFinder createPluginDescriptorFinder() {
|
||||
return new ManifestPluginDescriptorFinder()
|
||||
}
|
||||
|
||||
@Override
|
||||
protected PluginLoader createPluginLoader() {
|
||||
return new DefaultPluginLoader(this)
|
||||
}
|
||||
|
||||
@Override
|
||||
protected PluginRepository createPluginRepository() {
|
||||
return new DefaultPluginRepository(getPluginsRoot())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.plugin
|
||||
|
||||
|
||||
import java.net.http.HttpRequest
|
||||
import java.net.http.HttpResponse
|
||||
|
||||
import nextflow.serde.gson.GsonEncoder
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import io.seqera.http.HxClient
|
||||
import io.seqera.npr.api.schema.v1.ListDependenciesResponse
|
||||
import io.seqera.npr.api.schema.v1.PluginDependency
|
||||
import nextflow.BuildInfo
|
||||
import nextflow.util.RetryConfig
|
||||
import org.pf4j.PluginRuntimeException
|
||||
import org.pf4j.update.FileDownloader
|
||||
import org.pf4j.update.FileVerifier
|
||||
import org.pf4j.update.PluginInfo
|
||||
import org.pf4j.update.PluginInfo.PluginRelease
|
||||
import org.pf4j.update.SimpleFileDownloader
|
||||
import org.pf4j.update.verifier.CompoundVerifier
|
||||
/**
|
||||
* Represents an update repository served via an HTTP api.
|
||||
*
|
||||
* It implements PrefetchUpdateRepository so that all relevant
|
||||
* plugin metadata can be loaded with a single HTTP request, rather
|
||||
* than a request-per-plugin.
|
||||
*
|
||||
* Metadata is prefetched into memory when Nextflow starts and expires
|
||||
* upon termination (or when 'refresh()' is called).
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class HttpPluginRepository implements PrefetchUpdateRepository {
|
||||
private final String id
|
||||
private final URI url
|
||||
private final HxClient httpClient
|
||||
|
||||
private Map<String, PluginInfo> plugins
|
||||
|
||||
HttpPluginRepository(String id, URI url) {
|
||||
this.id = id
|
||||
// ensure url ends with a slash
|
||||
this.url = !url.toString().endsWith("/")
|
||||
? URI.create(url.toString() + "/")
|
||||
: url
|
||||
this.httpClient = HxClient.newBuilder()
|
||||
.retryConfig(RetryConfig.config())
|
||||
.build()
|
||||
}
|
||||
|
||||
// NOTE ON PREFETCHING
|
||||
//
|
||||
// The prefetch mechanism is used to work around a limitation in the
|
||||
// UpdateRepository interface from pf4j.
|
||||
//
|
||||
// Specifically, p4fj expects that getPlugins() returns a Map<> of all
|
||||
// metadata about all plugins. To implement this for an HTTP repository
|
||||
// would require either downloading the entire contents of the remote
|
||||
// repository or implementing a lazy map and making an HTTP request for
|
||||
// each required plugin.
|
||||
//
|
||||
// Instead we can use the list of configured plugins to load all relevant
|
||||
// metadata in a single HTTP request at startup, and use this to populate
|
||||
// the map. Once the prefetch is complete, this repository will behave
|
||||
// like any other implementation of UpdateRepository.
|
||||
@Override
|
||||
void prefetch(List<PluginRef> plugins) {
|
||||
if (plugins && !plugins.isEmpty()) {
|
||||
this.plugins = fetchMetadata(plugins)
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
String getId() {
|
||||
return id
|
||||
}
|
||||
|
||||
@Override
|
||||
URL getUrl() {
|
||||
return url.toURL()
|
||||
}
|
||||
|
||||
@Override
|
||||
Map<String, PluginInfo> getPlugins() {
|
||||
if (plugins==null) {
|
||||
log.warn "getPlugins() called before prefetch() - plugins map will be empty"
|
||||
return Map.of()
|
||||
}
|
||||
return Collections.unmodifiableMap(plugins)
|
||||
}
|
||||
|
||||
@Override
|
||||
PluginInfo getPlugin(String id) {
|
||||
return plugins.computeIfAbsent(id) { key -> fetchMetadataByIds([key]).get(key) }
|
||||
}
|
||||
|
||||
@Override
|
||||
void refresh() {
|
||||
plugins = fetchMetadataByIds(plugins.keySet())
|
||||
}
|
||||
|
||||
@Override
|
||||
FileDownloader getFileDownloader() {
|
||||
return new SimpleFileDownloader()
|
||||
}
|
||||
|
||||
@Override
|
||||
FileVerifier getFileVerifier() {
|
||||
return new CompoundVerifier()
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// http handling
|
||||
|
||||
private Map<String, PluginInfo> fetchMetadataByIds(Collection<String> ids) {
|
||||
def specs = ids.collect(id -> new PluginRef(id, null))
|
||||
return fetchMetadata(specs)
|
||||
}
|
||||
|
||||
private Map<String, PluginInfo> fetchMetadata(Collection<PluginRef> specs) {
|
||||
final ordered = specs.sort(false)
|
||||
return fetchMetadata0(ordered)
|
||||
}
|
||||
|
||||
private Map<String, PluginInfo> fetchMetadata0(List<PluginRef> specs) {
|
||||
def pluginsParam = specs.collect { "${it.id}${it.version ? '@' + it.version : ''}" }.join(',')
|
||||
def uri = url.resolve("v1/plugins/dependencies?plugins=${URLEncoder.encode(pluginsParam, 'UTF-8')}&nextflowVersion=${URLEncoder.encode(BuildInfo.version, 'UTF-8')}")
|
||||
def req = HttpRequest.newBuilder()
|
||||
.uri(uri)
|
||||
.GET()
|
||||
.build()
|
||||
try {
|
||||
return sendAndParse(req)
|
||||
}
|
||||
catch (PluginRuntimeException e) {
|
||||
throw e
|
||||
}
|
||||
catch (Exception e) {
|
||||
throw new PluginRuntimeException(e, "Unable to connect to %s - cause: %s", uri, e.message)
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, PluginInfo> sendAndParse(HttpRequest req) {
|
||||
final encoder = new GsonEncoder<ListDependenciesResponse>() {}
|
||||
final resp = httpClient.send(req, HttpResponse.BodyHandlers.ofString())
|
||||
final body = resp.body()
|
||||
log.debug "Registry request: ${resp.uri()}\n- code: ${resp.statusCode()}\n- body: ${body}"
|
||||
if( resp.statusCode() != 200 ) {
|
||||
final msg = "Invalid response while fetching plugin metadata from: ${req.uri()}\n- http status: ${resp.statusCode()}\n- response : ${body}"
|
||||
throw new PluginRuntimeException(msg)
|
||||
}
|
||||
try {
|
||||
final ListDependenciesResponse decoded = encoder.decode(body)
|
||||
if( decoded.plugins == null ) {
|
||||
throw new PluginRuntimeException("Failed to download plugin metadata: Failed to parse response body")
|
||||
}
|
||||
final result = new HashMap<String, PluginInfo>()
|
||||
for( PluginDependency plugin : decoded.plugins ) {
|
||||
if( plugin.releases ) {
|
||||
final pluginInfo = mapToPluginInfo(plugin)
|
||||
result.put(plugin.id, pluginInfo)
|
||||
}
|
||||
else
|
||||
log.debug "Registry ${resp.uri().host} has no releases for plugin: ${plugin}"
|
||||
}
|
||||
return result
|
||||
}
|
||||
catch( Exception e ) {
|
||||
final msg = "Unexpected error while fetching plugin metadata from: ${req.uri()}\n- message : ${e.message}\n- response: ${body}"
|
||||
throw new PluginRuntimeException(msg)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps a Plugin object from the repository API to a PluginInfo object for pf4j compatibility.
|
||||
* Handles conversion of OffsetDateTime to Date and ensures the releases collection is never null.
|
||||
*
|
||||
* @param plugin The Plugin object from the repository API response
|
||||
* @return A PluginInfo object compatible with pf4j's update repository interface
|
||||
*/
|
||||
static protected PluginInfo mapToPluginInfo(PluginDependency plugin) {
|
||||
assert plugin.releases, "Plugin releases cannot be empty"
|
||||
|
||||
final pluginInfo = new PluginInfo()
|
||||
pluginInfo.id = plugin.id
|
||||
pluginInfo.projectUrl = plugin.projectUrl
|
||||
pluginInfo.provider = plugin.provider
|
||||
|
||||
// Map releases to PluginInfo.PluginRelease
|
||||
pluginInfo.releases = new ArrayList<>()
|
||||
for (def release : plugin.releases) {
|
||||
final pluginRelease = new PluginRelease()
|
||||
pluginRelease.version = release.version
|
||||
pluginRelease.date = release.date ? Date.from(release.date.toInstant()) : null
|
||||
pluginRelease.url = release.url
|
||||
pluginRelease.sha512sum = release.sha512sum
|
||||
pluginRelease.requires = release.requires
|
||||
pluginInfo.releases.add(pluginRelease)
|
||||
}
|
||||
|
||||
return pluginInfo
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.plugin
|
||||
|
||||
import java.nio.file.FileAlreadyExistsException
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.Const
|
||||
import nextflow.extension.FilesEx
|
||||
import org.pf4j.DefaultPluginLoader
|
||||
import org.pf4j.DefaultPluginManager
|
||||
import org.pf4j.ManifestPluginDescriptorFinder
|
||||
import org.pf4j.PluginDescriptorFinder
|
||||
import org.pf4j.PluginLoader
|
||||
import org.pf4j.PluginRepository
|
||||
import org.pf4j.PluginWrapper
|
||||
/**
|
||||
* Custom plugin manager creating tracking plugins with symlinks to
|
||||
* the parent repository
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class LocalPluginManager extends CustomPluginManager {
|
||||
|
||||
static private Path PLUGINS_LOCAL_ROOT = Const.appCacheDir.resolve('plr')
|
||||
|
||||
private Path repository
|
||||
|
||||
LocalPluginManager(Path repository) {
|
||||
super(makeLocalRoot())
|
||||
if( !localRoot ) throw new IllegalArgumentException("Missing Local plugins root directory")
|
||||
if( !repository ) throw new IllegalArgumentException("Missing plugins repository directory")
|
||||
this.repository = repository
|
||||
}
|
||||
|
||||
static protected Path makeLocalRoot() {
|
||||
final result = PLUGINS_LOCAL_ROOT.resolve(UUID.randomUUID().toString())
|
||||
if( !FilesEx.mkdirs(result) )
|
||||
throw new IOException("Unable to create plugins local directory: $result -- Make sure you have write permissions in this directory path")
|
||||
Runtime.addShutdownHook { FilesEx.deleteDir(result) }
|
||||
return result
|
||||
}
|
||||
|
||||
protected Path getLocalRoot() { getPluginsRoot() }
|
||||
|
||||
@Override
|
||||
protected PluginDescriptorFinder createPluginDescriptorFinder() {
|
||||
return new ManifestPluginDescriptorFinder()
|
||||
}
|
||||
|
||||
@Override
|
||||
protected PluginLoader createPluginLoader() {
|
||||
return new DefaultPluginLoader(this)
|
||||
}
|
||||
|
||||
@Override
|
||||
protected PluginRepository createPluginRepository() {
|
||||
return new LocalPluginRepository(localRoot)
|
||||
}
|
||||
|
||||
/**
|
||||
* Override parent {@link DefaultPluginManager#loadPlugin(java.nio.file.Path)} method
|
||||
* creating a symlink the plugin path directory in the current plugin root
|
||||
*
|
||||
* @param pluginPath
|
||||
* The file path where the plugin is stored unzipped) in the
|
||||
* central plugins directory
|
||||
*/
|
||||
@Override
|
||||
String loadPlugin(Path pluginPath) {
|
||||
final symlink = createLinkFromPath(pluginPath)
|
||||
super.loadPlugin(symlink)
|
||||
}
|
||||
|
||||
@Override
|
||||
PluginWrapper loadPluginFromPath(Path pluginPath) {
|
||||
final symlink = createLinkFromPath(pluginPath)
|
||||
return super.loadPluginFromPath(symlink)
|
||||
}
|
||||
|
||||
private Path createLinkFromPath(Path pluginPath) {
|
||||
if( pluginPath.startsWith(localRoot))
|
||||
return pluginPath
|
||||
if( !pluginPath )
|
||||
throw new IllegalArgumentException("Plugin path cannot be null")
|
||||
if( !Files.isDirectory(pluginPath) )
|
||||
throw new IllegalArgumentException("Plugin path must be a directory: $pluginPath")
|
||||
|
||||
// create a symlink relative to the current root
|
||||
final symlink = localRoot.resolve(pluginPath.getFileName())
|
||||
createLink0(symlink, pluginPath)
|
||||
return symlink
|
||||
}
|
||||
|
||||
private void createLink0(Path symlink, Path pluginPath) {
|
||||
try {
|
||||
log.trace "Creating local plugins root link: $symlink → $pluginPath"
|
||||
Files.createSymbolicLink(symlink, pluginPath)
|
||||
}
|
||||
catch (FileAlreadyExistsException e) {
|
||||
log.debug "Deleting existing local plugins root link: $symlink"
|
||||
if( !sameTarget(symlink, pluginPath) ) {
|
||||
Files.delete(symlink)
|
||||
Files.createSymbolicLink(symlink, pluginPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private boolean sameTarget(Path link, Path target) {
|
||||
try {
|
||||
return Files.readSymbolicLink(link) == target
|
||||
}
|
||||
catch (IOException e) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.plugin
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import org.pf4j.DefaultPluginRepository
|
||||
|
||||
/**
|
||||
* Extends the default plugin repository to avoid the deletion
|
||||
* of the plugin directory and delete instead the local symbolic link
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class LocalPluginRepository extends DefaultPluginRepository {
|
||||
|
||||
LocalPluginRepository(Path pluginsRoot) {
|
||||
super(pluginsRoot)
|
||||
}
|
||||
|
||||
/**
|
||||
* Override {@link DefaultPluginRepository#deletePluginPath(java.nio.file.Path)}
|
||||
* to prevent deleting the real plugin directory
|
||||
*
|
||||
* @param pluginPath
|
||||
* @return
|
||||
*/
|
||||
@Override
|
||||
boolean deletePluginPath(Path pluginPath) {
|
||||
if(Files.isSymbolicLink(pluginPath))
|
||||
try {
|
||||
Files.delete(pluginPath)
|
||||
return true
|
||||
}
|
||||
catch (Exception e) {
|
||||
log.debug "Unable to delete plugin path: $pluginPath"
|
||||
return false
|
||||
}
|
||||
else
|
||||
return super.deletePluginPath(pluginPath)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.plugin
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.extension.FilesEx
|
||||
import org.pf4j.ManifestPluginDescriptorFinder
|
||||
import org.pf4j.PluginDescriptor
|
||||
import org.pf4j.update.FileDownloader
|
||||
import org.pf4j.update.FileVerifier
|
||||
import org.pf4j.update.PluginInfo
|
||||
import org.pf4j.update.UpdateRepository
|
||||
import org.pf4j.update.verifier.CompoundVerifier
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
/**
|
||||
* Implementation of UpdateRepository which looks in a local directory of already-downloaded
|
||||
* plugins to find available versions.
|
||||
*/
|
||||
@CompileStatic
|
||||
@Slf4j
|
||||
class LocalUpdateRepository implements UpdateRepository {
|
||||
private final String id
|
||||
private final Path dir
|
||||
private Map<String, PluginInfo> plugins
|
||||
|
||||
LocalUpdateRepository(String id, Path dir) {
|
||||
this.id = id
|
||||
this.dir = dir
|
||||
}
|
||||
|
||||
@Override
|
||||
String getId() {
|
||||
return id
|
||||
}
|
||||
|
||||
@Override
|
||||
URL getUrl() {
|
||||
return dir.toUri().toURL()
|
||||
}
|
||||
|
||||
@Override
|
||||
Map<String, PluginInfo> getPlugins() {
|
||||
if( !plugins )
|
||||
plugins = loadPlugins(dir)
|
||||
return plugins
|
||||
}
|
||||
|
||||
@Override
|
||||
PluginInfo getPlugin(String id) {
|
||||
return getPlugins().get(id)
|
||||
}
|
||||
|
||||
@Override
|
||||
void refresh() {
|
||||
this.plugins = null
|
||||
}
|
||||
|
||||
@Override
|
||||
FileDownloader getFileDownloader() {
|
||||
// plugins in this repo are already downloaded, so treat any download url as a file path
|
||||
return (URL url) -> Path.of(url.toURI())
|
||||
}
|
||||
|
||||
@Override
|
||||
FileVerifier getFileVerifier() {
|
||||
return new CompoundVerifier()
|
||||
}
|
||||
|
||||
private static Map<String, PluginInfo> loadPlugins(Path dir) {
|
||||
// each plugin is stored in a dir called $id-$version; grab the descriptor from each
|
||||
final manifestReader = new ManifestPluginDescriptorFinder()
|
||||
final descriptors = FilesEx.listFiles(dir)
|
||||
.collect { plugin -> new LocalPlugin(plugin, manifestReader.find(plugin)) }
|
||||
|
||||
// now group the descriptors by id, to create a PluginInfo with list of versions
|
||||
return descriptors
|
||||
.groupBy { d -> d.getPluginId() }
|
||||
.collectEntries { id, versions -> Map.entry(id, toPluginInfo(id, versions)) }
|
||||
}
|
||||
|
||||
private static PluginInfo toPluginInfo(String id, List<LocalPlugin> versions) {
|
||||
final info = new PluginInfo()
|
||||
info.id = id
|
||||
info.releases = versions.collect { v ->
|
||||
final release = new PluginInfo.PluginRelease()
|
||||
release.version = v.version
|
||||
release.requires = v.requires
|
||||
release.url = v.path.toUri().toURL()
|
||||
if( !info.provider && v.provider )
|
||||
info.provider = v.provider
|
||||
return release
|
||||
}
|
||||
return info
|
||||
}
|
||||
|
||||
private static class LocalPlugin {
|
||||
Path path
|
||||
|
||||
@Delegate
|
||||
PluginDescriptor descriptor
|
||||
|
||||
LocalPlugin(Path path, PluginDescriptor descriptor) {
|
||||
this.path = path
|
||||
this.descriptor = descriptor
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.plugin
|
||||
|
||||
import com.google.common.hash.Hasher
|
||||
import groovy.transform.Canonical
|
||||
import nextflow.util.CacheFunnel
|
||||
import nextflow.util.CacheHelper
|
||||
/**
|
||||
* Model a plugin Id and version
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Canonical
|
||||
class PluginRef implements CacheFunnel, Comparable<PluginRef> {
|
||||
|
||||
/**
|
||||
* Plugin unique ID
|
||||
*/
|
||||
String id
|
||||
|
||||
/**
|
||||
* The plugin version
|
||||
*/
|
||||
String version
|
||||
|
||||
/**
|
||||
* Parse a plugin fully-qualified ID eg. nf-amazon@1.2.0
|
||||
*
|
||||
* @param fqid The fully qualified plugin id
|
||||
* @return A {@link PluginRef} representing the plugin
|
||||
*/
|
||||
static PluginRef parse(String fqid, DefaultPlugins defaultPlugins=null) {
|
||||
final tokens = fqid.tokenize('@') as List<String>
|
||||
final id = tokens[0]
|
||||
final ver = tokens[1]
|
||||
if( ver || defaultPlugins==null )
|
||||
return new PluginRef(id, ver)
|
||||
if( defaultPlugins.hasPlugin(id) )
|
||||
return defaultPlugins.getPlugin(id)
|
||||
return new PluginRef(id)
|
||||
}
|
||||
|
||||
@Override
|
||||
Hasher funnel(Hasher hasher, CacheHelper.HashMode mode) {
|
||||
hasher.putUnencodedChars(id)
|
||||
if( version )
|
||||
hasher.putUnencodedChars(version)
|
||||
return hasher
|
||||
}
|
||||
|
||||
@Override
|
||||
String toString() {
|
||||
version ? "${id}@${version}" : id
|
||||
}
|
||||
|
||||
@Override
|
||||
int compareTo(PluginRef that) {
|
||||
return this.toString() <=> that.toString()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,505 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.plugin
|
||||
|
||||
import static java.nio.file.StandardCopyOption.*
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.util.function.Predicate
|
||||
import java.util.regex.Pattern
|
||||
|
||||
import com.github.zafarkhaja.semver.Version
|
||||
import dev.failsafe.Failsafe
|
||||
import dev.failsafe.RetryPolicy
|
||||
import dev.failsafe.event.ExecutionAttemptedEvent
|
||||
import dev.failsafe.function.CheckedSupplier
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.BuildInfo
|
||||
import nextflow.SysEnv
|
||||
import nextflow.extension.FilesEx
|
||||
import nextflow.file.FileHelper
|
||||
import nextflow.file.FileMutex
|
||||
import org.pf4j.InvalidPluginDescriptorException
|
||||
import org.pf4j.PluginDependency
|
||||
import org.pf4j.PluginRuntimeException
|
||||
import org.pf4j.PluginState
|
||||
import org.pf4j.PluginWrapper
|
||||
import org.pf4j.RuntimeMode
|
||||
import org.pf4j.update.DefaultUpdateRepository
|
||||
import org.pf4j.update.PluginInfo
|
||||
import org.pf4j.update.UpdateManager
|
||||
import org.pf4j.update.UpdateRepository
|
||||
import org.pf4j.util.FileUtils
|
||||
/**
|
||||
* Implements the download/install/update for nextflow plugins
|
||||
*
|
||||
* Plugins are downloaded and stored uncompressed in the
|
||||
* directory defined by the variable `NXF_PLUGINS_DIR`, $NXF_HOME/nextflow
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class PluginUpdater extends UpdateManager {
|
||||
|
||||
static final public Pattern META_REGEX = ~/(.+)-(\d+\.\d+\.\d+\S*)-meta\.json/
|
||||
|
||||
private CustomPluginManager pluginManager
|
||||
|
||||
private Path pluginsStore
|
||||
|
||||
private boolean pullOnly
|
||||
|
||||
private boolean offline
|
||||
|
||||
private DefaultPlugins defaultPlugins = DefaultPlugins.INSTANCE
|
||||
|
||||
protected PluginUpdater(CustomPluginManager pluginManager) {
|
||||
super(pluginManager)
|
||||
this.pluginManager = pluginManager
|
||||
}
|
||||
|
||||
PluginUpdater(CustomPluginManager pluginManager, Path pluginsRoot, URL repo, boolean offline) {
|
||||
super(pluginManager, wrap(repo, pluginsRoot, offline))
|
||||
this.offline = offline
|
||||
this.pluginsStore = pluginsRoot
|
||||
this.pluginManager = pluginManager
|
||||
}
|
||||
|
||||
static private List<UpdateRepository> wrap(URL remote, Path local, boolean offline) {
|
||||
List<UpdateRepository> result = new ArrayList<>(1)
|
||||
if( offline ) {
|
||||
log.debug "Using local update repository: ${local}"
|
||||
result.add(new LocalUpdateRepository('downloaded', local))
|
||||
}
|
||||
else {
|
||||
def remoteRepo = remote.path.endsWith('.json')
|
||||
? new DefaultUpdateRepository('nextflow.io', remote)
|
||||
: new HttpPluginRepository('registry', remote.toURI())
|
||||
|
||||
log.debug "Using plugin repository: ${remoteRepo.getClass().getSimpleName()} [${remoteRepo.id}]; url=${remote}"
|
||||
result.add(remoteRepo)
|
||||
result.addAll(customRepos())
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
static private List<DefaultUpdateRepository> customRepos() {
|
||||
final repos = SysEnv.get('NXF_PLUGINS_TEST_REPOSITORY')
|
||||
if( !repos )
|
||||
return List.of()
|
||||
// warn the user that a custom
|
||||
final msg = """\
|
||||
=======================================================================
|
||||
= WARNING =
|
||||
= You are running this script using a un-official plugin repository. =
|
||||
= =
|
||||
= ${repos}
|
||||
= =
|
||||
= This is only meant to be used for plugin testing purposes. =
|
||||
=============================================================================
|
||||
""".stripIndent(true)
|
||||
log.warn(msg)
|
||||
final result = new ArrayList<DefaultUpdateRepository>(10)
|
||||
// the repos string can contain one or more plugin repository uri separated by comma
|
||||
for( String it : repos.tokenize(',') )
|
||||
result.add(customRepo(it))
|
||||
return result
|
||||
}
|
||||
|
||||
static private DefaultUpdateRepository customRepo(String uri) {
|
||||
// Check if it's a plugin meta file. The name must match the pattern `<plugin id>-X.Y.Z-meta.json`
|
||||
final matcher = META_REGEX.matcher(uri.tokenize('/')[-1])
|
||||
if( matcher.matches() ) {
|
||||
try {
|
||||
final pluginId = matcher.group(1)
|
||||
final temp = File.createTempFile('nxf-','json')
|
||||
temp.deleteOnExit()
|
||||
temp.text = /[{"id":"${pluginId}", "releases":[ ${new URL(uri).text} ]}]/
|
||||
uri = 'file://' + temp.absolutePath
|
||||
}
|
||||
catch (FileNotFoundException e) {
|
||||
throw new IllegalArgumentException("Provided repository URL does not exist or cannot be accessed: $uri")
|
||||
}
|
||||
}
|
||||
// create the update repository instance
|
||||
final fileName = uri.tokenize('/')[-1]
|
||||
return new DefaultUpdateRepository('uri', new URL(uri), fileName)
|
||||
}
|
||||
|
||||
/**
|
||||
* Prefetch metadata for plugins. This gives an opportunity for certain
|
||||
* repository types to perform some data-loading optimisations.
|
||||
*/
|
||||
void prefetchMetadata(List<PluginRef> plugins) {
|
||||
// use direct field access to avoid the refresh() call in getRepositories()
|
||||
// which could fail anything which hasn't had a chance to prefetch yet
|
||||
for( def repo : this.@repositories ) {
|
||||
if( repo instanceof PrefetchUpdateRepository ) {
|
||||
log.trace "Prefetching plugin metadata from repository: ${repo.getClass().getSimpleName()} [${repo.id}]; plugins=${plugins}"
|
||||
repo.prefetch(plugins)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a plugin installing or updating the dependencies if necessary
|
||||
* and start the plugin
|
||||
*
|
||||
* @param pluginId The plugin Id
|
||||
* @param version The required version, can be null for the latest version
|
||||
*/
|
||||
void prepareAndStart(String pluginId, String version) {
|
||||
final PluginWrapper current = pluginManager.getPlugin(pluginId)
|
||||
if( !current ) {
|
||||
log.debug "Installing plugin ${pluginId} version: ${version ?: 'latest'}"
|
||||
// install & start the plugin
|
||||
installPlugin(pluginId, version)
|
||||
}
|
||||
else if( shouldUpdate(pluginId, version, current) ) {
|
||||
log.debug "Updating plugin ${pluginId} version: ${version ?: 'latest'} [current version: $current.descriptor.version]"
|
||||
// update & start the plugin
|
||||
updatePlugin(pluginId, version)
|
||||
}
|
||||
else {
|
||||
if( !version ) version = current.descriptor.version
|
||||
log.debug "Starting plugin ${pluginId} version: ${version}"
|
||||
pluginManager.startPlugin(pluginId)
|
||||
}
|
||||
}
|
||||
|
||||
void pullPlugins(List<String> plugins) {
|
||||
pullOnly=true
|
||||
try {
|
||||
final specs = plugins.collect(it -> PluginRef.parse(it,defaultPlugins))
|
||||
prefetchMetadata(specs)
|
||||
for( PluginRef spec : specs ) {
|
||||
pullPlugin0(spec.id, spec.version)
|
||||
}
|
||||
}
|
||||
finally {
|
||||
pullOnly=false
|
||||
}
|
||||
}
|
||||
|
||||
void pullPlugin0(String pluginId, String version) {
|
||||
final PluginWrapper current = pluginManager.getPlugin(pluginId)
|
||||
if( !current ) {
|
||||
log.debug "Installing plugin ${pluginId} version: ${version ?: 'latest'}"
|
||||
// install & start the plugin
|
||||
installPlugin(pluginId, version)
|
||||
}
|
||||
else if( shouldUpdate(pluginId, version, current) ) {
|
||||
log.debug "Updating plugin ${pluginId} version: ${version ?: 'latest'} [current version: $current.descriptor.version]"
|
||||
// update & start the plugin
|
||||
updatePlugin(pluginId, version)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Install a new plugin downloading the artifact from the remote source if needed
|
||||
*
|
||||
* @param id The plugin Id
|
||||
* @param version The plugin version
|
||||
* @return {@code true} when the plugin is correctly download, installed and started, {@code false} otherwise
|
||||
*/
|
||||
@Override
|
||||
boolean installPlugin(String id, String version) {
|
||||
if( !pluginsStore )
|
||||
throw new IllegalStateException("Missing pluginStore attribute")
|
||||
|
||||
return load0(id, version)
|
||||
}
|
||||
|
||||
private Path download0(String id, String version) {
|
||||
// 0. check if version is specified
|
||||
if( !version )
|
||||
throw new InvalidPluginDescriptorException("Missing version for plugin $id")
|
||||
log.info "Downloading plugin ${id}@${version}"
|
||||
|
||||
// 1. check if already exists
|
||||
final pluginPath = pluginsStore.resolve("$id-$version")
|
||||
if( FilesEx.exists(pluginPath) ) {
|
||||
return pluginPath
|
||||
}
|
||||
|
||||
// 2. download to temporary location
|
||||
Path downloaded = safeDownloadPlugin(id, version);
|
||||
|
||||
// 3. unzip the content and delete downloaded file
|
||||
Path dir = FileUtils.expandIfZip(downloaded)
|
||||
FileHelper.deletePath(downloaded)
|
||||
|
||||
// 4. move the final destination the plugin directory
|
||||
assert pluginPath.getFileName() == dir.getFileName()
|
||||
try {
|
||||
safeMove(dir, pluginPath)
|
||||
}
|
||||
catch (IOException e) {
|
||||
throw new PluginRuntimeException(e, "Failed to write file '$pluginPath' to plugins folder")
|
||||
}
|
||||
|
||||
return pluginPath
|
||||
}
|
||||
|
||||
protected Path safeDownloadPlugin(String id, String version) {
|
||||
final CheckedSupplier<Path> supplier = () -> downloadPlugin(id, version)
|
||||
final policy = retryPolicy(id,version)
|
||||
return Failsafe.with(policy).get(supplier)
|
||||
}
|
||||
|
||||
protected <T> RetryPolicy<T> retryPolicy(String id, String version) {
|
||||
final listener = new dev.failsafe.event.EventListener<ExecutionAttemptedEvent<T>>() {
|
||||
@Override
|
||||
void accept(ExecutionAttemptedEvent<T> event) throws Throwable {
|
||||
log.debug("Failed to download plugin: $id; version: $version - attempt: ${event.attemptCount}", event.lastFailure)
|
||||
}
|
||||
}
|
||||
|
||||
final condition = new Predicate<Throwable>() {
|
||||
@Override
|
||||
boolean test(Throwable error) {
|
||||
return error?.cause instanceof ConnectException
|
||||
}
|
||||
}
|
||||
|
||||
return RetryPolicy.<T>builder()
|
||||
.handleIf(condition)
|
||||
.withMaxAttempts(3)
|
||||
.onRetry(listener)
|
||||
.build()
|
||||
}
|
||||
|
||||
protected void safeMove(Path source, Path target) {
|
||||
try {
|
||||
Files.move(source, target, ATOMIC_MOVE, REPLACE_EXISTING)
|
||||
}
|
||||
catch (IOException e) {
|
||||
log.debug "Failed atomic move for plugin $source -> $target - Reason: ${e.message ?: e} - Fallback on safe move"
|
||||
safeMove0(source, target)
|
||||
}
|
||||
}
|
||||
|
||||
protected void safeMove0(Path source, Path target) {
|
||||
// make sure to the target path does not exist
|
||||
FileHelper.deletePath(target)
|
||||
// copy the source to the more
|
||||
FileHelper.copyPath(source, target)
|
||||
// finally remove the source
|
||||
try {
|
||||
FileHelper.deletePath(source)
|
||||
}
|
||||
catch (IOException e) {
|
||||
log.warn("Unable to delete plugin directory: $source", e)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Race condition safe plugin download. Multiple instances are synchronised
|
||||
* using a file system lock created in the tmp directory
|
||||
*
|
||||
* @param id The plugin Id
|
||||
* @param version The plugin version string
|
||||
* @return The uncompressed plugin directory path
|
||||
*/
|
||||
private Path safeDownload(String id, String version) {
|
||||
final sentinel = lockFile(id,version)
|
||||
final mutex = new FileMutex(target: sentinel, timeout: '10 min')
|
||||
try {
|
||||
return mutex.lock { download0(id, version) }
|
||||
}
|
||||
finally {
|
||||
sentinel.delete()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the file to be used a lock to synchronise concurrent downloaded from multiple Nextflow launches
|
||||
*
|
||||
* @param id The plugin Id
|
||||
* @param version The plugin version
|
||||
* @return The lock the path
|
||||
*/
|
||||
private File lockFile(String id, String version) {
|
||||
def tmp = System.getProperty('java.io.tmpdir')
|
||||
new File(tmp, "nextflow-plugin-${id}-${version}.lock")
|
||||
}
|
||||
|
||||
private boolean load0(String id, String requestedVersion) {
|
||||
assert id, "Missing plugin Id"
|
||||
|
||||
if( offline && !requestedVersion ) {
|
||||
throw new IllegalStateException("Cannot find version for $id plugin -- plugin versions MUST be specified in offline mode")
|
||||
}
|
||||
|
||||
def version = requestedVersion
|
||||
if( !version ) {
|
||||
version = getLastPluginRelease(id)?.version
|
||||
}
|
||||
else if( !Version.isValid(version) ) {
|
||||
// a version is 'valid' if it's an exact semver version "major.minor.patch" so
|
||||
// if it's not that, treat it as a version constraint and look for matches
|
||||
version = findNewestMatchingRelease(id, version)?.version
|
||||
}
|
||||
|
||||
if( !version ) {
|
||||
final msg = requestedVersion
|
||||
? "Cannot find version of $id plugin matching $requestedVersion"
|
||||
: "Cannot find latest version of $id plugin"
|
||||
throw new IllegalStateException(msg)
|
||||
}
|
||||
|
||||
if( version != requestedVersion ) {
|
||||
log.debug "Plugin $id version $requestedVersion resolved to: $version"
|
||||
}
|
||||
|
||||
def pluginPath = pluginsStore.resolve("$id-$version")
|
||||
if( !FilesEx.exists(pluginPath) ) {
|
||||
pluginPath = safeDownload(id, version)
|
||||
}
|
||||
|
||||
// verify the plugin install path contains the expected manifest path
|
||||
if( !FilesEx.exists(pluginPath.resolve('classes/META-INF/MANIFEST.MF')) ) {
|
||||
log.warn("Plugin '${pluginPath.getFileName()}' installation looks corrupted - Delete the following directory and run nextflow again: $pluginPath")
|
||||
}
|
||||
|
||||
// load the plugin from the file system
|
||||
PluginWrapper wrapper = pluginManager.loadPluginFromPath(pluginPath)
|
||||
|
||||
// pull all required required deps
|
||||
final deps = wrapper.descriptor.dependencies ?: Collections.<PluginDependency>emptyList()
|
||||
for( PluginDependency it : deps ) {
|
||||
// 1. check for installed version satisfying req
|
||||
final installed = checkInstalled(it.pluginId, it.pluginVersionSupport)
|
||||
if( installed ) {
|
||||
log.debug "Plugin $id requires $it.pluginId supported version: $it.pluginVersionSupport - found version: $installed"
|
||||
continue
|
||||
}
|
||||
|
||||
// 2. find latest satisfying req
|
||||
// -- if it's a core nextflow plugin use the version expected by it
|
||||
def depVersion = defaultPlugins.getPlugin(it.pluginId)?.version
|
||||
// -- otherwise try to find the newest matching release
|
||||
if( !depVersion )
|
||||
depVersion = findNewestMatchingRelease(it.pluginId, it.pluginVersionSupport)?.version
|
||||
log.debug "Plugin $id requires $it.pluginId supported version: $it.pluginVersionSupport - available version: $depVersion"
|
||||
if( pullOnly )
|
||||
pullPlugin0(it.pluginId, depVersion)
|
||||
else
|
||||
prepareAndStart(it.pluginId, depVersion)
|
||||
}
|
||||
|
||||
if( pullOnly )
|
||||
return false
|
||||
|
||||
// resolve the plugins
|
||||
pluginManager.resolvePlugins()
|
||||
// finally start it
|
||||
PluginState state = pluginManager.startPlugin(id)
|
||||
return PluginState.STARTED == state
|
||||
}
|
||||
|
||||
/**
|
||||
* Update an existing plugin, unloading the current version and installing
|
||||
* the requested one downloading the artifact if needed
|
||||
*
|
||||
* @param id The plugin Id
|
||||
* @param version The plugin version
|
||||
* @return {@code true} when the plugin is updated and started or {@code false} otherwise
|
||||
*/
|
||||
@Override
|
||||
boolean updatePlugin(String id, String version) {
|
||||
if (pluginManager.getPlugin(id) == null) {
|
||||
throw new PluginRuntimeException("Plugin $id cannot be updated since it is not installed")
|
||||
}
|
||||
|
||||
PluginInfo pluginInfo = getPluginsMap().get(id)
|
||||
if (pluginInfo == null) {
|
||||
throw new PluginRuntimeException("Plugin $id does not exist in any repository")
|
||||
}
|
||||
|
||||
if (!pluginManager.deletePlugin(id)) {
|
||||
return false
|
||||
}
|
||||
|
||||
load0(id, version)
|
||||
}
|
||||
|
||||
protected boolean shouldUpdate(String pluginId, String version, PluginWrapper current) {
|
||||
if( pluginManager.runtimeMode == RuntimeMode.DEVELOPMENT ) {
|
||||
log.debug "Update not supported during development mode"
|
||||
return false
|
||||
}
|
||||
if( offline ) {
|
||||
log.debug "Update not supported in offline mode"
|
||||
return false
|
||||
}
|
||||
|
||||
if( !version )
|
||||
version = getLastPluginRelease(pluginId)?.version
|
||||
if( !version ) {
|
||||
log.warn "Cannot find latest version for plugin $pluginId [skip update]"
|
||||
return false
|
||||
}
|
||||
|
||||
return pluginManager
|
||||
.getVersionManager()
|
||||
.compareVersions(current.descriptor.version, version) < 0
|
||||
}
|
||||
|
||||
protected String checkInstalled(String id, String verConstraint) {
|
||||
final versionManager = pluginManager.getVersionManager()
|
||||
// check if the installed version satisfies the requirement
|
||||
final current = pluginManager.getPlugin(id)
|
||||
if( !current )
|
||||
return null
|
||||
|
||||
final found = versionManager.checkVersionConstraint(current.descriptor.version, verConstraint)
|
||||
return found ? current.descriptor.version : null
|
||||
}
|
||||
|
||||
/**
|
||||
* Find newest release version matching the requested plugin constraint and nextflow runtime
|
||||
*
|
||||
* @param id The plugin id
|
||||
* @param verConstraint The version constraint
|
||||
* @return The plugin release satisfying the requested criteria or null it none is found
|
||||
*/
|
||||
protected PluginInfo.PluginRelease findNewestMatchingRelease(String id, String verConstraint) {
|
||||
assert verConstraint
|
||||
|
||||
final versionManager = pluginManager.getVersionManager()
|
||||
|
||||
PluginInfo pluginInfo = getPluginsMap().get(id)
|
||||
if( !pluginInfo )
|
||||
throw new IllegalArgumentException("Unknown plugin id: $id")
|
||||
|
||||
// note: order releases list by descending version numbers ie. latest version comes first
|
||||
def releases = pluginInfo.releases.sort(false) { a,b -> Version.parse(b.version) <=> Version.parse(a.version) }
|
||||
for (PluginInfo.PluginRelease rel : releases ) {
|
||||
if( !versionManager.checkVersionConstraint(rel.version, verConstraint) || !rel.url )
|
||||
continue
|
||||
|
||||
if( versionManager.checkVersionConstraint(BuildInfo.version, rel.requires) )
|
||||
return rel
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.plugin
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import org.pf4j.PluginManager
|
||||
/**
|
||||
* Plugin manager specialized for Nextflow build environment
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class Plugins {
|
||||
|
||||
private final static PluginsFacade INSTANCE = new PluginsFacade()
|
||||
|
||||
static PluginManager getManager() { INSTANCE.manager }
|
||||
|
||||
static synchronized void init(boolean embeddedMode=false) {
|
||||
INSTANCE.init(embeddedMode)
|
||||
}
|
||||
|
||||
static synchronized void init(Path root, String mode, CustomPluginManager pluginManager) {
|
||||
INSTANCE.init(root, mode, pluginManager)
|
||||
}
|
||||
|
||||
/**
|
||||
* @param config
|
||||
*/
|
||||
static void load(Map config) {
|
||||
INSTANCE.load(config)
|
||||
}
|
||||
|
||||
static void start(String pluginId) {
|
||||
INSTANCE.start(pluginId)
|
||||
}
|
||||
|
||||
static synchronized void stop() {
|
||||
INSTANCE.stop()
|
||||
}
|
||||
|
||||
static <T> List<T> getExtensions(Class<T> type) {
|
||||
INSTANCE.getExtensions(type)
|
||||
}
|
||||
|
||||
static <T> List<T> getPriorityExtensions(Class<T> type, String group=null) {
|
||||
INSTANCE.getPriorityExtensions(type,group)
|
||||
}
|
||||
|
||||
static <T> T getExtension(Class<T> type) {
|
||||
final allExtensions = INSTANCE.getExtensions(type)
|
||||
return allExtensions ? allExtensions.first() : null
|
||||
}
|
||||
|
||||
static <T> List<T> getExtensionsInPluginId(Class<T> type, String pluginId) {
|
||||
final allExtensions = INSTANCE.getExtensions(type, pluginId)
|
||||
return allExtensions
|
||||
}
|
||||
|
||||
static void pull(List<String> ids) {
|
||||
INSTANCE.pullPlugins(ids)
|
||||
}
|
||||
|
||||
static boolean startIfMissing(String pluginId) {
|
||||
if( INSTANCE ) {
|
||||
return INSTANCE.startIfMissing(pluginId)
|
||||
} else {
|
||||
log.debug "Plugins subsystem not available - Ignoring installIfMissing('$pluginId')"
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
static boolean isStarted(String pluginId) {
|
||||
INSTANCE.isStarted(pluginId)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,608 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.plugin
|
||||
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.Memoized
|
||||
import groovy.transform.PackageScope
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.SysEnv
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.extension.Bolts
|
||||
import nextflow.extension.FilesEx
|
||||
import org.pf4j.DefaultPluginManager
|
||||
import org.pf4j.PluginManager
|
||||
import org.pf4j.PluginState
|
||||
import org.pf4j.PluginStateEvent
|
||||
import org.pf4j.PluginStateListener
|
||||
/**
|
||||
* Manage plugins installation and configuration
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class PluginsFacade implements PluginStateListener {
|
||||
|
||||
@PackageScope
|
||||
final static String LEGACY_PLUGINS_REPO = 'https://raw.githubusercontent.com/nextflow-io/plugins/main/plugins.json'
|
||||
|
||||
final static String NEXTFLOW_PLUGINS_REPO = 'https://registry.nextflow.io/api'
|
||||
|
||||
private static final String DEV_MODE = 'dev'
|
||||
private static final String PROD_MODE = 'prod'
|
||||
private Map<String,String> env = SysEnv.get()
|
||||
|
||||
private String mode
|
||||
private Path root
|
||||
private boolean offline
|
||||
private PluginUpdater updater
|
||||
private CustomPluginManager manager
|
||||
private DefaultPlugins defaultPlugins = DefaultPlugins.INSTANCE
|
||||
private String indexUrl
|
||||
private boolean embedded
|
||||
|
||||
PluginsFacade() {
|
||||
mode = getPluginsMode()
|
||||
root = getPluginsDir()
|
||||
indexUrl = getPluginsRegistryUrl()
|
||||
offline = env.get('NXF_OFFLINE') == 'true'
|
||||
if( mode==DEV_MODE && root.toString()=='plugins' ) {
|
||||
// In dev mode with default 'plugins' path, try to detect the actual plugins directory
|
||||
// by looking for the nextflow project root. This is needed when running tests via Gradle
|
||||
// where classes are loaded from JARs but the plugins are in the project directory.
|
||||
final detected = detectPluginsDevRoot0()
|
||||
if( detected )
|
||||
root = detected
|
||||
}
|
||||
System.setProperty('pf4j.mode', mode)
|
||||
}
|
||||
|
||||
PluginsFacade(Path root, String mode=PROD_MODE, boolean offline=false,
|
||||
String indexUrl=LEGACY_PLUGINS_REPO) {
|
||||
this.mode = mode
|
||||
this.root = root
|
||||
this.offline = offline
|
||||
this.indexUrl = indexUrl
|
||||
System.setProperty('pf4j.mode', mode)
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine if it's running from a JAR archive
|
||||
* @return {@code true} if the code is running from a JAR artifact, {@code false} otherwise
|
||||
*/
|
||||
protected String isRunningFromDistArchive() {
|
||||
final className = this.class.name.replace('.', '/');
|
||||
final classJar = this.class.getResource("/" + className + ".class").toString();
|
||||
return classJar.startsWith("jar:")
|
||||
}
|
||||
|
||||
protected Path getPluginsDir() {
|
||||
final dir = env.get('NXF_PLUGINS_DIR')
|
||||
if( dir ) {
|
||||
log.trace "Detected NXF_PLUGINS_DIR=$dir"
|
||||
return Paths.get(dir)
|
||||
}
|
||||
else if( env.containsKey('NXF_HOME') ) {
|
||||
log.trace "Detected NXF_HOME - Using ${env.NXF_HOME}/plugins"
|
||||
return Paths.get(env.NXF_HOME, 'plugins')
|
||||
}
|
||||
else {
|
||||
log.trace "Using local plugins directory"
|
||||
return Paths.get('plugins')
|
||||
}
|
||||
}
|
||||
|
||||
static protected boolean isSupportedIndex(String url) {
|
||||
if( !url ) {
|
||||
throw new IllegalArgumentException("Missing plugins registry URL")
|
||||
}
|
||||
if( !url.startsWith('https://') && !url.startsWith('http://') ) {
|
||||
throw new IllegalArgumentException("Plugins registry URL must start with 'http://' or 'https://': $url")
|
||||
}
|
||||
if( url == LEGACY_PLUGINS_REPO ) {
|
||||
return true
|
||||
}
|
||||
final hostname = URI.create(url).authority
|
||||
return hostname.endsWith('.nextflow.io')
|
||||
|| hostname.endsWith('.nextflow.com')
|
||||
|| hostname.endsWith('.seqera.io')
|
||||
}
|
||||
|
||||
protected String getPluginsRegistryUrl() {
|
||||
final url = env.get('NXF_PLUGINS_REGISTRY_URL')
|
||||
if( !url ) {
|
||||
log.trace "Using default plugins url: ${NEXTFLOW_PLUGINS_REPO}"
|
||||
return NEXTFLOW_PLUGINS_REPO
|
||||
}
|
||||
log.debug "Detected NXF_PLUGINS_REGISTRY_URL=$url"
|
||||
if( !isSupportedIndex(url) ) {
|
||||
// warn that this is experimental behaviour
|
||||
log.warn """\
|
||||
=======================================================================
|
||||
= WARNING =
|
||||
= This workflow run us using an unofficial plugins registry. =
|
||||
= =
|
||||
= ${url}
|
||||
= =
|
||||
= Its usage is unsupported and not recommended for production workloads. =
|
||||
=============================================================================
|
||||
""".stripIndent(true)
|
||||
}
|
||||
return url
|
||||
}
|
||||
|
||||
private boolean isNextflowDevRoot(File file) {
|
||||
file.name=='nextflow' && file.isDirectory() && new File(file, 'settings.gradle').isFile()
|
||||
}
|
||||
|
||||
private Path pluginsDevRoot(File path) {
|
||||
// main project root
|
||||
if( isNextflowDevRoot(path) )
|
||||
return path.toPath().resolve('plugins')
|
||||
// when nextflow is included into another build, check the sibling directory
|
||||
if( new File(path,'settings.gradle').exists() && isNextflowDevRoot(path=new File(path,'../nextflow')) )
|
||||
return path.toPath().resolve('plugins')
|
||||
else
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Try to detect the development plugin root without throwing an exception.
|
||||
*
|
||||
* @return The nextflow plugins project path, or null if not found
|
||||
*/
|
||||
protected Path detectPluginsDevRoot0() {
|
||||
def file = new File('.').canonicalFile
|
||||
while( file!=null ) {
|
||||
final root = pluginsDevRoot(file)
|
||||
if( root ) {
|
||||
log.debug "Detected dev plugins root: $root"
|
||||
return root
|
||||
}
|
||||
file = file.parentFile
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine the development plugin root. This is required to
|
||||
* allow running unit tests for plugin projects importing the
|
||||
* nextflow core runtime.
|
||||
*
|
||||
* The nextflow main project is expected to be cloned into
|
||||
* a sibling directory respect to the plugin project
|
||||
*
|
||||
* @return The nextflow plugins project path in the local file system
|
||||
* @throws IllegalStateException if the plugins root cannot be detected
|
||||
*/
|
||||
protected Path detectPluginsDevRoot() {
|
||||
final root = detectPluginsDevRoot0()
|
||||
if( root )
|
||||
return root
|
||||
throw new IllegalStateException("Unable to detect local plugins root")
|
||||
}
|
||||
|
||||
protected String getPluginsMode() {
|
||||
final mode = env.get('NXF_PLUGINS_MODE')
|
||||
if( mode ) {
|
||||
log.trace "Detected NXF_PLUGINS_MODE=$mode"
|
||||
return mode
|
||||
}
|
||||
else if( env.containsKey('NXF_HOME') ) {
|
||||
log.trace "Detected NXF_HOME - Using plugins mode=prod"
|
||||
return PROD_MODE
|
||||
}
|
||||
else {
|
||||
log.debug "Using dev plugins mode"
|
||||
return DEV_MODE
|
||||
}
|
||||
}
|
||||
|
||||
protected boolean getPluginsDefault() {
|
||||
if( env.containsKey('NXF_PLUGINS_DEFAULT')) {
|
||||
log.trace "Detected NXF_PLUGINS_DEFAULT=$env.NXF_PLUGINS_DEFAULT"
|
||||
return env.NXF_PLUGINS_DEFAULT!='false'
|
||||
}
|
||||
else if( env.containsKey('NXF_HOME') ) {
|
||||
log.trace "Detected NXF_HOME - Using plugin defaults"
|
||||
return true
|
||||
}
|
||||
else {
|
||||
log.trace "Disabling plugin defaults"
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
private CustomPluginManager newPluginManager(Path root, boolean embedded) {
|
||||
if( mode==DEV_MODE ) {
|
||||
// plugin manage for dev purposes
|
||||
return new DevPluginManager(root)
|
||||
}
|
||||
if( embedded ) {
|
||||
// use the custom plugin manager to by-pass the creation of a local plugin repository
|
||||
return new EmbeddedPluginManager(root)
|
||||
}
|
||||
return new LocalPluginManager(root)
|
||||
}
|
||||
|
||||
protected CustomPluginManager createManager(Path root, boolean embedded) {
|
||||
final result = newPluginManager(root, embedded)
|
||||
result.addPluginStateListener(this)
|
||||
return result
|
||||
}
|
||||
|
||||
protected PluginUpdater createUpdater(Path root, CustomPluginManager manager) {
|
||||
return ( mode!=DEV_MODE
|
||||
? new PluginUpdater(manager, root, new URL(indexUrl), offline)
|
||||
: new DevPluginUpdater(manager) )
|
||||
}
|
||||
|
||||
@Override
|
||||
void pluginStateChanged(PluginStateEvent ev) {
|
||||
final err = ev.plugin.failedException
|
||||
final dsc = ev.plugin.descriptor
|
||||
if( err ) {
|
||||
throw new IllegalStateException("Unable to start plugin id=${dsc.pluginId} version=${dsc.version} -- cause: ${err.message ?: err}", err)
|
||||
}
|
||||
}
|
||||
|
||||
PluginManager getManager() { manager }
|
||||
|
||||
void init(boolean embedded=false) {
|
||||
if( manager )
|
||||
throw new IllegalArgumentException("Plugin system already setup")
|
||||
|
||||
log.debug "Setting up plugin manager > mode=${mode}; embedded=$embedded; plugins-dir=$root; core-plugins: ${defaultPlugins.toSortedString()}"
|
||||
// make sure plugins dir exists
|
||||
if( mode!=DEV_MODE && !FilesEx.exists(root) && !FilesEx.mkdirs(root) )
|
||||
throw new IOException("Unable to create plugins dir: $root")
|
||||
|
||||
this.manager = createManager(root, embedded)
|
||||
this.updater = createUpdater(root, manager)
|
||||
manager.loadPlugins()
|
||||
if( embedded ) {
|
||||
manager.startPlugins()
|
||||
this.embedded = embedded
|
||||
}
|
||||
}
|
||||
|
||||
void init(Path root, String mode, CustomPluginManager pluginManager) {
|
||||
if( manager )
|
||||
throw new IllegalArgumentException("Plugin system already setup")
|
||||
this.root = root
|
||||
this.mode = mode
|
||||
// setup plugin manager
|
||||
this.manager = pluginManager
|
||||
this.manager.addPluginStateListener(this)
|
||||
// setup the updater
|
||||
this.updater = createUpdater(root, manager)
|
||||
// load plugins
|
||||
manager.loadPlugins()
|
||||
if( embedded ) {
|
||||
manager.startPlugins()
|
||||
this.embedded = embedded
|
||||
}
|
||||
}
|
||||
|
||||
void load(Map config) {
|
||||
if( !manager )
|
||||
throw new IllegalArgumentException("Plugin system has not been initialized")
|
||||
start(pluginsRequirement(config))
|
||||
}
|
||||
|
||||
synchronized void stop() {
|
||||
if( manager ) {
|
||||
manager.stopPlugins()
|
||||
manager = null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a list of extension matching the requested interface type
|
||||
*
|
||||
* @param type
|
||||
* The request extension interface
|
||||
* @return
|
||||
* The list of extensions matching the requested interface.
|
||||
*/
|
||||
def <T> List<T> getExtensions(Class<T> type) {
|
||||
if( manager ) {
|
||||
return manager.getExtensions(type)
|
||||
}
|
||||
else {
|
||||
// this should only be used to load system extensions
|
||||
// i.e. included in the app class path not provided by
|
||||
// a plugin extension
|
||||
log.debug "Using Default plugin manager"
|
||||
return defaultManager().getExtensions(type)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a list of extension matching the requested interface type in a plugin
|
||||
*
|
||||
* @param type
|
||||
* The request extension interface
|
||||
* @return
|
||||
* The list of extensions matching the requested interface.
|
||||
*/
|
||||
def <T> List<T> getExtensions(Class<T> type, String pluginId) {
|
||||
if( manager ) {
|
||||
return manager.getExtensions(type, pluginId)
|
||||
}
|
||||
else {
|
||||
return List.of()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a list of extension matching the requested type
|
||||
* ordered by a priority value. The element at the beginning
|
||||
* of the list (index 0) has higher priority
|
||||
*
|
||||
* @param type
|
||||
* The request extension interface
|
||||
* @return
|
||||
* The list of extensions matching the requested interface.
|
||||
* The extension with higher priority appears first (lower index)
|
||||
*/
|
||||
def <T> List<T> getPriorityExtensions(Class<T> type,String group=null) {
|
||||
def extensions = getExtensions(type)
|
||||
if( group )
|
||||
extensions = extensions.findAll(it -> group0(it)==group )
|
||||
final result = extensions.sort( it -> priority0(it) )
|
||||
if( log.isTraceEnabled() )
|
||||
log.trace "Discovered extensions for type ${type.getName()}: ${extensions.join(',')}"
|
||||
return result
|
||||
}
|
||||
|
||||
protected int priority0(Object it) {
|
||||
final annot = it.getClass().getAnnotation(Priority)
|
||||
return annot ? annot.value() : 0
|
||||
}
|
||||
|
||||
protected String group0(Object it) {
|
||||
final annot = it.getClass().getAnnotation(Priority)
|
||||
return annot && annot.group() ? annot.group() : null
|
||||
}
|
||||
|
||||
@Memoized
|
||||
private PluginManager defaultManager() {
|
||||
new DefaultPluginManager()
|
||||
}
|
||||
|
||||
void start(String pluginId) {
|
||||
start(List.of(PluginRef.parse(pluginId, defaultPlugins)))
|
||||
}
|
||||
|
||||
void start(List<PluginRef> specs) {
|
||||
// check if the plugins are allowed to start
|
||||
final disallow = specs.find(it-> !isAllowed(it))
|
||||
if( disallow ) {
|
||||
throw new AbortOperationException("Refuse to use plugin '${disallow.id}' - allowed plugins are: ${allowedPluginsString()}")
|
||||
}
|
||||
// when running in embedded mode, default plugins should not be started
|
||||
// the following split partition the "specs" collection in to list
|
||||
// - the first holding the plugins for which "start" is not required
|
||||
// - the second all remaining plugins that requires a start invocation
|
||||
final split = specs.split(plugin -> isEmbedded() && defaultPlugins.hasPlugin(plugin.id))
|
||||
final skippable = split[0]
|
||||
final startable = split[1]
|
||||
// just report a debug line for the skipped ones
|
||||
if( skippable ) {
|
||||
final skippedIds = skippable.collect{ plugin -> plugin.id }
|
||||
log.debug "Plugin 'start' is not required in embedded mode -- ignoring for plugins: $skippedIds"
|
||||
}
|
||||
// prefetch the plugins meta
|
||||
updater.prefetchMetadata(startable)
|
||||
// finally start the plugins
|
||||
for( PluginRef plugin : startable ) {
|
||||
updater.prepareAndStart(plugin.id, plugin.version)
|
||||
}
|
||||
}
|
||||
|
||||
boolean isStarted(String pluginId) {
|
||||
manager.getPlugin(pluginId)?.pluginState == PluginState.STARTED
|
||||
}
|
||||
|
||||
/**
|
||||
* @return {@code true} when running in embedded mode ie. the nextflow distribution
|
||||
* include also plugin libraries. When running is this mode, plugins should not be started
|
||||
* and cannot be updated.
|
||||
*/
|
||||
protected boolean isEmbedded() {
|
||||
return embedded
|
||||
}
|
||||
|
||||
protected List<PluginRef> pluginsRequirement(Map config) {
|
||||
def specs = parseConf(config)
|
||||
if( isEmbedded() && specs ) {
|
||||
// custom plugins are not allowed for nextflow self-contained package
|
||||
log.warn "Nextflow embedded mode only core plugins -- User config plugins will be ignored: ${specs.join(',')}"
|
||||
return Collections.emptyList()
|
||||
}
|
||||
if( specs ) {
|
||||
log.debug "Plugins declared=$specs"
|
||||
}
|
||||
if( getPluginsDefault() ){
|
||||
final defSpecs = defaultPluginsConf(config)
|
||||
specs = mergePluginSpecs(specs, defSpecs)
|
||||
log.debug "Plugins default=$defSpecs"
|
||||
}
|
||||
|
||||
// add tower plugin when config contains tower options
|
||||
if( (Bolts.navigate(config,'tower.enabled') || Bolts.navigate(config,'fusion.enabled') || env.TOWER_ACCESS_TOKEN ) && !specs.find {it.id == 'nf-tower' } ) {
|
||||
specs << defaultPlugins.getPlugin('nf-tower')
|
||||
}
|
||||
if( (Bolts.navigate(config,'wave.enabled') || Bolts.navigate(config,'fusion.enabled')) && !specs.find {it.id == 'nf-wave' } ) {
|
||||
specs << defaultPlugins.getPlugin('nf-wave')
|
||||
}
|
||||
if( Bolts.navigate(config,'process.executor')=='seqera') {
|
||||
specs << defaultPlugins.getPlugin('nf-seqera')
|
||||
}
|
||||
|
||||
// add cloudcache plugin when cloudcache is enabled in the config
|
||||
if( Bolts.navigate(config, 'cloudcache.enabled')==true ) {
|
||||
specs << defaultPlugins.getPlugin('nf-cloudcache')
|
||||
}
|
||||
|
||||
log.debug "Plugins resolved requirement=$specs"
|
||||
return specs
|
||||
}
|
||||
|
||||
protected List<PluginRef> defaultPluginsConf(Map config) {
|
||||
// retrieve the list from the env var
|
||||
final commaSepList = env.get('NXF_PLUGINS_DEFAULT')
|
||||
if( commaSepList && commaSepList !in ['true','false'] ) {
|
||||
// if the plugin id in the list does *not* contain the @version suffix, it picks the version
|
||||
// specified in the defaults list. Otherwise parse the provider id@version string to the corresponding spec
|
||||
return commaSepList
|
||||
.tokenize(',')
|
||||
.collect( it-> defaultPlugins.hasPlugin(it) ? defaultPlugins.getPlugin(it) : PluginRef.parse(it) )
|
||||
}
|
||||
|
||||
// infer from app config
|
||||
final plugins = new ArrayList<PluginRef>()
|
||||
final workDir = config.workDir as String
|
||||
final bucketDir = config.bucketDir as String
|
||||
final executor = Bolts.navigate(config, 'process.executor')
|
||||
|
||||
if( executor == 'awsbatch' || workDir?.startsWith('s3://') || bucketDir?.startsWith('s3://') || env.containsKey('NXF_ENABLE_AWS_SES') )
|
||||
plugins << defaultPlugins.getPlugin('nf-amazon')
|
||||
|
||||
if( executor == 'google-lifesciences' || executor == 'google-batch' || workDir?.startsWith('gs://') || bucketDir?.startsWith('gs://') )
|
||||
plugins << defaultPlugins.getPlugin('nf-google')
|
||||
|
||||
if( executor == 'azurebatch' || workDir?.startsWith('az://') || bucketDir?.startsWith('az://') )
|
||||
plugins << defaultPlugins.getPlugin('nf-azure')
|
||||
|
||||
if( executor == 'k8s' )
|
||||
plugins << defaultPlugins.getPlugin('nf-k8s')
|
||||
|
||||
if( Bolts.navigate(config, 'weblog.enabled'))
|
||||
plugins << new PluginRef('nf-weblog')
|
||||
|
||||
return plugins
|
||||
}
|
||||
|
||||
/**
|
||||
* Lookup for plugins declared in the nextflow.config using the `plugins` scope
|
||||
*
|
||||
* @param config The nextflow config as a Map object
|
||||
* @return The list of declared plugins
|
||||
*/
|
||||
protected List<PluginRef> parseConf(Map config) {
|
||||
final pluginsConf = config.plugins as List<String>
|
||||
final result = new ArrayList( pluginsConf?.size() ?: 0 )
|
||||
if(pluginsConf) for( String it : pluginsConf ) {
|
||||
result.add( PluginRef.parse(it, defaultPlugins) )
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
synchronized void pullPlugins(List<String> ids) {
|
||||
updater.pullPlugins(ids)
|
||||
}
|
||||
|
||||
boolean startIfMissing(String pluginId) {
|
||||
if( env.NXF_PLUGINS_DEFAULT == 'false' )
|
||||
return false
|
||||
if( isEmbedded() && defaultPlugins.hasPlugin(pluginId) )
|
||||
return false
|
||||
|
||||
if( isStarted(pluginId) )
|
||||
return false
|
||||
|
||||
synchronized (this) {
|
||||
if( isStarted(pluginId) )
|
||||
return false
|
||||
start(pluginId)
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge two lists of plugin requirements
|
||||
*
|
||||
* @param configPlugins
|
||||
* The list of plugins specified via the configuration file. This has higher priority
|
||||
* @param defaultPlugins
|
||||
* The list of plugins specified via the environment
|
||||
* @return
|
||||
* The list of plugins resulting from merging the two lists
|
||||
*/
|
||||
protected List<PluginRef> mergePluginSpecs(List<PluginRef> configPlugins, List<PluginRef> defaultPlugins) {
|
||||
final map = new LinkedHashMap<String,PluginRef>(10)
|
||||
// add all plugins in the 'configPlugins' argument
|
||||
for( PluginRef plugin : configPlugins ) {
|
||||
map.put(plugin.id, plugin)
|
||||
}
|
||||
// add the plugin in the 'defaultPlugins' argument
|
||||
// if the map already contains the plugin,
|
||||
// override it only if it does not specify a version
|
||||
for( PluginRef plugin : defaultPlugins ) {
|
||||
if( !map[plugin.id] || !map[plugin.id].version ) {
|
||||
map.put(plugin.id, plugin)
|
||||
}
|
||||
}
|
||||
return new ArrayList<PluginRef>(map.values())
|
||||
}
|
||||
|
||||
protected List<PluginRef> parseAllowedPlugins(Map<String,String> env) {
|
||||
final list = env.get('NXF_PLUGINS_ALLOWED')
|
||||
// note: empty string means no plugins is allowed
|
||||
return list!=null
|
||||
? list.tokenize(',').collect(it-> PluginRef.parse(it))
|
||||
: null
|
||||
}
|
||||
|
||||
/**
|
||||
* The list of allowed plugins to be used defined by the variable NXF_PLUGINS_ALLOWED
|
||||
*
|
||||
* @return
|
||||
* The list of allowed plugins. {@code null} mean all. Empty list means none
|
||||
*/
|
||||
@Memoized
|
||||
protected List<PluginRef> getAllowedPlugins() {
|
||||
return parseAllowedPlugins(env)
|
||||
}
|
||||
|
||||
protected boolean isAllowed(String pluginId) {
|
||||
final list = getAllowedPlugins()
|
||||
return list != null
|
||||
? list.any(it->it.id==pluginId)
|
||||
: true
|
||||
}
|
||||
|
||||
protected isAllowed(PluginRef plugin) {
|
||||
return isAllowed(plugin.id)
|
||||
}
|
||||
|
||||
private String allowedPluginsString() {
|
||||
final list = getAllowedPlugins()
|
||||
if( list == null )
|
||||
return '(all)'
|
||||
if( list.isEmpty() )
|
||||
return '(none)'
|
||||
return list.collect(it->it.id).join('')
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.plugin
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import org.pf4j.update.UpdateRepository
|
||||
|
||||
/**
|
||||
* Extension to pf4j's UpdateRepository which supports pre-fetching
|
||||
* metadata for a specified set of plugins.
|
||||
*
|
||||
* This gives the ability to avoid downloading metadata for unused
|
||||
* plugins.
|
||||
*/
|
||||
@CompileStatic
|
||||
interface PrefetchUpdateRepository extends UpdateRepository {
|
||||
/**
|
||||
* This will be called when Nextflow starts, before
|
||||
* initialising the plugins.
|
||||
*/
|
||||
void prefetch(List<PluginRef> plugins)
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.plugin
|
||||
|
||||
import java.lang.annotation.ElementType
|
||||
import java.lang.annotation.Retention
|
||||
import java.lang.annotation.RetentionPolicy
|
||||
import java.lang.annotation.Target
|
||||
|
||||
/**
|
||||
* Allow the definition of plugin priority order, smaller value is an higher priority
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
@Target(ElementType.TYPE)
|
||||
@interface Priority {
|
||||
int value()
|
||||
String group() default ''
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.plugin
|
||||
|
||||
import java.lang.annotation.ElementType
|
||||
import java.lang.annotation.Retention
|
||||
import java.lang.annotation.RetentionPolicy
|
||||
import java.lang.annotation.Target
|
||||
|
||||
/**
|
||||
* Allow the definition of plugin priority order, smaller value is an higher priority
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
|
||||
@Deprecated
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
@Target(ElementType.TYPE)
|
||||
@interface Scoped {
|
||||
String value()
|
||||
int priority() default 0
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.plugin.util
|
||||
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.extension.FilesEx
|
||||
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class PluginRefactor {
|
||||
|
||||
private String pluginName
|
||||
|
||||
private String providerName
|
||||
|
||||
private String packageName
|
||||
|
||||
private File pluginDir
|
||||
|
||||
private String pluginClassPrefix
|
||||
|
||||
private File gradleSettingsFile
|
||||
|
||||
private File gradleBuildFile
|
||||
|
||||
private Map<String,String> tokenMapping = new HashMap<>()
|
||||
|
||||
String getPluginName() {
|
||||
return pluginName
|
||||
}
|
||||
|
||||
String getPackageName() {
|
||||
return packageName
|
||||
}
|
||||
|
||||
String getProviderName() {
|
||||
return providerName
|
||||
}
|
||||
|
||||
File getPluginDir() {
|
||||
return pluginDir
|
||||
}
|
||||
|
||||
PluginRefactor withPluginDir(File directory) {
|
||||
this.pluginDir = directory.absoluteFile.canonicalFile
|
||||
return this
|
||||
}
|
||||
|
||||
PluginRefactor withPluginName(String name) {
|
||||
this.pluginName = normalizeToKebabCase(name)
|
||||
this.pluginClassPrefix = normalizeToClassName(name)
|
||||
if( pluginName.toLowerCase()=='plugin' )
|
||||
throw new IllegalStateException("Invalid plugin name: '$name'")
|
||||
if( !pluginClassPrefix )
|
||||
throw new IllegalStateException("Invalid plugin name: '$name'")
|
||||
return this
|
||||
}
|
||||
|
||||
PluginRefactor withProviderName(String name) {
|
||||
this.providerName = name?.trim()
|
||||
if( !providerName )
|
||||
throw new AbortOperationException("Provider name cannot be empty or blank")
|
||||
this.packageName = normalizeToPackageNameSegment(name)
|
||||
if( !packageName )
|
||||
throw new AbortOperationException("Invalid provider name: $name")
|
||||
return this
|
||||
}
|
||||
|
||||
protected void init() {
|
||||
if( !pluginName )
|
||||
throw new IllegalStateException("Missing plugin name")
|
||||
if( !providerName )
|
||||
throw new IllegalStateException("Missing provider name")
|
||||
// initial
|
||||
this.gradleBuildFile = new File(pluginDir, 'build.gradle')
|
||||
this.gradleSettingsFile = new File(pluginDir, 'settings.gradle')
|
||||
if( !gradleBuildFile.exists() )
|
||||
throw new AbortOperationException("Plugin file does not exist: $gradleBuildFile")
|
||||
if( !gradleSettingsFile.exists() )
|
||||
throw new AbortOperationException("Plugin file does not exist: $gradleSettingsFile")
|
||||
if( !providerName )
|
||||
throw new AbortOperationException("Plugin provider name is missing")
|
||||
if( !packageName )
|
||||
throw new AbortOperationException("Plugin package name is missing")
|
||||
// packages to be updates
|
||||
tokenMapping.put('acme', packageName)
|
||||
tokenMapping.put('provider-name', providerName)
|
||||
tokenMapping.put('nf-plugin-template', pluginName)
|
||||
}
|
||||
|
||||
void apply() {
|
||||
init()
|
||||
replacePrefixInFiles(pluginDir, pluginClassPrefix)
|
||||
renameDirectory(new File(pluginDir, "src/main/groovy/acme"), new File(pluginDir, "src/main/groovy/${packageName}"))
|
||||
renameDirectory(new File(pluginDir, "src/test/groovy/acme"), new File(pluginDir, "src/test/groovy/${packageName}"))
|
||||
updateClassNamesAndSymbols(pluginDir)
|
||||
}
|
||||
|
||||
protected void replacePrefixInFiles(File rootDir, String newPrefix) {
|
||||
if (!rootDir.exists() || !rootDir.isDirectory()) {
|
||||
throw new IllegalStateException("Invalid directory: $rootDir")
|
||||
}
|
||||
|
||||
rootDir.eachFileRecurse { file ->
|
||||
if (file.isFile() && file.name.startsWith('My') && FilesEx.getExtension(file) in ['groovy']) {
|
||||
final newName = file.name.replaceFirst(/^My/, newPrefix)
|
||||
final renamedFile = new File(file.parentFile, newName)
|
||||
if (file.renameTo(renamedFile)) {
|
||||
log.debug "Renamed: ${file.name} -> ${renamedFile.name}"
|
||||
final source = FilesEx.getBaseName(file)
|
||||
final target = FilesEx.getBaseName(renamedFile)
|
||||
tokenMapping.put(source, target)
|
||||
}
|
||||
else {
|
||||
throw new IllegalStateException("Failed to rename: ${file.name}")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected void updateClassNamesAndSymbols(File rootDir) {
|
||||
rootDir.eachFileRecurse { file ->
|
||||
if (file.isFile() && FilesEx.getExtension(file) in ['groovy','gradle','md']) {
|
||||
replaceTokensInFile(file, tokenMapping)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected void replaceTokensInFile(File inputFile, Map<String, String> replacements, File outputFile = inputFile) {
|
||||
def content = inputFile.text
|
||||
|
||||
// Replace each key with its corresponding value
|
||||
for( Map.Entry<String,String> entry : replacements ) {
|
||||
content = content.replaceAll(Pattern.quote(entry.key), Matcher.quoteReplacement(entry.value))
|
||||
}
|
||||
|
||||
outputFile.text = content
|
||||
log.debug "Replacements done in: ${outputFile.path}"
|
||||
}
|
||||
|
||||
protected void renameDirectory(File oldDir, File newDir) {
|
||||
if (!oldDir.exists() || !oldDir.isDirectory()) {
|
||||
throw new AbortOperationException("Plugin template directory to rename does not exist: $oldDir")
|
||||
}
|
||||
|
||||
if( oldDir==newDir ) {
|
||||
log.debug "Unneeded path rename: $oldDir -> $newDir"
|
||||
}
|
||||
|
||||
if (newDir.exists()) {
|
||||
throw new AbortOperationException("Plugin target directory already exists: $newDir")
|
||||
}
|
||||
|
||||
if (oldDir.renameTo(newDir)) {
|
||||
log.debug "Successfully renamed: $oldDir -> $newDir"
|
||||
}
|
||||
else {
|
||||
throw new AbortOperationException("Unable to replace plugin template path: $oldDir -> $newDir")
|
||||
}
|
||||
}
|
||||
|
||||
static String normalizeToClassName(String input) {
|
||||
// Replace non-alphanumeric characters with spaces (except underscores)
|
||||
final cleaned = input.replaceAll(/[^a-zA-Z0-9_]/, ' ')
|
||||
.replaceAll(/_/, ' ')
|
||||
.trim()
|
||||
// Split by whitespace, capitalize each word, join them
|
||||
final parts = cleaned.split(/\s+/).collect { it.capitalize() }
|
||||
final result = parts.join('').replace('Plugin','')
|
||||
// Remove "Nf" prefix only
|
||||
return result.startsWith('Nf') ? result.substring(2) : result
|
||||
}
|
||||
|
||||
static String normalizeToKebabCase(String input) {
|
||||
// Insert spaces before capital letters (handles CamelCase)
|
||||
def spaced = input.replaceAll(/([a-z])([A-Z])/, '$1 $2')
|
||||
.replaceAll(/([A-Z]+)([A-Z][a-z])/, '$1 $2')
|
||||
// Replace non-alphanumeric characters and underscores with spaces
|
||||
def cleaned = spaced.replaceAll(/[^a-zA-Z0-9]/, ' ')
|
||||
.trim()
|
||||
// Split, lowercase, and join with hyphens
|
||||
def parts = cleaned.split(/\s+/).collect { it.toLowerCase() }
|
||||
return parts.join('-')
|
||||
}
|
||||
|
||||
static String normalizeToPackageNameSegment(String input) {
|
||||
// Replace non-alphanumeric characters with spaces
|
||||
def cleaned = input.replaceAll(/[^a-zA-Z0-9]/, ' ')
|
||||
.trim()
|
||||
// Split into lowercase words and join
|
||||
def parts = cleaned.split(/\s+/).collect { it.toLowerCase() }
|
||||
def name = parts.join('')
|
||||
|
||||
// Strip leading digits
|
||||
name = name.replaceFirst(/^\d+/, '')
|
||||
return name ?: null
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.serde
|
||||
|
||||
/**
|
||||
* An interface for encoding and decoding objects between two types.
|
||||
*
|
||||
* @param <T> the type of the original object to be encoded.
|
||||
* @param <S> the type of the encoded representation.
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
interface Encoder<T,S> {
|
||||
|
||||
/**
|
||||
* Encodes an object of type {@code T} into its corresponding encoded representation of type {@code S}.
|
||||
*
|
||||
* @param object the object to encode
|
||||
* @return the encoded representation of the object
|
||||
*/
|
||||
S encode(T object)
|
||||
|
||||
/**
|
||||
* Decodes an encoded representation of type {@code S} back into its original form of type {@code T}.
|
||||
*
|
||||
* @param encoded the encoded representation to decode
|
||||
* @return the decoded object
|
||||
*/
|
||||
T decode(S encoded)
|
||||
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.serde
|
||||
|
||||
/**
|
||||
* Implements a marker interface for Json serialization objects.
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
interface JsonSerializable {
|
||||
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.serde.gson
|
||||
|
||||
import java.lang.reflect.Type
|
||||
|
||||
import com.google.gson.JsonElement
|
||||
import com.google.gson.JsonPrimitive
|
||||
import com.google.gson.JsonSerializationContext
|
||||
import com.google.gson.JsonSerializer
|
||||
import groovy.transform.CompileStatic
|
||||
/**
|
||||
* Implements a Gson serializer for Groovy GString
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class GStringSerializer implements JsonSerializer<GString> {
|
||||
|
||||
@Override
|
||||
JsonElement serialize(GString src, Type typeOfSrc, JsonSerializationContext context) {
|
||||
// Convert GString to plain String
|
||||
return new JsonPrimitive(src.toString())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.serde.gson
|
||||
|
||||
import java.lang.reflect.Type
|
||||
import java.nio.file.Path
|
||||
import java.time.Instant
|
||||
import java.time.OffsetDateTime
|
||||
|
||||
import com.google.gson.Gson
|
||||
import com.google.gson.GsonBuilder
|
||||
import com.google.gson.TypeAdapterFactory
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.serde.Encoder
|
||||
import nextflow.util.TypeHelper
|
||||
import org.codehaus.groovy.runtime.GStringImpl
|
||||
|
||||
/**
|
||||
* Implement a JSON encoder based on Google Gson
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
abstract class GsonEncoder<T> implements Encoder<T, String> {
|
||||
|
||||
private Type type
|
||||
|
||||
private TypeAdapterFactory factory
|
||||
|
||||
private boolean prettyPrint
|
||||
|
||||
private boolean serializeNulls
|
||||
|
||||
private volatile Gson gson
|
||||
|
||||
protected GsonEncoder() {
|
||||
this.type = TypeHelper.getGenericType(this, 0)
|
||||
}
|
||||
|
||||
GsonEncoder<T> withTypeAdapterFactory(TypeAdapterFactory factory) {
|
||||
this.factory = factory
|
||||
return this
|
||||
}
|
||||
|
||||
GsonEncoder<T> withPrettyPrint(boolean value) {
|
||||
this.prettyPrint = value
|
||||
return this
|
||||
}
|
||||
|
||||
GsonEncoder<T> withSerializeNulls(boolean value) {
|
||||
this.serializeNulls = value
|
||||
return this
|
||||
}
|
||||
|
||||
private Gson gson0() {
|
||||
if( gson )
|
||||
return gson
|
||||
synchronized (this) {
|
||||
if( gson )
|
||||
return gson
|
||||
return gson = create0()
|
||||
}
|
||||
}
|
||||
|
||||
private Gson create0() {
|
||||
final builder = new GsonBuilder()
|
||||
builder.registerTypeAdapter(Instant.class, new InstantAdapter())
|
||||
builder.registerTypeAdapter(OffsetDateTime.class, new OffsetDateTimeAdapter())
|
||||
builder.registerTypeAdapter(GStringImpl.class, new GStringSerializer())
|
||||
builder.registerTypeHierarchyAdapter(Path.class, new PathAdapter())
|
||||
if( factory )
|
||||
builder.registerTypeAdapterFactory(factory)
|
||||
if( prettyPrint )
|
||||
builder.setPrettyPrinting()
|
||||
if( serializeNulls )
|
||||
builder.serializeNulls()
|
||||
return builder.create()
|
||||
}
|
||||
|
||||
@Override
|
||||
String encode(T object) {
|
||||
return gson0().toJson(object, type)
|
||||
}
|
||||
|
||||
@Override
|
||||
T decode(String json) {
|
||||
gson0().fromJson(json, type)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.serde.gson
|
||||
|
||||
import java.time.Instant
|
||||
|
||||
import com.google.gson.TypeAdapter
|
||||
import com.google.gson.stream.JsonReader
|
||||
import com.google.gson.stream.JsonToken
|
||||
import com.google.gson.stream.JsonWriter
|
||||
import groovy.transform.CompileStatic
|
||||
|
||||
/**
|
||||
* Implements a Gson adapter for {@link Instant}
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class InstantAdapter extends TypeAdapter<Instant> {
|
||||
@Override
|
||||
void write(JsonWriter writer, Instant value) throws IOException {
|
||||
writer.value(value?.toString())
|
||||
}
|
||||
|
||||
@Override
|
||||
Instant read(JsonReader reader) throws IOException {
|
||||
if( reader.peek() == JsonToken.NULL ) {
|
||||
reader.nextNull()
|
||||
return null
|
||||
}
|
||||
return Instant.parse(reader.nextString())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.serde.gson
|
||||
|
||||
import java.time.Instant
|
||||
import java.time.OffsetDateTime
|
||||
|
||||
import com.google.gson.TypeAdapter
|
||||
import com.google.gson.stream.JsonReader
|
||||
import com.google.gson.stream.JsonToken
|
||||
import com.google.gson.stream.JsonWriter
|
||||
import groovy.transform.CompileStatic
|
||||
|
||||
/**
|
||||
* Implements a Gson adapter for {@link Instant}
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class OffsetDateTimeAdapter extends TypeAdapter<OffsetDateTime> {
|
||||
@Override
|
||||
void write(JsonWriter writer, OffsetDateTime value) throws IOException {
|
||||
writer.value(value?.toString())
|
||||
}
|
||||
|
||||
@Override
|
||||
OffsetDateTime read(JsonReader reader) throws IOException {
|
||||
if (reader.peek() == JsonToken.NULL) {
|
||||
reader.nextNull()
|
||||
return null
|
||||
}
|
||||
return OffsetDateTime.parse(reader.nextString())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.serde.gson
|
||||
|
||||
import com.google.gson.TypeAdapter
|
||||
import com.google.gson.stream.JsonReader
|
||||
import com.google.gson.stream.JsonWriter
|
||||
import nextflow.file.FileHelper
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
class PathAdapter extends TypeAdapter<Path> {
|
||||
@Override
|
||||
void write(JsonWriter writer, Path value) throws IOException {
|
||||
writer.value(value?.toUriString())
|
||||
}
|
||||
|
||||
@Override
|
||||
Path read(JsonReader reader) throws IOException {
|
||||
if (reader.peek() == JsonToken.NULL) {
|
||||
reader.nextNull()
|
||||
return null
|
||||
}
|
||||
return FileHelper.asPath(reader.nextString())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,354 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.serde.gson;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import com.google.errorprone.annotations.CanIgnoreReturnValue;
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.JsonElement;
|
||||
import com.google.gson.JsonObject;
|
||||
import com.google.gson.JsonParseException;
|
||||
import com.google.gson.JsonPrimitive;
|
||||
import com.google.gson.TypeAdapter;
|
||||
import com.google.gson.TypeAdapterFactory;
|
||||
import com.google.gson.reflect.TypeToken;
|
||||
import com.google.gson.stream.JsonReader;
|
||||
import com.google.gson.stream.JsonWriter;
|
||||
|
||||
/*
|
||||
* NOTE: this class is copied from Gson extra module which is not included in the default
|
||||
* library distribution.
|
||||
*
|
||||
* See
|
||||
* https://github.com/google/gson/blob/main/extras/src/main/java/com/google/gson/typeadapters/RuntimeTypeAdapterFactory.java
|
||||
*
|
||||
*/
|
||||
|
||||
|
||||
/**
|
||||
* Adapts values whose runtime type may differ from their declaration type. This is necessary when a
|
||||
* field's type is not the same type that GSON should create when deserializing that field. For
|
||||
* example, consider these types:
|
||||
*
|
||||
* <pre>{@code
|
||||
* abstract class Shape {
|
||||
* int x;
|
||||
* int y;
|
||||
* }
|
||||
* class Circle extends Shape {
|
||||
* int radius;
|
||||
* }
|
||||
* class Rectangle extends Shape {
|
||||
* int width;
|
||||
* int height;
|
||||
* }
|
||||
* class Diamond extends Shape {
|
||||
* int width;
|
||||
* int height;
|
||||
* }
|
||||
* class Drawing {
|
||||
* Shape bottomShape;
|
||||
* Shape topShape;
|
||||
* }
|
||||
* }</pre>
|
||||
*
|
||||
* <p>Without additional type information, the serialized JSON is ambiguous. Is the bottom shape in
|
||||
* this drawing a rectangle or a diamond?
|
||||
*
|
||||
* <pre>{@code
|
||||
* {
|
||||
* "bottomShape": {
|
||||
* "width": 10,
|
||||
* "height": 5,
|
||||
* "x": 0,
|
||||
* "y": 0
|
||||
* },
|
||||
* "topShape": {
|
||||
* "radius": 2,
|
||||
* "x": 4,
|
||||
* "y": 1
|
||||
* }
|
||||
* }
|
||||
* }</pre>
|
||||
*
|
||||
* This class addresses this problem by adding type information to the serialized JSON and honoring
|
||||
* that type information when the JSON is deserialized:
|
||||
*
|
||||
* <pre>{@code
|
||||
* {
|
||||
* "bottomShape": {
|
||||
* "type": "Diamond",
|
||||
* "width": 10,
|
||||
* "height": 5,
|
||||
* "x": 0,
|
||||
* "y": 0
|
||||
* },
|
||||
* "topShape": {
|
||||
* "type": "Circle",
|
||||
* "radius": 2,
|
||||
* "x": 4,
|
||||
* "y": 1
|
||||
* }
|
||||
* }
|
||||
* }</pre>
|
||||
*
|
||||
* Both the type field name ({@code "type"}) and the type labels ({@code "Rectangle"}) are
|
||||
* configurable.
|
||||
*
|
||||
* <h2>Registering Types</h2>
|
||||
*
|
||||
* Create a {@code RuntimeTypeAdapterFactory} by passing the base type and type field name to the
|
||||
* {@link #of} factory method. If you don't supply an explicit type field name, {@code "type"} will
|
||||
* be used.
|
||||
*
|
||||
* <pre>{@code
|
||||
* RuntimeTypeAdapterFactory<Shape> shapeAdapterFactory
|
||||
* = RuntimeTypeAdapterFactory.of(Shape.class, "type");
|
||||
* }</pre>
|
||||
*
|
||||
* Next register all of your subtypes. Every subtype must be explicitly registered. This protects
|
||||
* your application from injection attacks. If you don't supply an explicit type label, the type's
|
||||
* simple name will be used.
|
||||
*
|
||||
* <pre>{@code
|
||||
* shapeAdapterFactory.registerSubtype(Rectangle.class, "Rectangle");
|
||||
* shapeAdapterFactory.registerSubtype(Circle.class, "Circle");
|
||||
* shapeAdapterFactory.registerSubtype(Diamond.class, "Diamond");
|
||||
* }</pre>
|
||||
*
|
||||
* Finally, register the type adapter factory in your application's GSON builder:
|
||||
*
|
||||
* <pre>{@code
|
||||
* Gson gson = new GsonBuilder()
|
||||
* .registerTypeAdapterFactory(shapeAdapterFactory)
|
||||
* .create();
|
||||
* }</pre>
|
||||
*
|
||||
* Like {@code GsonBuilder}, this API supports chaining:
|
||||
*
|
||||
* <pre>{@code
|
||||
* RuntimeTypeAdapterFactory<Shape> shapeAdapterFactory = RuntimeTypeAdapterFactory.of(Shape.class)
|
||||
* .registerSubtype(Rectangle.class)
|
||||
* .registerSubtype(Circle.class)
|
||||
* .registerSubtype(Diamond.class);
|
||||
* }</pre>
|
||||
*
|
||||
* <h2>Serialization and deserialization</h2>
|
||||
*
|
||||
* In order to serialize and deserialize a polymorphic object, you must specify the base type
|
||||
* explicitly.
|
||||
*
|
||||
* <pre>{@code
|
||||
* Diamond diamond = new Diamond();
|
||||
* String json = gson.toJson(diamond, Shape.class);
|
||||
* }</pre>
|
||||
*
|
||||
* And then:
|
||||
*
|
||||
* <pre>{@code
|
||||
* Shape shape = gson.fromJson(json, Shape.class);
|
||||
* }</pre>
|
||||
*/
|
||||
public class RuntimeTypeAdapterFactory<T> implements TypeAdapterFactory {
|
||||
private final Class<?> baseType;
|
||||
private final String typeFieldName;
|
||||
private final Map<String, Class<?>> labelToSubtype = new LinkedHashMap<>();
|
||||
private final Map<Class<?>, String> subtypeToLabel = new LinkedHashMap<>();
|
||||
private final boolean maintainType;
|
||||
private boolean recognizeSubtypes;
|
||||
|
||||
protected RuntimeTypeAdapterFactory(Class<?> baseType, String typeFieldName, boolean maintainType) {
|
||||
if (typeFieldName == null || baseType == null) {
|
||||
throw new NullPointerException();
|
||||
}
|
||||
this.baseType = baseType;
|
||||
this.typeFieldName = typeFieldName;
|
||||
this.maintainType = maintainType;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new runtime type adapter for {@code baseType} using {@code typeFieldName} as the type
|
||||
* field name. Type field names are case sensitive.
|
||||
*
|
||||
* @param maintainType true if the type field should be included in deserialized objects
|
||||
*/
|
||||
public static <T> RuntimeTypeAdapterFactory<T> of(
|
||||
Class<T> baseType, String typeFieldName, boolean maintainType) {
|
||||
return new RuntimeTypeAdapterFactory<>(baseType, typeFieldName, maintainType);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new runtime type adapter for {@code baseType} using {@code typeFieldName} as the type
|
||||
* field name. Type field names are case sensitive.
|
||||
*/
|
||||
public static <T> RuntimeTypeAdapterFactory<T> of(Class<T> baseType, String typeFieldName) {
|
||||
return new RuntimeTypeAdapterFactory<>(baseType, typeFieldName, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new runtime type adapter for {@code baseType} using {@code "type"} as the type field
|
||||
* name.
|
||||
*/
|
||||
public static <T> RuntimeTypeAdapterFactory<T> of(Class<T> baseType) {
|
||||
return new RuntimeTypeAdapterFactory<>(baseType, "type", false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensures that this factory will handle not just the given {@code baseType}, but any subtype of
|
||||
* that type.
|
||||
*/
|
||||
@CanIgnoreReturnValue
|
||||
public RuntimeTypeAdapterFactory<T> recognizeSubtypes() {
|
||||
this.recognizeSubtypes = true;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Registers {@code type} identified by {@code label}. Labels are case sensitive.
|
||||
*
|
||||
* @throws IllegalArgumentException if either {@code type} or {@code label} have already been
|
||||
* registered on this type adapter.
|
||||
*/
|
||||
@CanIgnoreReturnValue
|
||||
public RuntimeTypeAdapterFactory<T> registerSubtype(Class<? extends T> type, String label) {
|
||||
if (type == null || label == null) {
|
||||
throw new NullPointerException();
|
||||
}
|
||||
if (subtypeToLabel.containsKey(type) || labelToSubtype.containsKey(label)) {
|
||||
throw new IllegalArgumentException("types and labels must be unique");
|
||||
}
|
||||
labelToSubtype.put(label, type);
|
||||
subtypeToLabel.put(type, label);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Registers {@code type} identified by its {@link Class#getSimpleName simple name}. Labels are
|
||||
* case sensitive.
|
||||
*
|
||||
* @throws IllegalArgumentException if either {@code type} or its simple name have already been
|
||||
* registered on this type adapter.
|
||||
*/
|
||||
@CanIgnoreReturnValue
|
||||
public RuntimeTypeAdapterFactory<T> registerSubtype(Class<? extends T> type) {
|
||||
return registerSubtype(type, type.getSimpleName());
|
||||
}
|
||||
|
||||
protected Class<?> getSubTypeFromLabel(String label){
|
||||
return labelToSubtype.get(label);
|
||||
}
|
||||
|
||||
protected String getLabelFromSubtype(Class<?> subType){
|
||||
return subtypeToLabel.get(subType);
|
||||
}
|
||||
|
||||
protected String getTypeFieldName(){
|
||||
return typeFieldName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public <R> TypeAdapter<R> create(Gson gson, TypeToken<R> type) {
|
||||
if (type == null) {
|
||||
return null;
|
||||
}
|
||||
Class<?> rawType = type.getRawType();
|
||||
boolean handle =
|
||||
recognizeSubtypes ? baseType.isAssignableFrom(rawType) : baseType.equals(rawType);
|
||||
if (!handle) {
|
||||
return null;
|
||||
}
|
||||
|
||||
TypeAdapter<JsonElement> jsonElementAdapter = gson.getAdapter(JsonElement.class);
|
||||
Map<String, TypeAdapter<?>> labelToDelegate = new LinkedHashMap<>();
|
||||
Map<Class<?>, TypeAdapter<?>> subtypeToDelegate = new LinkedHashMap<>();
|
||||
for (Map.Entry<String, Class<?>> entry : labelToSubtype.entrySet()) {
|
||||
TypeAdapter<?> delegate = gson.getDelegateAdapter(this, TypeToken.get(entry.getValue()));
|
||||
labelToDelegate.put(entry.getKey(), delegate);
|
||||
subtypeToDelegate.put(entry.getValue(), delegate);
|
||||
}
|
||||
|
||||
return new TypeAdapter<R>() {
|
||||
@Override
|
||||
public R read(JsonReader in) throws IOException {
|
||||
JsonElement jsonElement = jsonElementAdapter.read(in);
|
||||
JsonElement labelJsonElement;
|
||||
if (maintainType) {
|
||||
labelJsonElement = jsonElement.getAsJsonObject().get(typeFieldName);
|
||||
} else {
|
||||
labelJsonElement = jsonElement.getAsJsonObject().remove(typeFieldName);
|
||||
}
|
||||
|
||||
if (labelJsonElement == null) {
|
||||
throw new JsonParseException(
|
||||
"cannot deserialize "
|
||||
+ baseType
|
||||
+ " because it does not define a field named "
|
||||
+ typeFieldName);
|
||||
}
|
||||
String label = labelJsonElement.getAsString();
|
||||
@SuppressWarnings("unchecked") // registration requires that subtype extends T
|
||||
TypeAdapter<R> delegate = (TypeAdapter<R>) labelToDelegate.get(label);
|
||||
if (delegate == null) {
|
||||
throw new JsonParseException(
|
||||
"cannot deserialize "
|
||||
+ baseType
|
||||
+ " subtype named "
|
||||
+ label
|
||||
+ "; did you forget to register a subtype?");
|
||||
}
|
||||
return delegate.fromJsonTree(jsonElement);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(JsonWriter out, R value) throws IOException {
|
||||
Class<?> srcType = value.getClass();
|
||||
String label = subtypeToLabel.get(srcType);
|
||||
@SuppressWarnings("unchecked") // registration requires that subtype extends T
|
||||
TypeAdapter<R> delegate = (TypeAdapter<R>) subtypeToDelegate.get(srcType);
|
||||
if (delegate == null) {
|
||||
throw new JsonParseException(
|
||||
"cannot serialize " + srcType.getName() + "; did you forget to register a subtype?");
|
||||
}
|
||||
JsonObject jsonObject = delegate.toJsonTree(value).getAsJsonObject();
|
||||
|
||||
if (maintainType) {
|
||||
jsonElementAdapter.write(out, jsonObject);
|
||||
return;
|
||||
}
|
||||
|
||||
JsonObject clone = new JsonObject();
|
||||
|
||||
if (jsonObject.has(typeFieldName)) {
|
||||
throw new JsonParseException(
|
||||
"cannot serialize "
|
||||
+ srcType.getName()
|
||||
+ " because it already defines a field named "
|
||||
+ typeFieldName);
|
||||
}
|
||||
clone.add(typeFieldName, new JsonPrimitive(label));
|
||||
|
||||
for (Map.Entry<String, JsonElement> e : jsonObject.entrySet()) {
|
||||
clone.add(e.getKey(), e.getValue());
|
||||
}
|
||||
jsonElementAdapter.write(out, clone);
|
||||
}
|
||||
}.nullSafe();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.ui
|
||||
|
||||
|
||||
/**
|
||||
* A generic decorator for {@code TextLabel} class
|
||||
*/
|
||||
interface LabelDecorator {
|
||||
|
||||
String apply( TextLabel label, String value )
|
||||
|
||||
}
|
||||
@@ -0,0 +1,269 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.ui
|
||||
|
||||
|
||||
/**
|
||||
* Used to render text based table for UI purpose. Example:
|
||||
*
|
||||
* <pre>
|
||||
* def table = new TableBuilder().head('col1').head('col2').head('col3', 10)
|
||||
* table << x << y << z << table.closeRow()
|
||||
* table << p << q << w << table.closeRow()
|
||||
*
|
||||
* :
|
||||
* println table.toString()
|
||||
* </pre>
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class TableBuilder {
|
||||
|
||||
/**
|
||||
* The list defining the table header
|
||||
*/
|
||||
List<TextLabel> headers = []
|
||||
|
||||
List<TextLabel.Align> colsAlign = []
|
||||
|
||||
/**
|
||||
* All the rows
|
||||
*/
|
||||
List<List<TextLabel>> allRows = []
|
||||
|
||||
List<Integer> dim = []
|
||||
|
||||
List<Integer> maxColsWidth = []
|
||||
|
||||
String cellSeparator = ' '
|
||||
|
||||
String rowSeparator = '\n'
|
||||
|
||||
List<TextLabel> currentRow = []
|
||||
|
||||
/**
|
||||
* Defines a single column header, for example:
|
||||
* <pre>
|
||||
* def table = new TableBuilder().head('col1').head('col2').head('col3', 10)
|
||||
* table << x << y << z
|
||||
* </pre>
|
||||
*
|
||||
* @param name The string value to be used as column header
|
||||
* @param maxWidth The column max width
|
||||
* @return The table itself, to enable method chaining
|
||||
*/
|
||||
TableBuilder head( String name, int maxWidth = 0 ) {
|
||||
def label = new TextLabel(name)
|
||||
|
||||
headers << label
|
||||
maxColsWidth << maxWidth
|
||||
colsAlign << null
|
||||
|
||||
def widths = headers .collect { it?.toString()?.size() }
|
||||
trackWidths(widths)
|
||||
return this
|
||||
}
|
||||
|
||||
TableBuilder head( String name, TextLabel.Align align ) {
|
||||
assert align
|
||||
head(name)
|
||||
colsAlign[-1] = align
|
||||
|
||||
return this
|
||||
}
|
||||
|
||||
/**
|
||||
* Use the specific string array as the table header definition
|
||||
* @param cols
|
||||
* @return
|
||||
*/
|
||||
TableBuilder setHeaders( String... cols ) {
|
||||
assert cols != null
|
||||
setHeaders( cols.collect { new TextLabel(it) } as TextLabel[] )
|
||||
}
|
||||
|
||||
TableBuilder setHeaders( TextLabel... cols ) {
|
||||
assert cols != null
|
||||
// copy the header
|
||||
this.headers = new ArrayList<>(cols as List<TextLabel>)
|
||||
// keep track of the columns width
|
||||
def widths = cols .collect { it?.toString()?.size() }
|
||||
trackWidths(widths)
|
||||
//return the object itself
|
||||
return this
|
||||
}
|
||||
|
||||
|
||||
TableBuilder setMaxColsWidth( int...colsWidth ) {
|
||||
assert colsWidth != null
|
||||
|
||||
maxColsWidth = new ArrayList<>(colsWidth as List<Integer>)
|
||||
|
||||
return this
|
||||
}
|
||||
|
||||
TableBuilder append( Object... values ) {
|
||||
append( values as List )
|
||||
}
|
||||
|
||||
/**
|
||||
* Append the specified list of values as the next row in the table, the columns width
|
||||
* are adapted accordingly
|
||||
*
|
||||
* @param values
|
||||
* @return
|
||||
*/
|
||||
TableBuilder append( List values ) {
|
||||
assert values != null
|
||||
|
||||
def row = new ArrayList<TextLabel>(values.size())
|
||||
def len = new ArrayList<Integer>(values.size())
|
||||
values.each{ it ->
|
||||
row << ( it instanceof TextLabel ? it : new TextLabel(it) )
|
||||
len << it?.toString()?.size()
|
||||
}
|
||||
|
||||
trackWidths(len)
|
||||
allRows << row
|
||||
|
||||
this
|
||||
}
|
||||
|
||||
/**
|
||||
* Defines the left-shift operator useful to build the table using the following syntax
|
||||
* <pre>
|
||||
* def table = new TableBuilder()
|
||||
* table << col1 << col2 << col3 << table.closeRow()
|
||||
* :
|
||||
* </pre>
|
||||
*
|
||||
*
|
||||
* @param value The value to be added in the table at the current row
|
||||
* @return The table instance itself
|
||||
*/
|
||||
TableBuilder leftShift( def value ) {
|
||||
if( value == this ) {
|
||||
return this
|
||||
}
|
||||
|
||||
if( value instanceof TextLabel ) {
|
||||
currentRow << value
|
||||
}
|
||||
else {
|
||||
currentRow << new TextLabel(value)
|
||||
}
|
||||
|
||||
return this
|
||||
}
|
||||
|
||||
/**
|
||||
* Close a row in the
|
||||
* @return
|
||||
*/
|
||||
TableBuilder closeRow() {
|
||||
append(currentRow)
|
||||
currentRow = []
|
||||
return this
|
||||
}
|
||||
|
||||
protected void trackWidths( List<Integer> newDim ) {
|
||||
def size = Math.min(dim.size(), newDim.size())
|
||||
|
||||
for( int i=0; i<size; i++ ) {
|
||||
if ( dim[i] < newDim[i]) {
|
||||
dim[i] = newDim[i]
|
||||
}
|
||||
}
|
||||
|
||||
if( newDim.size() > size ) {
|
||||
for( int i=size; i<newDim.size(); i++ ) {
|
||||
dim << newDim[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Render the final table and return the string
|
||||
*/
|
||||
String toString() {
|
||||
|
||||
// check if there's some rows no closed
|
||||
if( currentRow ) {
|
||||
closeRow()
|
||||
}
|
||||
|
||||
|
||||
StringBuilder result = new StringBuilder()
|
||||
|
||||
def count=0
|
||||
|
||||
/*
|
||||
* render the header
|
||||
*/
|
||||
if( headers ) {
|
||||
count++
|
||||
headers.eachWithIndex { TextLabel cell, int index ->
|
||||
renderCell( result, cell, index )
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* render the table
|
||||
*/
|
||||
allRows.each{ List<TextLabel> row ->
|
||||
// render the 'rowSeparator' (only after the first row
|
||||
if( count++ && rowSeparator!=null ) { result.append(rowSeparator) }
|
||||
// render the row
|
||||
row.eachWithIndex { TextLabel cell, int index ->
|
||||
renderCell( result, cell, index )
|
||||
}
|
||||
}
|
||||
|
||||
result.toString()
|
||||
}
|
||||
|
||||
/**
|
||||
* Render a cell in the table
|
||||
*
|
||||
* @param result The {@code StringBuilder} collecting the result table text
|
||||
* @param cell The cell to the rendered
|
||||
* @param index The current index in the row of the cell to be rendered
|
||||
*/
|
||||
private void renderCell( StringBuilder result, TextLabel cell, int index ) {
|
||||
|
||||
// the 'cellSeparator' only after the first col
|
||||
if( index && cellSeparator != null ) result.append(cellSeparator)
|
||||
|
||||
// set the max col width
|
||||
if( maxColsWidth && index<maxColsWidth.size() && maxColsWidth[index] ) {
|
||||
cell.max( maxColsWidth[index] )
|
||||
}
|
||||
|
||||
// set the max
|
||||
cell.width( dim[index] )
|
||||
|
||||
if( colsAlign[index] ) {
|
||||
cell.setAlign( colsAlign[index] )
|
||||
}
|
||||
|
||||
// render the cell
|
||||
result.append( cell.toString() )
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.ui
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
|
||||
@EqualsAndHashCode(includes='value')
|
||||
class TextLabel {
|
||||
|
||||
enum Align { RIGHT, LEFT }
|
||||
|
||||
int width
|
||||
|
||||
Align align = Align.LEFT
|
||||
|
||||
boolean active
|
||||
|
||||
def value
|
||||
|
||||
def List<LabelDecorator> decorators = []
|
||||
|
||||
def Integer max
|
||||
|
||||
/**
|
||||
* Create a label with the provided value
|
||||
*/
|
||||
TextLabel( def value ) {
|
||||
this.value = value
|
||||
this.align = ( value instanceof Number || value?.toString()?.isNumber() ) ? Align.RIGHT : Align.LEFT
|
||||
}
|
||||
|
||||
def TextLabel width( int num ) { this.width = num; this }
|
||||
|
||||
def TextLabel left() { this.align = Align.LEFT; this }
|
||||
|
||||
def TextLabel right() { this.align = Align.RIGHT; this }
|
||||
|
||||
def TextLabel number() { this.align = Align.RIGHT; this }
|
||||
|
||||
def TextLabel max( int value ) { this.max = value; this }
|
||||
|
||||
/**
|
||||
* Switch OFF the decorators rendering
|
||||
*/
|
||||
def TextLabel switchOff() { active = false; this }
|
||||
|
||||
/**
|
||||
* Turn ON the decorators rendering
|
||||
*/
|
||||
def TextLabel switchOn() {
|
||||
if ( !decorators ) {
|
||||
decorators << AnsiStyle.style().negative()
|
||||
}
|
||||
|
||||
active = true;
|
||||
return this
|
||||
}
|
||||
|
||||
|
||||
def TextLabel leftShift( LabelDecorator deco ) {
|
||||
this.decorators.add(deco)
|
||||
return this
|
||||
}
|
||||
|
||||
def TextLabel add( LabelDecorator deco ) {
|
||||
this.decorators.add(deco)
|
||||
return this
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Renders the string applying the provided decorators
|
||||
*/
|
||||
def String toString() {
|
||||
|
||||
String result = value != null ? value.toString() : '-'
|
||||
|
||||
if( width && align == Align.LEFT ) {
|
||||
result = result.padRight(width)
|
||||
}
|
||||
else if ( width && align == Align.RIGHT ) {
|
||||
result = result.padLeft(width)
|
||||
}
|
||||
else {
|
||||
result
|
||||
}
|
||||
|
||||
// apply the max rule
|
||||
if ( max != null ) {
|
||||
result = applyMax(result)
|
||||
}
|
||||
|
||||
// if not active return as it is
|
||||
if( !active ) {
|
||||
return result
|
||||
}
|
||||
|
||||
decorators.each {
|
||||
result = it.apply(this,result)
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
String applyMax( String str ) {
|
||||
assert str
|
||||
|
||||
if( str.length()<=max) {
|
||||
return str
|
||||
}
|
||||
|
||||
def cut
|
||||
if( align == Align.LEFT ) {
|
||||
cut = str.substring(max)
|
||||
str = str.substring(0,max)
|
||||
}
|
||||
else {
|
||||
int p = str.size()-max
|
||||
cut = str.substring( 0, p )
|
||||
str = str.substring( p, str.size() )
|
||||
}
|
||||
|
||||
if( str.size()>3 && !cut.isAllWhitespace() ) {
|
||||
if ( align == Align.LEFT ) {
|
||||
str = str[0..-3] + '..'
|
||||
}
|
||||
else if( align == Align.RIGHT ) {
|
||||
str = '..' + str[2..-1]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return str
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new {@code TextLabel} object with the specified value
|
||||
*/
|
||||
static TextLabel of( def value ) {
|
||||
new TextLabel( value )
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.ui.console
|
||||
|
||||
|
||||
import org.pf4j.ExtensionPoint
|
||||
/**
|
||||
* Define the extension interface for console app
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
interface ConsoleExtension extends ExtensionPoint {
|
||||
|
||||
void run(String...args)
|
||||
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.ListIterator;
|
||||
import nextflow.script.types.Tuple;
|
||||
|
||||
/**
|
||||
* Provides a basic tuple implementation extending an {@link ArrayList}
|
||||
* and not allowing any content change operation
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
public class ArrayTuple<E> extends ArrayList<E> implements Tuple {
|
||||
|
||||
private static final long serialVersionUID = - 4765828600345948947L;
|
||||
|
||||
public ArrayTuple() {}
|
||||
|
||||
public ArrayTuple(Collection<E> other) {
|
||||
super(other);
|
||||
}
|
||||
|
||||
public ArrayTuple(int initialCapacity) {
|
||||
super(initialCapacity);
|
||||
}
|
||||
|
||||
public boolean add(E e) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
public boolean remove(Object o) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
public boolean addAll(Collection<? extends E> coll) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
public boolean removeAll(Collection<?> coll) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
public boolean retainAll(Collection<?> coll) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
public void clear() {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
public E set(int index, E element) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
public void add(int index, E element) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
public E remove(int index) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
public boolean addAll(int index, Collection<? extends E> c) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
public Iterator<E> iterator() {
|
||||
return new Iterator<E>() {
|
||||
private final Iterator<? extends E> i = ArrayTuple.super.iterator();
|
||||
|
||||
public boolean hasNext() {return i.hasNext(); }
|
||||
public E next() {return i.next(); }
|
||||
public void remove() {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
public ListIterator<E> listIterator(final int index) {
|
||||
return new ListIterator<E>() {
|
||||
private final ListIterator<? extends E> i = ArrayTuple.super.listIterator(index);
|
||||
|
||||
public boolean hasNext() {return i.hasNext();}
|
||||
public E next() {return i.next();}
|
||||
public boolean hasPrevious() {return i.hasPrevious();}
|
||||
public E previous() {return i.previous();}
|
||||
public int nextIndex() {return i.nextIndex();}
|
||||
public int previousIndex() {return i.previousIndex();}
|
||||
|
||||
public void remove() {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
public void set(E e) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
public void add(E e) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
public List<E> subList(int fromIndex, int toIndex) {
|
||||
return new ArrayTuple<E>(super.subList(fromIndex, toIndex));
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import groovy.transform.CompileStatic;
|
||||
import static java.util.Objects.requireNonNull;
|
||||
|
||||
/**
|
||||
* Base62 encoder/decoder.
|
||||
* <p>
|
||||
* This is free and unencumbered public domain software
|
||||
* <p>
|
||||
* Source: https://github.com/opencoinage/opencoinage/blob/master/src/java/org/opencoinage/util/Base62.java
|
||||
*/
|
||||
@CompileStatic
|
||||
public class Base62 {
|
||||
|
||||
private static final BigInteger BASE = BigInteger.valueOf(62);
|
||||
private static final String DIGITS = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
|
||||
|
||||
private Base62() { }
|
||||
|
||||
/**
|
||||
* Encodes a number using Base62 encoding.
|
||||
*
|
||||
* @param number a positive integer
|
||||
* @return a Base62 string
|
||||
*
|
||||
* @throws IllegalArgumentException if <code>number</code> is a negative integer
|
||||
*/
|
||||
static String encode(BigInteger number) {
|
||||
if (number.compareTo(BigInteger.ZERO) < 0) {
|
||||
throwIllegalArgumentException("number must not be negative");
|
||||
}
|
||||
StringBuilder result = new StringBuilder();
|
||||
while (number.compareTo(BigInteger.ZERO) > 0) {
|
||||
BigInteger[] divmod = number.divideAndRemainder(BASE);
|
||||
number = divmod[0];
|
||||
int digit = divmod[1].intValue();
|
||||
result.insert(0, DIGITS.charAt(digit));
|
||||
}
|
||||
return (result.length() == 0) ? DIGITS.substring(0, 1) : result.toString();
|
||||
}
|
||||
|
||||
private static BigInteger throwIllegalArgumentException(String format, Object... args) {
|
||||
throw new IllegalArgumentException(String.format(format, args));
|
||||
}
|
||||
|
||||
/**
|
||||
* Decodes a string using Base62 encoding.
|
||||
*
|
||||
* @param string a Base62 string
|
||||
* @return a positive integer
|
||||
*
|
||||
* @throws IllegalArgumentException if <code>string</code> is empty
|
||||
*/
|
||||
static BigInteger decode(final String string) {
|
||||
return decode(string, 128);
|
||||
}
|
||||
|
||||
static BigInteger decode(final String string, int bitLimit) {
|
||||
requireNonNull(string, "Decoded string must not be null");
|
||||
if (string.length() == 0) {
|
||||
return throwIllegalArgumentException("String '%s' must not be empty", string);
|
||||
}
|
||||
|
||||
if (!Pattern.matches("[" + DIGITS + "]*", string)) {
|
||||
throwIllegalArgumentException("String '%s' contains illegal characters, only '%s' are allowed", string, DIGITS);
|
||||
}
|
||||
BigInteger result = BigInteger.ZERO;
|
||||
int digits = string.length();
|
||||
for (int index = 0; index < digits; index++) {
|
||||
int digit = DIGITS.indexOf(string.charAt(digits - index - 1));
|
||||
result = result.add(BigInteger.valueOf(digit).multiply(BASE.pow(index)));
|
||||
if (bitLimit > 0 && result.bitLength() > bitLimit) {
|
||||
throwIllegalArgumentException("String contains '%s' more than 128-bit information (%sbit)", string, result.bitLength());
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util
|
||||
|
||||
import com.google.common.hash.Hasher
|
||||
import groovy.transform.CompileStatic
|
||||
/**
|
||||
* Interface to delegate cache hashing to
|
||||
* a the implementing object
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
interface CacheFunnel {
|
||||
|
||||
Hasher funnel(Hasher hasher, CacheHelper.HashMode mode)
|
||||
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util;
|
||||
|
||||
import com.google.common.hash.HashFunction;
|
||||
import com.google.common.hash.Hasher;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* Provide helper method to handle caching
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
public class CacheHelper {
|
||||
|
||||
public enum HashMode {
|
||||
|
||||
STANDARD, DEEP, LENIENT, SHA256;
|
||||
|
||||
private static HashMode defaultValue;
|
||||
|
||||
static {
|
||||
if( System.getenv().containsKey("NXF_CACHE_MODE") )
|
||||
defaultValue = valueOf(System.getenv().get("NXF_CACHE_MODE"));
|
||||
}
|
||||
|
||||
public static HashMode DEFAULT() {
|
||||
return defaultValue != null ? defaultValue : STANDARD;
|
||||
}
|
||||
|
||||
public static HashMode of( Object obj ) {
|
||||
if( obj==null || obj instanceof Boolean )
|
||||
return null;
|
||||
if( obj instanceof CharSequence ) {
|
||||
if( "true".equals(obj) || "false".equals(obj) )
|
||||
return null;
|
||||
if( "standard".equals(obj) )
|
||||
return STANDARD;
|
||||
if( "lenient".equals(obj) )
|
||||
return LENIENT;
|
||||
if( "deep".equals(obj) )
|
||||
return DEEP;
|
||||
if( "sha256".equals(obj) )
|
||||
return SHA256;
|
||||
}
|
||||
LoggerFactory.getLogger(HashMode.class).warn("Unknown cache mode: {}", obj.toString());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public static Hasher hasher( Object value ) {
|
||||
return hasher(value, HashMode.STANDARD);
|
||||
}
|
||||
|
||||
public static Hasher hasher( Object value, HashMode mode ) {
|
||||
return hasher( HashBuilder.defaultHasher(), value, mode );
|
||||
}
|
||||
|
||||
public static Hasher hasher( HashFunction function, Object value, HashMode mode ) {
|
||||
return hasher( function.newHasher(), value, mode );
|
||||
}
|
||||
|
||||
public static Hasher hasher( Hasher hasher, Object value, HashMode mode ) {
|
||||
return HashBuilder.hasher(hasher, value, mode);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util
|
||||
import java.nio.charset.Charset
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class CharsetHelper {
|
||||
|
||||
static Charset getCharset( object ) {
|
||||
|
||||
if( object instanceof Map ) {
|
||||
if( object.containsKey('charset') )
|
||||
object = (object as Map).charset
|
||||
else
|
||||
return Charset.defaultCharset()
|
||||
}
|
||||
|
||||
if( object instanceof Charset )
|
||||
return (Charset)object
|
||||
|
||||
if( object instanceof String && Charset.isSupported(object) )
|
||||
return Charset.forName(object)
|
||||
|
||||
if( object != null )
|
||||
log.warn "Invalid charset object: $object -- using default: ${Charset.defaultCharset()}"
|
||||
|
||||
Charset.defaultCharset()
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.extension.Bolts
|
||||
|
||||
/**
|
||||
* Validation helper
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class CheckHelper {
|
||||
|
||||
/**
|
||||
* Valid a method named parameters map
|
||||
*
|
||||
* @param name The name of the method, only in the error reported message
|
||||
* @param params The actual parameters map
|
||||
* @param valid A map providing for each parameter name the valid values
|
||||
* @throws IllegalArgumentException when the parameter include an unexpected parameter name or value
|
||||
*/
|
||||
static void checkParams( String name, Map<String,?> params, Map<String,?> valid ) {
|
||||
|
||||
if( !params ) return
|
||||
|
||||
def allKeys = valid.keySet()
|
||||
for( String key : params.keySet() ) {
|
||||
if( !allKeys.contains(key) )
|
||||
throw new IllegalArgumentException("Unknown argument '${key}' for operator '$name' -- Possible arguments: ${allKeys.join(', ')}")
|
||||
|
||||
final value = params.get(key)
|
||||
final accepted = valid.get(key)
|
||||
if( accepted instanceof Collection ) {
|
||||
boolean ok = false
|
||||
final itr = accepted.iterator()
|
||||
while( !ok && itr.hasNext() )
|
||||
ok |= isValid(value, itr.next())
|
||||
if( !ok )
|
||||
throw new IllegalArgumentException("Value '${value}' cannot be used in in parameter '${key}' for operator '$name' -- Possible values: ${(accepted as Collection).join(', ')}")
|
||||
}
|
||||
|
||||
else if( !isValid(value, accepted) )
|
||||
throw new IllegalArgumentException("Value '${value}' cannot be used in in parameter '${key}' for operator '$name' -- Value don't match: ${accepted}")
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the provide value is included in the specified range
|
||||
*
|
||||
* @param value A value to verify
|
||||
* @param range The range it may be a {@link Class} a {@link Collection} of values, a regexp {@link Pattern} or a specific value
|
||||
* @return {@code true} is the match is satisfied or {@code false} otherwise
|
||||
*/
|
||||
static boolean isValid( value, range ) {
|
||||
if( range instanceof Class && value != null )
|
||||
return range.isAssignableFrom(value.class)
|
||||
|
||||
if( range instanceof Collection )
|
||||
return range.contains(value)
|
||||
|
||||
if( value != null && range instanceof Pattern )
|
||||
return range.matcher(value.toString()).matches()
|
||||
|
||||
value == range
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify that all method named parameters are included in the provided list
|
||||
*
|
||||
* @param name The method name used in the reported error message
|
||||
* @param params The list of accepted named parameters
|
||||
* @param valid The list of accepted parameter names
|
||||
* @throws IllegalArgumentException a parameter is not included the in valid names list
|
||||
*/
|
||||
static void checkParams( String name, Map<String,?> params, List<String> valid ) {
|
||||
if( !params ) return
|
||||
|
||||
for( String key : params.keySet() ) {
|
||||
if( !valid.contains(key) ) {
|
||||
def matches = Bolts.closest(valid,key) ?: valid
|
||||
def message = "Unknown argument '${key}' for operator '$name'. Did you mean one of these?\n" + matches.collect { " $it"}.join('\n')
|
||||
throw new IllegalArgumentException(message)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Verify that all method named parameters are included in the provided list
|
||||
*
|
||||
* @param name The method name used in the reported error message
|
||||
* @param params The list of accepted named parameters
|
||||
* @param valid The list of accepted parameter names
|
||||
* @throws IllegalArgumentException a parameter is not included the in valid names list
|
||||
*/
|
||||
static void checkParams( String name, Map<String,?> params, String... valid ) {
|
||||
checkParams(name, params, Arrays.asList(valid))
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* Implement command line parsing helpers
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class CmdLineHelper {
|
||||
|
||||
static private Pattern CLI_OPT = ~/--([a-zA-Z_-]+)(?:\W.*)?$|-([a-zA-Z])(?:\W.*)?$/
|
||||
|
||||
private List<String> args
|
||||
|
||||
CmdLineHelper( String cmdLineToBeParsed ) {
|
||||
args = splitter(cmdLineToBeParsed ?: '')
|
||||
}
|
||||
|
||||
private boolean contains(String argument) {
|
||||
return args.indexOf(argument) != -1
|
||||
}
|
||||
|
||||
private getArg( String argument ) {
|
||||
int pos = args.indexOf(argument)
|
||||
if( pos == -1 ) return null
|
||||
|
||||
List<String> result = []
|
||||
for( int i=pos+1; i<args.size(); i++ ) {
|
||||
if( args[i].startsWith('-') ) {
|
||||
break
|
||||
}
|
||||
result.add(args[i])
|
||||
}
|
||||
|
||||
if( result.size()==0 ) {
|
||||
return true
|
||||
}
|
||||
else if( result.size()==1 ) {
|
||||
return result[0]
|
||||
}
|
||||
else {
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Given a string the splitter method separate it by blank returning a list of string.
|
||||
* Tokens wrapped by a single quote or double quotes are considered as a contiguous string
|
||||
* and is added as a single element in the returned list.
|
||||
* <p>
|
||||
* For example the string: {@code "alpha beta 'delta gamma'"} will return the following result
|
||||
* {@code ["alpha", "beta", "delta gamma"]}
|
||||
*
|
||||
* @param cmdline The string to be splitted in single elements
|
||||
* @return A list of string on which each entry represent a command line argument, or an
|
||||
* empty list if the {@code cmdline} parameter is empty
|
||||
*/
|
||||
static List<String> splitter( String cmdline ) {
|
||||
|
||||
List<String> result = []
|
||||
|
||||
if( cmdline ) {
|
||||
QuoteStringTokenizer tokenizer = new QuoteStringTokenizer(cmdline);
|
||||
while( tokenizer.hasNext() ) {
|
||||
result.add(tokenizer.next())
|
||||
}
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
|
||||
static String toLine( String... args ) {
|
||||
toLine( args as List)
|
||||
}
|
||||
|
||||
static String toLine( List<String> args ) {
|
||||
def result = new ArrayList(args.size())
|
||||
args.each { result << (it.contains(' ') ? "'${it}'" : it) }
|
||||
return result.join(' ')
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse command line and returns the options and their values as a map object.
|
||||
*
|
||||
* @param cmdline
|
||||
* The command line as single string
|
||||
* @return
|
||||
* A map object holding the option key-value(s) associations
|
||||
*/
|
||||
static CmdLineOptionMap parseGnuArgs(String cmdline) {
|
||||
final BLANK = ' ' as char
|
||||
final result = new CmdLineOptionMap()
|
||||
|
||||
if( !cmdline )
|
||||
return result
|
||||
|
||||
final tokenizer = new QuoteStringTokenizer(cmdline, BLANK);
|
||||
String opt = null
|
||||
String last = null
|
||||
while( tokenizer.hasNext() ) {
|
||||
final String token = tokenizer.next()
|
||||
if( !token || token=='--')
|
||||
continue
|
||||
final matcher = CLI_OPT.matcher(token)
|
||||
if( matcher.matches() ) {
|
||||
if( opt ) {
|
||||
result.addOption(opt,'true')
|
||||
}
|
||||
opt = matcher.group(1) ?: matcher.group(2)
|
||||
}
|
||||
else {
|
||||
if( !opt ) {
|
||||
if( !last ) continue
|
||||
result.addOption(last, token)
|
||||
}
|
||||
else {
|
||||
result.addOption(opt, token)
|
||||
last = opt
|
||||
opt = null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if( opt )
|
||||
result.addOption(opt, 'true')
|
||||
|
||||
return result
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util
|
||||
|
||||
|
||||
import com.google.common.hash.Hasher
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import groovy.transform.ToString
|
||||
|
||||
/**
|
||||
* Holder for parsed command line options.
|
||||
*
|
||||
* @author Manuele Simi <manuele.simi@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
@ToString(includes = 'options', includeFields = true)
|
||||
@EqualsAndHashCode(includes = 'options', includeFields = true)
|
||||
class CmdLineOptionMap implements CacheFunnel {
|
||||
|
||||
final private Map<String, List<String>> options = new LinkedHashMap<String, List<String>>()
|
||||
final private static CmdLineOptionMap EMPTY = new CmdLineOptionMap()
|
||||
|
||||
protected CmdLineOptionMap addOption(String key, String value) {
|
||||
if ( !options.containsKey(key) )
|
||||
options[key] = new ArrayList<String>(10)
|
||||
options[key].add(value)
|
||||
return this
|
||||
}
|
||||
|
||||
boolean hasMultipleValues(String key) {
|
||||
options.containsKey(key) ? options[key].size() > 1 : false
|
||||
}
|
||||
|
||||
boolean hasOptions() {
|
||||
options.size()
|
||||
}
|
||||
|
||||
List<String> getValues(String key) {
|
||||
return options.containsKey(key) ? options[key] : new ArrayList<String>(10)
|
||||
}
|
||||
|
||||
def getFirstValue(String key) {
|
||||
getFirstValueOrDefault(key, null)
|
||||
}
|
||||
|
||||
boolean asBoolean() {
|
||||
return options.size()>0
|
||||
}
|
||||
|
||||
boolean exists(String key) {
|
||||
options.containsKey(key)
|
||||
}
|
||||
|
||||
def getFirstValueOrDefault(String key, String alternative) {
|
||||
options.containsKey(key) && options[key].get(0) ? options[key].get(0) : alternative
|
||||
}
|
||||
|
||||
static CmdLineOptionMap fromMap(final Map map) {
|
||||
def optionMap = new CmdLineOptionMap()
|
||||
map.each {
|
||||
optionMap.addOption(it.key as String, it.value as String)
|
||||
}
|
||||
return optionMap
|
||||
}
|
||||
|
||||
static CmdLineOptionMap emptyOption() {
|
||||
return EMPTY
|
||||
}
|
||||
|
||||
@Override
|
||||
String toString() {
|
||||
def serialized = []
|
||||
options.each {
|
||||
serialized << "option{${it.key}: ${it.value.each {it}}}"
|
||||
}
|
||||
return "[${serialized.join(', ')}]"
|
||||
}
|
||||
|
||||
@Override
|
||||
Hasher funnel(Hasher hasher, CacheHelper.HashMode mode) {
|
||||
return CacheHelper.hasher(hasher, options, mode)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class CollectionHelper {
|
||||
|
||||
|
||||
static List flatten( Collection collection ) {
|
||||
def result = []
|
||||
flatten(collection, { result << it })
|
||||
return result
|
||||
}
|
||||
|
||||
static void flatten( Collection list, Closure row ) {
|
||||
|
||||
def maxLen = 1
|
||||
list.each { if( it instanceof Collection ) { maxLen = Math.max(maxLen,((Collection)it).size()) } }
|
||||
|
||||
for( int i=0; i<maxLen; i++ ) {
|
||||
def set = new ArrayList(list.size())
|
||||
|
||||
for( int j=0; j<list.size(); j++ ) {
|
||||
def val = list[j]
|
||||
if( val instanceof Collection )
|
||||
set[j] = i < ((Collection)val).size() ? ((Collection)val)[i] : null
|
||||
else
|
||||
set[j] = val
|
||||
}
|
||||
|
||||
row.call(set)
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.PackageScope
|
||||
import groovy.transform.ToString
|
||||
import org.apache.commons.lang3.StringUtils
|
||||
/**
|
||||
* Simple comma-separated-values parser
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@ToString(includeFields = true, includeNames = true)
|
||||
@CompileStatic
|
||||
class CsvParser {
|
||||
|
||||
final static private char COMMA = ',' as char
|
||||
|
||||
private char quote
|
||||
|
||||
private char separator = COMMA
|
||||
|
||||
private String empty
|
||||
|
||||
private boolean strip
|
||||
|
||||
CsvParser setQuote(char ch) {
|
||||
this.quote = ch
|
||||
return this
|
||||
}
|
||||
|
||||
CsvParser setQuote(String ch) {
|
||||
this.quote = firstChar(ch)
|
||||
return this
|
||||
}
|
||||
|
||||
CsvParser setSeparator(char ch) {
|
||||
this.separator = ch ?: COMMA
|
||||
return this
|
||||
}
|
||||
|
||||
CsvParser setSeparator(String ch) {
|
||||
this.separator = firstChar(ch) ?: COMMA
|
||||
return this
|
||||
}
|
||||
|
||||
CsvParser setStrip(boolean value) {
|
||||
this.strip = value
|
||||
return this
|
||||
}
|
||||
|
||||
static private char firstChar(String str) {
|
||||
if( !str )
|
||||
return 0 as char
|
||||
if( str.size()>1 )
|
||||
throw new IllegalArgumentException("Not a valid CSV character: $str")
|
||||
str.charAt(0)
|
||||
}
|
||||
|
||||
List<String> parse( String line ) {
|
||||
List<String> result = []
|
||||
while( line != null ) {
|
||||
if( !line ) {
|
||||
result.add(empty)
|
||||
break
|
||||
}
|
||||
else if( quote && line.charAt(0)==quote ) {
|
||||
line = readQuotedValue(line, result)
|
||||
}
|
||||
else {
|
||||
line = readSimpleValue(line, result)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
private String readSimpleValue(String line, List<String> result) {
|
||||
def p = line.indexOf( (int)separator )
|
||||
if( p == -1 ) {
|
||||
result.add(stripBlanks(line))
|
||||
return null
|
||||
}
|
||||
else {
|
||||
result.add(stripBlanks(line.substring(0,p)) ?: empty)
|
||||
return line.substring(p+1)
|
||||
}
|
||||
}
|
||||
|
||||
private String readQuotedValue(String line, List<String> result) {
|
||||
def value = line.substring(1)
|
||||
def p = value.indexOf( (int)quote )
|
||||
if( p == -1 )
|
||||
throw new IllegalStateException("Missing double-quote termination in CSV value -- offending line: $line")
|
||||
result.add( stripBlanks(value.substring(0,p)) )
|
||||
|
||||
def next = p+1
|
||||
if( next<value.size() ) {
|
||||
if( value.charAt(next) != separator )
|
||||
throw new IllegalStateException("Invalid CSV value -- offending line: $line")
|
||||
return value.substring(next+1)
|
||||
}
|
||||
else {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
@PackageScope String stripBlanks(String str) {
|
||||
final value = strip ? StringUtils.strip(str) : str
|
||||
|
||||
if( !quote )
|
||||
return value
|
||||
|
||||
if( value.length()>1 && value.charAt(0)==quote && value.charAt(value.size()-1)==quote )
|
||||
return value.substring(1, value.length()-1)
|
||||
|
||||
else
|
||||
return value
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util
|
||||
|
||||
/**
|
||||
* Some utils for debugging
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
final class DebugUtils {
|
||||
|
||||
private static final String SEPARATOR = "\n";
|
||||
|
||||
private DebugUtils() {
|
||||
}
|
||||
|
||||
static String formatStackTrace(StackTraceElement[] stackTrace) {
|
||||
StringBuilder buffer = new StringBuilder();
|
||||
for (StackTraceElement element : stackTrace) {
|
||||
buffer.append(element).append(SEPARATOR);
|
||||
}
|
||||
return buffer.toString();
|
||||
}
|
||||
|
||||
static String formatCurrentStacktrace() {
|
||||
StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace();
|
||||
return formatStackTrace(stackTrace);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,434 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util
|
||||
|
||||
import java.time.temporal.ChronoUnit
|
||||
import java.time.temporal.Temporal
|
||||
import java.time.temporal.TemporalAmount
|
||||
import java.time.temporal.TemporalUnit
|
||||
import java.time.temporal.UnsupportedTemporalTypeException
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.script.types.Duration as IDuration
|
||||
import org.apache.commons.lang3.time.DurationFormatUtils
|
||||
/**
|
||||
* A simple time duration representation
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
@EqualsAndHashCode(includes = 'durationInMillis')
|
||||
class Duration implements IDuration, TemporalAmount, Comparable<Duration>, Serializable, Cloneable {
|
||||
|
||||
static private final List<TemporalUnit> SUPPORTED_UNITS = List.<TemporalUnit>of(ChronoUnit.DAYS, ChronoUnit.HOURS, ChronoUnit.MINUTES, ChronoUnit.SECONDS, ChronoUnit.MILLIS)
|
||||
|
||||
static private final FORMAT = ~/^(\d+\.?\d*)\s*([a-zA-Z]+)/
|
||||
|
||||
static private final LEGACY = ~/^(\d{1,2}):(\d{1,2}):(\d{1,2})$/
|
||||
|
||||
static private final List<String> MILLIS = ['ms','milli','millis']
|
||||
|
||||
static private final List<String> SECONDS = ['s','sec','second','seconds']
|
||||
|
||||
static private final List<String> MINUTES = ['m','min','minute','minutes']
|
||||
|
||||
static private final List<String> HOURS = ['h','hour','hours']
|
||||
|
||||
static private final List<String> DAYS = ['d','day','days']
|
||||
|
||||
static public final List<String> UNITS
|
||||
|
||||
static {
|
||||
UNITS = []
|
||||
UNITS.addAll(MILLIS)
|
||||
UNITS.addAll(SECONDS)
|
||||
UNITS.addAll(MINUTES)
|
||||
UNITS.addAll(HOURS)
|
||||
UNITS.addAll(DAYS)
|
||||
}
|
||||
|
||||
/**
|
||||
* Duration in millis
|
||||
*/
|
||||
final long durationInMillis
|
||||
|
||||
/**
|
||||
* Create e a duration object having the specified number of millis
|
||||
*
|
||||
* @param duration The duration as milliseconds
|
||||
*/
|
||||
Duration(long duration) {
|
||||
assert duration>=0, "Duration unit cannot be a negative number"
|
||||
this.durationInMillis = duration
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Default constructor is required by Kryo serializer
|
||||
* Do not removed or use it directly
|
||||
*/
|
||||
private Duration() { durationInMillis=0 }
|
||||
|
||||
/**
|
||||
* Create the object using a string 'duration' format.
|
||||
* Accepted prefix are:
|
||||
* <li>{@code ms}, {@code milli}, {@code millis}: for milliseconds
|
||||
* <li>{@code s}, {@code second}, {@code seconds}: for seconds
|
||||
* <li>{@code m}, {@code minute}, {@code minutes}: for minutes
|
||||
* <li>{@code h}, {@code hour}, {@code hours}: for hours
|
||||
* <li>{@code d}, {@code day}, {@code days}: for days
|
||||
*
|
||||
*
|
||||
* @param str
|
||||
*/
|
||||
Duration(String str) {
|
||||
|
||||
try {
|
||||
try {
|
||||
durationInMillis = parseSimple(str)
|
||||
}
|
||||
catch( IllegalArgumentException e ) {
|
||||
durationInMillis = parseLegacy(str)
|
||||
}
|
||||
}
|
||||
catch( IllegalArgumentException e ) {
|
||||
throw e
|
||||
}
|
||||
catch( Exception e ) {
|
||||
throw new IllegalArgumentException("Not a valid duration value: ${str}", e)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a duration string in legacy format i.e. hh:mm:ss
|
||||
*
|
||||
* @param str The string to be parsed e.g. {@code 05:10:30} (5 hours, 10 min, 30 seconds)
|
||||
* @return The duration in millisecond
|
||||
*/
|
||||
private long parseLegacy( String str ) {
|
||||
def matcher = (str =~ LEGACY)
|
||||
if( !matcher.matches() )
|
||||
new IllegalArgumentException("Not a valid duration value: ${str}")
|
||||
|
||||
def groups = (List<String>)matcher[0]
|
||||
def hh = groups[1].toInteger()
|
||||
def mm = groups[2].toInteger()
|
||||
def ss = groups[3].toInteger()
|
||||
|
||||
return TimeUnit.HOURS.toMillis(hh) + TimeUnit.MINUTES.toMillis(mm) + TimeUnit.SECONDS.toMillis(ss)
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a duration string
|
||||
*
|
||||
* @param str A duration string containing one or more component e.g. {@code 1d 3h 10mins}
|
||||
* @return The duration in millisecond
|
||||
*/
|
||||
private long parseSimple( String str ) {
|
||||
|
||||
long result=0
|
||||
for( int i=0; true; i++ ) {
|
||||
def matcher = (str =~ FORMAT)
|
||||
if( matcher.find() ) {
|
||||
def groups = (List<String>)matcher[0]
|
||||
def all = groups[0]
|
||||
def digit = groups[1]
|
||||
def unit = groups[2]
|
||||
|
||||
result += convert( digit.toDouble(), unit )
|
||||
str = str.substring(all.length()).trim()
|
||||
continue
|
||||
}
|
||||
|
||||
if( i == 0 || str )
|
||||
throw new IllegalArgumentException("Not a valid duration value: ${str}")
|
||||
break
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a single duration component
|
||||
*
|
||||
* @param digit
|
||||
* @param unit A valid duration unit e.g. {@code d}, {@code d}, {@code h}, {@code hour}, etc
|
||||
* @return The duration in millisecond
|
||||
*/
|
||||
private long convert( double digit, String unit ) {
|
||||
|
||||
if( unit in MILLIS ) {
|
||||
return Math.round(digit)
|
||||
}
|
||||
if ( unit in SECONDS ) {
|
||||
return Math.round(digit * 1_000)
|
||||
}
|
||||
if ( unit in MINUTES ) {
|
||||
return Math.round(digit * 60 * 1_000)
|
||||
}
|
||||
if ( unit in HOURS ) {
|
||||
return Math.round(digit * 60 * 60 * 1_000)
|
||||
}
|
||||
if ( unit in DAYS ) {
|
||||
return Math.round(digit * 24 * 60 * 60 * 1_000)
|
||||
}
|
||||
|
||||
throw new IllegalStateException()
|
||||
}
|
||||
|
||||
Duration(long value, TimeUnit unit) {
|
||||
assert value>=0, "Duration unit cannot be a negative number"
|
||||
assert unit, "Time unit cannot be null"
|
||||
this.durationInMillis = unit.toMillis(value)
|
||||
}
|
||||
|
||||
static Duration of( long value ) {
|
||||
new Duration(value)
|
||||
}
|
||||
|
||||
static Duration of( String str ) {
|
||||
new Duration(str)
|
||||
}
|
||||
|
||||
static Duration of( String str, Duration fallback ) {
|
||||
try {
|
||||
return new Duration(str)
|
||||
}
|
||||
catch( IllegalArgumentException e ) {
|
||||
log.debug "Not a valid duration value: $str -- Fallback on default value: $fallback"
|
||||
return fallback
|
||||
}
|
||||
}
|
||||
|
||||
static Duration between( Temporal start, Temporal end ) {
|
||||
new Duration(java.time.Duration.between(start, end).toMillis())
|
||||
}
|
||||
|
||||
@Override
|
||||
long toMillis() {
|
||||
durationInMillis
|
||||
}
|
||||
|
||||
long getMillis() {
|
||||
durationInMillis
|
||||
}
|
||||
|
||||
@Override
|
||||
long toSeconds() {
|
||||
TimeUnit.MILLISECONDS.toSeconds(durationInMillis)
|
||||
}
|
||||
|
||||
long getSeconds() {
|
||||
toSeconds()
|
||||
}
|
||||
|
||||
@Override
|
||||
long toMinutes() {
|
||||
TimeUnit.MILLISECONDS.toMinutes(durationInMillis)
|
||||
}
|
||||
|
||||
long getMinutes() {
|
||||
toMinutes()
|
||||
}
|
||||
|
||||
@Override
|
||||
long toHours() {
|
||||
TimeUnit.MILLISECONDS.toHours(durationInMillis)
|
||||
}
|
||||
|
||||
long getHours() {
|
||||
toHours()
|
||||
}
|
||||
|
||||
@Override
|
||||
long toDays() {
|
||||
TimeUnit.MILLISECONDS.toDays(durationInMillis)
|
||||
}
|
||||
|
||||
long getDays() {
|
||||
toDays()
|
||||
}
|
||||
|
||||
/**
|
||||
* Duration formatting utilities and constants. The following table describes the tokens used in the pattern language for formatting.
|
||||
* <p>
|
||||
* <pre>
|
||||
* character duration element
|
||||
* y years
|
||||
* d days
|
||||
* H hours
|
||||
* m minutes
|
||||
* s seconds
|
||||
* </pre>
|
||||
*
|
||||
* @param fmt
|
||||
* @return
|
||||
*/
|
||||
String format( String fmt ) {
|
||||
DurationFormatUtils.formatDuration(durationInMillis, fmt)
|
||||
}
|
||||
|
||||
String toString() {
|
||||
|
||||
// just prints the milliseconds
|
||||
if( durationInMillis < 1_000 ) {
|
||||
return durationInMillis + 'ms'
|
||||
}
|
||||
|
||||
// when less than 60 seconds round up to 100th of millis
|
||||
if( durationInMillis < 60_000 ) {
|
||||
return String.valueOf( Math.round(durationInMillis / 1_000 * 10 as double) / 10 ) + 's'
|
||||
}
|
||||
|
||||
def secs
|
||||
def mins
|
||||
def hours
|
||||
def days
|
||||
def result = []
|
||||
|
||||
// round up to seconds
|
||||
secs = Math.round( (double)(durationInMillis / 1_000) )
|
||||
|
||||
mins = secs.intdiv(60)
|
||||
secs = secs % 60
|
||||
if( secs )
|
||||
result.add( secs+'s' )
|
||||
|
||||
hours = mins.intdiv(60)
|
||||
mins = mins % 60
|
||||
if( mins )
|
||||
result.add(0, mins+'m' )
|
||||
|
||||
days = hours.intdiv(24)
|
||||
hours = hours % 24
|
||||
if( hours )
|
||||
result.add(0, hours+'h' )
|
||||
|
||||
if( days )
|
||||
result.add(0, days+'d')
|
||||
|
||||
return result.join(' ')
|
||||
}
|
||||
|
||||
def plus( Duration value ) {
|
||||
return new Duration( durationInMillis + value.durationInMillis )
|
||||
}
|
||||
|
||||
def minus( Duration value ) {
|
||||
return new Duration( durationInMillis - value.durationInMillis )
|
||||
}
|
||||
|
||||
def multiply( Number value ) {
|
||||
return new Duration( (long)(durationInMillis * value) )
|
||||
}
|
||||
|
||||
def div( Number value ) {
|
||||
return new Duration( Math.round((double)(durationInMillis / value)) )
|
||||
}
|
||||
|
||||
boolean asBoolean() {
|
||||
return durationInMillis != 0
|
||||
}
|
||||
|
||||
@Override
|
||||
int compareTo(Duration that) {
|
||||
return this.durationInMillis <=> that.durationInMillis
|
||||
}
|
||||
|
||||
static int compareTo(Duration left, Object right) {
|
||||
assert left
|
||||
|
||||
if( right==null )
|
||||
throw new IllegalArgumentException("Not a valid duration value: null")
|
||||
|
||||
if( right instanceof Duration )
|
||||
return left <=> (Duration)right
|
||||
|
||||
if( right instanceof Number )
|
||||
return left.durationInMillis <=> right.toLong()
|
||||
|
||||
if( right instanceof CharSequence )
|
||||
return left <=> Duration.of(right.toString())
|
||||
|
||||
throw new IllegalArgumentException("Not a valid duration value: $right")
|
||||
}
|
||||
|
||||
// TemporalAmount interface methods
|
||||
|
||||
/**
|
||||
* Gets the value of the requested unit.
|
||||
*
|
||||
* @param unit the TemporalUnit for which to return the value
|
||||
* @return the long value of the unit
|
||||
*/
|
||||
@Override
|
||||
long get(TemporalUnit unit) {
|
||||
if (unit == ChronoUnit.MILLIS) {
|
||||
return durationInMillis
|
||||
}
|
||||
if (unit == ChronoUnit.SECONDS) {
|
||||
return toSeconds()
|
||||
}
|
||||
if (unit == ChronoUnit.MINUTES) {
|
||||
return toMinutes()
|
||||
}
|
||||
if (unit == ChronoUnit.HOURS) {
|
||||
return toHours()
|
||||
}
|
||||
if (unit == ChronoUnit.DAYS) {
|
||||
return toDays()
|
||||
}
|
||||
throw new UnsupportedTemporalTypeException("Unsupported unit: " + unit)
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a list of units uniquely defining the value of this TemporalAmount.
|
||||
*
|
||||
* @return a list of the supported ChronoUnits, not null
|
||||
*/
|
||||
@Override
|
||||
List<TemporalUnit> getUnits() {
|
||||
return SUPPORTED_UNITS
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds this amount to the specified temporal object.
|
||||
*
|
||||
* @param temporal the temporal object to adjust, not null
|
||||
* @return an object of the same type with the adjustment made, not null
|
||||
*/
|
||||
@Override
|
||||
Temporal addTo(Temporal temporal) {
|
||||
return temporal.plus(durationInMillis, ChronoUnit.MILLIS)
|
||||
}
|
||||
|
||||
/**
|
||||
* Subtracts this amount from the specified temporal object.
|
||||
*
|
||||
* @param temporal the temporal object to adjust, not null
|
||||
* @return an object of the same type with the adjustment made, not null
|
||||
*/
|
||||
@Override
|
||||
Temporal subtractFrom(Temporal temporal) {
|
||||
return temporal.minus(durationInMillis, ChronoUnit.MILLIS)
|
||||
}
|
||||
|
||||
}
|
||||
112
nextflow/modules/nf-commons/src/main/nextflow/util/Escape.groovy
Normal file
112
nextflow/modules/nf-commons/src/main/nextflow/util/Escape.groovy
Normal file
@@ -0,0 +1,112 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.extension.FilesEx
|
||||
|
||||
/**
|
||||
* Escape helper class
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class Escape {
|
||||
|
||||
final private static List<String> SPECIAL_CHARS = ["'", '"', ' ', '(', ')', '\\', '!', '&', '|', '<', '>', '`', ':', ';']
|
||||
|
||||
final private static List<String> VAR_CHARS = ['$']
|
||||
|
||||
final private static List<String> WILDCARDS = ["*", "?", "{", "}", "[", "]", "'", '"', ' ', '(', ')', '\\', '!', '&', '|', '<', '>', '`', ':']
|
||||
|
||||
private static String replace(List<String> special, String str, boolean doNotEscapeComplement=false) {
|
||||
def copy = new StringBuilder(str.size() +10)
|
||||
for( int i=0; i<str.size(); i++) {
|
||||
def ch = str[i]
|
||||
def p = special.indexOf(ch)
|
||||
if( p != -1 ) {
|
||||
// when ! is the first character after a `[` it should not be escaped
|
||||
// see http://man7.org/linux/man-pages/man7/glob.7.html
|
||||
final isComplement = doNotEscapeComplement && ch=='!' && ( i>0 && str[i-1]=='[' && (i==1 || str[i-2]!='\\') && str.substring(i).contains(']'))
|
||||
if( !isComplement )
|
||||
copy.append('\\')
|
||||
}
|
||||
copy.append(str[i])
|
||||
}
|
||||
return copy.toString()
|
||||
}
|
||||
|
||||
static String wildcards(String str) {
|
||||
replace(WILDCARDS, str)
|
||||
}
|
||||
|
||||
static String path(String val) {
|
||||
replace(SPECIAL_CHARS, val, true)
|
||||
}
|
||||
|
||||
static String path(Path val) {
|
||||
path(val.toString())
|
||||
}
|
||||
|
||||
static String path(File val) {
|
||||
path(val.toString())
|
||||
}
|
||||
|
||||
static String path(GString val) {
|
||||
path(val.toString())
|
||||
}
|
||||
|
||||
static String cli(String[] args) {
|
||||
args.collect { cli(it) }.join(' ')
|
||||
}
|
||||
|
||||
static String cli(String arg) {
|
||||
if( arg.contains("'") )
|
||||
return wildcards(arg)
|
||||
def x = wildcards(arg)
|
||||
x == arg ? arg : "'" + arg + "'"
|
||||
}
|
||||
|
||||
static String uriPath(Path file) {
|
||||
final scheme = FilesEx.getScheme(file)
|
||||
if( scheme == 'file' )
|
||||
return path(file)
|
||||
|
||||
final uri = FilesEx.toUriString(file)
|
||||
final prefix = "$scheme://"
|
||||
assert uri.startsWith(prefix)
|
||||
def loc = uri.substring(prefix.length())
|
||||
while( loc.size()>1 && loc.endsWith('/') )
|
||||
loc = loc.substring(0,loc.length()-1)
|
||||
return prefix + path(loc)
|
||||
}
|
||||
|
||||
static String blanks(String str) {
|
||||
str
|
||||
.replaceAll('\n',/\\n/)
|
||||
.replaceAll('\t',/\\t/)
|
||||
.replaceAll('\r',/\\r/)
|
||||
.replaceAll('\f',/\\f/)
|
||||
|
||||
}
|
||||
|
||||
static String variable(String val) {
|
||||
replace(VAR_CHARS, val, false)
|
||||
}
|
||||
}
|
||||
243
nextflow/modules/nf-commons/src/main/nextflow/util/HashBag.java
Normal file
243
nextflow/modules/nf-commons/src/main/nextflow/util/HashBag.java
Normal file
@@ -0,0 +1,243 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
|
||||
import nextflow.script.types.Bag;
|
||||
|
||||
/**
|
||||
* Implementation of a bag based on a frequency map.
|
||||
*
|
||||
* @author Ben Sherman <bentshermann@gmail.com>
|
||||
*/
|
||||
public class HashBag<E> implements Bag<E> {
|
||||
|
||||
private Map<E, Integer> counts;
|
||||
|
||||
public HashBag() {
|
||||
this(10);
|
||||
}
|
||||
|
||||
public HashBag(int initialCapacity) {
|
||||
counts = new HashMap<>(initialCapacity);
|
||||
}
|
||||
|
||||
public HashBag(Collection<? extends E> c) {
|
||||
this(c.size());
|
||||
addAll(c);
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensures that this collection contains the specified element.
|
||||
*
|
||||
* @param e
|
||||
*/
|
||||
@Override
|
||||
public boolean add(E e) {
|
||||
var v = counts.computeIfAbsent(e, (k) -> 0);
|
||||
counts.put(e, v + 1);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds all of the elements in the specified collection to this collection.
|
||||
*
|
||||
* @param c
|
||||
*/
|
||||
@Override
|
||||
public boolean addAll(Collection<? extends E> c) {
|
||||
for( var e : c )
|
||||
add(e);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes all of the elements from this collection.
|
||||
*/
|
||||
@Override
|
||||
public void clear() {
|
||||
counts.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if this collection contains the specified element.
|
||||
*
|
||||
* @param o
|
||||
*/
|
||||
@Override
|
||||
public boolean contains(Object o) {
|
||||
return counts.containsKey(o);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if this collection contains all of the elements in the specified collection.
|
||||
*
|
||||
* @param c
|
||||
*/
|
||||
@Override
|
||||
public boolean containsAll(Collection<?> c) {
|
||||
for( var e : c ) {
|
||||
if( !contains(e) )
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compares the specified object with this collection for equality.
|
||||
*
|
||||
* @param o
|
||||
*/
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
return o instanceof HashBag that && this.counts.equals(that.counts);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the hash code value for this collection.
|
||||
*/
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return counts.hashCode();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if this collection contains no elements.
|
||||
*/
|
||||
@Override
|
||||
public boolean isEmpty() {
|
||||
return counts.isEmpty();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns an iterator over the elements in this collection.
|
||||
*/
|
||||
@Override
|
||||
public Iterator<E> iterator() {
|
||||
return new Iterator<E>() {
|
||||
private Iterator<Map.Entry<E,Integer>> itor = counts.entrySet().iterator();
|
||||
private E currentElement;
|
||||
private int currentCount;
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
return currentCount > 0 || itor.hasNext();
|
||||
}
|
||||
|
||||
@Override
|
||||
public E next() {
|
||||
if( currentCount == 0 ) {
|
||||
var entry = itor.next();
|
||||
currentElement = entry.getKey();
|
||||
currentCount = entry.getValue();
|
||||
}
|
||||
currentCount--;
|
||||
return currentElement;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes a single instance of the specified element from this collection, if it is present.
|
||||
*
|
||||
* @param o
|
||||
*/
|
||||
@Override
|
||||
public boolean remove(Object o) {
|
||||
var count = counts.get(o);
|
||||
if( count == null )
|
||||
return false;
|
||||
if( count.intValue() == 1 )
|
||||
counts.remove(o);
|
||||
else
|
||||
counts.put((E) o, count.intValue() - 1);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes all of this collection's elements that are also contained in the specified collection.
|
||||
*
|
||||
* @param c
|
||||
*/
|
||||
@Override
|
||||
public boolean removeAll(Collection<?> c) {
|
||||
var changed = false;
|
||||
for( var e : c ) {
|
||||
if( remove(e) )
|
||||
changed = true;
|
||||
}
|
||||
return changed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retains only the elements in this collection that are contained in the specified collection.
|
||||
*
|
||||
* @param c
|
||||
*/
|
||||
@Override
|
||||
public boolean retainAll(Collection<?> c) {
|
||||
var changed = false;
|
||||
for( var e : new HashSet<>(counts.keySet()) ) {
|
||||
if( !c.contains(e) ) {
|
||||
counts.remove(e);
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
return changed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the number of elements in this collection.
|
||||
*/
|
||||
@Override
|
||||
public int size() {
|
||||
var result = 0;
|
||||
for( var count : counts.values() )
|
||||
result += count;
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object[] toArray() {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> T[] toArray(T[] a) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
var builder = new StringBuilder();
|
||||
builder.append('[');
|
||||
var itor = iterator();
|
||||
while( itor.hasNext() ) {
|
||||
builder.append(itor.next().toString());
|
||||
if( itor.hasNext() )
|
||||
builder.append(", ");
|
||||
}
|
||||
builder.append(']');
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,553 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.file.FileVisitResult;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.ProviderMismatchException;
|
||||
import java.nio.file.SimpleFileVisitor;
|
||||
import java.nio.file.attribute.BasicFileAttributes;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
|
||||
import com.google.common.cache.CacheBuilder;
|
||||
import com.google.common.cache.CacheLoader;
|
||||
import com.google.common.cache.LoadingCache;
|
||||
import com.google.common.hash.Funnels;
|
||||
import com.google.common.hash.HashCode;
|
||||
import com.google.common.hash.HashFunction;
|
||||
import com.google.common.hash.Hasher;
|
||||
import com.google.common.hash.Hashing;
|
||||
import com.google.common.io.ByteStreams;
|
||||
import nextflow.Global;
|
||||
import nextflow.ISession;
|
||||
import nextflow.extension.Bolts;
|
||||
import nextflow.extension.FilesEx;
|
||||
import nextflow.io.SerializableMarker;
|
||||
import nextflow.script.types.Bag;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import static nextflow.Const.DEFAULT_ROOT;
|
||||
import static nextflow.util.CacheHelper.HashMode;
|
||||
|
||||
|
||||
/**
|
||||
* Implements the hashing of objects
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
public class HashBuilder {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(HashBuilder.class);
|
||||
|
||||
private static final HashFunction DEFAULT_HASHING = Hashing.murmur3_128();
|
||||
|
||||
private static final int HASH_BITS = DEFAULT_HASHING.bits();
|
||||
|
||||
private static final int HASH_BYTES = HASH_BITS / 8;
|
||||
|
||||
private static final Map<String,Object> FIRST_ONLY;
|
||||
|
||||
static {
|
||||
FIRST_ONLY = new HashMap<>(1);
|
||||
FIRST_ONLY.put("firstOnly", Boolean.TRUE);
|
||||
}
|
||||
|
||||
public static Hasher defaultHasher() {
|
||||
return DEFAULT_HASHING.newHasher();
|
||||
}
|
||||
|
||||
private Hasher hasher = defaultHasher();
|
||||
|
||||
private HashMode mode = HashMode.STANDARD;
|
||||
|
||||
private Path basePath;
|
||||
|
||||
public HashBuilder() {}
|
||||
|
||||
public HashBuilder withHasher(Hasher hasher) {
|
||||
this.hasher = hasher;
|
||||
return this;
|
||||
}
|
||||
|
||||
public HashBuilder withMode(HashMode mode) {
|
||||
this.mode = mode;
|
||||
return this;
|
||||
}
|
||||
|
||||
public HashBuilder withBasePath(Path basePath) {
|
||||
this.basePath = basePath;
|
||||
return this;
|
||||
}
|
||||
|
||||
public HashBuilder with(Object value) {
|
||||
|
||||
if( value == null )
|
||||
return this;
|
||||
|
||||
else if( value instanceof Boolean )
|
||||
hasher.putBoolean((Boolean) value);
|
||||
|
||||
else if( value instanceof Short )
|
||||
hasher.putShort((Short) value);
|
||||
|
||||
else if( value instanceof Integer)
|
||||
hasher.putInt((Integer) value);
|
||||
|
||||
else if( value instanceof Long )
|
||||
hasher.putLong((Long) value);
|
||||
|
||||
else if( value instanceof Float )
|
||||
hasher.putFloat((Float) value);
|
||||
|
||||
else if( value instanceof Double )
|
||||
hasher.putDouble( (Double)value );
|
||||
|
||||
else if( value instanceof Byte )
|
||||
hasher.putByte( (Byte)value );
|
||||
|
||||
else if( value instanceof Number )
|
||||
// reduce all other number types (BigInteger, BigDecimal, AtomicXxx, etc) to string equivalent
|
||||
hasher.putUnencodedChars(value.toString());
|
||||
|
||||
else if( value instanceof Character )
|
||||
hasher.putChar( (Character)value );
|
||||
|
||||
else if( value instanceof CharSequence )
|
||||
hasher.putUnencodedChars( (CharSequence)value );
|
||||
|
||||
else if( value instanceof byte[] )
|
||||
hasher.putBytes( (byte[])value );
|
||||
|
||||
else if( value instanceof Object[]) {
|
||||
for( Object item : ((Object[])value) )
|
||||
with(item);
|
||||
}
|
||||
|
||||
else if( value instanceof CacheFunnel )
|
||||
((CacheFunnel)value).funnel(hasher, mode);
|
||||
|
||||
else if( value instanceof Map )
|
||||
hashUnorderedCollection(hasher, ((Map) value).entrySet(), mode);
|
||||
|
||||
else if( value instanceof Map.Entry ) {
|
||||
Map.Entry entry = (Map.Entry)value;
|
||||
with(entry.getKey());
|
||||
with(entry.getValue());
|
||||
}
|
||||
|
||||
else if( value instanceof Bag || value instanceof Set )
|
||||
hashUnorderedCollection(hasher, (Collection) value, mode);
|
||||
|
||||
else if( value instanceof Collection)
|
||||
for( Object item : ((Collection)value) )
|
||||
with(item);
|
||||
|
||||
else if( value instanceof Path )
|
||||
hashFile(hasher, (Path)value, mode, basePath);
|
||||
|
||||
else if( value instanceof java.io.File )
|
||||
hashFile(hasher, (java.io.File)value, mode, basePath);
|
||||
|
||||
else if( value instanceof UUID ) {
|
||||
UUID uuid = (UUID)value;
|
||||
hasher.putLong(uuid.getMostSignificantBits()).putLong(uuid.getLeastSignificantBits());
|
||||
}
|
||||
|
||||
else if( value instanceof VersionNumber )
|
||||
hasher.putInt( value.hashCode() );
|
||||
|
||||
else if( value instanceof SerializableMarker)
|
||||
hasher.putInt( value.hashCode() );
|
||||
|
||||
else if( value instanceof Enum )
|
||||
hasher.putUnencodedChars( value.getClass().getName() + "." + value );
|
||||
|
||||
else {
|
||||
Bolts.debug1(log, FIRST_ONLY, "[WARN] Unknown hashing type: " + value.getClass());
|
||||
hasher.putInt( value.hashCode() );
|
||||
}
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
public Hasher getHasher() {
|
||||
return hasher;
|
||||
}
|
||||
|
||||
public HashCode build() {
|
||||
return hasher.hash();
|
||||
}
|
||||
|
||||
public static Hasher hasher( Hasher hasher, Object value, HashMode mode ) {
|
||||
|
||||
return new HashBuilder()
|
||||
.withHasher(hasher)
|
||||
.withMode(mode)
|
||||
.with(value)
|
||||
.getHasher();
|
||||
}
|
||||
|
||||
/**
|
||||
* Hash a file using only the relative file name instead of
|
||||
* the absolute file path.
|
||||
*
|
||||
* @param path
|
||||
* @param basePath
|
||||
* @param mode
|
||||
*/
|
||||
public static HashCode hashPath(Path path, Path basePath, HashMode mode) {
|
||||
return new HashBuilder().withMode(mode).withBasePath(basePath).with(path).build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Hashes the specified file
|
||||
*
|
||||
* @param hasher The current {@code Hasher} object
|
||||
* @param file The {@code File} object to hash
|
||||
* @param mode When {@code mode} is equals to the string {@code deep} is used the file content
|
||||
* in order to create the hash key for this file, otherwise just the file metadata information
|
||||
* (full name, size and last update timestamp)
|
||||
* @return The updated {@code Hasher} object
|
||||
*/
|
||||
static private Hasher hashFile( Hasher hasher, java.io.File file, HashMode mode, Path basePath ) {
|
||||
return hashFile(hasher, file.toPath(), mode, basePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Hashes the specified file
|
||||
*
|
||||
* @param hasher The current {@code Hasher} object
|
||||
* @param path The {@code Path} object to hash
|
||||
* @param mode When {@code mode} is equals to the string {@code deep} is used the file content
|
||||
* in order to create the hash key for this file, otherwise just the file metadata information
|
||||
* (full name, size and last update timestamp)
|
||||
* @return The updated {@code Hasher} object
|
||||
*/
|
||||
static private Hasher hashFile( Hasher hasher, Path path, HashMode mode, Path basePath ) {
|
||||
BasicFileAttributes attrs=null;
|
||||
try {
|
||||
attrs = Files.readAttributes(path, BasicFileAttributes.class);
|
||||
}
|
||||
catch(IOException e) {
|
||||
log.debug("Unable to get file attributes file: {} -- Cause: {}", FilesEx.toUriString(path), e.toString());
|
||||
}
|
||||
catch(ProviderMismatchException e) {
|
||||
// see https://github.com/nextflow-io/nextflow/pull/1382
|
||||
log.warn("File system is unable to get file attributes file: {} -- Cause: {}", FilesEx.toUriString(path), e.toString());
|
||||
}
|
||||
catch(Exception e) {
|
||||
log.warn("Unable to get file attributes file: {} -- Cause: {}", FilesEx.toUriString(path), e.toString());
|
||||
}
|
||||
|
||||
if( (mode==HashMode.STANDARD || mode==HashMode.LENIENT) && isAssetFile(path, DEFAULT_ROOT) ) {
|
||||
if( attrs==null ) {
|
||||
// when file attributes are not avail, or it's a directory
|
||||
// hash the file using the file name path and the repository
|
||||
log.warn("Unable to fetch attribute for file: {} - Hash is inferred from Git repository commit Id", FilesEx.toUriString(path));
|
||||
return hashFileAsset(hasher, path);
|
||||
}
|
||||
final Path base = Global.getSession().getBaseDir();
|
||||
if( attrs.isDirectory() ) {
|
||||
// hash all the directory content
|
||||
return hashDirSha256(hasher, path, base);
|
||||
}
|
||||
else {
|
||||
// hash the content being an asset file
|
||||
// (i.e. included in the project repository) it's expected to small file
|
||||
// which makes the content hashing doable
|
||||
return hashFileSha256(hasher, path, base);
|
||||
}
|
||||
}
|
||||
|
||||
if( mode==HashMode.DEEP && attrs!=null && attrs.isRegularFile() )
|
||||
return hashFileContent(hasher, path);
|
||||
if( mode==HashMode.SHA256 && attrs!=null && attrs.isRegularFile() )
|
||||
return hashFileSha256(hasher, path, null);
|
||||
// default
|
||||
return hashFileMetadata(hasher, path, attrs, mode, basePath);
|
||||
}
|
||||
|
||||
|
||||
static private LoadingCache<Path,String> sha256Cache = CacheBuilder
|
||||
.newBuilder()
|
||||
.maximumSize(10_000)
|
||||
.build(new CacheLoader<Path, String>() {
|
||||
@Override
|
||||
public String load(Path key) throws Exception {
|
||||
return hashFileSha256Impl0(key);
|
||||
}
|
||||
});
|
||||
|
||||
static protected Hasher hashFileSha256( Hasher hasher, Path path, Path base ) {
|
||||
try {
|
||||
log.trace("Hash sha-256 file content path={} - base={}", path, base);
|
||||
// the file relative base
|
||||
if( base!=null )
|
||||
hasher.putUnencodedChars(base.relativize(path).toString());
|
||||
// file content hash
|
||||
String sha256 = sha256Cache.get(path);
|
||||
hasher.putUnencodedChars(sha256);
|
||||
}
|
||||
catch (ExecutionException t) {
|
||||
Throwable err = t.getCause()!=null ? t.getCause() : t;
|
||||
String msg = err.getMessage()!=null ? err.getMessage() : err.toString();
|
||||
log.warn("Unable to compute sha-256 hashing for file: {} - Cause: {}", FilesEx.toUriString(path), msg);
|
||||
}
|
||||
return hasher;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute an, order independent, hash of a directory path traversing recursively the directory content.
|
||||
*
|
||||
* @param hasher
|
||||
* The {@link Hasher} object to which the resulting directory hash will be added.
|
||||
* @param dir
|
||||
* The target directory path to be hashed.
|
||||
* @param base
|
||||
* The "base" directory path against which resolve relative paths.
|
||||
* @return
|
||||
* The resulting {@link Hasher} object updated with the directory path.
|
||||
*/
|
||||
static protected Hasher hashDirSha256( Hasher hasher, Path dir, Path base ) {
|
||||
if( base==null )
|
||||
throw new IllegalArgumentException("Argument 'base' cannot be null");
|
||||
// the byte array used as "accumulator" for
|
||||
final byte[] resultBytes = new byte[HASH_BYTES];
|
||||
try {
|
||||
Files.walkFileTree(dir, new SimpleFileVisitor<Path>() {
|
||||
public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IOException {
|
||||
log.trace("Hash sha-256 dir content [FILE] path={} - base={}", path, base);
|
||||
try {
|
||||
// the file relative base
|
||||
final String relPath = base.relativize(path).toString();
|
||||
// the file content sha-256 checksum
|
||||
final String sha256 = sha256Cache.get(path);
|
||||
// compute the file path hash and sum to the result hash
|
||||
// since the sum is commutative, the traverse order does not matter
|
||||
// compute a hash of the (file path, file hash) pair.
|
||||
// since the sum is commutative, the resulting hash in `resultBytes` is invariant to the file traversal order.
|
||||
// however, the file path and file hash do need to be processed together,
|
||||
// otherwise this introduces an edge case with directories with similar contents with have the same sha (see nextflow-io/nextflow#6198)
|
||||
sumBytes(resultBytes, hashBytes(Map.entry(relPath, sha256), HashMode.STANDARD));
|
||||
return FileVisitResult.CONTINUE;
|
||||
}
|
||||
catch (ExecutionException t) {
|
||||
throw new IOException(t);
|
||||
}
|
||||
}
|
||||
|
||||
public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes attrs) {
|
||||
log.trace("Hash sha-256 dir content [DIR] path={} - base={}", path, base);
|
||||
// the file relative base
|
||||
final String relPath = base.relativize(path).toString();
|
||||
// compute the file path hash and sum to the result hash
|
||||
// since the sum is commutative, the traverse order does not matter
|
||||
sumBytes(resultBytes, hashBytes(relPath, HashMode.STANDARD));
|
||||
return FileVisitResult.CONTINUE;
|
||||
}
|
||||
});
|
||||
// finally put the result bytes in the hashing
|
||||
hasher.putBytes(resultBytes);
|
||||
}
|
||||
catch (IOException t) {
|
||||
Throwable err = t.getCause()!=null ? t.getCause() : t;
|
||||
String msg = err.getMessage()!=null ? err.getMessage() : err.toString();
|
||||
log.warn("Unable to compute sha-256 hashing for directory: {} - Cause: {}", FilesEx.toUriString(dir), msg);
|
||||
}
|
||||
return hasher;
|
||||
}
|
||||
|
||||
static protected String hashFileSha256Impl0(Path path) throws IOException {
|
||||
log.debug("Hash asset file sha-256: {}", path);
|
||||
Hasher hasher = Hashing.sha256().newHasher();
|
||||
ByteStreams.copy(Files.newInputStream(path), Funnels.asOutputStream(hasher));
|
||||
return hasher.hash().toString();
|
||||
}
|
||||
|
||||
static private Hasher hashFileAsset( Hasher hasher, Path path ) {
|
||||
log.debug("Hash asset file: {}", path);
|
||||
hasher.putUnencodedChars( Global.getSession().getCommitId() );
|
||||
return hasher;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hashes the file by using the metadata information: full path string, size and last update timestamp
|
||||
*
|
||||
* @param hasher The current {@code Hasher} object
|
||||
* @param file file The {@code Path} object to hash
|
||||
* @return The updated {@code Hasher} object
|
||||
*/
|
||||
static private Hasher hashFileMetadata( Hasher hasher, Path file, BasicFileAttributes attrs, HashMode mode, Path basePath ) {
|
||||
|
||||
String filename = basePath != null && file.startsWith(basePath)
|
||||
? basePath.relativize(file).toString()
|
||||
: file.toAbsolutePath().toString();
|
||||
|
||||
hasher = hasher.putUnencodedChars( filename );
|
||||
if( attrs != null ) {
|
||||
hasher = hasher.putLong(attrs.size());
|
||||
if( attrs.lastModifiedTime() != null && mode != HashMode.LENIENT ) {
|
||||
hasher = hasher.putLong( attrs.lastModifiedTime().toMillis() );
|
||||
}
|
||||
}
|
||||
|
||||
if( log.isTraceEnabled() ) {
|
||||
log.trace("Hashing file meta: path={}; size={}, lastModified={}, mode={}",
|
||||
file.toAbsolutePath().toString(),
|
||||
attrs!=null ? attrs.size() : "--",
|
||||
attrs!=null && attrs.lastModifiedTime() != null && mode != HashMode.LENIENT ? attrs.lastModifiedTime().toMillis() : "--",
|
||||
mode
|
||||
);
|
||||
}
|
||||
return hasher;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hashes the file by reading file content
|
||||
*
|
||||
* @param hasher The current {@code Hasher} object
|
||||
* @param path file The {@code Path} object to hash
|
||||
* @return The updated {@code Hasher} object
|
||||
*/
|
||||
static private Hasher hashFileContent( Hasher hasher, Path path ) {
|
||||
|
||||
OutputStream output = Funnels.asOutputStream(hasher);
|
||||
try {
|
||||
Files.copy(path, output);
|
||||
}
|
||||
catch( IOException e ) {
|
||||
throw new IllegalStateException("Unable to hash content: " + FilesEx.toUriString(path), e);
|
||||
}
|
||||
finally {
|
||||
FilesEx.closeQuietly(output);
|
||||
}
|
||||
|
||||
return hasher;
|
||||
}
|
||||
|
||||
static HashCode hashContent( Path file ) {
|
||||
return hashContent(file, null);
|
||||
}
|
||||
|
||||
static HashCode hashContent( Path file, HashFunction function ) {
|
||||
|
||||
if( function == null )
|
||||
function = DEFAULT_HASHING;
|
||||
|
||||
Hasher hasher = function.newHasher();
|
||||
return hashFileContent(hasher, file).hash();
|
||||
}
|
||||
|
||||
static private Hasher hashUnorderedCollection(Hasher hasher, Collection collection, HashMode mode) {
|
||||
byte[] resultBytes = new byte[HASH_BYTES];
|
||||
for (Object item : collection) {
|
||||
// hash ghe collection item
|
||||
byte[] nextBytes = hashBytes(item, mode);
|
||||
// sum the hash bytes to the "resultBytes" accumulator
|
||||
// since the sum is a commutative operation the order does not matter
|
||||
sumBytes(resultBytes, nextBytes);
|
||||
}
|
||||
// add the result bytes and return the resulting object
|
||||
return hasher.putBytes(resultBytes);
|
||||
}
|
||||
|
||||
static private byte[] hashBytes(Object item, HashMode mode) {
|
||||
return hasher(defaultHasher(), item, mode).hash().asBytes();
|
||||
}
|
||||
|
||||
/**
|
||||
* Sum two arras of bytes having the same length, required to compute hash of unordered collections.
|
||||
*
|
||||
* - For each byte position, add the corresponding byte from nextBytes into resultBytes
|
||||
* - Order doesn't matter: addition is commutative (a + b = b + a), so the final result is
|
||||
* the same no matter the order of items.
|
||||
* - This is what makes it suitable for unordered collections
|
||||
*
|
||||
* @param resultBytes
|
||||
* The first argument to be summed. This array is used as the accumulator array (i.e. the result)
|
||||
* @param nextBytes
|
||||
* The second argument to be summed.
|
||||
* @return
|
||||
* The array resulting adding the bytes in the second array to the first one. Note,
|
||||
* the result array instance is the same object passed as first argument.
|
||||
*
|
||||
*/
|
||||
static private byte[] sumBytes(byte[] resultBytes, byte[] nextBytes) {
|
||||
if( nextBytes.length != resultBytes.length )
|
||||
throw new IllegalStateException("All hash codes must have the same bit length");
|
||||
for (int i = 0; i < nextBytes.length; i++) {
|
||||
resultBytes[i] += nextBytes[i];
|
||||
}
|
||||
return resultBytes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the argument is an asset file i.e. a file that makes part of the
|
||||
* pipeline Git repository.
|
||||
*
|
||||
* <p>Asset files are hashed using their content (SHA-256) rather than metadata
|
||||
* to maintain cache validity across different clones where timestamps may differ
|
||||
* on remote executors like batch processing systems.
|
||||
*
|
||||
* <p>This method checks two locations:
|
||||
* <ol>
|
||||
* <li>Files under {@code session.getBaseDir()} - the script's working directory</li>
|
||||
* <li>Files under {@code assetRoot} - the repository root (typically ~/.nextflow/assets)</li>
|
||||
* </ol>
|
||||
*
|
||||
* The distinction is important when executing workflows from subdirectories using
|
||||
* the main-script parameter, as repository assets may exist outside the script's
|
||||
* directory but still be part of the repository.
|
||||
*
|
||||
* @param path
|
||||
* The item to check.
|
||||
* @param assetRoot
|
||||
* Location where assets are being stored (the repository root).
|
||||
* @return
|
||||
* {@code true} if the path is included in the pipeline Git repository,
|
||||
* {@code false} otherwise.
|
||||
*
|
||||
* @see <a href="https://github.com/nextflow-io/nextflow/issues/6604">Issue #6604</a>
|
||||
* @see <a href="https://github.com/nextflow-io/nextflow/pull/6605">PR #6605</a>
|
||||
*/
|
||||
static protected boolean isAssetFile(Path path, File assetRoot) {
|
||||
final ISession session = Global.getSession();
|
||||
if( session==null )
|
||||
return false;
|
||||
// if the commit ID is null the current run is not launched from a repo
|
||||
if( session.getCommitId()==null )
|
||||
return false;
|
||||
// if the file belong to different file system, cannot be a file belonging to the repo
|
||||
if( session.getBaseDir().getFileSystem()!=path.getFileSystem() )
|
||||
return false;
|
||||
// Check both the script's base directory and the repository root.
|
||||
// This handles cases where a workflow is executed from a subdirectory
|
||||
// (using the main-script parameter) but references assets elsewhere in the repo.
|
||||
// The assetRoot check ensures these non-sibling assets are still recognized.
|
||||
return path.startsWith(session.getBaseDir()) || path.startsWith(assetRoot.toPath());
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
|
||||
/**
|
||||
* Read a INI file
|
||||
*
|
||||
* See http://stackoverflow.com/a/15638381/395921
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class IniFile {
|
||||
|
||||
private Pattern fSection = Pattern.compile( "\\s*\\[([^]]*)\\]\\s*" );
|
||||
private Pattern fKeyValue = Pattern.compile( "\\s*([^=]*)=(.*)" );
|
||||
private Map<String, Map<String, String>> fEntries = new HashMap<>();
|
||||
|
||||
private Path fFile;
|
||||
|
||||
IniFile() {}
|
||||
|
||||
IniFile( Path path ) throws IOException {
|
||||
load( path );
|
||||
}
|
||||
|
||||
IniFile(String path) {
|
||||
load(path)
|
||||
}
|
||||
|
||||
IniFile load( String path ) {
|
||||
assert path
|
||||
load(path as Path)
|
||||
}
|
||||
|
||||
IniFile load( File file ) {
|
||||
assert file
|
||||
load(file.toPath())
|
||||
}
|
||||
|
||||
IniFile load( Path path, cs = null ) {
|
||||
assert path
|
||||
|
||||
this.fFile = path
|
||||
if( Files.exists(path) ) {
|
||||
final charset = CharsetHelper.getCharset(cs)
|
||||
final reader = Files.newBufferedReader(path, charset)
|
||||
try {
|
||||
load(reader)
|
||||
}
|
||||
finally{
|
||||
reader.close()
|
||||
}
|
||||
}
|
||||
|
||||
return this
|
||||
}
|
||||
|
||||
IniFile load( Reader br ) {
|
||||
assert br
|
||||
|
||||
String line;
|
||||
String section = null;
|
||||
while(( line = br.readLine()) != null ) {
|
||||
Matcher m = fSection.matcher( line );
|
||||
if( m.matches()) {
|
||||
section = m.group( 1 ).trim();
|
||||
}
|
||||
else if( section != null ) {
|
||||
m = fKeyValue.matcher( line );
|
||||
if( m.matches()) {
|
||||
String key = m.group( 1 ).trim();
|
||||
String value = m.group( 2 ).trim();
|
||||
Map< String, String > kv = fEntries.get( section );
|
||||
if( kv == null ) {
|
||||
fEntries.put( section, kv = new HashMap<>());
|
||||
}
|
||||
kv.put( key, value );
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return this
|
||||
}
|
||||
|
||||
String getString( String section, String key, String defValue = null ) {
|
||||
Map< String, String > kv = fEntries.get( section );
|
||||
if( kv == null ) {
|
||||
return defValue;
|
||||
}
|
||||
return kv.get(key) ?: defValue
|
||||
}
|
||||
|
||||
int getInt( String section, String key, int defValue = 0) {
|
||||
Map< String, String > kv = fEntries.get( section );
|
||||
if( kv == null ) {
|
||||
return defValue;
|
||||
}
|
||||
|
||||
kv.containsKey(key) ? Integer.parseInt( kv.get( key )) : defValue;
|
||||
}
|
||||
|
||||
float getFloat( String section, String key, float defValue = 0 ) {
|
||||
Map< String, String > kv = fEntries.get( section );
|
||||
if( kv == null ) {
|
||||
return defValue;
|
||||
}
|
||||
|
||||
kv.containsKey(key) ? Float.parseFloat( kv.get( key )) : defValue
|
||||
}
|
||||
|
||||
double getDouble( String section, String key, double defValue = 0 ) {
|
||||
Map< String, String > kv = fEntries.get( section );
|
||||
if( kv == null ) {
|
||||
return defValue;
|
||||
}
|
||||
|
||||
kv.containsKey(key) ? Double.parseDouble( kv.get( key )) : defValue
|
||||
}
|
||||
|
||||
boolean getBool( String section, String key, boolean defValue = false ) {
|
||||
Map< String, String > kv = fEntries.get( section );
|
||||
if( kv == null ) {
|
||||
return defValue;
|
||||
}
|
||||
|
||||
kv.containsKey(key) ? Boolean.parseBoolean( kv.get( key )) : defValue
|
||||
}
|
||||
|
||||
Map<String,String> section(String section) {
|
||||
def result = fEntries.get(section)
|
||||
return result != null ? Collections.unmodifiableMap(result) : Collections.<String,String>emptyMap()
|
||||
}
|
||||
|
||||
def propertyMissing(String name) {
|
||||
if( fEntries.containsKey(name) )
|
||||
return section(name)
|
||||
|
||||
throw new MissingPropertyException(name,IniFile)
|
||||
}
|
||||
|
||||
def getFile() {
|
||||
return fFile
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
|
||||
/**
|
||||
* A object deserializer that allows you to specify a custom class loader
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class InputStreamDeserializer extends ObjectInputStream {
|
||||
|
||||
ClassLoader loader
|
||||
|
||||
InputStreamDeserializer( InputStream input, ClassLoader loader ) {
|
||||
super(input)
|
||||
this.loader = loader
|
||||
}
|
||||
|
||||
protected Class<?> resolveClass(ObjectStreamClass desc)
|
||||
throws IOException, ClassNotFoundException
|
||||
{
|
||||
try {
|
||||
return super.resolveClass(desc)
|
||||
}
|
||||
catch( ClassNotFoundException e ) {
|
||||
log.debug "Trying to load class: ${desc.getName()}"
|
||||
return Class.forName( desc.getName(), false, loader )
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Deserialize an object given its bytes representation and a custom class-loader
|
||||
*
|
||||
* @param bytes
|
||||
* @param loader
|
||||
* @return
|
||||
*/
|
||||
static <T> T deserialize( byte[] bytes, ClassLoader loader ) {
|
||||
assert bytes
|
||||
|
||||
def buffer = new ByteArrayInputStream(bytes)
|
||||
(T)new InputStreamDeserializer(buffer, loader).readObject()
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
|
||||
/**
|
||||
* A {@link Map} that handles keys in a case insensitive manner
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class InsensitiveMap<K,V> implements Map<K,V> {
|
||||
|
||||
@Delegate
|
||||
private Map<K,V> target
|
||||
|
||||
private InsensitiveMap(Map<K,V> map) {
|
||||
this.target = map
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean containsKey(Object key) {
|
||||
target.any( it -> key?.toString()?.toLowerCase() == it.key?.toString()?.toLowerCase())
|
||||
}
|
||||
|
||||
@Override
|
||||
V get(Object key) {
|
||||
target.find(it -> key?.toString()?.toLowerCase() == it.key?.toString()?.toLowerCase())?.value
|
||||
}
|
||||
|
||||
static <K,V> Map<K,V> of(Map<K,V> target) {
|
||||
new InsensitiveMap(target)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
|
||||
/**
|
||||
* Math helper functions
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class MathHelper {
|
||||
|
||||
static int ceilDiv(int x, int y) {
|
||||
return -Math.floorDiv(-x,y)
|
||||
}
|
||||
|
||||
static long ceilDiv(long x, long y){
|
||||
return -Math.floorDiv(-x,y)
|
||||
}
|
||||
|
||||
static long ceilDiv(long x, int y){
|
||||
return -Math.floorDiv(-x,y)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util
|
||||
|
||||
import java.text.DecimalFormat
|
||||
import java.text.DecimalFormatSymbols
|
||||
import java.util.regex.Pattern
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import nextflow.script.types.MemoryUnit as IMemoryUnit
|
||||
/**
|
||||
* Represent a memory unit
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
@EqualsAndHashCode(includes = 'size', includeFields = true)
|
||||
class MemoryUnit implements IMemoryUnit, Comparable<MemoryUnit>, Serializable, Cloneable {
|
||||
|
||||
final static public MemoryUnit ZERO = new MemoryUnit(0)
|
||||
|
||||
final static private Pattern FORMAT = ~/([0-9\.]+)\s*(\S)?B?/
|
||||
|
||||
final static public List UNITS = [ "B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB" ]
|
||||
|
||||
private long size
|
||||
|
||||
static final private DecimalFormatSymbols formatSymbols
|
||||
|
||||
static {
|
||||
formatSymbols = new DecimalFormatSymbols()
|
||||
formatSymbols.setDecimalSeparator('.' as char)
|
||||
}
|
||||
|
||||
/**
|
||||
* Default constructor is required by Kryo serializer
|
||||
* Do not remove of use directly
|
||||
*/
|
||||
private MemoryUnit() { this.size=0 }
|
||||
|
||||
/**
|
||||
* Create a memory unit instance
|
||||
*
|
||||
* @param value The number of bytes it represent
|
||||
*/
|
||||
MemoryUnit( long value ) {
|
||||
assert value>=0, "Memory unit cannot be a negative number"
|
||||
this.size = value
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a memory unit instance with the given semantic string
|
||||
*
|
||||
* @param str A string using the following of of the following units: B, KB, MB, GB, TB, PB, EB, ZB
|
||||
*/
|
||||
MemoryUnit( String str ) {
|
||||
|
||||
def matcher = FORMAT.matcher(str)
|
||||
if( !matcher.matches() ) {
|
||||
throw new IllegalArgumentException("Not a valid FileSize value: '$str'")
|
||||
}
|
||||
|
||||
final value = matcher.group(1)
|
||||
final unit = matcher.group(2)?.toUpperCase()
|
||||
|
||||
if ( !unit || unit == "B" ) {
|
||||
size = Long.parseLong(value)
|
||||
}
|
||||
else {
|
||||
int p = UNITS.indexOf(unit)
|
||||
if ( p == -1 ) {
|
||||
// try adding a 'B' specified
|
||||
p = UNITS.indexOf(unit+'B')
|
||||
if( p == -1 ) {
|
||||
throw new IllegalArgumentException("Not a valid file size unit: ${str}")
|
||||
}
|
||||
}
|
||||
|
||||
size = Math.round( Double.parseDouble(value) * Math.pow(1024, p) )
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
long toBytes() {
|
||||
size
|
||||
}
|
||||
|
||||
long getBytes() { size }
|
||||
|
||||
@Override
|
||||
long toKilo() { size >> 10 }
|
||||
|
||||
long getKilo() { size >> 10 }
|
||||
|
||||
@Override
|
||||
long toMega() { size >> 20 }
|
||||
|
||||
long getMega() { size >> 20 }
|
||||
|
||||
@Override
|
||||
long toGiga() { size >> 30 }
|
||||
|
||||
long getGiga() { size >> 30 }
|
||||
|
||||
MemoryUnit plus( MemoryUnit value ) {
|
||||
return value != null ? new MemoryUnit( size + value.size ) : this
|
||||
}
|
||||
|
||||
MemoryUnit minus( MemoryUnit value ) {
|
||||
return value != null ? new MemoryUnit( size - value.size ) : this
|
||||
}
|
||||
|
||||
MemoryUnit multiply( Number value ) {
|
||||
return new MemoryUnit( (long)(size * value) )
|
||||
}
|
||||
|
||||
MemoryUnit div( Number value ) {
|
||||
return new MemoryUnit( Math.round((double)(size / value)) )
|
||||
}
|
||||
|
||||
String toString() {
|
||||
if(size <= 0) {
|
||||
return "0"
|
||||
}
|
||||
|
||||
// see http://stackoverflow.com/questions/2510434/format-bytes-to-kilobytes-megabytes-gigabytes
|
||||
int digitGroups = (int) (Math.log10(size) / Math.log10(1024))
|
||||
final formatter = new DecimalFormat("0.#", formatSymbols)
|
||||
formatter.setGroupingUsed(false)
|
||||
formatter.format(size / Math.pow(1024, digitGroups)) + " " + UNITS[digitGroups]
|
||||
}
|
||||
|
||||
@Override
|
||||
int compareTo(MemoryUnit that) {
|
||||
return this.size <=> that.size
|
||||
}
|
||||
|
||||
static int compareTo(MemoryUnit left, Object right) {
|
||||
assert left
|
||||
|
||||
if( right==null )
|
||||
throw new IllegalArgumentException("Not a valid memory value: null")
|
||||
|
||||
if( right instanceof MemoryUnit )
|
||||
return left <=> (MemoryUnit)right
|
||||
|
||||
if( right instanceof Number )
|
||||
return left.size <=> right.toLong()
|
||||
|
||||
if( right instanceof CharSequence )
|
||||
return left <=> MemoryUnit.of(right.toString())
|
||||
|
||||
throw new IllegalArgumentException("Not a valid memory value: $right")
|
||||
}
|
||||
|
||||
static MemoryUnit of( String value ) {
|
||||
new MemoryUnit(value)
|
||||
}
|
||||
|
||||
static MemoryUnit of( long value ) {
|
||||
new MemoryUnit(value)
|
||||
}
|
||||
|
||||
boolean asBoolean() {
|
||||
return size != 0
|
||||
}
|
||||
|
||||
/**
|
||||
* Function to parse/convert given memory unit
|
||||
*
|
||||
* @param unit String expressing memory unit in bytes, e.g. KB, MB, GB
|
||||
*/
|
||||
@Override
|
||||
long toUnit(String unit){
|
||||
int p = UNITS.indexOf(unit)
|
||||
if (p==-1)
|
||||
throw new IllegalArgumentException("Not a valid memory unit: $unit")
|
||||
return size.intdiv(Math.round(Math.pow(1024,p)))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util
|
||||
|
||||
import java.lang.management.ManagementFactory
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
|
||||
/**
|
||||
* Implements helper methods for {@link Process} objects
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class ProcessHelper {
|
||||
|
||||
static long pid(Process process) {
|
||||
try {
|
||||
// with Java 9 and later pid can be access with `.toHandle().getPid()` method
|
||||
final toHandle = Process.class.getMethod("toHandle")
|
||||
final handle = toHandle.invoke(process)
|
||||
return (Long) Class.forName("java.lang.ProcessHandle").getMethod("pid").invoke(handle)
|
||||
}
|
||||
catch (Exception e) {
|
||||
// ignore it and fallback on next
|
||||
}
|
||||
|
||||
// Fallback for Java6+ on unix. This is known not to work on Windows.
|
||||
try {
|
||||
final pidField = process.class.getDeclaredField("pid")
|
||||
pidField.setAccessible(true)
|
||||
return pidField.getLong(process)
|
||||
}
|
||||
catch(Exception e) {
|
||||
throw new UnsupportedOperationException("Unable to access process pid for class: ${process.getClass().getName()}",e )
|
||||
}
|
||||
}
|
||||
|
||||
static long selfPid() {
|
||||
try {
|
||||
Long.parseLong(ManagementFactory.getRuntimeMXBean().getName().split("@")[0])
|
||||
}
|
||||
catch(Exception e) {
|
||||
throw new UnsupportedOperationException("Unable to find current process pid",e)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util
|
||||
|
||||
|
||||
/**
|
||||
* Split a string into literal tokens i.e. sequence of chars that does not contain a blank
|
||||
* or wrapper by a quote or double quote
|
||||
*
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*
|
||||
*/
|
||||
class QuoteStringTokenizer implements Iterator<String>, Iterable<String> {
|
||||
|
||||
private List<Character> chars = Arrays.<Character>asList(' ' as Character);
|
||||
|
||||
private List<String> tokens = new ArrayList<String>();
|
||||
|
||||
private Iterator<String> itr;
|
||||
|
||||
public QuoteStringTokenizer( String value ) {
|
||||
this(value, ' ' as Character);
|
||||
}
|
||||
|
||||
public QuoteStringTokenizer( String value, char... separators ) {
|
||||
|
||||
if( separators == null || separators.length==0 ) {
|
||||
chars = new ArrayList<Character>(3);
|
||||
chars .add(' ' as Character);
|
||||
}
|
||||
else {
|
||||
chars = new ArrayList<Character>(3);
|
||||
for( char ch : separators ) { chars.add(ch); }
|
||||
}
|
||||
|
||||
chars.add("\"" as char);
|
||||
chars.add("'" as char);
|
||||
|
||||
/* start parsing */
|
||||
parseNext(value != null ? value.trim() : "");
|
||||
|
||||
}
|
||||
|
||||
void parseNext( String value ) {
|
||||
for( int i=0; i<value.length(); i++ ) {
|
||||
|
||||
char ch=value.charAt(i);
|
||||
if( chars.contains(ch) ) {
|
||||
if( i>0 ) {
|
||||
tokens.add(value.substring(0,i));
|
||||
}
|
||||
|
||||
if( ch=='"' as char || ch=='\'' as char ) {
|
||||
parseQuote(value.substring(i+1), ch);
|
||||
}
|
||||
else if( chars.contains(ch) ) {
|
||||
parseNext(value.substring(i+1));
|
||||
}
|
||||
break;
|
||||
}
|
||||
// and of the string
|
||||
else if( i+1 == value.length() ) {
|
||||
tokens.add(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void parseQuote( String value, char delim ) {
|
||||
|
||||
int p = value.indexOf(delim as String);
|
||||
if( p == -1 ) {
|
||||
tokens.add(value);
|
||||
return;
|
||||
}
|
||||
|
||||
tokens.add( value.substring(0,p) );
|
||||
parseNext(value.substring(p+1));
|
||||
}
|
||||
|
||||
public boolean hasNext() {
|
||||
return itr().hasNext();
|
||||
}
|
||||
|
||||
public String next() {
|
||||
return itr().next();
|
||||
}
|
||||
|
||||
public void remove() {
|
||||
throw new UnsupportedOperationException("Remove not supported");
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
return tokens.toString();
|
||||
}
|
||||
|
||||
public Iterator<String> iterator() {
|
||||
return itr();
|
||||
}
|
||||
|
||||
/*
|
||||
* lazy iterator creator
|
||||
*/
|
||||
private Iterator<String> itr() {
|
||||
if( itr == null ) {
|
||||
itr = tokens.iterator();
|
||||
}
|
||||
return itr;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util
|
||||
|
||||
import com.google.common.util.concurrent.RateLimiter
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
|
||||
/**
|
||||
* Model an execution rate limit measure unit
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
@EqualsAndHashCode(includeFields=true,includes='rate')
|
||||
class RateUnit {
|
||||
|
||||
private static String RATE_FORMAT = ~/^(\d+\.?\d*)\s*([a-zA-Z]*)/
|
||||
|
||||
private double rate
|
||||
|
||||
static RateUnit of(String str) { new RateUnit(str) }
|
||||
|
||||
/**
|
||||
* Create a rate unit by a double value representing the number of events per second
|
||||
*
|
||||
* @param rate The rate unit as a double value
|
||||
*/
|
||||
RateUnit(double rate) {
|
||||
assert rate>0, "Rate unit must be greater than zero"
|
||||
this.rate = rate
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a rate unit using a string annotation as `10 sec` ie. 10 per second
|
||||
* or `100 / 5 min` 100 events each 5 minutes
|
||||
*
|
||||
* @param str The rate string
|
||||
*/
|
||||
RateUnit(String str) {
|
||||
this.rate = parse(str)
|
||||
}
|
||||
|
||||
/**
|
||||
* @return The actual rate unit as a double
|
||||
*/
|
||||
double getRate() { rate }
|
||||
|
||||
/**
|
||||
* Create a {@link RateLimiter} object for rate unit
|
||||
*
|
||||
* @return An instance of {@link RateLimiter}
|
||||
*/
|
||||
RateLimiter getRateLimiter() { RateLimiter.create(rate) }
|
||||
|
||||
protected double parse(String limit) {
|
||||
|
||||
def tokens = limit.tokenize('/')
|
||||
if( tokens.size() == 2 ) {
|
||||
/*
|
||||
* the rate limit is provide num of task over a duration
|
||||
* - eg. 100 / 5 min
|
||||
* - ie. max 100 task per 5 minutes
|
||||
*/
|
||||
|
||||
final X = tokens[0].trim()
|
||||
final Y = tokens[1].trim()
|
||||
|
||||
return parse0(X, Y, limit)
|
||||
}
|
||||
|
||||
/*
|
||||
* the rate limit is provide as a duration
|
||||
* - eg. 200 min
|
||||
* - ie. max 200 task per minutes
|
||||
*/
|
||||
|
||||
final matcher = (limit =~ RATE_FORMAT)
|
||||
if( !matcher.matches() )
|
||||
throw new IllegalArgumentException("Invalid submit-rate-limit value: $limit -- It must be formatted as `num request sec|min|hour` eg. 10 sec ie. max 10 tasks per second")
|
||||
|
||||
final num = matcher.group(1) ?: '_'
|
||||
final unit = matcher.group(2) ?: 'sec'
|
||||
|
||||
return parse0(num, "1 $unit", limit)
|
||||
}
|
||||
|
||||
|
||||
private double parse0(String X, String Y, String limit ) {
|
||||
if( !X.isInteger() )
|
||||
throw new IllegalArgumentException("Invalid submit-rate-limit value: $limit -- It must be formatted as `num request / duration` eg. 10/1s")
|
||||
|
||||
final num = Integer.parseInt(X)
|
||||
final duration = Y.isInteger() ? Duration.of( Y+'sec' ) : ( Y[0].isInteger() ? Duration.of(Y) : Duration.of('1'+Y) )
|
||||
long seconds = duration.toSeconds()
|
||||
if( !seconds )
|
||||
throw new IllegalArgumentException("Invalid submit-rate-limit value: $limit -- The interval must be at least 1 second")
|
||||
|
||||
num / seconds as double
|
||||
}
|
||||
|
||||
String toString() {
|
||||
String.format(Locale.ROOT, "%.2f", rate) + '/sec'
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
/*
|
||||
* Copyright 2013-2025, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import nextflow.script.types.Record;
|
||||
import org.codehaus.groovy.runtime.DefaultGroovyMethods;
|
||||
|
||||
/**
|
||||
* Implements Record as an immutable map.
|
||||
*
|
||||
* @author Ben Sherman <bentshermann@gmail.com>
|
||||
*/
|
||||
public class RecordMap extends LinkedHashMap<String,Object> implements Record {
|
||||
|
||||
public RecordMap() {}
|
||||
|
||||
public RecordMap(Map<String,Object> props) {
|
||||
super(props);
|
||||
}
|
||||
|
||||
public RecordMap(int initialCapacity) {
|
||||
super(initialCapacity);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void clear() {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object put(String key, Object value) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void putAll(Map<? extends String, ? extends Object> m) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object remove(Object key) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
public Record subMap(Collection<String> keys) {
|
||||
return new RecordMap(DefaultGroovyMethods.subMap(this, keys));
|
||||
}
|
||||
|
||||
public RecordMap plus(RecordMap other) {
|
||||
var result = new LinkedHashMap<String,Object>();
|
||||
result.putAll(this);
|
||||
result.putAll(other);
|
||||
return new RecordMap(result);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util
|
||||
|
||||
import com.google.common.base.CaseFormat
|
||||
import groovy.transform.CompileDynamic
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import groovy.transform.Memoized
|
||||
import groovy.transform.ToString
|
||||
import groovy.util.logging.Slf4j
|
||||
import io.seqera.util.retry.Retryable
|
||||
import nextflow.Global
|
||||
import nextflow.SysEnv
|
||||
/**
|
||||
* Models retry policy configuration
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@ToString(includePackage = false, includeNames = true)
|
||||
@EqualsAndHashCode
|
||||
@CompileStatic
|
||||
class RetryConfig implements Retryable.Config {
|
||||
|
||||
public static final Duration DEFAULT_DELAY = Duration.of('350ms')
|
||||
public static final Duration DEFAULT_MAX_DELAY = Duration.of('90s')
|
||||
public static final Integer DEFAULT_MAX_ATTEMPTS = 5
|
||||
public static final Double DEFAULT_JITTER = 0.25
|
||||
public static final double DEFAULT_MULTIPLIER = 2.0
|
||||
|
||||
private final static String ENV_PREFIX = 'NXF_RETRY_POLICY_'
|
||||
|
||||
final private Duration delay
|
||||
final private Duration maxDelay
|
||||
final private int maxAttempts
|
||||
final private double jitter
|
||||
final private double multiplier
|
||||
|
||||
RetryConfig() {
|
||||
this(Collections.emptyMap())
|
||||
}
|
||||
|
||||
RetryConfig(Map config) {
|
||||
delay =
|
||||
valueOf(config, 'delay', ENV_PREFIX, DEFAULT_DELAY, Duration)
|
||||
maxDelay =
|
||||
valueOf(config, 'maxDelay', ENV_PREFIX, DEFAULT_MAX_DELAY, Duration)
|
||||
maxAttempts =
|
||||
valueOf(config, 'maxAttempts', ENV_PREFIX, DEFAULT_MAX_ATTEMPTS, Integer)
|
||||
jitter =
|
||||
valueOf(config, 'jitter', ENV_PREFIX, DEFAULT_JITTER, Double)
|
||||
multiplier =
|
||||
valueOf(config, 'multiplier', ENV_PREFIX, DEFAULT_MULTIPLIER, Double)
|
||||
}
|
||||
|
||||
Duration getDelay() {
|
||||
return delay
|
||||
}
|
||||
|
||||
Duration getMaxDelay() {
|
||||
return maxDelay
|
||||
}
|
||||
|
||||
@Override
|
||||
int getMaxAttempts() {
|
||||
return maxAttempts
|
||||
}
|
||||
|
||||
@Override
|
||||
double getJitter() {
|
||||
return jitter
|
||||
}
|
||||
|
||||
@Override
|
||||
double getMultiplier() {
|
||||
return multiplier
|
||||
}
|
||||
|
||||
static RetryConfig config() {
|
||||
config(Global.config)
|
||||
}
|
||||
|
||||
@Memoized
|
||||
static RetryConfig config(Map config) {
|
||||
if( config==null ) {
|
||||
log.debug "Missing nextflow session - using default retry config"
|
||||
config = Collections.emptyMap()
|
||||
}
|
||||
|
||||
return new RetryConfig(getNestedConfig(config, 'nextflow', 'retryPolicy') ?: Collections.emptyMap())
|
||||
}
|
||||
|
||||
private static Map getNestedConfig(Map config, String... keys) {
|
||||
def current = config
|
||||
for (String key : keys) {
|
||||
if (current instanceof Map && current.containsKey(key)) {
|
||||
current = current.get(key)
|
||||
} else {
|
||||
return null
|
||||
}
|
||||
}
|
||||
return current instanceof Map ? current : null
|
||||
}
|
||||
|
||||
static <T> T valueOf(Map config, String name, String prefix, T defValue, Class<T> type) {
|
||||
assert name, "Argument 'name' cannot be null or empty"
|
||||
assert type, "Argument 'type' cannot be null"
|
||||
|
||||
// try to get the value from the config map
|
||||
final cfg = config?.get(name)
|
||||
if( cfg != null ) {
|
||||
return toType(cfg, type)
|
||||
}
|
||||
// try to fallback to the sys environment
|
||||
if( !prefix.endsWith('_') )
|
||||
prefix += '_'
|
||||
final key = prefix.toUpperCase() + CaseFormat.LOWER_CAMEL.to(CaseFormat.UPPER_UNDERSCORE, name)
|
||||
final env = SysEnv.get(key)
|
||||
if( env != null ) {
|
||||
return toType(env, type)
|
||||
}
|
||||
// return the default value
|
||||
return defValue
|
||||
}
|
||||
|
||||
@CompileDynamic
|
||||
static protected <T> T toType(Object value, Class<T> type) {
|
||||
if( value == null )
|
||||
return null
|
||||
if( type==Boolean.class ) {
|
||||
return type.cast(Boolean.valueOf(value.toString()))
|
||||
}
|
||||
else {
|
||||
return value.asType(type)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util
|
||||
|
||||
import java.security.SecureRandom
|
||||
|
||||
/**
|
||||
* Rnd key generator
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class Rnd {
|
||||
|
||||
private static final BigInteger B = BigInteger.ONE.shiftLeft(80); // 2^64
|
||||
|
||||
private static final SecureRandom random = new SecureRandom();
|
||||
|
||||
private Rnd() {}
|
||||
|
||||
/**
|
||||
* Generate random key base62 encoded ie. containing only [0-9a-zA-Z] characters
|
||||
* guaranteed to be unique.
|
||||
*
|
||||
* Tested with 100 mln iteration -> 0 collision
|
||||
*
|
||||
* @return A random generated alphanumeric between 9-14 characters
|
||||
*/
|
||||
static String key() {
|
||||
byte[] buffer = new byte[10]
|
||||
random.nextBytes(buffer)
|
||||
def big = new BigInteger(buffer)
|
||||
if (big.signum() < 0) {
|
||||
big = big.add(B)
|
||||
}
|
||||
|
||||
return Base62.encode(big)
|
||||
}
|
||||
|
||||
static String hex() {
|
||||
byte[] buffer = new byte[10]
|
||||
random.nextBytes(buffer)
|
||||
return buffer.encodeHex()
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import org.pf4j.ExtensionPoint
|
||||
|
||||
/**
|
||||
* Register a serializer class in the Kryo serializers list
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
interface SerializerRegistrant extends ExtensionPoint {
|
||||
|
||||
/**
|
||||
* Serializer should implement this method adding the
|
||||
* serialized class and the serializer class to the
|
||||
* map passed as argument
|
||||
*
|
||||
* @param
|
||||
* serializers The serializer map, where the key element
|
||||
* represent the class to be serialized and the value
|
||||
* the serialization object
|
||||
*/
|
||||
void register(Map<Class,Object> serializers)
|
||||
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
|
||||
/**
|
||||
* A service loader inspired to {@link ServiceLoader}
|
||||
* that allows to load only service names or classes without instantiating
|
||||
* the actual instances
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class ServiceDiscover<S> {
|
||||
|
||||
private static final String PREFIX = "META-INF/services/";
|
||||
|
||||
private Class<S> service
|
||||
|
||||
private ClassLoader classLoader
|
||||
|
||||
static <S> List<Class<S>> load(Class<S> service) {
|
||||
new ServiceDiscover<S>(service).load()
|
||||
}
|
||||
|
||||
ServiceDiscover(Class<S> service) {
|
||||
assert service
|
||||
this.service = service
|
||||
this.classLoader = Thread.currentThread().getContextClassLoader()
|
||||
}
|
||||
|
||||
ServiceDiscover(Class<S> service, ClassLoader classLoader) {
|
||||
assert service
|
||||
assert classLoader
|
||||
this.service = service
|
||||
this.classLoader = classLoader
|
||||
}
|
||||
|
||||
private int parseLine(Class service, URL u, BufferedReader r, int lc, List<String> names)
|
||||
throws IOException, ServiceConfigurationError
|
||||
{
|
||||
String ln = r.readLine();
|
||||
if (ln == null) {
|
||||
return -1
|
||||
}
|
||||
|
||||
int ci = ln.indexOf('#')
|
||||
if (ci >= 0)
|
||||
ln = ln.substring(0, ci)
|
||||
|
||||
ln = ln.trim()
|
||||
int n = ln.length()
|
||||
|
||||
if (n != 0) {
|
||||
if ((ln.indexOf(' ') >= 0) || (ln.indexOf('\t') >= 0))
|
||||
fail(service, u, lc, "Illegal configuration file syntax")
|
||||
|
||||
int cp = ln.codePointAt(0)
|
||||
if (!Character.isJavaIdentifierStart(cp))
|
||||
fail(service, u, lc, "Illegal provider-class name: " + ln)
|
||||
|
||||
for (int i = Character.charCount(cp); i < n; i += Character.charCount(cp)) {
|
||||
cp = ln.codePointAt(i)
|
||||
if (!Character.isJavaIdentifierPart(cp) && (cp != '.'))
|
||||
fail(service, u, lc, "Illegal provider-class name: " + ln)
|
||||
}
|
||||
|
||||
if ( !names.contains(ln) )
|
||||
names.add(ln)
|
||||
}
|
||||
|
||||
return lc + 1
|
||||
}
|
||||
|
||||
|
||||
private List<String> parse(Class service, URL u) throws ServiceConfigurationError
|
||||
{
|
||||
InputStream stream = null;
|
||||
BufferedReader reader = null;
|
||||
ArrayList<String> names = new ArrayList<>();
|
||||
try {
|
||||
stream = u.openStream();
|
||||
reader = new BufferedReader(new InputStreamReader(stream, "utf-8"));
|
||||
int lc = 1;
|
||||
while ((lc = parseLine(service, u, reader, lc, names)) >= 0) { /*empty*/ }
|
||||
}
|
||||
catch (IOException x) {
|
||||
fail(service, "Error reading configuration file", x);
|
||||
}
|
||||
finally {
|
||||
try {
|
||||
if (reader != null) reader.close();
|
||||
if (stream != null) stream.close();
|
||||
}
|
||||
catch (IOException y) {
|
||||
fail(service, "Error closing configuration file", y);
|
||||
}
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
private static void fail(Class service, String msg, Throwable cause)
|
||||
throws ServiceConfigurationError
|
||||
{
|
||||
throw new ServiceConfigurationError(service.getName() + ": " + msg,
|
||||
cause);
|
||||
}
|
||||
|
||||
private static void fail(Class service, String msg)
|
||||
throws ServiceConfigurationError
|
||||
{
|
||||
throw new ServiceConfigurationError(service.getName() + ": " + msg);
|
||||
}
|
||||
|
||||
private static void fail(Class service, URL u, int line, String msg)
|
||||
throws ServiceConfigurationError
|
||||
{
|
||||
fail(service, u.toString() + ":" + line + ": " + msg);
|
||||
}
|
||||
|
||||
private Class<S> classForName(String clazzName) {
|
||||
|
||||
Class<?> result = null
|
||||
try {
|
||||
result = Class.forName(clazzName, false, classLoader);
|
||||
}
|
||||
catch (ClassNotFoundException x) {
|
||||
fail(service, "Provider $clazzName not found");
|
||||
}
|
||||
if (!service.isAssignableFrom(result)) {
|
||||
fail(service, "Provider $clazzName not a subtype");
|
||||
}
|
||||
try {
|
||||
return (Class<S>)result;
|
||||
}
|
||||
catch (Throwable x) {
|
||||
fail(service, "Provider $clazzName could not be instantiated: " + x, x);
|
||||
}
|
||||
throw new Error()
|
||||
}
|
||||
|
||||
List<String> names() {
|
||||
|
||||
Enumeration<URL> configs = null
|
||||
|
||||
try {
|
||||
configs = classLoader.getResources(PREFIX + service.getName())
|
||||
|
||||
} catch (IOException x) {
|
||||
fail(service, "Error locating configuration files", x);
|
||||
}
|
||||
|
||||
List<String> result = []
|
||||
for( URL url : configs ) {
|
||||
for( String name : parse(service, url) ) {
|
||||
result << name
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
List<Class<S>> load() {
|
||||
names().collect{ classForName(it) }
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util
|
||||
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
import com.google.common.net.InetAddresses
|
||||
import groovy.transform.CompileStatic
|
||||
|
||||
/**
|
||||
* String helper routines
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class StringUtils {
|
||||
|
||||
static final public Pattern URL_PROTOCOL = ~/^([a-zA-Z0-9]*):\\/\\/(.+)/
|
||||
static final private Pattern URL_PASSWORD = ~/^[a-zA-Z][a-zA-Z0-9]*:\\/\\/(.+)@.+/
|
||||
|
||||
/**
|
||||
* Deprecated. Use {@link nextflow.file.FileHelper#getUrlProtocol(java.lang.String)} instead
|
||||
*/
|
||||
@Deprecated
|
||||
static String getUrlProtocol(String str) {
|
||||
final m = URL_PROTOCOL.matcher(str)
|
||||
return m.matches() ? m.group(1) : null
|
||||
}
|
||||
|
||||
static final private Pattern BASE_URL = ~/(?i)((?:[a-z][a-zA-Z0-9]*)?:\/\/[^:|\/]+(?::\d*)?)(?:$|\/.*)/
|
||||
|
||||
/**
|
||||
* Deprecated. Use {@link nextflow.file.FileHelper#baseUrl(java.lang.String)} instead
|
||||
*/
|
||||
@Deprecated
|
||||
static String baseUrl(String url) {
|
||||
if( !url )
|
||||
return null
|
||||
final m = BASE_URL.matcher(url)
|
||||
return m.matches() ? m.group(1).toLowerCase() : null
|
||||
}
|
||||
|
||||
static private Pattern multilinePattern = ~/["']?(password|token|secret|license)["']?\s?[:=]\s?["']?(\w+)["']?/
|
||||
|
||||
static String stripSecrets(String message) {
|
||||
if (message == null) {
|
||||
return message
|
||||
}
|
||||
StringBuilder sb = new StringBuilder(message)
|
||||
Matcher matcher = multilinePattern.matcher(sb)
|
||||
while (matcher.find()) {
|
||||
for(int idx=0; idx<matcher.groupCount(); idx+=2){
|
||||
int ini = matcher.start(idx+2)
|
||||
int end = matcher.end(idx+2)
|
||||
sb.delete(ini, end)
|
||||
sb.insert(ini, '********')
|
||||
}
|
||||
matcher.reset()
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
static private boolean isSensitive(Object key) {
|
||||
final str = key.toString().toLowerCase()
|
||||
return str.contains('password') \
|
||||
|| str.contains('token') \
|
||||
|| str.contains('secret') \
|
||||
|| str.contains('license') \
|
||||
|| str.contains('auth')
|
||||
}
|
||||
|
||||
|
||||
static Map stripSecrets(Map map) {
|
||||
if( map==null )
|
||||
return null
|
||||
final copy = new HashMap(map.size())
|
||||
for( Map.Entry entry : map.entrySet() ) {
|
||||
if( entry.value instanceof Map ) {
|
||||
copy.put( entry.key, stripSecrets((Map)entry.value))
|
||||
}
|
||||
else if( isSensitive(entry.key) ) {
|
||||
copy.put(entry.key, redact(entry.value))
|
||||
}
|
||||
else {
|
||||
copy.put(entry.key, redactUrlPassword(entry.value))
|
||||
}
|
||||
}
|
||||
return copy
|
||||
}
|
||||
|
||||
static String redact(Object value) {
|
||||
if( value==null )
|
||||
return '(null)'
|
||||
if( !value )
|
||||
return '(empty)'
|
||||
final str = value.toString()
|
||||
return str.length()>=10 ? str[0..2] + '****' : '****'
|
||||
}
|
||||
|
||||
static String redactUrlPassword(value) {
|
||||
final str = value.toString()
|
||||
final m = URL_PASSWORD.matcher(str)
|
||||
if( m.matches() ) {
|
||||
return replaceGroup(m, str, 1, redact(m.group(1)))
|
||||
}
|
||||
return str
|
||||
}
|
||||
|
||||
static String replaceGroup(Matcher matcher, String source, int groupToReplace, String replacement) {
|
||||
return new StringBuilder(source)
|
||||
.replace(matcher.start(groupToReplace), matcher.end(groupToReplace), replacement)
|
||||
.toString()
|
||||
}
|
||||
|
||||
static boolean isIpV6String(String address) {
|
||||
if( !address || !address.contains(':') )
|
||||
return false
|
||||
try {
|
||||
InetAddresses.forString(address).getAddress().length==16
|
||||
}
|
||||
catch (IllegalArgumentException e) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
static String formatHostName(String host, String port) {
|
||||
if( !port || !host )
|
||||
return host
|
||||
final ipv6 = isIpV6String(host)
|
||||
return ipv6 ? "[$host]:$port" : "$host:$port"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util
|
||||
|
||||
|
||||
import java.lang.management.ManagementFactory
|
||||
import java.text.SimpleDateFormat
|
||||
import java.time.OffsetDateTime
|
||||
|
||||
import com.sun.management.OperatingSystemMXBean
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.Memoized
|
||||
import groovy.transform.PackageScope
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.SysEnv
|
||||
import nextflow.file.FileHelper
|
||||
/**
|
||||
* System helper methods
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class SysHelper {
|
||||
|
||||
public static final String DEFAULT_DOCKER_PLATFORM = 'linux/amd64'
|
||||
|
||||
private static final String DATE_FORMAT = 'dd-MMM-yyyy HH:mm:ss'
|
||||
|
||||
/**
|
||||
* Given a timestamp as epoch time convert to a string representation
|
||||
* using the {@link #DATE_FORMAT}
|
||||
*
|
||||
* @param dateInMillis
|
||||
* The date as number of milliseconds from 1 Jan 1970
|
||||
* @param tz
|
||||
* The {@link TimeZone} to use to format the date, or {@code null} to use the default time-zone
|
||||
* @return
|
||||
* The formatted date string
|
||||
*/
|
||||
static String fmtDate(long dateInMillis, TimeZone tz=null) {
|
||||
fmtDate(new Date(dateInMillis), tz)
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a {@link Date} object convert to a string representation
|
||||
* according the {@link #DATE_FORMAT}
|
||||
*
|
||||
* @param date
|
||||
* The date to render as a string
|
||||
* @param tz
|
||||
* The {@link TimeZone} to use to format the date, or {@code null} to use the default time-zone
|
||||
* @return
|
||||
* The formatted date string
|
||||
*/
|
||||
static String fmtDate(Date date, TimeZone tz=null) {
|
||||
final formatter=new SimpleDateFormat(fmtEnv(), Locale.ENGLISH)
|
||||
if(tz) formatter.setTimeZone(tz)
|
||||
formatter.format(date)
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a {@link java.time.OffsetDateTime} object convert to a string representation
|
||||
* according the {@link #DATE_FORMAT}
|
||||
*
|
||||
* @param dateTime
|
||||
* The OffsetDateTime to render as a string
|
||||
* @return
|
||||
* The formatted date string
|
||||
*/
|
||||
static String fmtDate(OffsetDateTime dateTime) {
|
||||
// Convert to Date while preserving the original timezone
|
||||
final date = Date.from(dateTime.toInstant())
|
||||
final timezone = TimeZone.getTimeZone(dateTime.getOffset())
|
||||
fmtDate(date, timezone)
|
||||
}
|
||||
|
||||
static private String fmtEnv() {
|
||||
final result = SysEnv.get('NXF_DATE_FORMAT', DATE_FORMAT)
|
||||
return result.toLowerCase() == 'iso'
|
||||
? "yyyy-MM-dd'T'HH:mm:ssXXX"
|
||||
: result
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the system uptime as returned by the {@code uptime} Linux tool
|
||||
*
|
||||
* @return The {@code uptime} stdout text
|
||||
* @throws IllegalStateException If the {@code uptime} command return a non-zero exit status
|
||||
*/
|
||||
static String getUptimeText() throws IllegalStateException {
|
||||
|
||||
def proc = new ProcessBuilder('uptime').start()
|
||||
def status = proc.waitFor()
|
||||
if( status == 0 ) {
|
||||
return proc.text.trim()
|
||||
}
|
||||
|
||||
throw new IllegalStateException("Unable to run system `uptime` command: exit=$status")
|
||||
}
|
||||
|
||||
/**
|
||||
* The system uptime
|
||||
*
|
||||
* @return A {@link Duration} object representing the uptime of the system
|
||||
*/
|
||||
static Duration getUptimeDuration() {
|
||||
def text = getUptimeText()
|
||||
def result = parseUptimeText(text)
|
||||
log.trace "Uptime $result -- parsed text: $text"
|
||||
return result
|
||||
}
|
||||
|
||||
static long getBootTimeMillis() {
|
||||
System.currentTimeMillis() - getUptimeDuration().toMillis()
|
||||
}
|
||||
|
||||
static String getBootTimeString() {
|
||||
fmtDate(new Date(getBootTimeMillis()))
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse `uptime` command line tool output
|
||||
*
|
||||
* @param str The text stdout produced by the {@code uptime} system tool
|
||||
* @return
|
||||
* @throws IllegalArgumentException
|
||||
*/
|
||||
@PackageScope
|
||||
static Duration parseUptimeText( String str ) throws IllegalArgumentException {
|
||||
try {
|
||||
def p = str.indexOf(' up ')
|
||||
def items = str.substring(p+4).tokenize(',')
|
||||
def uptime = items[0].trim().replace('mins','min').replace('hrs','hour')
|
||||
if( uptime.contains(':') ) {
|
||||
uptime = reformatHourAndMin(uptime)
|
||||
}
|
||||
else if( items.size()>1 && items[1].contains(":")) {
|
||||
uptime += " ${reformatHourAndMin(items[1])}"
|
||||
}
|
||||
return Duration.of(uptime)
|
||||
|
||||
}
|
||||
catch( Exception e ) {
|
||||
throw new IllegalArgumentException("Not a valid uptime text: `$str`", e)
|
||||
}
|
||||
}
|
||||
|
||||
private static String reformatHourAndMin(String text) {
|
||||
def items = text.tokenize(':')
|
||||
"${items[0]}h ${items[1]}m"
|
||||
}
|
||||
|
||||
/**
|
||||
* @return The actual free space in the node local storage
|
||||
*/
|
||||
static MemoryUnit getAvailDisk() {
|
||||
final free = FileHelper.getLocalTempPath().toFile().getFreeSpace()
|
||||
new MemoryUnit(free)
|
||||
}
|
||||
|
||||
@Memoized
|
||||
static private OperatingSystemMXBean getSystemMXBean() {
|
||||
(OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean()
|
||||
}
|
||||
|
||||
/**
|
||||
* @return The number of CPUs available
|
||||
*/
|
||||
static int getAvailCpus() {
|
||||
final int result = getSystemMXBean().getAvailableProcessors()
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* @return The total system memory available
|
||||
*/
|
||||
static MemoryUnit getAvailMemory() {
|
||||
new MemoryUnit(getSystemMXBean().getTotalPhysicalMemorySize())
|
||||
}
|
||||
|
||||
/**
|
||||
* @return The system host name
|
||||
*/
|
||||
static String getHostName() {
|
||||
System.getenv('HOSTNAME') ?: InetAddress.getLocalHost().getHostName()
|
||||
}
|
||||
|
||||
static String getArch() {
|
||||
final os = (java.lang.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean()
|
||||
return os.getArch()
|
||||
}
|
||||
|
||||
/**
|
||||
* Dump th stack trace of current running threads
|
||||
* @return
|
||||
*/
|
||||
static String dumpThreads() {
|
||||
|
||||
def buffer = new StringBuffer()
|
||||
Map<Thread, StackTraceElement[]> m = Thread.getAllStackTraces();
|
||||
for(Map.Entry<Thread, StackTraceElement[]> e : m.entrySet()) {
|
||||
buffer.append('\n').append(e.getKey().toString()).append('\n')
|
||||
for (StackTraceElement s : e.getValue()) {
|
||||
buffer.append(" " + s).append('\n')
|
||||
}
|
||||
}
|
||||
|
||||
return buffer.toString()
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util
|
||||
|
||||
import static java.lang.annotation.ElementType.*
|
||||
|
||||
import java.lang.annotation.Retention
|
||||
import java.lang.annotation.RetentionPolicy
|
||||
import java.lang.annotation.Target
|
||||
|
||||
/**
|
||||
* Marker interface to annotate types and methods that are meant to be used only
|
||||
* for testing purpose.
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Retention(RetentionPolicy.SOURCE)
|
||||
@Target(value=[CONSTRUCTOR, FIELD, LOCAL_VARIABLE, METHOD, PACKAGE, PARAMETER, TYPE])
|
||||
@interface TestOnly {
|
||||
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.SysEnv
|
||||
/**
|
||||
* Helper class for threads handling
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class Threads {
|
||||
|
||||
static boolean useVirtual() {
|
||||
SysEnv.get('NXF_ENABLE_VIRTUAL_THREADS')=='true'
|
||||
}
|
||||
|
||||
static Thread start(Closure action) {
|
||||
return useVirtual()
|
||||
? Thread.startVirtualThread(action)
|
||||
: Thread.startDaemon(action)
|
||||
}
|
||||
|
||||
static Thread start(String name, Closure action) {
|
||||
if( !useVirtual() )
|
||||
return Thread.startDaemon(name, action)
|
||||
// create a new virtual thread and change the name
|
||||
final result = Thread.startVirtualThread(action)
|
||||
result.setName(name)
|
||||
return result
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
|
||||
/**
|
||||
* Limit the execution of a code block on a specified time basis
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class Throttle {
|
||||
|
||||
static final Map<Integer,ThrottleObj> throttleMap = new ConcurrentHashMap<>()
|
||||
|
||||
@EqualsAndHashCode
|
||||
static class ThrottleObj {
|
||||
Object result
|
||||
long timestamp
|
||||
|
||||
ThrottleObj() {}
|
||||
|
||||
ThrottleObj( value, long timestamp ) {
|
||||
this.result = value
|
||||
this.timestamp = timestamp
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes the specified code block and returns the resulting value. All following
|
||||
* invocations within the specified time {@code period} are skipped and the cached
|
||||
* result value is returned instead.
|
||||
*
|
||||
* @param period
|
||||
* The time period in milliseconds, in which the throttler will allow at most to invoke
|
||||
* the specified closure
|
||||
* @param closure
|
||||
* The code block to execute
|
||||
* @return
|
||||
* The value returned by the closure execution
|
||||
*/
|
||||
static Object every( long period, Closure closure ) {
|
||||
throttle0( period, null, closure)
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes the specified code block and returns the resulting value. All following
|
||||
* invocations within the specified time {@code period} are skipped and the cached
|
||||
* result value is returned instead.
|
||||
*
|
||||
* @param period
|
||||
* The time duration, in which the throttler will allow at most to invoke
|
||||
* the specified closure
|
||||
* @param closure
|
||||
* The code block to execute
|
||||
* @return
|
||||
* The value returned by the closure execution
|
||||
*/
|
||||
static Object every( Duration period, Closure closure ) {
|
||||
throttle0( period.millis, null, closure)
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes the specified code block and returns the resulting value. All following
|
||||
* invocations within the specified time {@code period} are skipped and the cached
|
||||
* result value is returned instead.
|
||||
*
|
||||
* @param period
|
||||
* The duration string, in which the throttler will allow at most to invoke
|
||||
* the specified closure
|
||||
* @param closure
|
||||
* The code block to execute
|
||||
* @return
|
||||
* The value returned by the closure execution
|
||||
*/
|
||||
static Object every( String period, Closure closure ) {
|
||||
throttle0( Duration.of(period).millis, null, closure)
|
||||
}
|
||||
|
||||
/**
|
||||
* Allows at most one execution after the specified time {@code delay} returning
|
||||
* {@code null} before the first execution, and the cached result value the
|
||||
* following times
|
||||
*
|
||||
* @param delay
|
||||
* @param closure
|
||||
* @return
|
||||
*/
|
||||
static Object after( long delay, Closure closure ) {
|
||||
final obj = new ThrottleObj( null, System.currentTimeMillis() )
|
||||
throttle0( delay, obj, closure)
|
||||
}
|
||||
|
||||
/**
|
||||
* Allows at most one execution after the specified time {@code delay} returning
|
||||
* {@code null} before the first execution, and the cached result value the
|
||||
* following times
|
||||
*
|
||||
* @param delay
|
||||
* @param closure
|
||||
* @return
|
||||
*/
|
||||
static Object after( Duration delay, Closure closure ) {
|
||||
def obj = new ThrottleObj( null, System.currentTimeMillis() )
|
||||
throttle0( delay.millis, obj, closure)
|
||||
}
|
||||
|
||||
/**
|
||||
* Allows at most one execution after the specified time {@code delay} returning
|
||||
* {@code null} before the first execution, and the cached result value the
|
||||
* following times
|
||||
*
|
||||
* @param delay
|
||||
* @param closure
|
||||
* @return
|
||||
*/
|
||||
static Object after( String delay, Closure closure ) {
|
||||
def obj = new ThrottleObj( null, System.currentTimeMillis() )
|
||||
throttle0( Duration.of(delay).millis, obj, closure)
|
||||
}
|
||||
|
||||
/**
|
||||
* Allows at most one execution after the specified time {@code delay} returning
|
||||
* {@code initialValue} before the first execution, and the cached result value the
|
||||
* following times
|
||||
*
|
||||
* @param delay
|
||||
* @param initialValue
|
||||
* @param closure
|
||||
* @return
|
||||
*/
|
||||
static Object after( long delay, initialValue, Closure closure ) {
|
||||
final obj = new ThrottleObj( initialValue, System.currentTimeMillis() )
|
||||
throttle0( delay, obj, closure)
|
||||
}
|
||||
|
||||
/**
|
||||
* Allows at most one execution after the specified time {@code delay} returning
|
||||
* {@code initialValue} before the first execution, and the cached result value the
|
||||
* following times
|
||||
*
|
||||
* @param delay
|
||||
* @param initialValue
|
||||
* @param closure
|
||||
* @return
|
||||
*/
|
||||
static Object after( Duration delay, initialValue, Closure closure ) {
|
||||
def obj = new ThrottleObj( initialValue, System.currentTimeMillis() )
|
||||
throttle0( delay.millis, obj, closure)
|
||||
}
|
||||
|
||||
/**
|
||||
* Allows at most one execution after the specified time {@code delay} returning
|
||||
* {@code initialValue} before the first execution, and the cached result value the
|
||||
* following times
|
||||
*
|
||||
* @param delay
|
||||
* @param initialValue
|
||||
* @param closure
|
||||
* @return
|
||||
*/
|
||||
static Object after( String delay, initialValue, Closure closure ) {
|
||||
def obj = new ThrottleObj( initialValue, System.currentTimeMillis() )
|
||||
throttle0( Duration.of(delay).millis, obj, closure)
|
||||
}
|
||||
|
||||
private static throttle0( long timeout, ThrottleObj initialValue, Closure closure ) {
|
||||
assert closure != null
|
||||
|
||||
def key = 17
|
||||
key = 31 * key + closure.class.name.hashCode()
|
||||
key = 31 * key + closure.owner.hashCode()
|
||||
key = 31 * key + closure.delegate?.hashCode() ?: 0
|
||||
|
||||
ThrottleObj obj = throttleMap.get(key)
|
||||
if( obj == null ) {
|
||||
obj = initialValue ?: new ThrottleObj()
|
||||
throttleMap.put(key,obj)
|
||||
}
|
||||
|
||||
if( System.currentTimeMillis() - obj.timestamp > timeout ) {
|
||||
obj.timestamp = System.currentTimeMillis()
|
||||
obj.result = closure.call()
|
||||
}
|
||||
|
||||
obj.result
|
||||
}
|
||||
|
||||
static <V> V cache( Object key, Duration eviction, Closure<V> action ) {
|
||||
cache(key, eviction.toMillis(), action)
|
||||
}
|
||||
|
||||
static <V> V cache( Object key, long eviction, Closure<V> closure ) {
|
||||
final int hash = key?.hashCode() ?: 0
|
||||
|
||||
ThrottleObj obj = throttleMap.get(hash)
|
||||
if( obj == null ) {
|
||||
obj = new ThrottleObj()
|
||||
obj.result = closure.call()
|
||||
obj.timestamp = System.currentTimeMillis()
|
||||
throttleMap.put(hash,obj)
|
||||
}
|
||||
|
||||
else if( System.currentTimeMillis() - obj.timestamp > eviction ) {
|
||||
obj.timestamp = System.currentTimeMillis()
|
||||
obj.result = closure.call()
|
||||
}
|
||||
|
||||
(V)obj.result
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.util
|
||||
|
||||
import java.lang.reflect.Field
|
||||
import java.lang.reflect.ParameterizedType
|
||||
import java.lang.reflect.Type
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.Memoized
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.file.FileHelper
|
||||
import nextflow.script.dsl.Nullable
|
||||
import nextflow.script.types.Bag
|
||||
import nextflow.script.types.Record
|
||||
import nextflow.util.HashBag
|
||||
import nextflow.util.RecordMap
|
||||
import org.codehaus.groovy.runtime.typehandling.DefaultTypeTransformation
|
||||
import org.codehaus.groovy.runtime.typehandling.GroovyCastException
|
||||
|
||||
/**
|
||||
* Utility functions for working with types at runtime.
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
* @author Ben Sherman <bentshermann@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class TypeHelper {
|
||||
|
||||
/**
|
||||
* Returns the given type argument from the given value's superclass.
|
||||
*
|
||||
* This method assumes that the value's class extends a parameterized type.
|
||||
*
|
||||
* @param value
|
||||
* @param index
|
||||
*
|
||||
* @example
|
||||
* <pre>
|
||||
* class ExampleClass extends GenericBase<String, Integer> {}
|
||||
*
|
||||
* Type type = TypeHelper.getGenericType(new ExampleClass(), 1);
|
||||
* System.out.println(type); // Output: class java.lang.Integer
|
||||
* </pre>
|
||||
*/
|
||||
static Type getGenericType(Object value, int index) {
|
||||
final params = (ParameterizedType) value.getClass().getGenericSuperclass()
|
||||
return params.getActualTypeArguments()[index]
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the concrete Java class for a type.
|
||||
*
|
||||
* @param type
|
||||
*/
|
||||
static Class getRawType(Type type) {
|
||||
return \
|
||||
type instanceof Class ? type :
|
||||
type instanceof ParameterizedType ? type.getRawType() :
|
||||
Object
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine whether a type is a collection type (Bag, List, Set).
|
||||
*
|
||||
* @param type
|
||||
*/
|
||||
static boolean isCollectionType(Type type) {
|
||||
return Collection.class.isAssignableFrom(getRawType(type))
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine whether a type is a record type.
|
||||
*
|
||||
* @param type
|
||||
*/
|
||||
static boolean isRecordType(Type type) {
|
||||
return type instanceof Class && Record.class.isAssignableFrom(type)
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a value to the given type.
|
||||
*
|
||||
* @param value
|
||||
* @param type
|
||||
*/
|
||||
static Object asType(Object value, Type type) {
|
||||
if( value == null )
|
||||
return null
|
||||
|
||||
if( isCollectionType(type) )
|
||||
return asCollectionType(value as Collection, type)
|
||||
|
||||
if( isRecordType(type) )
|
||||
return asRecordType(value as Map, (Class) type)
|
||||
|
||||
if( type == Path )
|
||||
return TypeHelper.asPathType(value.toString())
|
||||
|
||||
return DefaultTypeTransformation.castToType(value, getRawType(type))
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a collection to the given collection type.
|
||||
*
|
||||
* If the type specifies an element type, each element in the
|
||||
* collection is converted to that type.
|
||||
*
|
||||
* @param collection
|
||||
* @param type
|
||||
*/
|
||||
static Collection asCollectionType(Collection collection, Type type) {
|
||||
if( type instanceof ParameterizedType ) {
|
||||
final elementType = type.getActualTypeArguments()[0]
|
||||
collection = collection.collect { el -> asType(el, elementType) }
|
||||
}
|
||||
|
||||
return switch( getRawType(type) ) {
|
||||
case Bag.class -> new HashBag<>(collection)
|
||||
case List.class -> collection as List
|
||||
case Set.class -> collection as Set
|
||||
default -> collection
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a string representing a file path to a Path.
|
||||
* Report an error if the path does not exist.
|
||||
*
|
||||
* @param str
|
||||
*/
|
||||
static Path asPathType(String str) {
|
||||
final result = FileHelper.asPath(str)
|
||||
if( !Files.exists(result) )
|
||||
throw new AbortOperationException("Input file '${str}' does not exist")
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a map to a record, validating it against the given
|
||||
* record type.
|
||||
*
|
||||
* @param map
|
||||
* @param type
|
||||
*/
|
||||
static Record asRecordType(Map<String,?> map, Class type) {
|
||||
final fields = recordFields(type)
|
||||
|
||||
for( final field : fields.values() ) {
|
||||
if( field.isAnnotationPresent(Nullable.class) )
|
||||
continue
|
||||
if( map.get(field.getName()) == null )
|
||||
throw new AbortOperationException("Input record ${map} is missing field '${field.getName()}' required by record type '${type.getSimpleName()}'")
|
||||
}
|
||||
|
||||
final result = new HashMap<String,Object>(map.size())
|
||||
for( final entry : map.entrySet() ) {
|
||||
final name = entry.key
|
||||
final value = fields.containsKey(name)
|
||||
? asType(entry.value, fields[name].getGenericType())
|
||||
: entry.value
|
||||
result.put(name, value)
|
||||
}
|
||||
return new RecordMap(result)
|
||||
}
|
||||
|
||||
@Memoized
|
||||
private static Map<String,Field> recordFields(Class type) {
|
||||
final fields = type.getDeclaredFields()
|
||||
final result = new HashMap<String,Field>(fields.size())
|
||||
for( final field : fields ) {
|
||||
if( field.isSynthetic() )
|
||||
continue
|
||||
result.put(field.getName(), field)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user