add nextflow d30e48d

This commit is contained in:
2026-04-29 23:01:54 +02:00
parent d0b12d668d
commit 97cc9058d3
2840 changed files with 730250 additions and 0 deletions

View File

@@ -0,0 +1,67 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage
import java.nio.file.Files
import java.nio.file.Path
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.extension.FilesEx
import static nextflow.lineage.fs.LinPath.*
/**
* File to store a history of the workflow executions and their corresponding LIDs
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
@Slf4j
@CompileStatic
class DefaultLinHistoryLog implements LinHistoryLog {
Path path
DefaultLinHistoryLog(Path folder) {
this.path = folder
if( !path.exists() )
Files.createDirectories(path)
}
void write(String name, UUID key, String runLid, Date date = null) {
assert name
assert key
def timestamp = date ?: new Date()
final recordFile = path.resolve(runLid.substring(LID_PROT.size()))
try {
recordFile.text = new LinHistoryRecord(timestamp, name, key, runLid).toString()
log.trace("Record for $key written in lineage history log ${FilesEx.toUriString(this.path)}")
}catch (Throwable e) {
log.warn("Can't write record $key file ${FilesEx.toUriString(recordFile)}", e.message)
}
}
List<LinHistoryRecord> getRecords(){
List<LinHistoryRecord> list = new LinkedList<LinHistoryRecord>()
try {
this.path.eachFile { Path file -> list.add(LinHistoryRecord.parse(file.text))}
}
catch (Throwable e) {
log.warn "Exception reading records from lineage history folder: ${FilesEx.toUriString(this.path)}", e.message
}
return list.sort {it.timestamp }
}
}

View File

@@ -0,0 +1,122 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage
import java.nio.file.Files
import java.nio.file.Path
import java.util.stream.Stream
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.exception.AbortOperationException
import nextflow.file.FileHelper
import nextflow.lineage.config.LineageConfig
import nextflow.lineage.serde.LinEncoder
import nextflow.lineage.serde.LinSerializable
/**
* Default Implementation for the a lineage store.
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
class DefaultLinStore implements LinStore {
private static String HISTORY_FILE_NAME = ".history"
private static final String METADATA_FILE = '.data.json'
private static final String DEFAULT_LOCATION = '.lineage'
private Path location
private LinHistoryLog historyLog
private LinEncoder encoder
DefaultLinStore open(LineageConfig config) {
location = toLocationPath(config.store.location)
encoder = new LinEncoder()
if( !Files.exists(location) && !Files.createDirectories(location) ) {
throw new AbortOperationException("Unable to create lineage store directory: $location")
}
historyLog = new DefaultLinHistoryLog(location.resolve(HISTORY_FILE_NAME))
return this
}
protected Path toLocationPath(String location) {
return location
? FileHelper.toCanonicalPath(location)
: Path.of('.').toAbsolutePath().normalize().resolve(DEFAULT_LOCATION)
}
@Override
void save(String key, LinSerializable value) {
final path = location.resolve("$key/$METADATA_FILE")
Files.createDirectories(path.parent)
log.debug "Save LID file path: $path"
path.text = encoder.encode(value)
}
@Override
LinSerializable load(String key) {
final path = location.resolve("$key/$METADATA_FILE")
log.debug("Loading from path $path")
if( path.exists() )
return encoder.decode(path.text) as LinSerializable
log.debug("File for key $key not found")
return null
}
Path getLocation() {
return location
}
@Override
LinHistoryLog getHistoryLog() {
return historyLog
}
@Override
void close() throws IOException {}
@Override
Stream<String> search(Map<String, List<String>> params) {
return Files.walk(location)
.filter { Path path ->
Files.isRegularFile(path) && path.fileName.toString().startsWith('.data.json')
}
.map { Path path ->
final obj = encoder.decode(path.text)
final key = location.relativize(path.parent).toString()
return new AbstractMap.SimpleEntry<String, LinSerializable>(key, obj)
}
.filter { entry ->
LinUtils.checkParams(entry.value, params)
}
.map {it-> it.key }
}
@Override
Stream<String> getSubKeys(String parentKey) {
final startPath = location.resolve(parentKey)
return Files.walk(startPath)
.filter { Path path ->
Files.isRegularFile(path) && path.fileName.toString().startsWith('.data.json') && path.parent != startPath
}
.map { Path path ->
location.relativize(path.parent).toString()
}
}
}

View File

@@ -0,0 +1,51 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage
import java.util.regex.Pattern
import groovy.transform.CompileStatic
import nextflow.lineage.config.LineageConfig
import nextflow.plugin.Priority
/**
* Default Factory for Lineage Store.
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
@CompileStatic
@Priority(0)
class DefaultLinStoreFactory extends LinStoreFactory {
private static final Pattern SCHEME = ~/^([a-zA-Z][a-zA-Z\d+\-.]*):/
private static final List<String> SUPPORTED_SCHEMES = List.of('file', 's3', 'gs', 'az')
@Override
boolean canOpen(LineageConfig config) {
final loc = config.store.location
if( !loc ) {
return true
}
final matcher = SCHEME.matcher(loc)
return matcher.find() ? matcher.group(1) in SUPPORTED_SCHEMES : true
}
@Override
protected LinStore newInstance(LineageConfig config) {
return new DefaultLinStore() .open(config)
}
}

View File

@@ -0,0 +1,73 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import groovyx.gpars.dataflow.DataflowWriteChannel
import nextflow.Channel
import nextflow.Session
import nextflow.extension.LinExtension
import nextflow.lineage.fs.LinPathFactory
import nextflow.lineage.model.v1beta1.FileOutput
import static nextflow.lineage.fs.LinPath.*
/**
* Lineage channel extensions
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
@CompileStatic
@Slf4j
class LinExtensionImpl implements LinExtension {
@Override
void fromLineage(Session session, DataflowWriteChannel channel, Map<String,?> opts) {
final queryParams = buildQueryParams(opts)
log.trace("Querying lineage with params: $queryParams")
new LinPropertyValidator().validateQueryParams(queryParams.keySet())
final store = getStore(session)
try( def stream = store.search(queryParams) ){
stream.forEach { channel.bind( LinPathFactory.create( asUriString(it) ) ) }
}
channel.bind(Channel.STOP)
}
private static Map<String, List<String>> buildQueryParams(Map<String,?> opts) {
final queryParams = [type: [FileOutput.class.simpleName] ]
if( opts.workflowRun )
queryParams['workflowRun'] = [opts.workflowRun as String]
if( opts.taskRun )
queryParams['taskRun'] = [opts.taskRun as String]
if( opts.label ) {
if( opts.label instanceof List )
queryParams['labels'] = opts.label as List<String>
else
queryParams['labels'] = [ opts.label.toString() ]
}
return queryParams
}
protected LinStore getStore(Session session) {
final store = LinStoreFactory.getOrCreate(session)
if( !store ) {
throw new Exception("Lineage store not found - Check Nextflow configuration")
}
return store
}
}

View File

@@ -0,0 +1,40 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage
/**
* Interface to log workflow executions and their corresponding Lineage IDs
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
interface LinHistoryLog {
/**
* Write a workflow execution linage history log record.
*
* @param name Workflow execution name.
* @param sessionId Workflow session ID.
* @param runLid Workflow run ID.
*/
void write(String name, UUID sessionId, String runLid)
/**
* Get the store records in the Lineage History Log.
*
* @return List of stored lineage history records.
*/
List<LinHistoryRecord> getRecords()
}

View File

@@ -0,0 +1,70 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage
import groovy.transform.CompileStatic
import groovy.transform.EqualsAndHashCode
import java.text.DateFormat
import java.text.SimpleDateFormat
/**
* Record of workflow executions and their corresponding Lineage IDs
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
@CompileStatic
@EqualsAndHashCode(includes = 'runName,sessionId')
class LinHistoryRecord {
static final public DateFormat TIMESTAMP_FMT = new SimpleDateFormat('yyyy-MM-dd HH:mm:ss z')
final Date timestamp
final String runName
final UUID sessionId
final String runLid
LinHistoryRecord(Date timestamp, String name, UUID sessionId, String runLid) {
this.timestamp = timestamp
this.runName = name
this.sessionId = sessionId
this.runLid = runLid
}
protected LinHistoryRecord() {}
List<String> toList() {
return List.of(
timestamp ? TIMESTAMP_FMT.format(timestamp) : '-',
runName ?: '-',
sessionId.toString(),
runLid ?: '-',
)
}
@Override
String toString() {
toList().join('\t')
}
static LinHistoryRecord parse(String line) {
final cols = line.tokenize('\t')
if (cols.size() == 4) {
return new LinHistoryRecord(TIMESTAMP_FMT.parse(cols[0]), cols[1], UUID.fromString(cols[2]), cols[3])
}
throw new IllegalArgumentException("Not a valid history entry: `$line`")
}
}

View File

@@ -0,0 +1,517 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage
import nextflow.NextflowMeta
import nextflow.extension.FilesEx
import nextflow.lineage.exception.OutputRelativePathException
import static nextflow.lineage.fs.LinPath.*
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.attribute.BasicFileAttributes
import java.time.OffsetDateTime
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.Session
import nextflow.lineage.model.v1beta1.Checksum
import nextflow.lineage.model.v1beta1.FileOutput
import nextflow.lineage.model.v1beta1.DataPath
import nextflow.lineage.model.v1beta1.Parameter
import nextflow.lineage.model.v1beta1.TaskOutput
import nextflow.lineage.model.v1beta1.Workflow
import nextflow.lineage.model.v1beta1.WorkflowOutput
import nextflow.lineage.model.v1beta1.WorkflowRun
import nextflow.file.FileHelper
import nextflow.file.FileHolder
import nextflow.processor.TaskHasher
import nextflow.processor.TaskRun
import nextflow.script.PlatformMetadata
import nextflow.script.ScriptMeta
import nextflow.script.params.BaseParam
import nextflow.script.params.CmdEvalParam
import nextflow.script.params.DefaultInParam
import nextflow.script.params.EachInParam
import nextflow.script.params.EnvInParam
import nextflow.script.params.EnvOutParam
import nextflow.script.params.FileInParam
import nextflow.script.params.FileOutParam
import nextflow.script.params.InParam
import nextflow.script.params.OutParam
import nextflow.script.params.StdInParam
import nextflow.script.params.StdOutParam
import nextflow.script.params.ValueInParam
import nextflow.script.params.ValueOutParam
import nextflow.trace.TraceObserverV2
import nextflow.trace.event.FilePublishEvent
import nextflow.trace.event.TaskEvent
import nextflow.trace.event.WorkflowOutputEvent
import nextflow.util.CacheHelper
import nextflow.util.PathNormalizer
import nextflow.util.SecretHelper
import nextflow.util.TestOnly
/**
* Observer to write the generated workflow metadata in a lineage store.
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
class LinObserver implements TraceObserverV2 {
private static Set<String> workflowMetadataPropertiesToRemove = Set.of(
"completed", "duration", "exitStatus", "errorMessage", "errorReport", "stats", "success" // Only existing at the end
)
private static Map<Class<? extends BaseParam>, String> taskParamToValue = [
(StdOutParam) : "stdout",
(StdInParam) : "stdin",
(FileInParam) : "path",
(FileOutParam) : "path",
(ValueInParam) : "val",
(ValueOutParam): "val",
(EnvInParam) : "env",
(EnvOutParam) : "env",
(CmdEvalParam) : "eval",
(EachInParam) : "each"
]
private String executionHash
private LinStore store
private Session session
private WorkflowOutput workflowOutput
private Map<String,String> outputsStoreDirLid = new HashMap<String,String>(10)
private PathNormalizer normalizer
LinObserver(Session session, LinStore store){
this.session = session
this.store = store
}
@TestOnly
String getExecutionHash(){ executionHash }
@TestOnly
String setExecutionHash(String hash){ this.executionHash = hash }
@TestOnly
String setNormalizer(PathNormalizer normalizer){ this.normalizer = normalizer }
@Override
void onFlowBegin() {
normalizer = new PathNormalizer(session.workflowMetadata)
executionHash = storeWorkflowRun(normalizer)
final executionUri = asUriString(executionHash)
workflowOutput = new WorkflowOutput(
OffsetDateTime.now(),
executionUri,
new LinkedList<Parameter>()
)
this.store.getHistoryLog().write(session.runName, session.uniqueId, executionUri)
}
@Override
void onFlowComplete(){
if(workflowOutput?.output ){
workflowOutput.createdAt = OffsetDateTime.now()
final key = executionHash + '#output'
this.store.save(key, workflowOutput)
}
}
protected Collection<Path> allScriptFiles() {
return ScriptMeta.allScriptNames().values()
}
protected List<DataPath> collectScriptDataPaths(PathNormalizer normalizer) {
final allScripts = allScriptFiles().sort()
final result = new ArrayList<DataPath>(allScripts.size()+1)
// the main script
result.add( new DataPath(
normalizer.normalizePath(session.workflowMetadata.scriptFile.normalize()),
Checksum.of(session.workflowMetadata.scriptId, "nextflow", CacheHelper.HashMode.DEFAULT())
) )
// all other scripts
for (Path it: allScripts) {
if( it==null || it == session.workflowMetadata.scriptFile )
continue
final dataPath = new DataPath(normalizer.normalizePath(it.normalize()), Checksum.ofNextflow(it.text))
result.add(dataPath)
}
return result
}
protected String storeWorkflowRun(PathNormalizer normalizer) {
// create the workflow object holding script files and repo tracking info
final workflow = new Workflow(
collectScriptDataPaths(normalizer),
session.workflowMetadata.repository,
session.workflowMetadata.commitId
)
// create the workflow run main object
final value = new WorkflowRun(
workflow,
session.uniqueId.toString(),
session.runName,
getNormalizedParams(session.params, normalizer),
SecretHelper.hideSecrets(session.config.deepClone()) as Map,
collectWorkflowMetadata(normalizer)
)
final executionHash = CacheHelper.hasher(value).hash().toString()
store.save(executionHash, value)
return executionHash
}
protected static List<Parameter> getNormalizedParams(Map<String, Object> params, PathNormalizer normalizer){
final normalizedParams = new LinkedList<Parameter>()
for( Map.Entry<String,Object> entry : params ) {
final key = entry.key
final val = entry.value
normalizedParams.add( new Parameter( getParameterType(val), key, normalizeValue(val, normalizer) ) )
}
return normalizedParams
}
@Override
void onTaskComplete(TaskEvent event) {
storeTaskInfo(event.handler.task)
}
protected void storeTaskInfo(TaskRun task) {
// store the task run entry
storeTaskRun(task, normalizer)
// store all task results
storeTaskResults(task, normalizer)
}
protected String storeTaskResults(TaskRun task, PathNormalizer normalizer){
final outputParams = getNormalizedTaskOutputs(task, normalizer)
final value = new TaskOutput( asUriString(task.hash.toString()), asUriString(executionHash), OffsetDateTime.now(), outputParams )
final key = task.hash.toString() + '#output'
store.save(key,value)
return key
}
private List<Parameter> getNormalizedTaskOutputs(TaskRun task, PathNormalizer normalizer){
final outputs = task.getOutputs()
final outputParams = new LinkedList<Parameter>()
for( Map.Entry<OutParam,Object> entry : outputs ) {
manageTaskOutputParameter(entry.key, outputParams, entry.value, task, normalizer)
}
return outputParams
}
private void manageTaskOutputParameter(OutParam key, LinkedList<Parameter> outputParams, value, TaskRun task, PathNormalizer normalizer) {
if (key instanceof FileOutParam) {
outputParams.add(new Parameter(getParameterType(key), key.name, manageFileOutParam(value, task)))
} else {
outputParams.add(new Parameter(getParameterType(key), key.name, normalizeValue(value, normalizer)))
}
}
private static Object normalizeValue(Object value, PathNormalizer normalizer) {
if (value instanceof Path)
return normalizer.normalizePath((Path)value)
else if (value instanceof CharSequence)
return normalizer.normalizePath(value.toString())
else
return value
}
private Object manageFileOutParam(Object value, TaskRun task) {
if (value == null) {
log.debug "Unexpected lineage File output value null"
return null
}
if (value instanceof Path) {
return asUriString(storeTaskOutput(task, (Path) value))
}
if (value instanceof Collection<Path>) {
final files = new LinkedList<String>()
for (Path it : value) {
files.add( asUriString(storeTaskOutput(task, (Path)it)) )
}
return files
}
// unexpected task output
throw new IllegalArgumentException("Unexpected output [${value.getClass().getName()}] '${value}' for task '${task.name}'")
}
protected String storeTaskRun(TaskRun task, PathNormalizer normalizer) {
final codeChecksum = Checksum.ofNextflow(session.stubRun ? task.stubSource : task.source)
final value = new nextflow.lineage.model.v1beta1.TaskRun(
session.uniqueId.toString(),
task.getName(),
codeChecksum,
task.script,
task.inputs ? manageTaskInputParameters(task.inputs, normalizer) : null,
task.isContainerEnabled() ? task.getContainerFingerprint() : null,
normalizer.normalizePath(task.getCondaEnv()),
normalizer.normalizePath(task.getSpackEnv()),
task.config?.getArchitecture()?.toString(),
getTaskGlobalVars(task),
getTaskBinEntries(task).collect { Path p -> new DataPath(
normalizer.normalizePath(p.normalize()),
Checksum.ofNextflow(p) )
},
asUriString(executionHash)
)
// store in the underlying persistence
final key = task.hash.toString()
store.save(key, value)
return key
}
protected Map<String,Object> getTaskGlobalVars(TaskRun task) {
return new TaskHasher(task).getTaskGlobalVars()
}
protected List<Path> getTaskBinEntries(TaskRun task) {
return new TaskHasher(task).getTaskBinEntries(task.source)
}
protected String storeTaskOutput(TaskRun task, Path path) {
try {
final attrs = readAttributes(path)
final key = getTaskOutputKey(task, path)
final checksum = Checksum.ofNextflow(path)
final value = new FileOutput(
path.toUriString(),
checksum,
asUriString(task.hash.toString()),
asUriString(executionHash),
asUriString(task.hash.toString()),
attrs.size(),
LinUtils.toDate(attrs?.creationTime()),
LinUtils.toDate(attrs?.lastModifiedTime()))
store.save(key, value)
return key
} catch (Throwable e) {
log.warn("Unexpected error storing lineage output '${path.toUriString()}' for task '${task.name}'", e)
return path.toUriString()
}
}
protected String getTaskOutputKey(TaskRun task, Path path) {
final rel = getTaskRelative(task, path)
return task.hash.toString() + SEPARATOR + rel
}
protected String getWorkflowOutputKey(Path target) {
final rel = getWorkflowRelative(target)
return executionHash + SEPARATOR + rel
}
protected String getTaskRelative(TaskRun task, Path path){
if (path.isAbsolute()) {
final rel = getTaskRelative0(task, path)
if (rel)
return rel
throw new IllegalArgumentException("Cannot access the relative path for output '${path.toUriString()}' and task '${task.name}'")
}
//Check if contains workdir or storeDir
final rel = getTaskRelative0(task, path.toAbsolutePath())
if (rel) return rel
if (path.normalize().getName(0).toString() == "..")
throw new IllegalArgumentException("Cannot access the relative path for output '${path.toUriString()}' and task '${task.name}'" )
return path.normalize().toString()
}
private String getTaskRelative0(TaskRun task, Path path){
final workDirAbsolute = task.workDir.toAbsolutePath()
if (path.startsWith(workDirAbsolute)) {
return workDirAbsolute.relativize(path).toString()
}
//If task output is not in the workDir check if output is stored in the task's storeDir
final storeDir = task.getConfig().getStoreDir().toAbsolutePath()
if( storeDir && path.startsWith(storeDir) ) {
final rel = storeDir.relativize(path)
//If output stored in storeDir, keep the path in case it is used as workflow output
this.outputsStoreDirLid.put(path.toString(), asUriString(task.hash.toString(),rel.toString()))
return rel
}
return null
}
protected BasicFileAttributes readAttributes(Path path) {
return Files.readAttributes(path, BasicFileAttributes)
}
@Override
void onFilePublish(FilePublishEvent event) {
try {
final checksum = Checksum.ofNextflow(event.target)
final key = getWorkflowOutputKey(event.target)
final sourceReference = event.source
? getSourceReference(event.source)
: asUriString(executionHash)
final attrs = readAttributes(event.target)
final value = new FileOutput(
event.target.toUriString(),
checksum,
sourceReference,
asUriString(executionHash),
null,
attrs.size(),
LinUtils.toDate(attrs?.creationTime()),
LinUtils.toDate(attrs?.lastModifiedTime()),
event.labels)
store.save(key, value)
}
catch (OutputRelativePathException ignored ){
log.warn1("Lineage for workflow output is not supported by publishDir directive")
}
catch (Throwable e) {
log.warn("Unexpected error storing published file '${event.target.toUriString()}' for workflow '${executionHash}'", e)
}
}
String getSourceReference(Path source){
final hash = FileHelper.getTaskHashFromPath(source, session.workDir)
if (hash) {
final target = FileHelper.getWorkFolder(session.workDir, hash).relativize(source).toString()
return asUriString(hash.toString(), target)
}
final storeDirReference = outputsStoreDirLid.get(source.toString())
return storeDirReference ? asUriString(storeDirReference) : null
}
@Override
void onWorkflowOutput(WorkflowOutputEvent event) {
final type = getParameterType(event.value)
final value = convertPathsToLidReferences(event.index ?: event.value)
workflowOutput.output.add(new Parameter(type, event.name, value))
}
protected static String getParameterType(Object param) {
if( param instanceof BaseParam )
return taskParamToValue.get(param.class)
// return generic types
if( param instanceof Path )
return Path.simpleName
if( param instanceof CharSequence )
return String.simpleName
if( param instanceof Collection )
return Collection.simpleName
if( param instanceof Map )
return Map.simpleName
if( param==null ) {
log.debug "Unexpected lineage param type null"
return null
}
return param.class.simpleName
}
private Object convertPathsToLidReferences(Object value){
if( value instanceof Path ) {
try {
final key = getWorkflowOutputKey(value)
return asUriString(key)
} catch (Throwable e){
//Workflow output key not found
return value
}
}
if( value instanceof Collection ) {
return value.collect { el -> convertPathsToLidReferences(el) }
}
if( value instanceof Map ) {
return value
.findAll { k, v -> v != null }
.collectEntries { k, v -> Map.entry(k, convertPathsToLidReferences(v)) }
}
return value
}
/**
* Relativizes a path from the workflow's output dir.
*
* @param path Path to relativize
* @return Path String with the relative path
* @throws IllegalArgumentException
*/
protected String getWorkflowRelative(Path path) throws IllegalArgumentException{
final outputDirAbs = session.outputDir.toAbsolutePath()
if (path.isAbsolute()) {
if (path.startsWith(outputDirAbs)) {
return outputDirAbs.relativize(path).toString()
}
log.debug("Cannot get relative path for workflow output '${path.toUriString()}'")
throw new OutputRelativePathException()
}
final pathAbs = path.toAbsolutePath()
if (pathAbs.startsWith(outputDirAbs)) {
return outputDirAbs.relativize(pathAbs).toString()
}
if (path.normalize().getName(0).toString() == "..") {
log.debug("Cannot get relative path for workflow output '${path.toUriString()}'")
throw new OutputRelativePathException()
}
return path.normalize().toString()
}
protected List<Parameter> manageTaskInputParameters(Map<InParam, Object> inputs, PathNormalizer normalizer) {
List<Parameter> managedInputs = new LinkedList<Parameter>()
inputs.forEach { param, value ->
if( param instanceof FileInParam )
managedInputs.add( new Parameter( getParameterType(param), param.name, manageFileInParam( (List<FileHolder>)value , normalizer) ) )
else if( !(param instanceof DefaultInParam) )
managedInputs.add( new Parameter( getParameterType(param), param.name, value) )
}
return managedInputs
}
private List<Object> manageFileInParam(List<FileHolder> files, PathNormalizer normalizer){
final paths = new LinkedList<Object>();
for( FileHolder it : files ) {
final path = it.sourcePath ?: it.storePath
final ref = getSourceReference(path)
paths.add(ref ?: new DataPath(
normalizer.normalizePath(path),
Checksum.ofNextflow(path))
)
}
return paths
}
/**
* Collects lineage data from workflow metadata applying the following transformations:
* - Normalizes paths against the original remote URL, or work directory and convert to URI strings
* - Remove transient properties (completed, duration, exitStatus, errorMessage, errorReport, stats, success)
* - Convert Nextflow metadata as Json Map
* @param normalizer
* @return Map with workflow metadata or null when error
*/
private Map collectWorkflowMetadata(PathNormalizer normalizer) {
try {
def metadata = session.workflowMetadata.toMap()
.collectEntries { it.value instanceof Path ? [it.key, FilesEx.toUriString(it.value as Path) ] : [it.key, it.value] }
metadata.removeAll {it.key.toString() in workflowMetadataPropertiesToRemove }
if( metadata.containsKey("nextflow") )
metadata["nextflow"] = (metadata["nextflow"] as NextflowMeta).toJsonMap()
if( metadata.containsKey("configFiles") )
metadata["configFiles"] = (metadata["configFiles"] as List<Path>).collect {normalizer.normalizePath(it)}
return metadata
} catch( Throwable e) {
log.debug("Error creating metadata", e)
return null
}
}
}

View File

@@ -0,0 +1,41 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage
import groovy.transform.CompileStatic
import nextflow.Session
import nextflow.trace.TraceObserverV2
import nextflow.trace.TraceObserverFactoryV2
/**
* Implements factory for {@link LinObserver} object
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class LinObserverFactory implements TraceObserverFactoryV2 {
@Override
Collection<TraceObserverV2> create(Session session) {
final result = new ArrayList<TraceObserverV2>(1)
final store = LinStoreFactory.getOrCreate(session)
if( store )
result.add( new LinObserver(session, store) )
return result
}
}

View File

@@ -0,0 +1,79 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage
import groovy.transform.CompileStatic
import nextflow.lineage.model.v1beta1.Checksum
import nextflow.lineage.model.v1beta1.DataPath
import nextflow.lineage.model.v1beta1.FileOutput
import nextflow.lineage.model.v1beta1.Parameter
import nextflow.lineage.model.v1beta1.TaskOutput
import nextflow.lineage.model.v1beta1.TaskRun
import nextflow.lineage.model.v1beta1.Workflow
import nextflow.lineage.model.v1beta1.WorkflowOutput
import nextflow.lineage.model.v1beta1.WorkflowRun
/**
* Class to validate if the string refers to a property in the classes of the Lineage Metadata model.
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
@CompileStatic
class LinPropertyValidator {
private static final List<Class> LIN_MODEL_CLASSES = [
Checksum,
DataPath,
FileOutput,
Parameter,
TaskOutput,
TaskRun,
Workflow,
WorkflowOutput,
WorkflowRun,
]
private Set<String> validProperties
LinPropertyValidator() {
this.validProperties = new HashSet<String>()
for( Class clazz : LIN_MODEL_CLASSES ) {
for( MetaProperty field : clazz.metaClass.getProperties() ) {
validProperties.add( field.name)
}
}
}
void validate(Collection<String> properties) {
for( String property : properties ) {
if( property !in this.validProperties ) {
def msg = "Property '$property' doesn't exist in the lineage model."
final matches = this.validProperties.closest(property)
if( matches )
msg += " -- Did you mean one of these?" + matches.collect { " $it"}.join(', ')
throw new IllegalArgumentException(msg)
}
}
}
void validateQueryParams(Collection<String> keys) {
for( final key : keys ) {
validate(key.tokenize('.'))
}
}
}

View File

@@ -0,0 +1,92 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage
import com.google.common.annotations.Beta
import groovy.transform.CompileStatic
import java.util.stream.Stream
import nextflow.lineage.config.LineageConfig
import nextflow.lineage.serde.LinSerializable
/**
* Interface for the lineage store
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Beta
@CompileStatic
interface LinStore extends Closeable {
/**
* Open the lineage store.
* @param config Configuration to open the lineage store.
*/
LinStore open(LineageConfig config)
/**
* Save a lineage entry in the store for in a given key.
*
* <p>The key is a hierarchical path-like identifier. Examples:
* <ul>
* <li>{@code "78ab04"} — a workflow run or task run (hash-based)</li>
* <li>{@code "78ab04#output"} — the output metadata for a workflow or task run</li>
* <li>{@code "78ab04/samples/file1.txt"} — an individual output file</li>
* </ul>
*
* @param key Entry key.
* @param value Entry object.
*/
void save(String key, LinSerializable value)
/**
* Load an entry for a given Lineage ID key.
*
* <p>The key is a hierarchical path-like identifier. Examples:
* <ul>
* <li>{@code "78ab04"} — a workflow run or task run (hash-based)</li>
* <li>{@code "78ab04#output"} — the output metadata for a workflow or task run</li>
* <li>{@code "78ab04/samples/file1.txt"} — an individual output file</li>
* </ul>
*
* @param key LID key.
* @return entry value, or null if key does not exists
*/
LinSerializable load(String key)
/**
* Get the {@link LinHistoryLog} object associated to the lineage store.
* @return {@link LinHistoryLog} object
*/
LinHistoryLog getHistoryLog()
/**
* Search for lineage entries.
* @param params Map of query params
* @return Stream with keys fulfilling the query params
*/
Stream<String> search(Map<String, List<String>> params)
/**
* Search for keys starting with a parent key.
* For example, if a LinStore contains the following keys: '123abc', '123abc/samples/file1.txt' and '123abc/summary',
* The execution of the function with parentKey='123abc' will return a stream with '123abc/samples/file1.txt' and '123abc/summary'.
* Similarly, the execution of the function with parentKey='123abc/samples' will just return '123abc/samples/file1.txt"
*
* @param parentKey
* @return Stream of keys starting with parentKey
*/
Stream<String> getSubKeys(String parentKey)
}

View File

@@ -0,0 +1,76 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.Session
import nextflow.lineage.config.LineageConfig
import nextflow.plugin.Plugins
import nextflow.util.TestOnly
import org.pf4j.ExtensionPoint
/**
* Factory for {@link LinStore} objects
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
@Slf4j
@CompileStatic
abstract class LinStoreFactory implements ExtensionPoint {
private static LinStore instance
private static boolean initialized
protected abstract boolean canOpen(LineageConfig config)
protected abstract LinStore newInstance(LineageConfig config)
static LinStore create(LineageConfig config){
final factory = Plugins
.getPriorityExtensions(LinStoreFactory)
.find( f-> f.canOpen(config))
if( !factory )
throw new IllegalStateException("Unable to find Nextflow Lineage store factory")
log.debug "Using Nextflow Lineage store factory: ${factory.getClass().getName()}"
return factory.newInstance(config)
}
static LinStore getOrCreate(Session session) {
if( instance || initialized )
return instance
synchronized (LinStoreFactory.class) {
if( instance || initialized )
return instance
initialized = true
final config = LineageConfig.create(session)
if( !config.enabled )
return null
return instance = create(config)
}
}
@TestOnly
static void reset(){
synchronized (LinStoreFactory.class) {
instance = null
initialized = false
}
}
}

View File

@@ -0,0 +1,264 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage
import static nextflow.lineage.fs.LinFileSystemProvider.*
import static nextflow.lineage.fs.LinPath.*
import java.nio.file.attribute.FileTime
import java.time.OffsetDateTime
import java.time.ZoneId
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.lineage.model.v1beta1.TaskRun
import nextflow.lineage.model.v1beta1.WorkflowRun
import nextflow.lineage.serde.LinEncoder
import nextflow.lineage.serde.LinSerializable
import nextflow.lineage.serde.LinTypeAdapterFactory
import nextflow.serde.gson.GsonEncoder
/**
* Utils class for Lineage IDs.
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
@Slf4j
@CompileStatic
class LinUtils {
/**
* Get a lineage record or fragment from the Lineage store.
*
* @param store Lineage store.
* @param uri Object or fragment to retrieve in URI-like format.
* Format 'lid://<key>[#fragment]' where:
* - Key: Metadata Element key
* - Fragment: Element fragment to retrieve.
* @return Lineage record or fragment.
*/
static Object getMetadataObject(LinStore store, URI uri) {
if( uri.scheme != SCHEME )
throw new IllegalArgumentException("Invalid LID URI - scheme is different for $SCHEME")
final key = uri.authority ? uri.authority + uri.path : uri.path
if( key == SEPARATOR )
throw new IllegalArgumentException("Cannot get record from the root LID URI")
if ( uri.query )
log.warn("Query string is not supported for Lineage URI: `$uri` -- it will be ignored")
return getMetadataObject0(store, key, uri.fragment )
}
private static Object getMetadataObject0(LinStore store, String key, String fragment) {
final record = store.load(key)
if( !record ) {
throw new FileNotFoundException("Lineage record $key not found")
}
if( fragment ) {
new LinPropertyValidator().validate(fragment.tokenize('.'))
return getSubObject(store, key, record, fragment)
}
return record
}
/**
* Get a lineage sub-record.
*
* If the requested sub-record is the workflow or task outputs, retrieves the outputs from the outputs description.
*
* @param store Store to retrieve lineage records.
* @param key Parent key.
* @param record Parent record.
* @param fragment String in indicating the properties to navigate to get the sub-record.
* @return Sub-record or null in it does not exist.
*/
static Object getSubObject(LinStore store, String key, LinSerializable record, String fragment) {
if( isSearchingOutputs(record, fragment) ) {
// When asking for a Workflow or task output retrieve the outputs description
final outputs = store.load("${key}#output")
if (!outputs)
return []
return navigate(outputs, fragment)
}
return navigate(record, fragment)
}
/**
* Check if the Lid pseudo path or query is for Task or Workflow outputs.
*
* @param record Parent lineage record
* @param fragment Fragment indicating the properties to navigate to get the sub-record.
* @return return 'true' if the parent is a Task/Workflow run and the first element in fragment is 'output'. Otherwise 'false'
*/
static boolean isSearchingOutputs(LinSerializable record, String fragment) {
return (record instanceof WorkflowRun || record instanceof TaskRun) && fragment && fragment.tokenize('.')[0] == 'output'
}
/**
* Evaluates record or the records in a collection matches a set of parameter-value pairs. It includes in the results collection in case of match.
*
* @param record Object or collection of records to evaluate
* @param params parameter-value pairs to evaluate in each record
* @param results results collection to include the matching records
*/
protected static void treatObject(def record, Map<String, List<String>> params, List<Object> results) {
if (params) {
if (record instanceof Collection) {
(record as Collection).forEach { treatObject(it, params, results) }
} else if (checkParams(record, params)) {
results.add(record)
}
} else {
results.add(record)
}
}
/**
* Check if an record fulfill the parameter-value
*
* @param record Object to evaluate
* @param params parameter-value pairs to evaluate
* @return true if all record parameters exist and matches with the value, otherwise false.
*/
static boolean checkParams(Object record, Map<String, List<String>> params) {
for( final entry : params.entrySet() ) {
final value = navigate(record, entry.key)
if( !checkParam(value, entry.value) ) {
return false
}
}
return true
}
private static boolean checkParam(Object value, List<String> expected) {
if( !value )
return false
// If value collection, convert to String and check all expected values are in the value.
if( value instanceof Collection ) {
final colValue = value as Collection
return colValue.collect { it.toString() }.containsAll(expected)
}
//Single record can't be compared with collection with one of more elements
if( expected.size() > 1 ) {
return false
}
return value.toString() == expected[0]
}
/**
* Retrieves the sub-record or value indicated by a path.
*
* @param obj Object to navigate
* @param path Elements path separated by '.' e.g. field.subfield
* @return sub-record / value
*/
static Object navigate(Object obj, String path) {
if (!obj)
return null
// type has been replaced by class when evaluating LidSerializable records
if (obj instanceof LinSerializable && path == 'type')
return obj.getClass()?.simpleName
try {
return path.tokenize('.').inject(obj) { current, key ->
getSubPath(current, key)
}
}
catch (Throwable e) {
log.debug("Error navigating to $path in record", e)
return null
}
}
private static Object getSubPath(current, String key) {
if (current == null) {
return null
}
if (current instanceof Map) {
return current[key] // Navigate Map properties
}
if (current instanceof Collection) {
return navigateCollection(current, key)
}
if (current.metaClass.hasProperty(current, key)) {
return current.getAt(key) // Navigate Object properties
}
log.debug("No property found for $key")
return null
}
private static Object navigateCollection(Collection collection, String key) {
final results = []
for (Object record : collection) {
final res = getSubPath(record, key)
if (res)
results.add(res)
}
if (results.isEmpty() ) {
log.trace("No property found for $key")
return null
}
// Return a single record if only ine results is found.
return results.size() == 1 ? results[0] : results
}
/**
* Helper function to convert from FileTime to ISO 8601 with offset
* of current timezone.
*
* @param time File time to convert
* @return The {@link OffsetDateTime} for the corresponding file time or null in case of not available (null)
*/
static OffsetDateTime toDate(FileTime time) {
return time != null
? time.toInstant().atZone(ZoneId.systemDefault()).toOffsetDateTime()
: null
}
/**
* Helper function to convert from String ISO 8601 to FileTime.
*
* @param date ISO formated time
* @return Converted FileTime or null if date is not available (null or 'N/A')
*/
static FileTime toFileTime(OffsetDateTime date) {
if (!date)
return null
return FileTime.from(date.toInstant())
}
/**
* Helper function to unify the encoding of outputs when querying and navigating the lineage pseudoFS.
* Outputs can include LinSerializable records, collections or parts of these records.
* LinSerializable records can be encoded with the LinEncoder, but collections or parts of
* these records require to extend the GsonEncoder.
*
* @param output Output to encode
* @return Output encoded as a JSON string
*/
static String encodeSearchOutputs(Object output, boolean prettyPrint = false) {
if (output instanceof LinSerializable) {
return new LinEncoder().withPrettyPrint(prettyPrint).encode(output)
} else {
return new GsonEncoder<Object>() {}
.withPrettyPrint(prettyPrint)
.withSerializeNulls(true)
.withTypeAdapterFactory(new LinTypeAdapterFactory())
.encode(output)
}
}
}

View File

@@ -0,0 +1,227 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.cli
import static nextflow.lineage.fs.LinPath.*
import java.nio.charset.StandardCharsets
import java.nio.file.Path
import groovy.transform.CompileStatic
import nextflow.Session
import nextflow.cli.CmdLineage
import nextflow.config.ConfigMap
import nextflow.exception.AbortOperationException
import nextflow.lineage.LinHistoryRecord
import nextflow.lineage.LinPropertyValidator
import nextflow.lineage.LinStore
import nextflow.lineage.LinStoreFactory
import nextflow.lineage.LinUtils
import nextflow.lineage.fs.LinPathFactory
import nextflow.lineage.serde.LinEncoder
import nextflow.ui.TableBuilder
import org.eclipse.jgit.diff.DiffAlgorithm
import org.eclipse.jgit.diff.DiffFormatter
import org.eclipse.jgit.diff.RawText
import org.eclipse.jgit.diff.RawTextComparator
/**
* Implements lineage command line operations
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class LinCommandImpl implements CmdLineage.LinCommand {
private static final Path DEFAULT_HTML_FILE = Path.of("lineage.html")
private static final String ERR_NOT_LOADED = 'Error lineage store not loaded - Check Nextflow configuration'
@Override
void list(ConfigMap config) {
final session = new Session(config)
final store = LinStoreFactory.getOrCreate(session)
if (store) {
printHistory(store)
} else {
println ERR_NOT_LOADED
}
}
private void printHistory(LinStore store) {
final records = store.historyLog?.records
if( !records ) {
println("No workflow runs found in lineage history log")
return
}
def table = new TableBuilder(cellSeparator: '\t')
.head('TIMESTAMP')
.head('RUN NAME')
.head('SESSION ID')
.head('LINEAGE ID')
for (LinHistoryRecord record : records) {
table.append(record.toList())
}
println table.toString()
}
@Override
void view(ConfigMap config, List<String> args) {
if( !isLidUri(args[0]) )
throw new Exception("Identifier is not a lineage URL")
final store = LinStoreFactory.getOrCreate(new Session(config))
if ( !store ) {
println ERR_NOT_LOADED
return
}
try {
def entry = LinUtils.getMetadataObject(store, new URI(args[0]))
if( entry == null ) {
println "No entry found for ${args[0]}"
return
}
println LinUtils.encodeSearchOutputs(entry, true)
} catch (Throwable e) {
println "Error loading ${args[0]} - ${e.message}"
}
}
@Override
void render(ConfigMap config, List<String> args) {
final store = LinStoreFactory.getOrCreate(new Session(config))
if( !store ) {
println ERR_NOT_LOADED
return
}
try {
final renderFile = args.size() > 1 ? Path.of(args[1]) : DEFAULT_HTML_FILE
new LinDagRenderer(store).render(args[0], renderFile)
println("Rendered lineage graph for ${args[0]} to $renderFile")
} catch (Throwable e) {
println("ERROR: rendering lineage graph - ${e.message}")
}
}
@Override
void diff(ConfigMap config, List<String> args) {
if (!isLidUri(args[0]) || !isLidUri(args[1]))
throw new Exception("Identifier is not a lineage URL")
final store = LinStoreFactory.getOrCreate(new Session(config))
if (!store) {
println ERR_NOT_LOADED
return
}
try {
final key1 = args[0].substring(LID_PROT.size())
final entry1 = store.load(key1)
if (!entry1) {
println "No entry found for ${args[0]}."
return
}
final key2 = args[1].substring(LID_PROT.size())
final entry2 = store.load(key2)
if (!entry2) {
println "No entry found for ${args[1]}."
return
}
final encoder = new LinEncoder().withPrettyPrint(true)
generateDiff(encoder.encode(entry1), key1, encoder.encode(entry2), key2)
} catch (Throwable e) {
println "Error generating diff between ${args[0]}: $e.message"
}
}
private static void generateDiff(String entry1, String key1, String entry2, String key2) {
// Convert strings to JGit RawText format
final text1 = new RawText(entry1.getBytes(StandardCharsets.UTF_8))
final text2 = new RawText(entry2.getBytes(StandardCharsets.UTF_8))
// Set up the diff algorithm (Git-style diff)
final diffAlgorithm = DiffAlgorithm.getAlgorithm(DiffAlgorithm.SupportedAlgorithm.MYERS)
final diffComparator = RawTextComparator.DEFAULT
// Compute the differences
final editList = diffAlgorithm.diff(diffComparator, text1, text2)
final output = new StringBuilder()
// Add header
output.append("diff --git ${key1} ${key2}\n")
output.append("--- ${key1}\n")
output.append("+++ ${key2}\n")
// Use DiffFormatter to display results in Git-style format
final outputStream = new ByteArrayOutputStream()
final diffFormatter = new DiffFormatter(outputStream)
diffFormatter.setOldPrefix(key1)
diffFormatter.setNewPrefix(key2)
diffFormatter.format(editList, text1, text2)
output.append(outputStream.toString(StandardCharsets.UTF_8))
println output.toString()
}
@Override
void find(ConfigMap config, List<String> args) {
final store = LinStoreFactory.getOrCreate(new Session(config))
if (!store) {
println ERR_NOT_LOADED
return
}
try {
final params = parseFindArgs(args)
new LinPropertyValidator().validateQueryParams(params.keySet())
try (def stream = store.search(params) ) {
stream.forEach { println asUriString(it) }
}
} catch (Throwable e){
println "Error searching for ${args[0]}. ${e.message}"
}
}
@Override
void check(ConfigMap config, List<String> args) {
final store = LinStoreFactory.getOrCreate(new Session(config))
if (!store) {
println ERR_NOT_LOADED
return
}
final valid = LinPathFactory.create(args[0]).validate()
if( !valid )
throw new AbortOperationException(valid.error)
else
println("Checksum validation succeed")
}
private Map<String, List<String>> parseFindArgs(List<String> args){
Map<String, List<String>> params = [:].withDefault { [] }
args.collectEntries { pair ->
final idx = pair.indexOf('=')
if( idx < 0 )
throw new IllegalArgumentException("Parameter $pair doesn't contain '=' separator")
def key = URLDecoder.decode(pair[0..<idx], 'UTF-8')
final value = URLDecoder.decode(pair[(idx + 1)..<pair.length()], 'UTF-8')
// Convert 'label' key in the CLI to 'labels' property in the model
if( key == 'label' )
key = 'labels'
params[key] << value
}
return params
}
}

View File

@@ -0,0 +1,209 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.cli
import java.nio.file.Path
import groovy.transform.Canonical
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.dag.MermaidHtmlRenderer
import nextflow.lineage.LinStore
import nextflow.lineage.model.v1beta1.FileOutput
import nextflow.lineage.model.v1beta1.TaskRun
import nextflow.lineage.model.v1beta1.WorkflowRun
import static nextflow.lineage.fs.LinPath.LID_PROT
import static nextflow.lineage.fs.LinPath.isLidUri
/**
* Renderer for the lineage graph.
*
* @author Ben Sherman <bentshermann@gmail.com>
*/
@Slf4j
@CompileStatic
class LinDagRenderer {
private final LinStore store
private Queue<String> queue
private List<Node> nodes
private List<Edge> edges
LinDagRenderer(LinStore store) {
this.store = store
}
private void enqueueLid(String lid) {
queue.add(lid)
}
private void addNode(String id, String label, NodeType type) {
nodes.add(new Node(id, label, type))
}
private void addEdge(String source, String target) {
edges.add(new Edge(source, target))
}
void render(String lid, Path file) {
// visit nodes
this.queue = new LinkedList<String>()
this.nodes = new LinkedList<Node>()
this.edges = new LinkedList<Edge>()
enqueueLid(lid)
while( !queue.isEmpty() )
visitLid(queue.remove())
// render Mermaid diagram
final lines = new LinkedList<String>()
lines << "flowchart TB"
for( final node : nodes ) {
if( node.type == NodeType.FILE )
lines << " ${node.id}[\"${node.label}\"]".toString()
if( node.type == NodeType.TASK )
lines << " ${node.id}([\"${node.label}\"])".toString()
}
for( final edge : edges )
lines << " ${edge.source} --> ${edge.target}".toString()
final template = MermaidHtmlRenderer.readTemplate()
file.text = template.replace('REPLACE_WITH_NETWORK_DATA', lines.join('\n'))
}
private void visitLid(String lid) {
if( !isLidUri(lid) )
throw new Exception("Identifier is not a lineage URL: ${lid}")
final record = store.load(rawLid(lid))
if( !record ) {
log.warn "Lineage record references an LID that does not exist: ${lid}"
return
}
if( record instanceof FileOutput )
visitFileOutput(lid, record)
else if( record instanceof TaskRun )
visitTaskRun(lid, record)
else if( record instanceof WorkflowRun )
visitWorkflowRun(lid, record)
else
throw new Exception("Cannot render lineage for type ${record.getClass().getSimpleName()} -- must be a FileOutput, TaskRun, or WorkflowRun")
}
private void visitFileOutput(String lid, FileOutput fileOutput) {
addNode(lid, lid, NodeType.FILE)
final source = fileOutput.source
if( !source )
return
if( isLidUri(source) ) {
enqueueLid(source)
addEdge(source, lid)
}
else {
final id = safeId(source)
final label = safeLabel(source)
addNode(id, label, NodeType.FILE)
addEdge(id, lid)
}
}
private void visitTaskRun(String lid, TaskRun taskRun) {
addNode(lid, "${taskRun.name} [${lid}]", NodeType.TASK)
for( final param : taskRun.input ) {
visitParameter(lid, param.value)
}
}
private void visitWorkflowRun(String lid, WorkflowRun workflowRun) {
addNode(lid, "${workflowRun.name} [${lid}]", NodeType.TASK)
for( final param : workflowRun.params ) {
visitParameter0(lid, param.value.toString())
}
}
private void visitParameter(String lid, Object value) {
if( value instanceof Collection ) {
for( final el : value )
visitParameter(lid, el)
}
else if( value instanceof CharSequence ) {
final source = value.toString()
if( isLidUri(source) ) {
enqueueLid(source)
addEdge(source, lid)
}
else {
visitParameter0(lid, source)
}
}
else if( value instanceof Map && value.path ) {
final path = value.path.toString()
if( isLidUri(path) ) {
enqueueLid(path)
addEdge(path, lid)
}
else {
visitParameter0(lid, path)
}
}
else {
visitParameter0(lid, value.toString())
}
}
private void visitParameter0(String lid, String value) {
final id = safeId(value)
final label = safeLabel(value)
addNode(id, label, NodeType.FILE)
addEdge(id, lid)
}
private static String rawLid(String lid) {
return lid.substring(LID_PROT.size())
}
private static String safeId(String rawId) {
return rawId.replaceAll(/[^a-zA-Z0-9_.:\/\-]/, '_')
}
private static String safeLabel(String label) {
return label.replace('http', 'h\u200Ettp')
}
@Canonical
private static class Node {
String id
String label
NodeType type
}
@Canonical
private static class Edge {
String source
String target
}
private static enum NodeType {
FILE,
TASK,
}
}

View File

@@ -0,0 +1,81 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.config
import groovy.transform.CompileStatic
import groovy.transform.ToString
import nextflow.Global
import nextflow.Session
import nextflow.config.spec.ConfigOption
import nextflow.config.spec.ConfigScope
import nextflow.config.spec.ScopeName
import nextflow.script.dsl.Description
/**
* Model workflow data lineage config
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@ScopeName("lineage")
@Description("""
The `lineage` scope controls the generation of lineage metadata.
""")
@ToString
@CompileStatic
class LineageConfig implements ConfigScope {
final LineageStoreOpts store
@ConfigOption
@Description("""
Enable generation of lineage metadata (default: `false`).
""")
final boolean enabled
/* required by extension point -- do not remove */
LineageConfig() {}
LineageConfig(Map opts) {
this.store = new LineageStoreOpts(opts.store as Map ?: Collections.emptyMap())
this.enabled = opts.enabled as boolean
}
static Map<String,Object> asMap() {
final result = session?.config?.navigate('lineage') as Map
return result != null ? result : new HashMap<String,Object>()
}
static LineageConfig create(Map config) {
new LineageConfig(config.lineage as Map ?: Collections.emptyMap())
}
static LineageConfig create(Session session) {
if( session ) {
return LineageConfig.create(session.config)
}
else
throw new IllegalStateException("Missing Nextflow session")
}
static LineageConfig create() {
create(getSession())
}
static private Session getSession() {
return Global.session as Session
}
}

View File

@@ -0,0 +1,44 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.config
import groovy.transform.CompileStatic
import groovy.transform.ToString
import nextflow.config.spec.ConfigOption
import nextflow.config.spec.ConfigScope
import nextflow.script.dsl.Description
/**
* Model data store options
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@ToString
@CompileStatic
class LineageStoreOpts implements ConfigScope {
@ConfigOption
@Description("""
The location of the lineage metadata store (default: `./.lineage`).
""")
final String location
LineageStoreOpts(Map opts) {
this.location = opts.location as String
}
}

View File

@@ -0,0 +1,25 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.exception
/**
* Exception to indicate the an output path is not relative to the output dir.
* It is used to detect the cases where publishDir is used with Data Lineage.
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
class OutputRelativePathException extends Exception {
}

View File

@@ -0,0 +1,138 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.fs
import com.google.common.collect.ImmutableSet
import nextflow.lineage.LinStore
import nextflow.lineage.LinStoreFactory
import java.nio.file.FileStore
import java.nio.file.FileSystem
import java.nio.file.Path
import java.nio.file.PathMatcher
import java.nio.file.WatchService
import java.nio.file.attribute.UserPrincipalLookupService
import java.nio.file.spi.FileSystemProvider
import groovy.transform.CompileStatic
import nextflow.lineage.LinStore
import nextflow.lineage.LinStoreFactory
import nextflow.lineage.config.LineageConfig
/**
* File system for LID Paths
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
@CompileStatic
class LinFileSystem extends FileSystem {
private LinFileSystemProvider provider
private LinStore store
/*
* Only needed to prevent serialization issues - see https://github.com/nextflow-io/nextflow/issues/5208
*/
protected LinFileSystem(){}
LinFileSystem(LinFileSystemProvider provider, LineageConfig config) {
this.provider = provider
this.store = LinStoreFactory.create(config)
}
LinStore getStore() {
return store
}
@Override
boolean equals( Object other ) {
if( this.class != other.class ) return false
final that = (LinFileSystem)other
this.provider == that.provider && this.store == that.store
}
@Override
int hashCode() {
Objects.hash(provider,store)
}
@Override
FileSystemProvider provider() {
return provider
}
@Override
void close() throws IOException {
}
@Override
boolean isOpen() {
return false
}
@Override
boolean isReadOnly() {
return true
}
@Override
String getSeparator() {
return LinPath.SEPARATOR
}
@Override
Iterable<Path> getRootDirectories() {
return null
}
@Override
Iterable<FileStore> getFileStores() {
return null
}
@Override
Set<String> supportedFileAttributeViews() {
return ImmutableSet.of("basic")
}
@Override
Path getPath(String first, String... more) {
final path = more ? LinPath.SEPARATOR + more.join(LinPath. SEPARATOR) : ''
return getPath(LinPath.asUri(LinPath.LID_PROT + first + path))
}
Path getPath(URI uri){
return new LinPath(this, uri)
}
@Override
PathMatcher getPathMatcher(String syntaxAndPattern) {
throw new UnsupportedOperationException();
}
@Override
UserPrincipalLookupService getUserPrincipalLookupService() {
throw new UnsupportedOperationException('User Principal Lookup Service not supported')
}
@Override
WatchService newWatchService() throws IOException {
throw new UnsupportedOperationException('Watch Service not supported')
}
}

View File

@@ -0,0 +1,400 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.fs
import java.nio.ByteBuffer
import java.nio.channels.NonWritableChannelException
import java.nio.channels.SeekableByteChannel
import java.nio.file.AccessDeniedException
import java.nio.file.AccessMode
import java.nio.file.CopyOption
import java.nio.file.DirectoryStream
import java.nio.file.FileStore
import java.nio.file.FileSystem
import java.nio.file.FileSystemNotFoundException
import java.nio.file.LinkOption
import java.nio.file.OpenOption
import java.nio.file.Path
import java.nio.file.ProviderMismatchException
import java.nio.file.StandardOpenOption
import java.nio.file.attribute.BasicFileAttributes
import java.nio.file.attribute.FileAttribute
import java.nio.file.attribute.FileAttributeView
import java.nio.file.spi.FileSystemProvider
import groovy.transform.CompileStatic
import nextflow.lineage.config.LineageConfig
import nextflow.util.TestOnly
/**
* File System Provider for LID Paths
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
@CompileStatic
class LinFileSystemProvider extends FileSystemProvider {
public static final String SCHEME = "lid"
private LinFileSystem fileSystem
@Override
String getScheme() {
return SCHEME
}
protected LinPath toLinPath(Path path) {
if (path !instanceof LinPath)
throw new ProviderMismatchException()
if (path instanceof LinMetadataPath)
return (LinMetadataPath) path
return (LinPath) path
}
private void checkScheme(URI uri) {
final scheme = uri.scheme.toLowerCase()
if (scheme != getScheme())
throw new IllegalArgumentException("Not a valid ${getScheme().toUpperCase()} scheme: $scheme")
}
@Override
synchronized FileSystem newFileSystem(URI uri, Map<String, ?> config) throws IOException {
checkScheme(uri)
if (fileSystem) {
return fileSystem
}
//Overwrite default values with provided configuration
final defaultConfig = LineageConfig.asMap()
if (config) {
for (Map.Entry<String, ?> e : config.entrySet()) {
defaultConfig.put(e.key, e.value)
}
}
return fileSystem = new LinFileSystem(this, new LineageConfig(defaultConfig))
}
@Override
FileSystem getFileSystem(URI uri) throws FileSystemNotFoundException {
if (!fileSystem)
throw new FileSystemNotFoundException()
return fileSystem
}
synchronized FileSystem getFileSystemOrCreate(URI uri) {
checkScheme(uri)
if (!fileSystem) {
fileSystem = (LinFileSystem) newFileSystem(uri, LineageConfig.asMap())
}
return fileSystem
}
@Override
LinPath getPath(URI uri) {
return (LinPath) ((LinFileSystem) getFileSystemOrCreate(uri)).getPath(uri)
}
@Override
OutputStream newOutputStream(Path path, OpenOption... options) throws IOException {
throw new UnsupportedOperationException("Write not supported by ${getScheme().toUpperCase()} file system provider")
}
@Override
InputStream newInputStream(Path path, OpenOption... options) throws IOException {
final lid = toLinPath(path)
if (lid instanceof LinMetadataPath)
return (lid as LinMetadataPath).newInputStream()
return newInputStream0(lid, options)
}
private static InputStream newInputStream0(LinPath lid, OpenOption... options) throws IOException {
final realPath = lid.getTargetOrMetadataPath()
if (realPath instanceof LinMetadataPath)
return (realPath as LinMetadataPath).newInputStream()
return realPath.fileSystem.provider().newInputStream(realPath, options)
}
@Override
SeekableByteChannel newByteChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException {
final lid = toLinPath(path)
validateOptions(options)
return newByteChannel0(lid, options, attrs)
}
@CompileStatic
private class LinPathSeekableByteChannel implements SeekableByteChannel {
SeekableByteChannel channel
LinPathSeekableByteChannel(SeekableByteChannel channel) {
this.channel = channel
}
@Override
int read(ByteBuffer dst) throws IOException {
return channel.read(dst)
}
@Override
int write(ByteBuffer src) throws IOException {
throw new NonWritableChannelException(){}
}
@Override
long position() throws IOException {
return channel.position()
}
@Override
SeekableByteChannel position(long newPosition) throws IOException {
channel.position(newPosition)
return this
}
@Override
long size() throws IOException {
return channel.size()
}
@Override
SeekableByteChannel truncate(long unused) throws IOException {
throw new NonWritableChannelException()
}
@Override
boolean isOpen() {
return channel.isOpen()
}
@Override
void close() throws IOException {
channel.close()
}
}
private static void validateOptions(Set<? extends OpenOption> options) {
if (!options || options.empty)
return
for (OpenOption opt : options) {
// All OpenOption values except for APPEND and WRITE are allowed
if (opt == StandardOpenOption.APPEND || opt == StandardOpenOption.WRITE)
throw new UnsupportedOperationException("'$opt' not allowed");
}
}
private SeekableByteChannel newByteChannel0(LinPath lid, Set<? extends OpenOption> options, FileAttribute<?>... attrs) {
if (lid instanceof LinMetadataPath) {
return (lid as LinMetadataPath).newSeekableByteChannel()
}
final realPath = lid.getTargetOrMetadataPath()
if (realPath instanceof LinMetadataPath) {
return (realPath as LinMetadataPath).newSeekableByteChannel()
} else {
SeekableByteChannel channel = realPath.fileSystem.provider().newByteChannel(realPath, options, attrs)
return new LinPathSeekableByteChannel(channel)
}
}
@Override
DirectoryStream<Path> newDirectoryStream(Path path, DirectoryStream.Filter<? super Path> filter) throws IOException {
final lid = toLinPath(path)
final real = lid.getTargetOrIntermediatePath()
if (real instanceof LinIntermediatePath)
return getDirectoryStreamFromSubPath(lid)
return getDirectoryStreamFromRealPath(real, lid)
}
private static DirectoryStream<Path> getDirectoryStreamFromSubPath(LinPath lid){
final paths = lid.getSubPaths()
if( !paths )
throw new FileNotFoundException("Sub paths for '$lid' do not exist")
return new DirectoryStream<Path>() {
Iterator<Path> iterator() {
return paths.iterator()
}
void close() {
paths.close()
}
}
}
private DirectoryStream<Path> getDirectoryStreamFromRealPath(Path real, LinPath lid) {
final stream = real
.getFileSystem()
.provider()
.newDirectoryStream(real, new LidFilter(fileSystem))
return new DirectoryStream<Path>() {
@Override
Iterator<Path> iterator() {
return new LidIterator(fileSystem, stream.iterator(), lid, real)
}
@Override
void close() throws IOException {
stream.close()
}
}
}
@CompileStatic
private class LidFilter implements DirectoryStream.Filter<Path> {
private final LinFileSystem fs
LidFilter(LinFileSystem fs) {
this.fs = fs
}
@Override
boolean accept(Path entry) throws IOException {
return true
}
}
private static LinPath fromRealToLinPath(Path toConvert, Path realBase, LinPath lidBase) {
if (toConvert.isAbsolute()) {
if (toConvert.class != realBase.class) {
throw new ProviderMismatchException()
}
final relative = realBase.relativize(toConvert)
return (LinPath) lidBase.resolve(relative.toString())
} else {
return (LinPath) lidBase.resolve(toConvert.toString())
}
}
private static class LidIterator implements Iterator<Path> {
private final LinFileSystem fs
private final Iterator<Path> target
private final LinPath parent
private final Path parentReal
LidIterator(LinFileSystem fs, Iterator<Path> itr, LinPath parent, Path real) {
this.fs = fs
this.target = itr
this.parent = parent
this.parentReal = real
}
@Override
boolean hasNext() {
return target.hasNext()
}
@Override
LinPath next() {
final path = target.next()
return path ? fromRealToLinPath(path, parentReal, parent) : null
}
}
@Override
void createDirectory(Path dir, FileAttribute<?>... attrs) throws IOException {
throw new UnsupportedOperationException("Create directory not supported by ${getScheme().toUpperCase()} file system provider")
}
@Override
void delete(Path path) throws IOException {
throw new UnsupportedOperationException("Delete not supported by ${getScheme().toUpperCase()} file system provider")
}
@Override
void copy(Path source, Path target, CopyOption... options) throws IOException {
throw new UnsupportedOperationException("Copy not supported by ${getScheme().toUpperCase()} file system provider")
}
@Override
void move(Path source, Path target, CopyOption... options) throws IOException {
throw new UnsupportedOperationException("Move not supported by ${getScheme().toUpperCase()} file system provider")
}
@Override
boolean isSameFile(Path path, Path path2) throws IOException {
return path == path2
}
@Override
boolean isHidden(Path path) throws IOException {
return toLinPath(path).getTargetOrMetadataPath().isHidden()
}
@Override
FileStore getFileStore(Path path) throws IOException {
throw new UnsupportedOperationException("File store not supported by ${getScheme().toUpperCase()} file system provider")
}
@Override
void checkAccess(Path path, AccessMode... modes) throws IOException {
validateAccessModes(modes)
final lid = toLinPath(path)
if (lid instanceof LinMetadataPath)
return
checkAccess0(lid, modes)
}
private void checkAccess0(LinPath lid, AccessMode... modes) {
final real = lid.getTargetOrMetadataPath()
if (real instanceof LinMetadataPath)
return
real.fileSystem.provider().checkAccess(real, modes)
}
private void validateAccessModes(AccessMode... modes) {
for (AccessMode m : modes) {
if (m == AccessMode.WRITE)
throw new AccessDeniedException("Write mode not supported")
if (m == AccessMode.EXECUTE)
throw new AccessDeniedException("Execute mode not supported")
}
}
@Override
<V extends FileAttributeView> V getFileAttributeView(Path path, Class<V> type, LinkOption... options) {
return null
}
@Override
<A extends BasicFileAttributes> A readAttributes(Path path, Class<A> type, LinkOption... options) throws IOException {
final lid = toLinPath(path)
if (lid instanceof LinMetadataPath)
return (lid as LinMetadataPath).readAttributes(type)
return readAttributes0(lid, type, options)
}
private <A extends BasicFileAttributes> A readAttributes0(LinPath lid, Class<A> type, LinkOption... options) throws IOException {
final target = lid.getTargetOrIntermediatePath()
if (target instanceof LinIntermediatePath)
return (target as LinIntermediatePath).readAttributes(type)
return target.fileSystem.provider().readAttributes(target, type, options)
}
@Override
Map<String, Object> readAttributes(Path path, String attributes, LinkOption... options) throws IOException {
throw new UnsupportedOperationException("Read file attributes not supported by ${getScheme().toUpperCase()} file system provider")
}
@Override
void setAttribute(Path path, String attribute, Object value, LinkOption... options) throws IOException {
throw new UnsupportedOperationException("Set file attributes not supported by ${getScheme().toUpperCase()} file system provider")
}
@TestOnly
void reset() {
fileSystem=null
}
}

View File

@@ -0,0 +1,69 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.fs
import java.nio.channels.SeekableByteChannel
import java.nio.file.attribute.BasicFileAttributes
import java.nio.file.attribute.FileTime
import java.time.Instant
class LinIntermediatePath extends LinMetadataPath{
LinIntermediatePath(LinFileSystem fs, String path) {
super("", FileTime.from(Instant.now()), fs, path, null)
}
@Override
InputStream newInputStream() {
throw new UnsupportedOperationException()
}
@Override
SeekableByteChannel newSeekableByteChannel(){
throw new UnsupportedOperationException()
}
@Override
<A extends BasicFileAttributes> A readAttributes(Class<A> type){
return (A) new BasicFileAttributes() {
@Override
long size() { return 0 }
@Override
FileTime lastModifiedTime() { FileTime.from(Instant.now()) }
@Override
FileTime lastAccessTime() { FileTime.from(Instant.now()) }
@Override
FileTime creationTime() { FileTime.from(Instant.now()) }
@Override
boolean isRegularFile() { return false }
@Override
boolean isDirectory() { return true }
@Override
boolean isSymbolicLink() { return false }
@Override
boolean isOther() { return false }
@Override
Object fileKey() { return null }
}
}
}

View File

@@ -0,0 +1,79 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.fs
import groovy.transform.CompileStatic
import java.nio.channels.SeekableByteChannel
import java.nio.file.attribute.BasicFileAttributes
import java.nio.file.attribute.FileTime
/**
* Class to model the metadata descriptions as a file.
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
@CompileStatic
class LinMetadataPath extends LinPath {
private byte[] results
private FileTime creationTime
LinMetadataPath(String resultsObject, FileTime creationTime, LinFileSystem fs, String path, String fragment) {
super(fs, "${path}${fragment ? '#'+ fragment : ''}")
this.results = resultsObject.getBytes("UTF-8")
this.creationTime = creationTime
}
InputStream newInputStream() {
return new ByteArrayInputStream(results)
}
SeekableByteChannel newSeekableByteChannel(){
return new LinMetadataSeekableByteChannel(results)
}
<A extends BasicFileAttributes> A readAttributes(Class<A> type){
return (A) new BasicFileAttributes() {
@Override
long size() { return results.length }
@Override
FileTime lastModifiedTime() { return creationTime }
@Override
FileTime lastAccessTime() { return creationTime }
@Override
FileTime creationTime() { return creationTime }
@Override
boolean isRegularFile() { return true }
@Override
boolean isDirectory() { return false }
@Override
boolean isSymbolicLink() { return false }
@Override
boolean isOther() { return false }
@Override
Object fileKey() { return null }
}
}
}

View File

@@ -0,0 +1,76 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.fs
import groovy.transform.CompileStatic
import java.nio.ByteBuffer
import java.nio.channels.ClosedChannelException
import java.nio.channels.NonWritableChannelException
import java.nio.channels.SeekableByteChannel
/**
* SeekableByteChannel for metadata results description as a file.
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
@CompileStatic
class LinMetadataSeekableByteChannel implements SeekableByteChannel {
private final ByteBuffer buffer
private boolean open
LinMetadataSeekableByteChannel(byte[] bytes){
this.open = true
this.buffer = ByteBuffer.wrap(bytes)
}
@Override
int read(ByteBuffer dst) {
if (!open) throw new ClosedChannelException()
if (!buffer.hasRemaining()) return -1
int remaining = Math.min(dst.remaining(), buffer.remaining())
byte[] temp = new byte[remaining]
buffer.get(temp)
dst.put(temp)
return remaining
}
@Override
int write(ByteBuffer src) { throw new NonWritableChannelException() }
@Override
long position() { return buffer.position() }
@Override
SeekableByteChannel position(long newPosition) {
if (newPosition < 0 || newPosition > buffer.limit()) throw new IllegalArgumentException()
buffer.position((int) newPosition)
return this
}
@Override
long size() { return buffer.limit() }
@Override
SeekableByteChannel truncate(long size) { throw new NonWritableChannelException() }
@Override
boolean isOpen() { return open }
@Override
void close() { open = false }
}

View File

@@ -0,0 +1,626 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.fs
import static nextflow.lineage.LinUtils.*
import static nextflow.lineage.fs.LinFileSystemProvider.*
import java.nio.file.FileSystem
import java.nio.file.LinkOption
import java.nio.file.Path
import java.nio.file.ProviderMismatchException
import java.nio.file.WatchEvent
import java.nio.file.WatchKey
import java.nio.file.WatchService
import java.time.OffsetDateTime
import java.util.stream.Stream
import groovy.transform.CompileStatic
import groovy.transform.EqualsAndHashCode
import groovy.transform.Memoized
import groovy.util.logging.Slf4j
import nextflow.file.FileHelper
import nextflow.file.LogicalDataPath
import nextflow.lineage.LinPropertyValidator
import nextflow.lineage.LinStore
import nextflow.lineage.model.v1beta1.Checksum
import nextflow.lineage.model.v1beta1.FileOutput
import nextflow.lineage.model.v1beta1.TaskRun
import nextflow.lineage.model.v1beta1.WorkflowRun
import nextflow.lineage.serde.LinSerializable
import nextflow.util.CacheHelper
import nextflow.util.TestOnly
/**
* LID file system path
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
@Slf4j
@CompileStatic
class LinPath implements Path, LogicalDataPath {
static public final List<String> SUPPORTED_CHECKSUM_ALGORITHMS = ["nextflow"]
static public final String SEPARATOR = '/'
public static final String LID_PROT = "${SCHEME}://"
static private final String[] EMPTY = new String[]{}
private LinFileSystem fileSystem
// String with the lineage file path
private String filePath
private String query
private String fragment
/*
* Only needed to prevent serialization issues - see https://github.com/nextflow-io/nextflow/issues/5208
*/
protected LinPath(){}
LinPath(LinFileSystem fs, URI uri) {
if( uri.scheme != SCHEME ) {
throw new IllegalArgumentException("Invalid LID URI - scheme is different for $SCHEME")
}
this.fileSystem = fs
setFieldsFormURI(uri)
// Check if query and fragment are with filePath
if( query == null && fragment == null )
setFieldsFormURI(new URI(toUriString()))
// Warn if query is specified
if( query )
log.warn("Query string is not supported for Lineage URI: `$uri` -- it will be ignored")
// Validate fragment
if( fragment )
new LinPropertyValidator().validate(fragment.tokenize('.'))
}
private void setFieldsFormURI(URI uri){
this.query = uri.query
this.fragment = uri.fragment
this.filePath = resolve0(fileSystem, norm0("${uri.authority?:''}${uri.path}") )
}
protected LinPath(String query, String fragment, String filepath, LinFileSystem fs) {
this.fileSystem = fs
this.query = query
this.fragment = fragment
this.filePath = filepath
}
LinPath(LinFileSystem fs, String path) {
this( fs, asUri( LID_PROT + norm0(path)) )
}
LinPath(LinFileSystem fs, String first, String[] more) {
this( fs, asUri( LID_PROT + buildPath(first, more) ) )
}
static String asUriString(String first, String... more) {
return LID_PROT + buildPath(first, more)
}
static boolean isLidUri(String path) {
return path && path.startsWith(LID_PROT)
}
private static String buildPath(String first, String[] more) {
first = norm0(first)
if( more ) {
final morePath = norm0(more).join(SEPARATOR)
return first.isEmpty() ? morePath : first + SEPARATOR + morePath
}
return first
}
@Memoized
protected static String validateDataOutput(FileOutput lidObject) {
final hashedPath = FileHelper.toCanonicalPath(lidObject.path as String)
if( !hashedPath.exists() )
throw new FileNotFoundException("Target path $lidObject.path does not exist")
return validateChecksum(lidObject.checksum, hashedPath)
}
protected static String validateChecksum(Checksum checksum, Path hashedPath) {
if( !checksum )
return null
if( !isAlgorithmSupported(checksum.algorithm) ) {
return "Checksum of '$hashedPath' can't be validated - algorithm '${checksum.algorithm}' is not supported"
}
final hash = checksum.mode
? CacheHelper.hasher(hashedPath, CacheHelper.HashMode.of(checksum.mode.toString().toLowerCase())).hash().toString()
: CacheHelper.hasher(hashedPath).hash().toString()
return hash != checksum.value
? "Checksum of '$hashedPath' does not match with lineage metadata"
: null
}
protected static isAlgorithmSupported(String algorithm) {
return algorithm && algorithm in SUPPORTED_CHECKSUM_ALGORITHMS
}
@TestOnly
protected String getFilePath() { this.filePath }
protected Stream<Path> getSubPaths(){
if( !fileSystem )
throw new IllegalArgumentException("Cannot get sub-paths for a relative lineage path")
if( filePath.isEmpty() || filePath == SEPARATOR )
throw new IllegalArgumentException("Cannot get sub-paths for an empty lineage path (lid:///)")
final store = fileSystem.getStore()
if( !store )
throw new Exception("Lineage store not found - Check Nextflow configuration")
return store.getSubKeys(filePath).map {new LinPath(fileSystem as LinFileSystem, it) as Path }
}
/**
* Finds the target path of a LinPath.
* This method return the different types depending on the type of metadata pointing:
* - When the LinPath point to FileOutput metadata or a subpath, it returns the real path.
* - When it points other lineage metadata or a fragment of a lineage metadata and asMetadata is true, it returns a LinMetadataPath
* which contains in memory the lineage metadata description or the requested fragment of this description.
* - When it points to a WorkflowRun or TaskRun metadata or subpath and asIntermediate is set to true. LinIntermediatePath, which is representing a directory
* In other cases it will return a FileNotFoundException
*
* @param fs LinFileSystem associated to the LinPath to find
* @param filePath Path to look for the target path
* @param fragment String with path to sub-object inside the description
* @param asMetadata Flag to indicate if other metadata descriptions must be returned as LinMetadataPath.
* @param asIntermediate Flag to indicate if WorkflowRun and TaskRun subpaths must be returned as LinIntermediatePath.
* @param subpath subpath associated to the target path to find. Used when looking for a parent path
* @return Real Path, LinMetadataPath or LinIntermediatePath path associated to the LinPath
* @throws Exception
* IllegalArgumentException if the filepath, filesystem or its LinStore are null.
* FileNotFoundException if the filePath, subpath and fragment is not found.
*/
protected static Path findTarget(LinFileSystem fs, String filePath, String fragment, boolean asMetadata, boolean asIntermediate) throws Exception {
if( !fs )
throw new IllegalArgumentException("Cannot get target path for a relative lineage path")
if( filePath.isEmpty() || filePath == SEPARATOR )
throw new IllegalArgumentException("Cannot get target path for an empty lineage path (lid:///)")
final store = fs.getStore()
if( !store )
throw new Exception("Lineage store not found - Check Nextflow configuration")
findTarget0(fs, store, filePath, fragment, asMetadata, asIntermediate, [])
}
private static Path findTarget0(LinFileSystem fs, LinStore store, String filePath, String fragment, boolean asMetadata, boolean asIntermediate, List<String> subpath) {
final object = store.load(filePath)
if( object ) {
return getTargetPathFromObject(object, fs, filePath, fragment, asMetadata, asIntermediate, subpath)
} else {
if( fragment ) {
// If object doesn't exit, it's not possible to get fragment.
throw new FileNotFoundException("Target path '$filePath#$fragment' does not exist")
}
return findTargetFromParent(fs, store, filePath, asIntermediate, subpath)
}
}
private static Path findTargetFromParent(LinFileSystem fs, LinStore store, String filePath, boolean asIntermediate, List<String> subpath) {
final currentPath = Path.of(filePath)
final parent = Path.of(filePath).getParent()
if( !parent ) {
throw new FileNotFoundException("Target path '$filePath/${subpath.join('/')} does not exist")
}
ArrayList<String> newChildren = new ArrayList<String>()
newChildren.add(currentPath.getFileName().toString())
newChildren.addAll(subpath)
//As Metadata set as false because parent path only inspected for FileOutput or intermediate.
return findTarget0(fs, store, parent.toString(), null, false, asIntermediate, newChildren)
}
private static Path getTargetPathFromObject(LinSerializable object, LinFileSystem fs, String filePath, String fragment, boolean asMetadataPath, boolean asIntermediatePath,List<String> subpath) {
// It's not possible to get a target path with both fragment and subpath
if( fragment && subpath ) {
throw new FileNotFoundException("Unable to get a target path for '$filePath' with fragments and subpath")
}
// If metadata flag is active and looks for a fragment returns the metadata despite the type of object
if( asMetadataPath && fragment ){
return getMetadataAsTargetPath(object, fs, filePath, fragment)
}
// Return real files when FileOutput sub-path
if( object instanceof FileOutput ) {
return getTargetPathFromOutput(object, subpath)
}
// Intermediate run case
if( asIntermediatePath && (object instanceof WorkflowRun || object instanceof TaskRun) ) {
return new LinIntermediatePath(fs, "$filePath/${subpath.join('/')}")
}
// It is not possible to get a metadata path with subpath. For other cases return metadata path if activated or throw exception
if( asMetadataPath && !subpath)
return getMetadataAsTargetPath(object, fs, filePath, fragment)
else
throw new FileNotFoundException("Target path '${filePath}/${subpath ? '/' + subpath.join('/') : ''}${fragment ? '#' + fragment : ''}' does not exist")
}
protected static Path getMetadataAsTargetPath(LinSerializable results, LinFileSystem fs, String filePath, String fragment) {
if( !results ) {
throw new FileNotFoundException("Target path '$filePath' does not exist")
}
if( fragment ) {
return getSubObjectAsPath(fs, filePath, results, fragment)
} else {
return generateLinMetadataPath(fs, filePath, results, fragment)
}
}
/**
* Get a metadata sub-object as LinMetadataPath.
* If the requested sub-object is the workflow or task outputs, retrieves the outputs from the outputs description.
*
* @param fs LinFilesystem for the te.
* @param key Parent metadata key.
* @param object Parent object.
* @param children Array of string in indicating the properties to navigate to get the sub-object.
* @return LinMetadataPath or null in it does not exist
*/
static LinMetadataPath getSubObjectAsPath(LinFileSystem fs, String key, LinSerializable object, String fragment) {
if( isSearchingOutputs(object, fragment) ) {
// When asking for a Workflow or task output retrieve the outputs description
final outputs = fs.store.load("${key}#output")
if( !outputs ) {
throw new FileNotFoundException("Target path '$key#output' does not exist")
}
return generateLinMetadataPath(fs, key, outputs, fragment)
} else {
return generateLinMetadataPath(fs, key, object, fragment)
}
}
private static LinMetadataPath generateLinMetadataPath(LinFileSystem fs, String key, Object object, String fragment) {
def creationTime = toFileTime(navigate(object, 'createdAt') as OffsetDateTime ?: OffsetDateTime.now())
final output = fragment ? navigate(object, fragment) : object
if( !output ) {
throw new FileNotFoundException("Target path '$key#${fragment}' does not exist")
}
return new LinMetadataPath(encodeSearchOutputs(output, true), creationTime, fs, key, fragment)
}
private static Path getTargetPathFromOutput(FileOutput object, List<String> children) {
final lidObject = object as FileOutput
// verify checksum validation
final violation = validateDataOutput(lidObject)
if( violation )
log.warn1(violation)
// return the real path stored in the metadata
def realPath = FileHelper.toCanonicalPath(lidObject.path as String)
if( children && children.size() > 0 )
realPath = realPath.resolve(children.join(SEPARATOR))
if( !realPath.exists() )
throw new FileNotFoundException("Target path '$realPath' does not exist")
return realPath
}
private static boolean isEmptyBase(LinFileSystem fs, String base) {
return !base || base == SEPARATOR || (fs && base == "..")
}
private static String resolve0(LinFileSystem fs, String base, String[] more) {
if( isEmptyBase(fs, base) ) {
return resolveEmptyPathCase(fs, more as List)
}
if( base.contains(SEPARATOR) ) {
final parts = base.tokenize(SEPARATOR)
final remain = parts[1..-1] + more.toList()
return resolve0(fs, parts[0], remain as String[])
}
final result = Path.of(base)
return more ? result.resolve(more.join(SEPARATOR)).toString() : result.toString()
}
private static String resolveEmptyPathCase(LinFileSystem fs, List<String> more) {
switch( more.size() ) {
case 0:
return "/"
case 1:
return resolve0(fs, more[0], EMPTY)
default:
return resolve0(fs, more[0], more[1..-1] as String[])
}
}
static private String norm0(String path) {
if( !path || path == SEPARATOR )
return ""
//Remove repeated elements
path = Path.of(path.trim()).normalize().toString()
//Remove initial and final separators
if( path.startsWith(SEPARATOR) )
path = path.substring(1)
if( path.endsWith(SEPARATOR) )
path = path.substring(0, path.size() - 1)
return path
}
static private String[] norm0(String... path) {
for( int i = 0; i < path.length; i++ ) {
path[i] = norm0(path[i])
}
return path
}
@Override
FileSystem getFileSystem() {
return fileSystem
}
@Override
boolean isAbsolute() {
return fileSystem != null
}
@Override
Path getRoot() {
return new LinPath(fileSystem, SEPARATOR)
}
@Override
Path getFileName() {
final result = Path.of(filePath).getFileName()?.toString()
return result ? new LinPath(query, fragment, result, null) : null
}
@Override
Path getParent() {
final c = getNameCount()
if( c > 1 )
return subpath(0, c - 1)
if( c == 1 )
return new LinPath(fileSystem, SEPARATOR)
return null
}
@Override
int getNameCount() {
return Path.of(filePath).nameCount
}
@Override
Path getName(int index) {
if( index < 0 )
throw new IllegalArgumentException("Path name index cannot be less than zero - offending value: $index")
final path = Path.of(filePath)
if( index == path.nameCount - 1 ) {
return new LinPath( query, fragment, path.getName(index).toString(), null)
}
return new LinPath(index == 0 ? fileSystem : null, path.getName(index).toString())
}
@Override
Path subpath(int beginIndex, int endIndex) {
if( beginIndex < 0 )
throw new IllegalArgumentException("subpath begin index cannot be less than zero - offending value: $beginIndex")
final path = Path.of(filePath)
return new LinPath(beginIndex == 0 ? fileSystem : null, path.subpath(beginIndex, endIndex).toString())
}
@Override
Path normalize() {
return new LinPath(fileSystem, Path.of(filePath).normalize().toString())
}
@Override
boolean startsWith(Path other) {
return startsWith(other.toString())
}
@Override
boolean startsWith(String other) {
return filePath.startsWith(other)
}
@Override
boolean endsWith(Path other) {
return endsWith(other.toString())
}
@Override
boolean endsWith(String other) {
return filePath.endsWith(other)
}
@Override
Path resolve(Path other) {
if( LinPath.class != other.class )
throw new ProviderMismatchException()
final that = (LinPath) other
if( that.fileSystem && this.fileSystem != that.fileSystem )
return other
if( that.isAbsolute() ) {
return that
} else {
final newPath = Path.of(filePath).resolve(that.toString())
return new LinPath(that.query, that.fragment, newPath.toString(), fileSystem)
}
}
@Override
Path resolve(String path) {
if( !path )
return this
final scheme = FileHelper.getUrlProtocol(path)
if( !scheme ) {
// consider the path as a lid relative path
return resolve(new LinPath(null, path))
}
if( scheme != SCHEME ) {
throw new ProviderMismatchException()
}
final that = fileSystem.provider().getPath(asUri(path))
return resolve(that)
}
@Override
Path relativize(Path other) {
if( LinPath.class != other.class ) {
throw new ProviderMismatchException()
}
LinPath lidOther = other as LinPath
if( this.isAbsolute() != lidOther.isAbsolute() )
throw new IllegalArgumentException("Cannot compare absolute with relative paths");
def path
if( this.isAbsolute() ) {
// Compare 'filePath' as absolute paths adding the root separator
path = Path.of(SEPARATOR + filePath).relativize(Path.of(SEPARATOR + lidOther.filePath))
} else {
// Compare 'filePath' as relative paths
path = Path.of(filePath).relativize(Path.of(lidOther.filePath))
}
return new LinPath(lidOther.query, lidOther.fragment, path.getNameCount() > 0 ? path.toString() : SEPARATOR, null)
}
@Override
URI toUri() {
return asUri("${SCHEME}://${filePath}${query ? '?' + query : ''}${fragment ? '#' + fragment : ''}")
}
String toUriString() {
return toUri().toString()
}
@Override
Path toAbsolutePath() {
return this
}
@Override
Path toRealPath(LinkOption... options) throws IOException {
return this.getTargetOrMetadataPath()
}
Path toTargetPath() {
return getTargetOrMetadataPath()
}
/**
* Get the path associated with a FileOutput record.
*
* @return Path associated with a FileOutput record
* @throws FileNotFoundException if the record does not exist or its type is not a FileOutput.
*/
protected Path getTargetPath() {
return findTarget(fileSystem, filePath, fragment, false, false)
}
/**
* Get the path associated with a FileOutput record or an intermediate subpath.
*
* @return Path associated with a FileOutput record or a LinIntermediatePath if LinPath points to a workflow and task run subpath.
* @throws FileNotFoundException if the record does not exist or its type is not a FileOutput or a intermediate directory
*/
protected Path getTargetOrIntermediatePath() {
return findTarget(fileSystem, filePath, fragment, false, true)
}
/**
* Get the path associated with a lineage record.
*
* @return Path associated with a FileOutput record or a LinMetadataFile with the lineage record for other types, or a intermediate directory
* @throws FileNotFoundException if the record does not exist
*/
protected Path getTargetOrMetadataPath() {
return findTarget(fileSystem, filePath, fragment,true, false)
}
@Override
File toFile() throws IOException {
throw new UnsupportedOperationException("toFile not supported by LinPath")
}
@Override
WatchKey register(WatchService watcher, WatchEvent.Kind<?>[] events, WatchEvent.Modifier... modifiers) throws IOException {
throw new UnsupportedOperationException("Register not supported by LinPath")
}
@Override
int compareTo(Path other) {
return toString().compareTo(other.toString());
}
@Override
boolean equals(Object other) {
if( LinPath.class != other.class ) {
return false
}
final that = (LinPath) other
return this.fileSystem == that.fileSystem && this.filePath.equals(that.filePath)
}
/**
* @return The unique hash code for this path
*/
@Override
int hashCode() {
return Objects.hash(fileSystem, filePath)
}
static URI asUri(String path) {
if( !path )
throw new IllegalArgumentException("Missing 'path' argument")
if( !path.startsWith(LID_PROT) )
throw new IllegalArgumentException("Invalid LID file system path URI - it must start with '${LID_PROT}' prefix - offendinf value: $path")
if( path.startsWith(LID_PROT + SEPARATOR) && path.length() > 7 )
throw new IllegalArgumentException("Invalid LID file system path URI - make sure the schema prefix does not container more than two slash characters or a query in the root '/' - offending value: $path")
if( path == LID_PROT ) //Empty path case
return new URI("lid:///")
return new URI(path)
}
@Override
String toString() {
return "$filePath${query ? '?' + query : ''}${fragment ? '#' + fragment : ''}".toString()
}
/**
* Validates the integrity of the LinPath. If there is a problem with the validation an exception is thrown.
* To validate just try to get the find target target path. It checks if lid exists, it is a FileOutput,
* the target path exists and the checksum is the same as the stored in the metadata.
*/
FileCheck validate() throws Exception{
final obj = fileSystem.store.load(filePath)
if( !obj )
return new FileCheck("File cannot be found")
if( obj instanceof FileOutput ) {
final res = validateDataOutput(obj as FileOutput)
return new FileCheck(res, obj)
}
return new FileCheck("Unexpected lineage object type: ${obj.getClass().getName()}")
}
@EqualsAndHashCode
static class FileCheck {
final String error
final FileOutput file
FileCheck(String error, FileOutput out=null) {
this.error = error
this.file = out
}
/**
* Implements groovy truth
*/
boolean asBoolean() {
return error==null && file!=null
}
}
}

View File

@@ -0,0 +1,59 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.fs
import static LinPath.*
import java.nio.file.Path
import groovy.transform.CompileStatic
import nextflow.lineage.config.LineageConfig
import nextflow.file.FileHelper
import nextflow.file.FileSystemPathFactory
/**
* Implements a {@link FileSystemPathFactory} for LID file system
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
@CompileStatic
class LinPathFactory extends FileSystemPathFactory {
@Override
protected Path parseUri(String uri) {
return isLidUri(uri) ? create(uri) : null
}
@Override
protected String toUriString(Path path) {
return path instanceof LinPath ? ((LinPath)path).toUriString() : null
}
@Override
protected String getBashLib(Path target) {
return null
}
@Override
protected String getUploadCmd(String source, Path target) {
return null
}
static LinPath create(String path) {
final uri = LinPath.asUri(path)
return (LinPath) FileHelper.getOrCreateFileSystemFor(uri, LineageConfig.asMap()).provider().getPath(uri)
}
}

View File

@@ -0,0 +1,47 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.model.v1beta1
import java.nio.file.Path
import groovy.transform.Canonical
import groovy.transform.CompileStatic
import nextflow.util.CacheHelper
/**
* Models a checksum including the value as well as the algortihm and mode used to compute it.
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io
*/
@Canonical
@CompileStatic
class Checksum {
String value
String algorithm
String mode
static Checksum of(String value, String algorithm, CacheHelper.HashMode mode) {
new Checksum(value, algorithm, mode.toString().toLowerCase())
}
static Checksum ofNextflow(String value) {
new Checksum(CacheHelper.hasher(value).hash().toString(), 'nextflow', CacheHelper.HashMode.DEFAULT().toString().toLowerCase())
}
static Checksum ofNextflow(Path path) {
new Checksum(CacheHelper.hasher(path).hash().toString(), 'nextflow', CacheHelper.HashMode.DEFAULT().toString().toLowerCase())
}
}

View File

@@ -0,0 +1,38 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.model.v1beta1
import groovy.transform.Canonical
import groovy.transform.CompileStatic
/**
* Models a data path which includes the path and a checksum to validate the content of the path.
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io
*/
@Canonical
@CompileStatic
class DataPath {
/**
* Real path of the output data.
*/
String path
/**
* Checksum of the output data.
*/
Checksum checksum
}

View File

@@ -0,0 +1,72 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.model.v1beta1
import groovy.transform.Canonical
import groovy.transform.CompileStatic
import nextflow.lineage.serde.LinSerializable
import java.time.OffsetDateTime
/**
* Model a base class for workflow and task outputs
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Canonical
@CompileStatic
class FileOutput implements LinSerializable {
/**
* Real path of the output data.
*/
String path
/**
* Checksum of the output data.
*/
Checksum checksum
/**
* Entity that generated the data. Possible entities are:
* - a FileOutput if the workflow published from a task data.
* - a TaskRun if the data is a task output.
* - a WorkflowRun if the data is generated by the workflow (e.g., an index file).
*/
String source
/**
* Reference to the WorkflowRun that generated the data.
*/
String workflowRun
/**
* Reference to the task that generated the data.
*/
String taskRun
/**
* Size of the data.
*/
long size
/**
* Data creation date.
*/
OffsetDateTime createdAt
/**
* Data last modified date.
*/
OffsetDateTime modifiedAt
/**
* Labels attached to the data
*/
List<String> labels
}

View File

@@ -0,0 +1,26 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.model.v1beta1
/**
* Marker interface holding lineage model common definitions
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
interface LinModel {
static final public String VERSION = 'lineage/v1beta1'
}

View File

@@ -0,0 +1,33 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.model.v1beta1
import groovy.transform.Canonical
import groovy.transform.CompileStatic
/**
* Model Workflow and Task Parameters.
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io
*/
@Canonical
@CompileStatic
class Parameter {
String type
String name
Object value
}

View File

@@ -0,0 +1,53 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.model.v1beta1
import groovy.transform.Canonical
import groovy.transform.CompileStatic
import nextflow.lineage.serde.LinSerializable
import java.time.OffsetDateTime
/**
* Models task results.
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
@Canonical
@CompileStatic
class TaskOutput implements LinSerializable {
/**
* Reference to the task that generated the output.
*/
String taskRun
/**
* Reference to the WorkflowRun that generated the output.
*/
String workflowRun
/**
* Creation date of this task output description
*/
OffsetDateTime createdAt
/**
* Output of the task
*/
List<Parameter> output
/**
* Labels attached to the task output
*/
List<String> labels
}

View File

@@ -0,0 +1,79 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.model.v1beta1
import groovy.transform.Canonical
import groovy.transform.CompileStatic
import nextflow.lineage.serde.LinSerializable
/**
* Models a task execution.
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Canonical
@CompileStatic
class TaskRun implements LinSerializable {
/**
* Execution session identifier
*/
String sessionId
/**
* Task name
*/
String name
/**
* Checksum of the task source code
*/
Checksum codeChecksum
/**
* Resolved task script
*/
String script
/**
* Task run input
*/
List<Parameter> input
/**
* Container used for the task run
*/
String container
/**
* Conda environment used for the task run
*/
String conda
/**
* Spack environment used for the task run
*/
String spack
/**
* Architecture defined in the Spack environment used for the task run
*/
String architecture
/**
* Global variables defined in the task run
*/
Map globalVars
/**
* Binaries used in the task run
*/
List<DataPath> binEntries
/**
* Workflow run associated to the task run
*/
String workflowRun
}

View File

@@ -0,0 +1,44 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.model.v1beta1
import groovy.transform.Canonical
import groovy.transform.CompileStatic
import nextflow.lineage.serde.LinSerializable
/**
* Models a workflow definition.
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io
*/
@Canonical
@CompileStatic
class Workflow implements LinSerializable {
/**
* List of script files used by a workflow, starting with the main script
*/
List<DataPath> scriptFiles
/**
* Workflow repository
*/
String repository
/**
* Workflow commit identifier
*/
String commitId
}

View File

@@ -0,0 +1,45 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.model.v1beta1
import groovy.transform.Canonical
import groovy.transform.CompileStatic
import nextflow.lineage.serde.LinSerializable
import java.time.OffsetDateTime
/**
* Models the results of a workflow execution.
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io
*/
@Canonical
@CompileStatic
class WorkflowOutput implements LinSerializable {
/**
* Creation date of the workflow output
*/
OffsetDateTime createdAt
/**
* Workflow run that generated the output
*/
String workflowRun
/**
* Workflow output
*/
List<Parameter> output
}

View File

@@ -0,0 +1,56 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.model.v1beta1
import groovy.transform.Canonical
import groovy.transform.CompileStatic
import nextflow.lineage.serde.LinSerializable
/**
* Models a Workflow Execution
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io
*/
@Canonical
@CompileStatic
class WorkflowRun implements LinSerializable {
/**
* Description of the workflow associated with the workflow run.
*/
Workflow workflow
/**
* Session identifier used in the workflow run
*/
String sessionId
/**
* Workflow run name
*/
String name
/**
* Workflow parameters
*/
List<Parameter> params
/**
* Resolved Configuration
*/
Map config
/**
* Raw metadata
*/
Map metadata
}

View File

@@ -0,0 +1,36 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.serde
import groovy.transform.CompileStatic
import nextflow.serde.gson.GsonEncoder
/**
* Implements a JSON encoder for lineage model objects
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class LinEncoder extends GsonEncoder<LinSerializable> {
LinEncoder() {
withTypeAdapterFactory(new LinTypeAdapterFactory())
// enable rendering of null values
withSerializeNulls(true)
}
}

View File

@@ -0,0 +1,29 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.serde
import groovy.transform.CompileStatic
import nextflow.serde.JsonSerializable
/**
* Marker interface for lineage serializable objects
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
interface LinSerializable extends JsonSerializable {
}

View File

@@ -0,0 +1,110 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.serde
import com.google.gson.Gson
import com.google.gson.JsonObject
import com.google.gson.JsonParseException
import com.google.gson.JsonParser
import com.google.gson.TypeAdapter
import com.google.gson.reflect.TypeToken
import com.google.gson.stream.JsonReader
import com.google.gson.stream.JsonWriter
import groovy.transform.CompileStatic
import nextflow.lineage.model.v1beta1.FileOutput
import nextflow.lineage.model.v1beta1.LinModel
import nextflow.lineage.model.v1beta1.TaskOutput
import nextflow.lineage.model.v1beta1.TaskRun
import nextflow.lineage.model.v1beta1.Workflow
import nextflow.lineage.model.v1beta1.WorkflowOutput
import nextflow.lineage.model.v1beta1.WorkflowRun
import nextflow.serde.gson.RuntimeTypeAdapterFactory
/**
* Class to serialize LiSerializable objects including the Lineage model version.
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
@CompileStatic
class LinTypeAdapterFactory<T> extends RuntimeTypeAdapterFactory<T> {
public static final String VERSION_FIELD = 'version'
public static final String SPEC_FIELD = 'spec'
public static final String CURRENT_VERSION = LinModel.VERSION
LinTypeAdapterFactory() {
super(LinSerializable.class, "kind", false)
this.registerSubtype(WorkflowRun, WorkflowRun.simpleName)
.registerSubtype(WorkflowOutput, WorkflowOutput.simpleName)
.registerSubtype(Workflow, Workflow.simpleName)
.registerSubtype(TaskRun, TaskRun.simpleName)
.registerSubtype(TaskOutput, TaskOutput.simpleName)
.registerSubtype(FileOutput, FileOutput.simpleName)
}
@Override
<R> TypeAdapter<R> create(Gson gson, TypeToken<R> type) {
if (!LinSerializable.class.isAssignableFrom(type.rawType)) {
return null
}
def delegate = super.create(gson, type as TypeToken<T>)
if (delegate == null) {
return null
}
return new TypeAdapter<R>() {
@Override
void write(JsonWriter out, R value) throws IOException {
final object = new JsonObject()
object.addProperty(VERSION_FIELD, CURRENT_VERSION)
String label = getLabelFromSubtype(value.class)
if (!label)
throw new JsonParseException("Not registered class ${value.class}")
object.addProperty(getTypeFieldName(), label)
def json = gson.toJsonTree(value)
object.add(SPEC_FIELD, json)
gson.toJson(object, out)
}
@Override
R read(JsonReader reader) throws IOException {
final obj = JsonParser.parseReader(reader)?.getAsJsonObject()
if( obj==null )
throw new JsonParseException("Parsed JSON object is null")
final versionEl = obj.get(VERSION_FIELD)
if (versionEl == null || versionEl.asString != CURRENT_VERSION) {
throw new JsonParseException("Invalid or missing '${VERSION_FIELD}' JSON property")
}
final typeEl = obj.get(getTypeFieldName())
if( typeEl==null )
throw new JsonParseException("JSON property '${getTypeFieldName()}' not found")
// Check if this is the new format (has 'spec' field) or old format (data at root level)
final specEl = obj.get(SPEC_FIELD)?.asJsonObject
if ( specEl != null ) {
// New format: data is wrapped in 'spec' field
specEl.add(getTypeFieldName(), typeEl)
return (R) delegate.fromJsonTree(specEl)
} else {
// Old format: data is at root level, just remove version field
obj.remove(VERSION_FIELD)
return (R) delegate.fromJsonTree(obj)
}
}
}
}
}

View File

@@ -0,0 +1,21 @@
#
# Copyright 2013-2025, Seqera Labs
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
nextflow.lineage.DefaultLinStoreFactory
nextflow.lineage.LinExtensionImpl
nextflow.lineage.LinObserverFactory
nextflow.lineage.cli.LinCommandImpl
nextflow.lineage.config.LineageConfig

View File

@@ -0,0 +1,17 @@
#
# Copyright 2013-2026, Seqera Labs
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
nextflow.lineage.fs.LinFileSystemProvider

View File

@@ -0,0 +1,79 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage
import spock.lang.Specification
import java.nio.file.Files
import java.nio.file.Path
/**
* Lineage History file tests
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
class DefaultLinHistoryLogTest extends Specification {
Path tempDir
Path historyFile
DefaultLinHistoryLog linHistoryLog
def setup() {
tempDir = Files.createTempDirectory("wdir")
historyFile = tempDir.resolve("lin-history")
linHistoryLog = new DefaultLinHistoryLog(historyFile)
}
def cleanup(){
tempDir?.deleteDir()
}
def "write should add a new file to the history folder"() {
given:
UUID sessionId = UUID.randomUUID()
String runName = "TestRun"
String runLid = "lid://123"
when:
linHistoryLog.write(runName, sessionId, runLid)
then:
def files = historyFile.listFiles()
files.size() == 1
def parsedRecord = LinHistoryRecord.parse(files[0].text)
parsedRecord.sessionId == sessionId
parsedRecord.runName == runName
parsedRecord.runLid == runLid
}
def 'should get records' () {
given:
UUID sessionId = UUID.randomUUID()
String runName = "Run1"
String runLid = "lid://123"
and:
linHistoryLog.write(runName, sessionId, runLid)
when:
def records = linHistoryLog.getRecords()
then:
records.size() == 1
records[0].sessionId == sessionId
records[0].runName == runName
records[0].runLid == runLid
}
}

View File

@@ -0,0 +1,49 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage
import nextflow.lineage.config.LineageConfig
import spock.lang.Specification
import spock.lang.Unroll
/**
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
class DefaultLinStoreFactoryTest extends Specification {
@Unroll
def 'should validate can open' () {
given:
def factory = new DefaultLinStoreFactory()
def config = new LineageConfig(CONFIG)
expect:
factory.canOpen(config) == EXPECTED
where:
EXPECTED | CONFIG
true | [:]
true | [store:[location:'/some/path']]
true | [store:[location:'some/rel/path']]
true | [store:[location:'file:/this/that']]
true | [store:[location:'s3://some/path']]
false | [store:[location:'http://some/path']]
false | [store:[location:'jdbc:foo']]
}
}

View File

@@ -0,0 +1,161 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage
import java.nio.file.Files
import java.nio.file.Path
import java.time.Instant
import java.time.OffsetDateTime
import java.time.ZoneOffset
import nextflow.lineage.model.v1beta1.Checksum
import nextflow.lineage.model.v1beta1.DataPath
import nextflow.lineage.model.v1beta1.FileOutput
import nextflow.lineage.model.v1beta1.Parameter
import nextflow.lineage.model.v1beta1.Workflow
import nextflow.lineage.model.v1beta1.WorkflowRun
import nextflow.lineage.serde.LinEncoder
import nextflow.lineage.config.LineageConfig
import spock.lang.Specification
import spock.lang.TempDir
/**
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
class DefaultLinStoreTest extends Specification {
@TempDir
Path tempDir
Path storeLocation
LineageConfig config
def setup() {
storeLocation = tempDir.resolve("store")
def configMap = [enabled: true, store: [location: storeLocation.toString(), logLocation: storeLocation.resolve(".log").toString()]]
config = new LineageConfig(configMap)
}
def 'should open store'() {
given:
def store = new DefaultLinStore()
when:
store.open(config)
def historyLog = store.getHistoryLog()
then:
store.getLocation() == storeLocation
historyLog != null
historyLog instanceof DefaultLinHistoryLog
}
def "save should store value in the correct file location"() {
given:
def key = "testKey"
def value = new FileOutput("/path/to/file", new Checksum("hash_value", "hash_algorithm", "standard"), "lid://source", "lid://workflow", "lid://task", 1234)
def lidStore = new DefaultLinStore()
lidStore.open(config)
when:
lidStore.save(key, value)
then:
def filePath = storeLocation.resolve("$key/.data.json")
Files.exists(filePath)
filePath.text == new LinEncoder().encode(value)
}
def "load should retrieve stored value correctly"() {
given:
def key = "testKey"
def value = new FileOutput("/path/to/file", new Checksum("hash_value", "hash_algorithm", "standard"), "lid://source", "lid://workflow", "lid://task", 1234)
def lidStore = new DefaultLinStore()
lidStore.open(config)
lidStore.save(key, value)
expect:
lidStore.load(key).toString() == value.toString()
}
def "load should return null if key does not exist"() {
given:
def lidStore = new DefaultLinStore()
lidStore.open(config)
expect:
lidStore.load("nonexistentKey") == null
}
def 'should query' () {
given:
def uniqueId = UUID.randomUUID()
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(1234567), ZoneOffset.UTC)
def mainScript = new DataPath("file://path/to/main.nf", new Checksum("78910", "nextflow", "standard"))
def workflow = new Workflow([mainScript],"https://nextflow.io/nf-test/", "123456" )
def key = "testKey"
def value1 = new WorkflowRun(workflow, uniqueId.toString(), "test_run", [ new Parameter("String", "param1", "value1"), new Parameter("String", "param2", "value2")] )
def key2 = "testKey2"
def value2 = new FileOutput("/path/tp/file1", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", null, 1234, time, time, ["value1", "value2"])
def key3 = "testKey3"
def value3 = new FileOutput("/path/tp/file2", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", null, 1234, time, time, ["value2", "value3"])
def key4 = "testKey4"
def value4 = new FileOutput("/path/tp/file", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", null, 1234, time, time, ["value4", "value3"])
def lidStore = new DefaultLinStore()
lidStore.open(config)
lidStore.save(key, value1)
lidStore.save(key2, value2)
lidStore.save(key3, value3)
lidStore.save(key4, value4)
when:
def results = lidStore.search( [type:['FileOutput'], labels:['value2']]).toList()
then:
results.size() == 2
results.containsAll([key2,key3])
}
def 'should search subkeys' () {
given:
def uniqueId = UUID.randomUUID()
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(1234567), ZoneOffset.UTC)
def mainScript = new DataPath("file://path/to/main.nf", new Checksum("78910", "nextflow", "standard"))
def workflow = new Workflow([mainScript], "https://nextflow.io/nf-test/", "123456")
def key = "testKey"
def value1 = new WorkflowRun(workflow, uniqueId.toString(), "test_run", [new Parameter("String", "param1", "value1"), new Parameter("String", "param2", "value2")])
def key2 = "testKey/file1"
def value2 = new FileOutput("/path/file1", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", null, 1234, time, time, ["value1", "value2"])
def key3 = "testKey/subfolder/file3"
def value3 = new FileOutput("/path//file2", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", null, 1234, time, time, ["value2", "value3"])
def key4 = "testKey2/file2"
def value4 = new FileOutput("/path/tp/file", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", null, 1234, time, time, ["value4", "value3"])
def lidStore = new DefaultLinStore()
lidStore.open(config)
lidStore.save(key, value1)
lidStore.save(key2, value2)
lidStore.save(key3, value3)
lidStore.save(key4, value4)
when:
def results = lidStore.getSubKeys("testKey").toList()
then:
results.size() == 2
results.containsAll([key2, key3])
}
}

View File

@@ -0,0 +1,112 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage
import java.nio.file.Path
import java.time.Instant
import java.time.OffsetDateTime
import java.time.ZoneOffset
import groovyx.gpars.dataflow.DataflowQueue
import nextflow.Channel
import nextflow.Session
import nextflow.lineage.config.LineageConfig
import nextflow.lineage.fs.LinPathFactory
import nextflow.lineage.model.v1beta1.Checksum
import nextflow.lineage.model.v1beta1.DataPath
import nextflow.lineage.model.v1beta1.FileOutput
import nextflow.lineage.model.v1beta1.Parameter
import nextflow.lineage.model.v1beta1.Workflow
import nextflow.lineage.model.v1beta1.WorkflowRun
import spock.lang.Specification
import spock.lang.TempDir
import static nextflow.lineage.fs.LinPath.*
import static test.ScriptHelper.runDataflow
/**
* Lineage channel extensions tests
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
class LinExtensionImplTest extends Specification {
@TempDir
Path tempDir
Path storeLocation
Map configMap
def setup() {
storeLocation = tempDir.resolve("store")
configMap = [linage: [enabled: true, store: [location: storeLocation.toString()]]]
}
def 'should return global query results' () {
given:
def uniqueId = UUID.randomUUID()
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(1234567), ZoneOffset.UTC)
def mainScript = new DataPath("file://path/to/main.nf", new Checksum("78910", "nextflow", "standard"))
def workflow = new Workflow([mainScript],"https://nextflow.io/nf-test/", "123456" )
def key = "testKey"
def value1 = new WorkflowRun(workflow, uniqueId.toString(), "test_run", [ new Parameter("String", "param1", "value1"), new Parameter("String", "param2", "value2")] )
def key2 = "testKey2"
def value2 = new FileOutput("/path/tp/file1", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", "taskid", 1234, time, time, ["value1","value2"])
def key3 = "testKey3"
def value3 = new FileOutput("/path/tp/file2", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", null, 1234, time, time, ["value2", "value3"])
def key4 = "testKey4"
def value4 = new FileOutput("/path/tp/file", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", "taskid", 1234, time, time, ["value4","value3"])
def lidStore = new DefaultLinStore()
def session = Mock(Session) {
getConfig() >> configMap
}
lidStore.open(LineageConfig.create(session))
lidStore.save(key, value1)
lidStore.save(key2, value2)
lidStore.save(key3, value3)
lidStore.save(key4, value4)
def linExt = Spy(new LinExtensionImpl())
when:
def results = new DataflowQueue()
linExt.fromLineage(session, results, [label: ["value2", "value3"]])
then:
linExt.getStore(session) >> lidStore
and:
results.val == LinPathFactory.create( asUriString(key3) )
results.val == Channel.STOP
when:
results = new DataflowQueue()
linExt.fromLineage(session, results, [taskRun: "taskid", label: ["value4"]])
then:
linExt.getStore(session) >> lidStore
and:
results.val == LinPathFactory.create( asUriString(key4) )
results.val == Channel.STOP
when:
results = new DataflowQueue()
linExt.fromLineage(session, results, [workflowRun: "testkey", taskRun: "taskid", label: "value2"])
then:
linExt.getStore(session) >> lidStore
and:
results.val == LinPathFactory.create( asUriString(key2) )
results.val == Channel.STOP
}
}

View File

@@ -0,0 +1,61 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage
import spock.lang.Specification
/**
* Lineage History Record tests
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
class LinHistoryRecordTest extends Specification {
def "LinHistoryRecord parse should throw for invalid record"() {
when:
LinHistoryRecord.parse("invalid-record")
then:
thrown(IllegalArgumentException)
}
def "LinHistoryRecord parse should handle 4-column record"() {
given:
def timestamp = new Date()
def formattedTimestamp = LinHistoryRecord.TIMESTAMP_FMT.format(timestamp)
def line = "${formattedTimestamp}\trun-1\t${UUID.randomUUID()}\tlid://123"
when:
def record = LinHistoryRecord.parse(line)
then:
record.timestamp != null
record.runName == "run-1"
record.runLid == "lid://123"
}
def "LinHistoryRecord toString should produce tab-separated format"() {
given:
UUID sessionId = UUID.randomUUID()
def record = new LinHistoryRecord(new Date(), "TestRun", sessionId, "lid://123")
when:
def line = record.toString()
then:
line.contains("\t")
line.split("\t").size() == 4
}
}

View File

@@ -0,0 +1,983 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage
import nextflow.extension.FilesEx
import nextflow.lineage.exception.OutputRelativePathException
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.attribute.BasicFileAttributes
import com.google.common.hash.HashCode
import nextflow.NextflowMeta
import nextflow.Session
import nextflow.file.FileHolder
import nextflow.lineage.model.v1beta1.Checksum
import nextflow.lineage.model.v1beta1.FileOutput
import nextflow.lineage.model.v1beta1.DataPath
import nextflow.lineage.model.v1beta1.Parameter
import nextflow.lineage.model.v1beta1.TaskOutput
import nextflow.lineage.model.v1beta1.Workflow
import nextflow.lineage.model.v1beta1.WorkflowOutput
import nextflow.lineage.model.v1beta1.WorkflowRun
import nextflow.lineage.serde.LinEncoder
import nextflow.lineage.config.LineageConfig
import nextflow.processor.TaskConfig
import nextflow.processor.TaskHandler
import nextflow.processor.TaskId
import nextflow.processor.TaskRun
import nextflow.script.ScriptBinding
import nextflow.script.PlatformMetadata
import nextflow.script.ScriptMeta
import nextflow.script.TokenVar
import nextflow.script.WorkflowMetadata
import nextflow.script.params.EnvOutParam
import nextflow.script.params.FileInParam
import nextflow.script.params.FileOutParam
import nextflow.script.params.InParam
import nextflow.script.params.OutParam
import nextflow.script.params.StdInParam
import nextflow.script.params.StdOutParam
import nextflow.script.params.ValueInParam
import nextflow.script.params.ValueOutParam
import nextflow.trace.event.FilePublishEvent
import nextflow.trace.event.TaskEvent
import nextflow.trace.event.WorkflowOutputEvent
import nextflow.util.CacheHelper
import nextflow.util.PathNormalizer
import spock.lang.Shared
import spock.lang.Specification
import spock.lang.Unroll
import static nextflow.lineage.fs.LinPath.*
/**
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
class LinObserverTest extends Specification {
@Shared
Path lidFolder = Files.createTempDirectory("lid")
def cleanupSpec(){
lidFolder.deleteDir()
}
def 'should normalize paths' (){
given:
def folder = Files.createTempDirectory('test')
def workDir = folder.resolve("workDir")
def projectDir = folder.resolve("projectDir")
def metadata = Mock(WorkflowMetadata){
getRepository() >> "https://nextflow.io/nf-test/"
getCommitId() >> "123456"
getScriptId() >> "78910"
getProjectDir() >> projectDir
getWorkDir() >> workDir
}
def params = [path: workDir.resolve("path/file.txt"), sequence: projectDir.resolve("file2.txt").toString(), value: 12]
when:
def results = LinObserver.getNormalizedParams(params, new PathNormalizer(metadata))
then:
results.size() == 3
results.get(0).name == "path"
results.get(0).type == Path.simpleName
results.get(0).value == "work/path/file.txt"
results.get(1).name == "sequence"
results.get(1).type == "String"
results.get(1).value == projectDir.resolve("file2.txt").toString()
results.get(2).name == "value"
results.get(2).type == "Integer"
results.get(2).value == 12
cleanup:
ScriptMeta.reset()
folder?.deleteDir()
}
def 'should collect script files' () {
given:
def folder = Files.createTempDirectory('test')
and:
def config = [workflow:[lineage:[enabled: true, store:[location:folder.toString()]]]]
def store = new DefaultLinStore();
def uniqueId = UUID.randomUUID()
def scriptFile = folder.resolve("main.nf")
def module1 = folder.resolve("a_script1.nf"); module1.text = 'hola'
def module2 = folder.resolve("b_script2.nf"); module2.text = 'world'
and:
def metadata = Mock(WorkflowMetadata){
getRepository() >> "https://nextflow.io/nf-test/"
getCommitId() >> "123456"
getScriptId() >> "78910"
getScriptFile() >> scriptFile
getProjectDir() >> folder.resolve("projectDir")
getWorkDir() >> folder.resolve("workDir")
}
def session = Mock(Session) {
getConfig() >> config
getUniqueId() >> uniqueId
getRunName() >> "test_run"
getWorkflowMetadata() >> metadata
getParams() >> new ScriptBinding.ParamsMap()
}
store.open(LineageConfig.create(session))
def observer = Spy(new LinObserver(session, store))
when:
def files = observer.collectScriptDataPaths(new PathNormalizer(metadata))
then:
observer.allScriptFiles() >> [ module2, scriptFile, module1 ]
and:
files.size() == 3
and:
files[0].path == "file://${scriptFile.toString()}"
files[0].checksum == new Checksum("78910", "nextflow", "standard")
and:
files[1].path == "file://$module1"
files[1].checksum == Checksum.ofNextflow(module1.text)
and:
files[2].path == "file://$module2"
files[2].checksum == Checksum.ofNextflow(module2.text)
cleanup:
ScriptMeta.reset()
folder?.deleteDir()
}
def 'should save workflow' (){
given:
def folder = Files.createTempDirectory('test')
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
def store = new DefaultLinStore();
def uniqueId = UUID.randomUUID()
def scriptFile = folder.resolve("main.nf")
def map = [
repository: "https://nextflow.io/nf-test/",
commitId: "123456",
scriptId: "78910",
scriptFile: scriptFile,
projectDir: folder.resolve("projectDir"),
revision: "main",
projectName: "nextflow.io/nf-test",
workDir: folder.resolve("workDir")
]
def metadata = Mock(WorkflowMetadata){
getRepository() >> map.repository
getCommitId() >> map.commitId
getScriptId() >> map.scriptId
getScriptFile() >> map.scriptFile
getProjectDir() >> map.projectDir
getRevision() >> map.revision
getProjectName() >> map.projectName
getWorkDir() >> map.workDir
toMap() >> map
}
def session = Mock(Session) {
getConfig() >> config
getUniqueId() >> uniqueId
getRunName() >> "test_run"
getWorkflowMetadata() >> metadata
getParams() >> new ScriptBinding.ParamsMap()
}
store.open(LineageConfig.create(session))
def observer = new LinObserver(session, store)
def mainScript = new DataPath("file://${scriptFile.toString()}", new Checksum("78910", "nextflow", "standard"))
def workflow = new Workflow([mainScript], map.repository, map.commitId)
def workflowRun = new WorkflowRun(workflow, uniqueId.toString(), "test_run", [], config, map)
when:
observer.onFlowCreate(session)
observer.onFlowBegin()
then:
folder.resolve("${observer.executionHash}/.data.json").text == new LinEncoder().encode(workflowRun)
cleanup:
folder?.deleteDir()
}
def 'should strip sensitive user data from platform metadata in lineage' () {
given:
def folder = Files.createTempDirectory('test')
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
def store = new DefaultLinStore()
def uniqueId = UUID.randomUUID()
def scriptFile = folder.resolve("main.nf")
and:
def platformMeta = new PlatformMetadata()
platformMeta.user = new PlatformMetadata.User([
id: 'u1234', userName: 'john-smith', email: 'john.smith@acme.com',
firstName: 'John', lastName: 'Smith', organization: 'ACME'
])
platformMeta.workspace = new PlatformMetadata.Workspace([
workspaceId: '1234', workspaceName: 'my-ws', workspaceFullName: 'My WS', orgName: 'ACME'
])
def map = [
repository: "https://nextflow.io/nf-test/",
commitId: "123456",
scriptId: "78910",
scriptFile: scriptFile,
projectDir: folder.resolve("projectDir"),
revision: "main",
projectName: "nextflow.io/nf-test",
workDir: folder.resolve("workDir"),
platform: platformMeta
]
def metadata = Mock(WorkflowMetadata) {
getRepository() >> map.repository
getCommitId() >> map.commitId
getScriptId() >> map.scriptId
getScriptFile() >> map.scriptFile
getProjectDir() >> map.projectDir
getWorkDir() >> map.workDir
toMap() >> map
}
def session = Mock(Session) {
getConfig() >> config
getUniqueId() >> uniqueId
getRunName() >> "test_run"
getWorkflowMetadata() >> metadata
getParams() >> new ScriptBinding.ParamsMap()
}
store.open(LineageConfig.create(session))
def observer = new LinObserver(session, store)
when:
observer.onFlowCreate(session)
observer.onFlowBegin()
def stored = store.load(observer.executionHash) as WorkflowRun
then:
def storedPlatform = stored.metadata.platform as Map
storedPlatform.user.id == 'u1234'
storedPlatform.user.userName == 'john-smith'
storedPlatform.user.organization == 'ACME'
!storedPlatform.user.containsKey('email')
!storedPlatform.user.containsKey('firstName')
!storedPlatform.user.containsKey('lastName')
cleanup:
folder?.deleteDir()
}
def 'should convert Path values to URI strings in workflow metadata' () {
given:
def folder = Files.createTempDirectory('test')
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
def store = new DefaultLinStore()
def uniqueId = UUID.randomUUID()
def scriptFile = folder.resolve("main.nf")
def workDir = folder.resolve("workDir")
def somePath = folder.resolve("data/input.txt")
def map = [
repository : "https://nextflow.io/nf-test/",
commitId : "abc123",
scriptId : "78910",
scriptFile : scriptFile,
projectDir : folder.resolve("projectDir"),
workDir : workDir,
somePathKey : somePath,
someStringKey: "hello"
]
def metadata = Mock(WorkflowMetadata) {
getRepository() >> map.repository
getCommitId() >> map.commitId
getScriptId() >> map.scriptId
getScriptFile() >> map.scriptFile
getProjectDir() >> map.projectDir
getWorkDir() >> map.workDir
toMap() >> map
}
def session = Mock(Session) {
getConfig() >> config
getUniqueId() >> uniqueId
getRunName() >> "test_run"
getWorkflowMetadata() >> metadata
getParams() >> new ScriptBinding.ParamsMap()
}
store.open(LineageConfig.create(session))
def observer = new LinObserver(session, store)
when:
observer.onFlowCreate(session)
observer.onFlowBegin()
def stored = store.load(observer.executionHash) as WorkflowRun
then:
stored.metadata.somePathKey == FilesEx.toUriString(somePath)
stored.metadata.someStringKey == "hello"
cleanup:
folder?.deleteDir()
}
def 'should remove transient properties from workflow metadata' () {
given:
def folder = Files.createTempDirectory('test')
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
def store = new DefaultLinStore()
def uniqueId = UUID.randomUUID()
def scriptFile = folder.resolve("main.nf")
def map = [
repository : "https://nextflow.io/nf-test/",
commitId : "abc123",
scriptId : "78910",
scriptFile : scriptFile,
projectDir : folder.resolve("projectDir"),
workDir : folder.resolve("workDir"),
// transient properties that must be removed
completed : new Date(),
duration : 1000L,
exitStatus : 0,
errorMessage: "none",
errorReport : "none",
stats : [:],
success : true
]
def metadata = Mock(WorkflowMetadata) {
getRepository() >> map.repository
getCommitId() >> map.commitId
getScriptId() >> map.scriptId
getScriptFile() >> map.scriptFile
getProjectDir() >> map.projectDir
getWorkDir() >> map.workDir
toMap() >> map
}
def session = Mock(Session) {
getConfig() >> config
getUniqueId() >> uniqueId
getRunName() >> "test_run"
getWorkflowMetadata() >> metadata
getParams() >> new ScriptBinding.ParamsMap()
}
store.open(LineageConfig.create(session))
def observer = new LinObserver(session, store)
when:
observer.onFlowCreate(session)
observer.onFlowBegin()
def stored = store.load(observer.executionHash) as WorkflowRun
then:
!stored.metadata.containsKey('completed')
!stored.metadata.containsKey('duration')
!stored.metadata.containsKey('exitStatus')
!stored.metadata.containsKey('errorMessage')
!stored.metadata.containsKey('errorReport')
!stored.metadata.containsKey('stats')
!stored.metadata.containsKey('success')
stored.metadata.containsKey('commitId')
cleanup:
folder?.deleteDir()
}
def 'should convert NextflowMeta to JSON map in workflow metadata' () {
given:
def folder = Files.createTempDirectory('test')
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
def store = new DefaultLinStore()
def uniqueId = UUID.randomUUID()
def scriptFile = folder.resolve("main.nf")
def nfMeta = new NextflowMeta("24.10.0", 9999, "01-01-2024 00:00 UTC")
def map = [
repository : "https://nextflow.io/nf-test/",
commitId : "abc123",
scriptId : "78910",
scriptFile : scriptFile,
projectDir : folder.resolve("projectDir"),
workDir : folder.resolve("workDir"),
nextflow : nfMeta
]
def metadata = Mock(WorkflowMetadata) {
getRepository() >> map.repository
getCommitId() >> map.commitId
getScriptId() >> map.scriptId
getScriptFile() >> map.scriptFile
getProjectDir() >> map.projectDir
getWorkDir() >> map.workDir
toMap() >> map
}
def session = Mock(Session) {
getConfig() >> config
getUniqueId() >> uniqueId
getRunName() >> "test_run"
getWorkflowMetadata() >> metadata
getParams() >> new ScriptBinding.ParamsMap()
}
store.open(LineageConfig.create(session))
def observer = new LinObserver(session, store)
when:
observer.onFlowCreate(session)
observer.onFlowBegin()
def stored = store.load(observer.executionHash) as WorkflowRun
then:
stored.metadata.nextflow instanceof Map
(stored.metadata.nextflow as Map).version == "24.10.0"
cleanup:
folder?.deleteDir()
}
def 'should normalize configFiles in workflow metadata' () {
given:
def folder = Files.createTempDirectory('test')
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
def store = new DefaultLinStore()
def uniqueId = UUID.randomUUID()
def scriptFile = folder.resolve("main.nf")
def workDir = folder.resolve("workDir")
def projectDir = folder.resolve("projectDir")
def configFile = projectDir.resolve("nextflow.config")
def map = [
repository : "https://nextflow.io/nf-test/",
commitId : "abc123",
scriptId : "78910",
scriptFile : scriptFile,
projectDir : projectDir,
workDir : workDir,
configFiles : [configFile]
]
def metadata = Mock(WorkflowMetadata) {
getRepository() >> map.repository
getCommitId() >> map.commitId
getScriptId() >> map.scriptId
getScriptFile() >> map.scriptFile
getProjectDir() >> map.projectDir
getWorkDir() >> map.workDir
toMap() >> map
}
def session = Mock(Session) {
getConfig() >> config
getUniqueId() >> uniqueId
getRunName() >> "test_run"
getWorkflowMetadata() >> metadata
getParams() >> new ScriptBinding.ParamsMap()
}
store.open(LineageConfig.create(session))
def observer = new LinObserver(session, store)
when:
observer.onFlowCreate(session)
observer.onFlowBegin()
def stored = store.load(observer.executionHash) as WorkflowRun
then:
def normalizer = new PathNormalizer(metadata)
(stored.metadata.configFiles as List) == [normalizer.normalizePath(configFile)]
cleanup:
folder?.deleteDir()
}
def 'should handle null user in platform metadata' () {
given:
def folder = Files.createTempDirectory('test')
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
def store = new DefaultLinStore()
def uniqueId = UUID.randomUUID()
def scriptFile = folder.resolve("main.nf")
def platformMeta = new PlatformMetadata()
platformMeta.workflowId = 'wf-999'
platformMeta.workflowUrl = 'https://tower.nf/orgs/org/workspaces/ws/watch/wf-999'
// user is intentionally left null
def map = [
repository : "https://nextflow.io/nf-test/",
commitId : "abc123",
scriptId : "78910",
scriptFile : scriptFile,
projectDir : folder.resolve("projectDir"),
workDir : folder.resolve("workDir"),
platform : platformMeta
]
def metadata = Mock(WorkflowMetadata) {
getRepository() >> map.repository
getCommitId() >> map.commitId
getScriptId() >> map.scriptId
getScriptFile() >> map.scriptFile
getProjectDir() >> map.projectDir
getWorkDir() >> map.workDir
toMap() >> map
}
def session = Mock(Session) {
getConfig() >> config
getUniqueId() >> uniqueId
getRunName() >> "test_run"
getWorkflowMetadata() >> metadata
getParams() >> new ScriptBinding.ParamsMap()
}
store.open(LineageConfig.create(session))
def observer = new LinObserver(session, store)
when:
observer.onFlowCreate(session)
observer.onFlowBegin()
def stored = store.load(observer.executionHash) as WorkflowRun
then:
def storedPlatform = stored.metadata.platform as Map
storedPlatform.workflowId == 'wf-999'
storedPlatform.workflowUrl == 'https://tower.nf/orgs/org/workspaces/ws/watch/wf-999'
storedPlatform.user == null
cleanup:
folder?.deleteDir()
}
def 'should return empty metadata when toMap throws' () {
given:
def folder = Files.createTempDirectory('test')
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
def store = new DefaultLinStore()
def uniqueId = UUID.randomUUID()
def scriptFile = folder.resolve("main.nf")
def metadata = Mock(WorkflowMetadata) {
getRepository() >> "https://nextflow.io/nf-test/"
getCommitId() >> "abc123"
getScriptId() >> "78910"
getScriptFile() >> scriptFile
getProjectDir() >> folder.resolve("projectDir")
getWorkDir() >> folder.resolve("workDir")
toMap() >> { throw new RuntimeException("metadata error") }
}
def session = Mock(Session) {
getConfig() >> config
getUniqueId() >> uniqueId
getRunName() >> "test_run"
getWorkflowMetadata() >> metadata
getParams() >> new ScriptBinding.ParamsMap()
}
store.open(LineageConfig.create(session))
def observer = new LinObserver(session, store)
when:
observer.onFlowCreate(session)
observer.onFlowBegin()
def stored = store.load(observer.executionHash) as WorkflowRun
then:
stored.metadata == null
cleanup:
folder?.deleteDir()
}
@Unroll
def 'should get parameter type' () {
expect:
LinObserver.getParameterType(PARAM) == STRING
where:
PARAM | STRING
null | null
new FileInParam(null, []) | "path"
new ValueOutParam(null, []) | "val"
new EnvOutParam(null, []) | "env"
new StdInParam(null, []) | "stdin"
new StdOutParam(null, []) | "stdout"
Path.of("test") | "Path"
["test"] | "Collection"
[key:"value"] | "Map"
}
def 'should save task run' () {
given:
def folder = Files.createTempDirectory('test').toRealPath()
def config = [workflow:[lineage:[enabled: true, store:[location:folder.toString()]]]]
def uniqueId = UUID.randomUUID()
def workDir = folder.resolve("work")
def session = Mock(Session) {
getConfig()>>config
getUniqueId()>>uniqueId
getRunName()>>"test_run"
getWorkDir() >> workDir
}
def metadata = Mock(WorkflowMetadata){
getRepository() >> "https://nextflow.io/nf-test/"
getCommitId() >> "123456"
getScriptId() >> "78910"
getProjectDir() >> folder.resolve("projectDir")
getWorkDir() >> workDir
}
and:
def store = new DefaultLinStore();
store.open(LineageConfig.create(session))
and:
def observer = Spy(new LinObserver(session, store))
def normalizer = new PathNormalizer(metadata)
observer.executionHash = "hash"
observer.normalizer = normalizer
and:
def hash = HashCode.fromString("1234567890")
def taskWd = workDir.resolve('12/34567890')
Files.createDirectories(taskWd)
and:
observer.getTaskGlobalVars(_) >> [:]
observer.getTaskBinEntries(_) >> []
and: 'Task Inputs'
def inputs = new LinkedHashMap<InParam, Object>()
// File from task
inputs.put(new FileInParam(null, []).bind("file1"), [new FileHolder(workDir.resolve('78/567890/file1.txt'))])
// Normal file
def file = folder.resolve("file2.txt")
file.text = "this is a test file"
def fileHash = CacheHelper.hasher(file).hash().toString()
inputs.put(new FileInParam(null, []).bind("file2"), [new FileHolder(file)])
//Value input
inputs.put(new ValueInParam(null, []).bind("id"), "value")
and: 'Task Outputs'
def outputs = new LinkedHashMap<OutParam, Object>()
// Single Path output
def outFile1 = taskWd.resolve('fileOut1.txt')
outFile1.text = 'some data'
def fileHash1 = CacheHelper.hasher(outFile1).hash().toString()
def attrs1 = Files.readAttributes(outFile1, BasicFileAttributes)
outputs.put(new FileOutParam(null, []).bind(new TokenVar("file1")), outFile1)
// Collection Path output
def outFile2 = taskWd.resolve('fileOut2.txt')
outFile2.text = 'some other data'
def fileHash2 = CacheHelper.hasher(outFile2).hash().toString()
def attrs2 = Files.readAttributes(outFile2, BasicFileAttributes)
outputs.put(new FileOutParam(null, []).bind(new TokenVar("file2")), [outFile2])
outputs.put(new ValueOutParam(null, []).bind(new TokenVar("id")), "value")
and: 'Task description'
def task = Mock(TaskRun) {
getId() >> TaskId.of(100)
getName() >> 'foo'
getHash() >> hash
getSource() >> 'echo task source'
getScript() >> 'this is the script'
getInputs() >> inputs
getOutputs() >> outputs
getWorkDir() >> taskWd
}
def handler = Mock(TaskHandler){
getTask() >> task
}
and: 'Expected LID objects'
def sourceHash = CacheHelper.hasher('echo task source').hash().toString()
def script = 'this is the script'
def taskDescription = new nextflow.lineage.model.v1beta1.TaskRun(uniqueId.toString(), "foo",
new Checksum(sourceHash, "nextflow", "standard"),
script,
[
new Parameter("path", "file1", ['lid://78567890/file1.txt']),
new Parameter("path", "file2", [[path: normalizer.normalizePath(file), checksum: [value:fileHash, algorithm: "nextflow", mode: "standard"]]]),
new Parameter("val", "id", "value")
], null, null, null, null, [:], [], "lid://hash")
def dataOutput1 = new FileOutput(outFile1.toString(), new Checksum(fileHash1, "nextflow", "standard"),
"lid://1234567890", "lid://hash", "lid://1234567890", attrs1.size(), LinUtils.toDate(attrs1.creationTime()), LinUtils.toDate(attrs1.lastModifiedTime()) )
def dataOutput2 = new FileOutput(outFile2.toString(), new Checksum(fileHash2, "nextflow", "standard"),
"lid://1234567890", "lid://hash", "lid://1234567890", attrs2.size(), LinUtils.toDate(attrs2.creationTime()), LinUtils.toDate(attrs2.lastModifiedTime()) )
when:
observer.onTaskComplete(new TaskEvent(handler, null))
def taskRunResult = store.load("${hash.toString()}")
def dataOutputResult1 = store.load("${hash}/fileOut1.txt") as FileOutput
def dataOutputResult2 = store.load("${hash}/fileOut2.txt") as FileOutput
def taskOutputsResult = store.load("${hash}#output") as TaskOutput
then:
taskRunResult == taskDescription
dataOutputResult1 == dataOutput1
dataOutputResult2 == dataOutput2
taskOutputsResult.taskRun == "lid://1234567890"
taskOutputsResult.workflowRun == "lid://hash"
taskOutputsResult.output.size() == 3
taskOutputsResult.output.get(0).type == "path"
taskOutputsResult.output.get(0).name == "file1"
taskOutputsResult.output.get(0).value == "lid://1234567890/fileOut1.txt"
taskOutputsResult.output.get(1).type == "path"
taskOutputsResult.output.get(1).name == "file2"
taskOutputsResult.output.get(1).value == ["lid://1234567890/fileOut2.txt"]
taskOutputsResult.output.get(2).type == "val"
taskOutputsResult.output.get(2).name == "id"
taskOutputsResult.output.get(2).value == "value"
cleanup:
folder?.deleteDir()
}
def 'should save task data output' () {
given:
def folder = Files.createTempDirectory('test')
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
def store = new DefaultLinStore();
def session = Mock(Session) {
getConfig()>>config
}
store.open(LineageConfig.create(session))
def observer = Spy(new LinObserver(session, store))
observer.executionHash = "hash"
and:
def workDir = folder.resolve('12/34567890')
Files.createDirectories(workDir)
and:
def outFile = workDir.resolve('foo/bar/file.bam')
Files.createDirectories(outFile.parent)
outFile.text = 'some data'
def fileHash = CacheHelper.hasher(outFile).hash().toString()
and:
def hash = HashCode.fromInt(123456789)
and:
def task = Mock(TaskRun) {
getId() >> TaskId.of(100)
getName() >> 'foo'
getHash() >> hash
getWorkDir() >> workDir
}
and:
def attrs = Files.readAttributes(outFile, BasicFileAttributes)
def output = new FileOutput(outFile.toString(), new Checksum(fileHash, "nextflow", "standard"),
"lid://15cd5b07", "lid://hash", "lid://15cd5b07", attrs.size(), LinUtils.toDate(attrs.creationTime()), LinUtils.toDate(attrs.lastModifiedTime()) )
and:
observer.readAttributes(outFile) >> attrs
when:
observer.storeTaskOutput(task, outFile)
then:
folder.resolve("${hash}/foo/bar/file.bam/.data.json").text == new LinEncoder().encode(output)
cleanup:
folder?.deleteDir()
}
def 'should relativise task output dirs' (){
when:
def config = [workflow:[lineage:[enabled: true, store:[location:lidFolder.toString()]]]]
def store = new DefaultLinStore();
def session = Mock(Session) {
getConfig()>>config
}
def hash = HashCode.fromInt(123456789)
def taskConfig = Mock(TaskConfig){
getStoreDir() >> STORE_DIR
}
def task = Mock(TaskRun) {
getId() >> TaskId.of(100)
getName() >> 'foo'
getHash() >> hash
getWorkDir() >> WORK_DIR
getConfig() >> taskConfig
}
store.open(LineageConfig.create(session))
def observer = new LinObserver(session, store)
then:
observer.getTaskRelative(task, PATH) == EXPECTED
where:
WORK_DIR | STORE_DIR | PATH | EXPECTED
Path.of('/path/to/work/12/3456789') | Path.of('/path/to/storeDir') | Path.of('/path/to/work/12/3456789/relative') | "relative"
Path.of('/path/to/work/12/3456789') | Path.of('/path/to/storeDir') | Path.of('/path/to/storeDir/relative') | "relative"
Path.of('work/12/3456789') | Path.of('storeDir') | Path.of('work/12/3456789/relative') | "relative"
Path.of('work/12/3456789') | Path.of('storeDir') | Path.of('storeDir/relative') | "relative"
Path.of('work/12/3456789') | Path.of('storeDir') | Path.of('results/relative') | "results/relative"
Path.of('/path/to/work/12/3456789') | Path.of('storeDir') | Path.of('./relative') | "relative"
}
@Unroll
def 'should return exception when relativize task output dirs'() {
when:
def config = [workflow:[lineage:[enabled: true, store:[location:lidFolder.toString()]]]]
def store = new DefaultLinStore();
def session = Mock(Session) {
getConfig()>>config
}
def hash = HashCode.fromInt(123456789)
def taskConfig = Mock(TaskConfig){
getStoreDir() >> STORE_DIR
}
def task = Mock(TaskRun) {
getId() >> TaskId.of(100)
getName() >> 'foo'
getHash() >> hash
getWorkDir() >> WORK_DIR
getConfig() >> taskConfig
}
store.open(LineageConfig.create(session))
def observer = new LinObserver(session, store)
observer.getTaskRelative(task, PATH)
then:
def e = thrown(IllegalArgumentException)
e.message == "Cannot access the relative path for output '$PATH' and task '${task.name}'".toString()
where:
WORK_DIR | STORE_DIR | PATH
Path.of('/path/to/work/12/3456789') | Path.of('/path/to/storeDir') | Path.of('/another/path/relative')
Path.of('/path/to/work/12/3456789') | Path.of('/path/to/storeDir') | Path.of('../path/to/storeDir/relative')
}
def 'should relativize workflow output dirs' (){
when:
def config = [workflow:[lineage:[enabled: true, store:[location:lidFolder.toString()]]]]
def store = new DefaultLinStore();
def session = Mock(Session) {
getOutputDir()>>OUTPUT_DIR
getConfig()>>config
}
store.open(LineageConfig.create(session))
def observer = new LinObserver(session, store)
then:
observer.getWorkflowRelative(PATH) == EXPECTED
where:
OUTPUT_DIR | PATH | EXPECTED
Path.of('/path/to/outDir') | Path.of('/path/to/outDir/relative') | "relative"
Path.of('outDir') | Path.of('outDir/relative') | "relative"
Path.of('/path/to/outDir') | Path.of('results/relative') | "results/relative"
Path.of('/path/to/outDir') | Path.of('./relative') | "relative"
}
@Unroll
def 'should return exception when relativize workflow output dirs' (){
when:
def config = [workflow:[lineage:[enabled: true, store:[location:lidFolder.toString()]]]]
def store = new DefaultLinStore();
def session = Mock(Session) {
getOutputDir()>>OUTPUT_DIR
getConfig()>>config
}
def observer = new LinObserver(session, store)
observer.getWorkflowRelative(PATH)
then:
thrown(OutputRelativePathException)
where:
OUTPUT_DIR | PATH | EXPECTED
Path.of('/path/to/outDir') | Path.of('/another/path/') | "relative"
Path.of('/path/to/outDir') | Path.of('../relative') | "relative"
}
def 'should save workflow output'() {
given:
def folder = Files.createTempDirectory('test')
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
def store = new DefaultLinStore();
def outputDir = folder.resolve('results')
def uniqueId = UUID.randomUUID()
def scriptFile = folder.resolve("main.nf")
def workDir= folder.resolve("work")
def metadata = Mock(WorkflowMetadata){
getRepository() >> "https://nextflow.io/nf-test/"
getCommitId() >> "123456"
getScriptId() >> "78910"
getScriptFile() >> scriptFile
getProjectDir() >> folder.resolve("projectDir")
getWorkDir() >> workDir
}
def session = Mock(Session) {
getConfig()>>config
getOutputDir()>>outputDir
getWorkDir() >> workDir
getWorkflowMetadata()>>metadata
getUniqueId()>>uniqueId
getRunName()>>"test_run"
getParams() >> new ScriptBinding.ParamsMap()
}
store.open(LineageConfig.create(session))
def observer = new LinObserver(session, store)
def encoder = new LinEncoder()
when: 'Starting workflow'
observer.onFlowCreate(session)
observer.onFlowBegin()
then: 'History file should contain execution hash'
def lid = LinHistoryRecord.parse(folder.resolve(".history/${observer.executionHash}").text)
lid.runLid == asUriString(observer.executionHash)
lid.sessionId == uniqueId
lid.runName == "test_run"
when: ' publish output with source file'
def outFile1 = outputDir.resolve('foo/file.bam')
Files.createDirectories(outFile1.parent)
outFile1.text = 'some data1'
def sourceFile1 = workDir.resolve('12/3987/file.bam')
Files.createDirectories(sourceFile1.parent)
sourceFile1.text = 'some data1'
observer.onFilePublish(new FilePublishEvent(sourceFile1, outFile1))
observer.onWorkflowOutput(new WorkflowOutputEvent("a", outFile1))
then: 'check file 1 output metadata in lid store'
def attrs1 = Files.readAttributes(outFile1, BasicFileAttributes)
def fileHash1 = CacheHelper.hasher(outFile1).hash().toString()
def output1 = new FileOutput(outFile1.toString(), new Checksum(fileHash1, "nextflow", "standard"),
"lid://123987/file.bam", "$LID_PROT${observer.executionHash}", null,
attrs1.size(), LinUtils.toDate(attrs1.creationTime()), LinUtils.toDate(attrs1.lastModifiedTime()) )
folder.resolve("${observer.executionHash}/foo/file.bam/.data.json").text == encoder.encode(output1)
when: 'publish without source path'
def outFile2 = outputDir.resolve('foo/file2.bam')
Files.createDirectories(outFile2.parent)
outFile2.text = 'some data2'
def attrs2 = Files.readAttributes(outFile2, BasicFileAttributes)
def fileHash2 = CacheHelper.hasher(outFile2).hash().toString()
observer.onFilePublish(new FilePublishEvent(null, outFile2))
observer.onWorkflowOutput(new WorkflowOutputEvent("b", outFile2))
then: 'Check outFile2 metadata in lid store'
def output2 = new FileOutput(outFile2.toString(), new Checksum(fileHash2, "nextflow", "standard"),
"lid://${observer.executionHash}" , "lid://${observer.executionHash}", null,
attrs2.size(), LinUtils.toDate(attrs2.creationTime()), LinUtils.toDate(attrs2.lastModifiedTime()) )
folder.resolve("${observer.executionHash}/foo/file2.bam/.data.json").text == encoder.encode(output2)
when: 'Workflow complete'
observer.onFlowComplete()
then: 'Check WorkflowOutput is written in the lid store'
def resultsRetrieved = store.load("${observer.executionHash}#output") as WorkflowOutput
resultsRetrieved.output == [new Parameter(Path.simpleName, "a", "lid://${observer.executionHash}/foo/file.bam"), new Parameter(Path.simpleName, "b", "lid://${observer.executionHash}/foo/file2.bam")]
cleanup:
folder?.deleteDir()
}
def 'should not save workflow output entry when no outputs'() {
given:
def folder = Files.createTempDirectory('test')
def config = [lineage: [enabled: true, store: [location: folder.toString()]]]
def store = new DefaultLinStore();
def outputDir = folder.resolve('results')
def uniqueId = UUID.randomUUID()
def scriptFile = folder.resolve("main.nf")
def workDir = folder.resolve("work")
def metadata = Mock(WorkflowMetadata) {
getRepository() >> "https://nextflow.io/nf-test/"
getCommitId() >> "123456"
getScriptId() >> "78910"
getScriptFile() >> scriptFile
getProjectDir() >> folder.resolve("projectDir")
getWorkDir() >> workDir
}
def session = Mock(Session) {
getConfig() >> config
getOutputDir() >> outputDir
getWorkDir() >> workDir
getWorkflowMetadata() >> metadata
getUniqueId() >> uniqueId
getRunName() >> "test_run"
getParams() >> new ScriptBinding.ParamsMap()
}
store.open(LineageConfig.create(session))
def observer = new LinObserver(session, store)
when:
observer.onFlowCreate(session)
observer.onFlowBegin()
observer.onFlowComplete()
def resultFile = folder.resolve("${observer.executionHash}#output")
then:
!resultFile.exists()
}
}

View File

@@ -0,0 +1,39 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage
import spock.lang.Specification
/**
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
class LinPropertyValidationTest extends Specification{
def 'should throw exception when property does not exist'(){
when:
new LinPropertyValidator().validate(['value', 'not_existing'])
then:
def e = thrown(IllegalArgumentException)
e.message.startsWith( "Property 'not_existing' doesn't exist in the lineage model")
}
def 'should not throw exception when property exist'(){
when:
new LinPropertyValidator().validate(['value', 'output'])
then:
noExceptionThrown()
}
}

View File

@@ -0,0 +1,202 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage
import java.time.ZoneId
import nextflow.lineage.model.v1beta1.Checksum
import nextflow.lineage.model.v1beta1.DataPath
import nextflow.lineage.model.v1beta1.Parameter
import nextflow.lineage.model.v1beta1.Workflow
import nextflow.lineage.model.v1beta1.WorkflowOutput
import nextflow.lineage.model.v1beta1.WorkflowRun
import nextflow.lineage.config.LineageConfig
import spock.lang.Specification
import spock.lang.TempDir
import java.nio.file.Path
import java.nio.file.attribute.FileTime
import java.time.Instant
import java.time.OffsetDateTime
import java.time.ZoneOffset
class LinUtilsTest extends Specification{
@TempDir
Path tempDir
Path storeLocation
LineageConfig config
def setup() {
storeLocation = tempDir.resolve("store")
def configMap = [enabled: true, store: [location: storeLocation.toString()]]
config = new LineageConfig(configMap)
}
def 'should convert to Date'(){
expect:
LinUtils.toDate(FILE_TIME) == DATE
where:
FILE_TIME | DATE
null | null
FileTime.fromMillis(1234) | Instant.ofEpochMilli(1234).atZone(ZoneId.systemDefault())?.toOffsetDateTime()
}
def 'should convert to FileTime'(){
expect:
LinUtils.toFileTime(DATE) == FILE_TIME
where:
FILE_TIME | DATE
null | null
FileTime.fromMillis(1234) | OffsetDateTime.ofInstant(Instant.ofEpochMilli(1234), ZoneOffset.UTC)
}
def 'should get lineage record'() {
given:
def uniqueId = UUID.randomUUID()
def mainScript = new DataPath("file://path/to/main.nf", new Checksum("78910", "nextflow", "standard"))
def workflow = new Workflow([mainScript], "https://nextflow.io/nf-test/", "123456")
def key1 = "testKey"
def value1 = new WorkflowRun(workflow, uniqueId.toString(), "test_run", [new Parameter("String", "param1", "value1"), new Parameter("String", "param2", "value2")])
def outputs1 = new WorkflowOutput(OffsetDateTime.now(), "lid://testKey", [new Parameter( "String", "output", "name")] )
def key2 = "testKey2"
def value2 = new WorkflowRun(workflow, uniqueId.toString(), "test_run2", [new Parameter("String", "param1", "value1"), new Parameter("String", "param2", "value2")])
def lidStore = new DefaultLinStore()
lidStore.open(config)
lidStore.save(key1, value1)
lidStore.save("$key1#output", outputs1)
lidStore.save(key2, value2)
when:
def params = LinUtils.getMetadataObject(lidStore, new URI('lid://testKey#params'))
then:
params instanceof List<Parameter>
(params as List<Parameter>).size() == 2
when:
def outputs = LinUtils.getMetadataObject(lidStore, new URI('lid://testKey#output'))
then:
outputs instanceof List<Parameter>
def param = (outputs as List)[0] as Parameter
param.name == "output"
when:
outputs = LinUtils.getMetadataObject(lidStore, new URI('lid://testKey2#output'))
then:
outputs instanceof List
(outputs as List).isEmpty()
when:
LinUtils.getMetadataObject(lidStore, new URI('lid://testKey#no-exist'))
then:
thrown(IllegalArgumentException)
when:
LinUtils.getMetadataObject(lidStore, new URI('lid://testKey#outputs.no-exist'))
then:
thrown(IllegalArgumentException)
when:
LinUtils.getMetadataObject(lidStore, new URI('lid://no-exist#something'))
then:
thrown(FileNotFoundException)
}
def "should check params in an object"() {
given:
def obj = [ "type": "value", "workflow": ["repository": "subvalue"], "output" : [ ["path":"/to/file"],["path":"file2"] ], "labels": ["a","b"] ]
expect:
LinUtils.checkParams(obj, PARAMS) == EXPECTED
where:
PARAMS | EXPECTED
["type": ["value"]] | true
["type": ["wrong"]] | false
["workflow.repository": ["subvalue"]] | true
["workflow.repository": ["wrong"]] | false
["output.path": ["wrong"]] | false
["output.path": ["/to/file"]] | true
["output.path": ["file2"]] | true
["labels": ["a"]] | true
["labels": ["c"]] | false
["labels": ["a","b"]] | true
["labels": ["a","b","c"]] | false
["type" : ["value", "value2"]] | false
}
def "should navigate in object params"() {
given:
def obj = [
"key1": "value1",
"nested": [
"subkey": "subvalue"
]
]
expect:
LinUtils.navigate(obj, PATH) == EXPECTED
where:
PATH | EXPECTED
"key1" | "value1"
"nested.subkey" | "subvalue"
"wrongKey" | null
}
def "should add objects matching parameters"() {
given:
def results = []
when:
LinUtils.treatObject(OBJECT, PARAMS, results)
then:
results == EXPECTED
where:
OBJECT | PARAMS | EXPECTED
["field": "value"] | ["field": ["value"]] | [["field": "value"]]
["field": "wrong"] | ["field": ["value"]] | []
[["field": "value"], ["field": "x"]] | ["field": ["value"]] | [["field": "value"]]
"string" | [:] | ["string"]
["nested": ["subfield": "match"]] | ["nested.subfield": ["match"]] | [["nested": ["subfield": "match"]]]
["nested": ["subfield": "nomatch"]] | ["nested.subfield": ["match"]] | []
[["nested": ["subfield": "match"]], ["nested": ["subfield": "other"]]] | ["nested.subfield": ["match"]] | [["nested": ["subfield": "match"]]]
}
def 'should navigate' (){
def uniqueId = UUID.randomUUID()
def mainScript = new DataPath("file://path/to/main.nf", new Checksum("78910", "nextflow", "standard"))
def workflow = new Workflow([mainScript], "https://nextflow.io/nf-test/", "123456")
def wfRun = new WorkflowRun(workflow, uniqueId.toString(), "test_run", [new Parameter("String", "param1", [key: "value1"]), new Parameter("String", "param2", "value2")])
expect:
LinUtils.navigate(wfRun, "workflow.commitId") == "123456"
LinUtils.navigate(wfRun, "params.name") == ["param1", "param2"]
LinUtils.navigate(wfRun, "params.value.key") == "value1"
LinUtils.navigate(wfRun, "params.value.no-exist") == null
LinUtils.navigate(wfRun, "params.no-exist") == null
LinUtils.navigate(wfRun, "no-exist") == null
LinUtils.navigate(null, "something") == null
}
}

View File

@@ -0,0 +1,478 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.cli
import java.nio.file.Files
import java.nio.file.Path
import java.time.Instant
import java.time.OffsetDateTime
import java.time.ZoneOffset
import nextflow.SysEnv
import nextflow.config.ConfigMap
import nextflow.dag.MermaidHtmlRenderer
import nextflow.exception.AbortOperationException
import nextflow.file.FileHelper
import nextflow.lineage.DefaultLinHistoryLog
import nextflow.lineage.LinHistoryRecord
import nextflow.lineage.LinStoreFactory
import nextflow.lineage.fs.LinFileSystemProvider
import nextflow.lineage.model.v1beta1.Checksum
import nextflow.lineage.model.v1beta1.DataPath
import nextflow.lineage.model.v1beta1.FileOutput
import nextflow.lineage.model.v1beta1.Parameter
import nextflow.lineage.model.v1beta1.TaskRun
import nextflow.lineage.model.v1beta1.Workflow
import nextflow.lineage.model.v1beta1.WorkflowRun
import nextflow.lineage.serde.LinEncoder
import nextflow.plugin.Plugins
import nextflow.util.CacheHelper
import org.junit.Rule
import spock.lang.Shared
import spock.lang.Specification
import test.OutputCapture
import static test.TestHelper.filterLogNoise
class LinCommandImplTest extends Specification{
@Shared
Path tmpDir
@Shared
Path storeLocation
@Shared
ConfigMap configMap
def reset() {
def provider = FileHelper.getProviderFor('lid') as LinFileSystemProvider
provider?.reset()
LinStoreFactory.reset()
}
def setup() {
reset()
// clear the environment to avoid the local env pollute the test env
SysEnv.push([:])
tmpDir = Files.createTempDirectory('tmp')
storeLocation = tmpDir.resolve("store")
configMap = new ConfigMap([lineage: [enabled: true, store: [location: storeLocation.toString(), logLocation: storeLocation.resolve(".log").toString()]]])
}
def cleanup() {
Plugins.stop()
LinStoreFactory.reset()
SysEnv.pop()
tmpDir?.deleteDir()
}
def setupSpec() {
reset()
}
/*
* Read more http://mrhaki.blogspot.com.es/2015/02/spocklight-capture-and-assert-system.html
*/
@Rule
OutputCapture capture = new OutputCapture()
def 'should print executions lids' (){
given:
def historyFile = storeLocation.resolve(".history")
def lidLog = new DefaultLinHistoryLog(historyFile)
def uniqueId = UUID.randomUUID()
def date = new Date();
def recordEntry = "${LinHistoryRecord.TIMESTAMP_FMT.format(date)}\trun_name\t${uniqueId}\tlid://123456".toString()
lidLog.write("run_name", uniqueId, "lid://123456", date)
when:
new LinCommandImpl().list(configMap)
def stdout = filterLogNoise(capture)
then:
stdout.size() == 2
stdout[1] == recordEntry
}
def 'should print no history' (){
given:
def historyFile = storeLocation.resolve(".meta/.history")
Files.createDirectories(historyFile.parent)
when:
new LinCommandImpl().list(configMap)
def stdout = filterLogNoise(capture, true)
then:
stdout.size() == 1
stdout[0] == "No workflow runs found in lineage history log"
}
def 'should show lid content' (){
given:
def lidFile = storeLocation.resolve("12345/.data.json")
Files.createDirectories(lidFile.parent)
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
def encoder = new LinEncoder().withPrettyPrint(true)
def entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
"lid://123987/file.bam","lid://123987/", null, 1234, time, time, null)
def jsonSer = encoder.encode(entry)
def expectedOutput = jsonSer
lidFile.text = jsonSer
when:
new LinCommandImpl().view(configMap, ["lid://12345"])
def stdout = filterLogNoise(capture)
then:
stdout.size() == expectedOutput.readLines().size()
stdout.join('\n') == expectedOutput
}
def 'should show empty lists content' (){
given:
def lidFile = storeLocation.resolve("12345/.data.json")
Files.createDirectories(lidFile.parent)
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
def encoder = new LinEncoder().withPrettyPrint(true)
def entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
"lid://123987/file.bam","lid://123987/", null, 1234, time, time, [])
def jsonSer = encoder.encode(entry)
def expectedOutput = '[]'
lidFile.text = jsonSer
when:
new LinCommandImpl().view(configMap, ["lid://12345#labels"])
def stdout = filterLogNoise(capture)
then:
stdout.size() == expectedOutput.readLines().size()
stdout.join('\n') == expectedOutput
}
def 'should show empty lists when no outputs' () {
def lidFile = storeLocation.resolve("12345/.data.json")
Files.createDirectories(lidFile.parent)
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
def encoder = new LinEncoder().withPrettyPrint(true)
def expectedOutput = '[]'
def wf = new Workflow([new DataPath("/path/to/main.nf)")], "hello-nf", "aasdklk")
def entry = new WorkflowRun(wf, "sessionId", "run_name",
[new Parameter("String", "sample_id", "ggal_gut"),
new Parameter("Integer", "reads", 2)])
lidFile.text = encoder.encode(entry)
when:
new LinCommandImpl().view(configMap, ["lid://12345#output"])
def stdout = filterLogNoise(capture)
then:
stdout.size() == expectedOutput.readLines().size()
stdout.join('\n') == expectedOutput
}
def 'should warn if no lid content' (){
given:
when:
new LinCommandImpl().view(configMap, ["lid://12345"])
def stdout = filterLogNoise(capture)
then:
stdout.size() == 1
stdout[0] == "Error loading lid://12345 - Lineage record 12345 not found"
}
def 'should get lineage lid content' (){
given:
def outputHtml = tmpDir.resolve('lineage.html')
def lidFile = storeLocation.resolve("12345/file.bam/.data.json")
def lidFile2 = storeLocation.resolve("123987/file.bam/.data.json")
def lidFile3 = storeLocation.resolve("123987/.data.json")
def lidFile4 = storeLocation.resolve("45678/output.txt/.data.json")
def lidFile5 = storeLocation.resolve("45678/.data.json")
Files.createDirectories(lidFile.parent)
Files.createDirectories(lidFile2.parent)
Files.createDirectories(lidFile3.parent)
Files.createDirectories(lidFile4.parent)
Files.createDirectories(lidFile5.parent)
def encoder = new LinEncoder()
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
def entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
"lid://123987/file.bam", "lid://45678", null, 1234, time, time, null)
lidFile.text = encoder.encode(entry)
entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
"lid://123987", "lid://45678", "lid://123987", 1234, time, time, null)
lidFile2.text = encoder.encode(entry)
entry = new TaskRun("u345-2346-1stw2", "foo",
new Checksum("abcde2345","nextflow","standard"),
'this is a script',
[new Parameter( "val", "sample_id","ggal_gut"),
new Parameter("path","reads", ["lid://45678/output.txt"] ),
new Parameter("path","input", [new DataPath("path/to/file",new Checksum("45372qe","nextflow","standard"))])
],
null, null, null, null, [:],[])
lidFile3.text = encoder.encode(entry)
entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
"lid://45678", "lid://45678", null, 1234, time, time, null)
lidFile4.text = encoder.encode(entry)
entry = new TaskRun("u345-2346-1stw2", "bar",
new Checksum("abfs2556","nextflow","standard"),
'this is a script',
null,null, null, null, null, [:],[])
lidFile5.text = encoder.encode(entry)
final network = """\
flowchart TB
lid://12345/file.bam["lid://12345/file.bam"]
lid://123987/file.bam["lid://123987/file.bam"]
lid://123987(["foo [lid://123987]"])
ggal_gut["ggal_gut"]
path/to/file["path/to/file"]
lid://45678/output.txt["lid://45678/output.txt"]
lid://45678(["bar [lid://45678]"])
lid://123987/file.bam --> lid://12345/file.bam
lid://123987 --> lid://123987/file.bam
ggal_gut --> lid://123987
lid://45678/output.txt --> lid://123987
path/to/file --> lid://123987
lid://45678 --> lid://45678/output.txt""".stripIndent()
final template = MermaidHtmlRenderer.readTemplate()
def expectedOutput = template.replace('REPLACE_WITH_NETWORK_DATA', network)
when:
new LinCommandImpl().render(configMap, ["lid://12345/file.bam", outputHtml.toString()])
def stdout = filterLogNoise(capture)
then:
stdout.size() == 1
stdout[0] == "Rendered lineage graph for lid://12345/file.bam to ${outputHtml}"
outputHtml.exists()
outputHtml.text == expectedOutput
}
def 'should get lineage from workflow lid content' (){
given:
def outputHtml = tmpDir.resolve('lineage.html')
def lidFile = storeLocation.resolve("12345/file.bam/.data.json")
def lidFile3 = storeLocation.resolve("12345/.data.json")
Files.createDirectories(lidFile.parent)
Files.createDirectories(lidFile3.parent)
def encoder = new LinEncoder()
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
def entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
"lid://12345", "lid://12345", null, 1234, time, time, null)
lidFile.text = encoder.encode(entry)
def wf = new Workflow([new DataPath("/path/to/main.nf)")], "hello-nf", "aasdklk")
entry = new WorkflowRun(wf,"sessionId","run_name",
[new Parameter( "String", "sample_id","ggal_gut"),
new Parameter("Integer","reads",2)])
lidFile3.text = encoder.encode(entry)
final network = """\
flowchart TB
lid://12345/file.bam["lid://12345/file.bam"]
lid://12345(["run_name [lid://12345]"])
ggal_gut["ggal_gut"]
2.0["2.0"]
lid://12345 --> lid://12345/file.bam
ggal_gut --> lid://12345
2.0 --> lid://12345""".stripIndent()
final template = MermaidHtmlRenderer.readTemplate()
def expectedOutput = template.replace('REPLACE_WITH_NETWORK_DATA', network)
when:
new LinCommandImpl().render(configMap, ["lid://12345/file.bam", outputHtml.toString()])
def stdout = filterLogNoise(capture)
then:
stdout.size() == 1
stdout[0] == "Rendered lineage graph for lid://12345/file.bam to ${outputHtml}"
outputHtml.exists()
outputHtml.text == expectedOutput
}
def 'should show an error if trying to do a query'(){
given:
def lidFile = storeLocation.resolve("12345/.data.json")
Files.createDirectories(lidFile.parent)
def encoder = new LinEncoder().withPrettyPrint(true)
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
def entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
"lid://123987/file.bam", "lid://123987/", null, 1234, time, time, null)
def jsonSer = encoder.encode(entry)
def expectedOutput = "Error loading lid:///?type=FileOutput - Cannot get record from the root LID URI"
lidFile.text = jsonSer
when:
new LinCommandImpl().view(configMap, ["lid:///?type=FileOutput"])
def stdout = filterLogNoise(capture)
then:
stdout.size() == expectedOutput.readLines().size()
stdout.join('\n') == expectedOutput
}
def 'should diff'(){
given:
def lidFile = storeLocation.resolve("12345/.data.json")
Files.createDirectories(lidFile.parent)
def lidFile2 = storeLocation.resolve("67890/.data.json")
Files.createDirectories(lidFile2.parent)
def encoder = new LinEncoder().withPrettyPrint(true)
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
def entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
"lid://123987/file.bam", "lid://123987/", null, 1234, time, time, null)
def entry2 = new FileOutput("path/to/file2",new Checksum("42472qet","nextflow","standard"),
"lid://123987/file2.bam", "lid://123987/", null, 1235, time, time, null)
lidFile.text = encoder.encode(entry)
lidFile2.text = encoder.encode(entry2)
def expectedOutput = '''diff --git 12345 67890
--- 12345
+++ 67890
@@ -2,16 +2,16 @@
"version": "lineage/v1beta1",
"kind": "FileOutput",
"spec": {
- "path": "path/to/file",
+ "path": "path/to/file2",
"checksum": {
- "value": "45372qe",
+ "value": "42472qet",
"algorithm": "nextflow",
"mode": "standard"
},
- "source": "lid://123987/file.bam",
+ "source": "lid://123987/file2.bam",
"workflowRun": "lid://123987/",
"taskRun": null,
- "size": 1234,
+ "size": 1235,
"createdAt": "1970-01-02T10:17:36.789Z",
"modifiedAt": "1970-01-02T10:17:36.789Z",
"labels": null
'''
when:
new LinCommandImpl().diff(configMap, ["lid://12345", "lid://67890"])
def stdout = filterLogNoise(capture)
then:
stdout.join('\n') == expectedOutput
}
def 'should print error if no entry found diff'(){
given:
def lidFile = storeLocation.resolve("12345/.data.json")
Files.createDirectories(lidFile.parent)
def encoder = new LinEncoder().withPrettyPrint(true)
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
def entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
"lid://123987/file.bam", "lid://123987/", null, 1234, time, time, null)
lidFile.text = encoder.encode(entry)
when:
new LinCommandImpl().diff(configMap, ["lid://89012", "lid://12345"])
new LinCommandImpl().diff(configMap, ["lid://12345", "lid://67890"])
def stdout = filterLogNoise(capture)
then:
stdout.size() == 2
stdout[0] == "No entry found for lid://89012."
stdout[1] == "No entry found for lid://67890."
}
def 'should print error store is not found in diff'(){
when:
def config = new ConfigMap()
new LinCommandImpl().list(config)
new LinCommandImpl().view(config, ["lid:///12345"])
new LinCommandImpl().render(config, ["lid://12345", "output.html"])
new LinCommandImpl().diff(config, ["lid://89012", "lid://12345"])
def stdout = filterLogNoise(capture)
def expectedOutput = "Error lineage store not loaded - Check Nextflow configuration"
then:
stdout.size() == 4
stdout[0] == expectedOutput
stdout[1] == expectedOutput
stdout[2] == expectedOutput
stdout[3] == expectedOutput
}
def 'should find metadata descriptions'(){
given:
def lidFile = storeLocation.resolve("123987/file.bam/.data.json")
Files.createDirectories(lidFile.parent)
def lidFile2 = storeLocation.resolve("123987/file2.bam/.data.json")
Files.createDirectories(lidFile2.parent)
def lidFile3 = storeLocation.resolve(".meta/123987/file3.bam/.data.json")
Files.createDirectories(lidFile3.parent)
def encoder = new LinEncoder().withPrettyPrint(true)
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
def entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
"lid://123987/file.bam", "lid://123987/", null, 1234, time, time, ["experiment=test"])
def entry2 = new FileOutput("path/to/file2",new Checksum("42472qet","nextflow","standard"),
"lid://123987/file2.bam", "lid://123987/", null, 1235, time, time, ["experiment=test"])
def entry3 = new FileOutput("path/to/file3",new Checksum("42472qet","nextflow","standard"),
"lid://123987/file2.bam", "lid://123987/", null, 1235, time, time, null)
def expectedOutput1 = 'lid://123987/file.bam\nlid://123987/file2.bam'
def expectedOutput2 = 'lid://123987/file2.bam\nlid://123987/file.bam'
lidFile.text = encoder.encode(entry)
lidFile2.text = encoder.encode(entry2)
lidFile3.text = encoder.encode(entry3)
when:
new LinCommandImpl().find(configMap, ["type=FileOutput", "label=experiment=test"])
def stdout = filterLogNoise(capture)
then:
stdout.join('\n') == expectedOutput1 || stdout.join('\n') == expectedOutput2
}
def 'should print correct validate path' () {
given:
def outputFolder = tmpDir.resolve('output')
Files.createDirectories(outputFolder)
def outputFile = outputFolder.resolve('file1.txt')
outputFile.text = "this is file1 == "
and:
def encoder = new LinEncoder().withPrettyPrint(true)
def hash = CacheHelper.hasher(outputFile).hash().toString()
def correctData = new FileOutput(outputFile.toString(), new Checksum(hash,"nextflow", "standard"))
def incorrectData = new FileOutput(outputFile.toString(), new Checksum("incorrectHash","nextflow", "standard"))
def lid1 = storeLocation.resolve('12345/output/file1.txt/.data.json')
Files.createDirectories(lid1.parent)
lid1.text = encoder.encode(correctData)
def lid2 = storeLocation.resolve('12345/output/file2.txt/.data.json')
Files.createDirectories(lid2.parent)
lid2.text = encoder.encode(incorrectData)
when:
new LinCommandImpl().check(configMap, ["lid://12345/output/file1.txt"])
def stdout = filterLogNoise(capture)
def expectedOutput1 = "Checksum validation succeed"
then:
stdout.size() == 1
stdout[0] == expectedOutput1
when:
new LinCommandImpl().check(configMap, ["lid://12345/output/file2.txt"])
then:
def err = thrown(AbortOperationException)
err.message == "Checksum of '${outputFile}' does not match with lineage metadata"
}
}

View File

@@ -0,0 +1,50 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.config
import spock.lang.Specification
/**
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
class LineageConfigTest extends Specification {
def 'should create default config' () {
when:
def config = new LineageConfig(Map.of())
then:
!config.enabled
!config.store.location
}
def 'should create default with enable' () {
when:
def config = new LineageConfig([enabled: true])
then:
config.enabled
!config.store.location
}
def 'should create data config with location' () {
when:
def config = new LineageConfig(enabled: true, store: [location: "/some/data/store"])
then:
config.enabled
config.store.location == '/some/data/store'
}
}

View File

@@ -0,0 +1,496 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.fs
import java.nio.ByteBuffer
import java.nio.channels.NonWritableChannelException
import java.nio.file.AccessDeniedException
import java.nio.file.AccessMode
import java.nio.file.FileSystemNotFoundException
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.ProviderMismatchException
import java.nio.file.StandardOpenOption
import java.nio.file.attribute.BasicFileAttributes
import nextflow.Global
import nextflow.Session
import nextflow.lineage.DefaultLinStore
import spock.lang.Shared
import spock.lang.Specification
/**
* LID File system provider tests
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
class LinFileSystemProviderTest extends Specification {
@Shared def wdir = Files.createTempDirectory('wdir')
@Shared def data = Files.createTempDirectory('work')
def cleanupSpec(){
wdir.deleteDir()
data.deleteDir()
}
def 'should return lid scheme' () {
given:
def provider = new LinFileSystemProvider()
expect:
provider.getScheme() == 'lid'
}
def 'should get lid path' () {
given:
def lid = Mock(LinPath)
and:
def provider = new LinFileSystemProvider()
expect:
provider.toLinPath(lid) == lid
when:
provider.toLinPath(Path.of('foo'))
then:
thrown(ProviderMismatchException)
}
def 'should create new file system' () {
given:
def provider = new LinFileSystemProvider()
def config = [store:[location:data.toString()]]
def lid = LinPath.asUri('lid://12345')
when:
def fs = provider.newFileSystem(lid, config) as LinFileSystem
then:
(fs.store as DefaultLinStore).location == data
}
def 'should get a file system' () {
given:
def provider = new LinFileSystemProvider()
def config = [store:[location: data.toString()]]
def uri = LinPath.asUri('lid://12345')
when:
provider.getFileSystem(uri)
then:
thrown(FileSystemNotFoundException)
when:
provider.newFileSystem(uri, config) as LinFileSystem
and:
def fs = provider.getFileSystem(uri) as LinFileSystem
then:
(fs.store as DefaultLinStore).location == data
}
def 'should get or create a file system' () {
given:
def config = [lineage:[store:[location: data.toString()]]]
Global.session = Mock(Session) { getConfig()>>config }
and:
def uri = LinPath.asUri('lid://12345')
def provider = new LinFileSystemProvider()
when:
def fs = provider.getFileSystemOrCreate(uri) as LinFileSystem
then:
(fs.store as DefaultLinStore).location == data
when:
def fs2 = provider.getFileSystemOrCreate(uri) as LinFileSystem
then:
fs2.is(fs)
}
def 'should create new byte channel' () {
given:
def config = [lineage:[store:[location:wdir.toString()]]]
def outputMeta = wdir.resolve("12345/output.txt")
def output = data.resolve("output.txt")
output.text = "Hello, World!"
outputMeta.mkdirs()
outputMeta.resolve(".data.json").text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path":"'+output.toString()+'"}}'
Global.session = Mock(Session) { getConfig()>>config }
and:
def provider = new LinFileSystemProvider()
def lid = provider.getPath(LinPath.asUri('lid://12345/output.txt'))
def opts = Set.of(StandardOpenOption.READ)
when:
def channel = provider.newByteChannel(lid, opts)
then:
channel.isOpen()
channel.position() == 0
channel.size() == "Hello, World!".getBytes().size()
when:
channel.truncate(25)
then:
thrown(NonWritableChannelException)
when:
def buffer = ByteBuffer.allocate(1000);
def read = channel.read(buffer)
def bytes = new byte[read]
buffer.get(0,bytes)
then:
bytes == "Hello, World!".getBytes()
when:
channel.position(2)
then:
channel.position() == 2
when:
channel.write(buffer)
then:
thrown(NonWritableChannelException)
when:
provider.newByteChannel(lid, Set.of(StandardOpenOption.WRITE))
then:
thrown(UnsupportedOperationException)
when:
provider.newByteChannel(lid, Set.of(StandardOpenOption.APPEND))
then:
thrown(UnsupportedOperationException)
cleanup:
channel.close()
outputMeta.deleteDir()
output.delete()
}
def 'should create new byte channel for LinMetadata' () {
given:
def config = [lineage:[store:[location:wdir.toString()]]]
def outputMeta = wdir.resolve("12345")
outputMeta.mkdirs()
outputMeta.resolve(".data.json").text = '{"version":"lineage/v1beta1","kind":"WorkflowRun","spec":{"sessionId":"session","name":"run_name","params":[{"type":"String","name":"param1","value":"value1"}]}}'
Global.session = Mock(Session) { getConfig()>>config }
and:
def provider = new LinFileSystemProvider()
def lid = provider.getPath(LinPath.asUri('lid://12345#name'))
when:
def channel = provider.newByteChannel(lid, Set.of(StandardOpenOption.READ))
then:
channel.isOpen()
channel.position() == 0
channel.size() == '"run_name"'.getBytes().size()
when:
channel.truncate(25)
then:
thrown(NonWritableChannelException)
when:
def buffer = ByteBuffer.allocate(1000);
def read = channel.read(buffer)
def bytes = new byte[read]
buffer.get(0,bytes)
then:
bytes =='"run_name"'.getBytes()
when:
channel.position(2)
then:
channel.position() == 2
when:
channel.write(buffer)
then:
thrown(NonWritableChannelException)
when:
provider.newByteChannel(lid, Set.of(StandardOpenOption.WRITE))
then:
thrown(UnsupportedOperationException)
when:
provider.newByteChannel(lid, Set.of(StandardOpenOption.APPEND))
then:
thrown(UnsupportedOperationException)
cleanup:
channel.close()
outputMeta.deleteDir()
}
def 'should read lid' () {
given:
def config = [lineage:[store:[location:wdir.toString()]]]
def outputMeta = wdir.resolve("12345/output.txt")
def output = data.resolve("output.txt")
output.text = "Hello, World!"
outputMeta.mkdirs()
outputMeta.resolve(".data.json").text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path":"'+output.toString()+'"}}'
Global.session = Mock(Session) { getConfig()>>config }
and:
def provider = new LinFileSystemProvider()
def lid = provider.getPath(LinPath.asUri('lid://12345/output.txt'))
def opts = Set.of(StandardOpenOption.READ)
expect:
lid.text == "Hello, World!"
cleanup:
outputMeta.deleteDir()
output.delete()
}
def 'should not create a directory' () {
given:
def config = [lineage:[store:[location:wdir.toString()]]]
Global.session = Mock(Session) { getConfig()>>config }
and:
def provider = new LinFileSystemProvider()
def lid = provider.getPath(LinPath.asUri('lid://12345'))
when:
provider.createDirectory(lid)
then:
thrown(UnsupportedOperationException)
}
def 'should create directory stream' () {
given:
def output1 = data.resolve('path')
output1.mkdir()
output1.resolve('file1.txt').text = 'file1'
output1.resolve('file2.txt').text = 'file2'
output1.resolve('file3.txt').text = 'file3'
wdir.resolve('12345/output1').mkdirs()
wdir.resolve('12345/output2').mkdirs()
wdir.resolve('12345/.data.json').text = '{"version":"lineage/v1beta1","kind":"TaskRun","spec":{"name":"dummy"}}'
wdir.resolve('12345/output1/.data.json').text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path": "' + output1.toString() + '"}}'
and:
def config = [lineage:[store:[location:wdir.toString()]]]
Global.session = Mock(Session) { getConfig()>>config }
and:
def provider = new LinFileSystemProvider()
def lid = provider.getPath(LinPath.asUri('lid://12345/output1'))
def lid2 = provider.getPath(LinPath.asUri('lid://12345'))
def lid3 = provider.getPath(LinPath.asUri('lid://678'))
expect:
Files.exists(lid)
Files.exists(lid.resolve('file1.txt'))
Files.exists(lid.resolve('file2.txt'))
Files.exists(lid.resolve('file3.txt'))
when:
provider.newDirectoryStream(lid3, (p) -> true)
then:
thrown(FileNotFoundException)
when:
def stream = provider.newDirectoryStream(lid, (p) -> true)
and:
def result = stream.toList()
then:
result.toSet() == [
lid.resolve('file1.txt'),
lid.resolve('file2.txt'),
lid.resolve('file3.txt')
] as Set
when:
stream = provider.newDirectoryStream(lid2, (p) -> true)
and:
result = stream.toList()
then:
result.size() == 1
result[0] == lid2.resolve('output1')
cleanup:
wdir.resolve('12345').deleteDir()
output1.deleteDir()
}
def 'should not delete a file' () {
given:
def config = [lineage:[store:[location:wdir.toString()]]]
Global.session = Mock(Session) { getConfig()>>config }
and:
def provider = new LinFileSystemProvider()
def lid = provider.getPath(LinPath.asUri('lid://12345'))
when:
provider.delete(lid)
then:
thrown(UnsupportedOperationException)
}
def 'should not copy a file' () {
given:
def config = [lineage:[store:[location:wdir.toString()]]]
Global.session = Mock(Session) { getConfig()>>config }
and:
def provider = new LinFileSystemProvider()
def lid1 = provider.getPath(LinPath.asUri('lid://12345/abc'))
def lid2 = provider.getPath(LinPath.asUri('lid://54321/foo'))
when:
provider.copy(lid1, lid2)
then:
thrown(UnsupportedOperationException)
}
def 'should not move a file' () {
given:
def config = [lineage:[store:[location:wdir.toString()]]]
Global.session = Mock(Session) { getConfig()>>config }
and:
def provider = new LinFileSystemProvider()
def lid1 = provider.getPath(LinPath.asUri('lid://12345/abc'))
def lid2 = provider.getPath(LinPath.asUri('lid://54321/foo'))
when:
provider.move(lid1, lid2)
then:
thrown(UnsupportedOperationException)
}
def 'should check is same file' () {
given:
def folder = Files.createTempDirectory('test')
def config = [lineage:[store:[location:folder.toString()]]]
Global.session = Mock(Session) { getConfig()>>config }
and:
def provider = new LinFileSystemProvider()
def lid1 = provider.getPath(LinPath.asUri('lid://12345/abc'))
def lid2 = provider.getPath(LinPath.asUri('lid://54321/foo'))
def lid3 = provider.getPath(LinPath.asUri('lid://54321/foo'))
expect:
!provider.isSameFile(lid1, lid2)
!provider.isSameFile(lid1, lid3)
and:
provider.isSameFile(lid2, lid3)
cleanup:
folder?.deleteDir()
}
def 'should check is hidden file' () {
given:
def folder = Files.createTempDirectory('test')
def config = [lineage:[store:[location:wdir.toString()]]]
Global.session = Mock(Session) { getConfig()>>config }
and:
def output = folder.resolve('path')
output.mkdir()
output.resolve('abc').text = 'file1'
output.resolve('.foo').text = 'file2'
wdir.resolve('12345/output').mkdirs()
wdir.resolve('12345/output/.data.json').text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path": "' + output.toString() + '"}}'
and:
def provider = new LinFileSystemProvider()
def lid1 = provider.getPath(LinPath.asUri('lid://12345/output/abc'))
def lid2 = provider.getPath(LinPath.asUri('lid://12345/output/.foo'))
expect:
!provider.isHidden(lid1)
provider.isHidden(lid2)
cleanup:
folder?.deleteDir()
}
def 'should read file attributes' () {
given:
def config = [lineage:[store:[location:wdir.toString()]]]
def file = data.resolve('abc')
file.text = 'Hello'
wdir.resolve('12345/abc').mkdirs()
wdir.resolve('12345/abc/.data.json').text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path":"' + file.toString() + '"}}'
and:
Global.session = Mock(Session) { getConfig()>>config }
and:
def provider = new LinFileSystemProvider()
def lid1 = provider.getPath(LinPath.asUri('lid://12345/abc'))
when:
def attr1 = provider.readAttributes(lid1, BasicFileAttributes)
def real1= Files.readAttributes(file,BasicFileAttributes)
then:
!attr1.directory
attr1.isRegularFile()
attr1.size() == real1.size()
attr1.creationTime() == real1.creationTime()
attr1.lastModifiedTime() == real1.lastModifiedTime()
attr1.lastAccessTime() == real1.lastAccessTime()
cleanup:
file?.delete()
wdir.resolve('12345').deleteDir()
}
def 'should throw exception in unsupported methods'() {
given:
def config = [lineage:[store:[location:wdir.toString()]]]
Global.session = Mock(Session) { getConfig()>>config }
def provider = new LinFileSystemProvider()
when:
provider.newOutputStream(null)
then:
thrown(UnsupportedOperationException)
when:
provider.getFileStore(null)
then:
thrown(UnsupportedOperationException)
when:
provider.readAttributes(null, "attrib")
then:
thrown(UnsupportedOperationException)
when:
provider.setAttribute(null, "attrib", null)
then:
thrown(UnsupportedOperationException)
}
def 'should throw exception when checking access mode'(){
given:
def config = [lineage:[store:[location:wdir.toString()]]]
Global.session = Mock(Session) { getConfig()>>config }
def provider = new LinFileSystemProvider()
def lid1 = provider.getPath(LinPath.asUri('lid://12345/abc'))
when:
provider.checkAccess(lid1, AccessMode.WRITE)
then:
def ex1 = thrown(AccessDeniedException)
ex1.message == "Write mode not supported"
when:
provider.checkAccess(lid1, AccessMode.EXECUTE)
then:
def ex2 = thrown(AccessDeniedException)
ex2.message == "Execute mode not supported"
}
}

View File

@@ -0,0 +1,89 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.fs
import java.nio.file.Files
import java.nio.file.Path
import nextflow.Global
import nextflow.Session
import spock.lang.Specification
import spock.lang.Unroll
/**
* LID Path Factory tests.
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
class LinPathFactoryTest extends Specification {
Path tmp
def setup() {
tmp = Files.createTempDirectory("data")
Global.session = Mock(Session) { getConfig()>> [workflow:[lineage:[store:[location: tmp.toString()]]]] }
}
def cleanup() {
Global.session = null
tmp.deleteDir()
}
def 'should create lin path' () {
given:
def factory = new LinPathFactory()
expect:
factory.parseUri('foo') == null
when:
def p1 = factory.parseUri('lid://12345')
then:
p1.toUriString() == 'lid://12345'
when:
def p2 = factory.parseUri('lid://12345/x/y/z')
then:
p2.toUriString() == 'lid://12345/x/y/z'
when:
def p3 = factory.parseUri('lid://12345//x///y/z//')
then:
p3.toUriString() == 'lid://12345/x/y/z'
when:
factory.parseUri('lid:///12345')
then:
thrown(IllegalArgumentException)
}
@Unroll
def 'should convert get lid uri string' () {
given:
def factory = new LinPathFactory()
when:
def lid = LinPathFactory.create(EXPECTED)
then:
factory.toUriString(lid) == EXPECTED
where:
_ | EXPECTED
_ | 'lid://123'
_ | 'lid://123/a/b/c'
}
}

View File

@@ -0,0 +1,718 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.fs
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.ProviderMismatchException
import java.time.OffsetDateTime
import nextflow.lineage.LinUtils
import nextflow.lineage.model.v1beta1.Checksum
import nextflow.lineage.model.v1beta1.FileOutput
import nextflow.lineage.model.v1beta1.Parameter
import nextflow.lineage.model.v1beta1.Workflow
import nextflow.lineage.model.v1beta1.WorkflowOutput
import nextflow.lineage.model.v1beta1.WorkflowRun
import nextflow.lineage.serde.LinEncoder
import nextflow.util.CacheHelper
import org.junit.Rule
import spock.lang.Shared
import spock.lang.Specification
import spock.lang.Unroll
import test.OutputCapture
import static test.TestHelper.filterLogNoise
/**
* LID Path Tests
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
class LinPathTest extends Specification {
@Shared
Path wdir
@Shared
Path data
@Shared
def fs = Mock(LinFileSystem)
def setupSpec(){
wdir = Files.createTempDirectory("wdir")
data = wdir.resolve('work')
}
def cleanupSpec(){
wdir.deleteDir()
}
@Rule
OutputCapture capture = new OutputCapture()
def 'should create from URI' () {
when:
def path = new LinPath(fs, new URI( URI_STRING ))
then:
path.filePath == PATH
path.fragment == FRAGMENT
path.query == QUERY
where:
URI_STRING | PATH | QUERY | FRAGMENT
"lid://1234/hola" | "1234/hola" | null | null
"lid://1234/hola#workflow.repository" | "1234/hola" | null | "workflow.repository"
"lid://1234/#workflow.repository" | "1234" | null | "workflow.repository"
"lid://1234/?q=a&b=c" | "1234" | "q=a&b=c" | null
"lid://1234/?q=a&b=c#workflow.repository" | "1234" | "q=a&b=c" | "workflow.repository"
"lid:///" | "/" | null | null
}
def 'should throw exception if fragment contains an unknown property'() {
when:
new LinPath(fs, new URI ("lid://1234/hola#no-exist"))
then:
thrown(IllegalArgumentException)
}
def 'should warn if query is specified'() {
when:
new LinPath(fs, new URI("lid://1234/hola?query"))
def stdout = filterLogNoise(capture)
then:
stdout.size() == 1
stdout[0].endsWith("Query string is not supported for Lineage URI: `lid://1234/hola?query` -- it will be ignored")
}
def 'should create correct lid Path' () {
when:
def lid = new LinPath(FS, PATH, MORE)
then:
lid.filePath == EXPECTED_FILE
where:
FS | PATH | MORE | EXPECTED_FILE
fs | '/' | [] as String[] | '/'
null | '/' | [] as String[] | '/'
fs | '/' | ['a','b'] as String[] | 'a/b'
null | '/' | ['a','b'] as String[] | 'a/b'
fs | '' | [] as String[] | '/'
null | '' | [] as String[] | '/'
fs | '' | ['a','b'] as String[] | 'a/b'
null | '' | ['a','b'] as String[] | 'a/b'
fs | '1234' | [] as String[] | '1234'
null | '1234' | [] as String[] | '1234'
fs | '1234' | ['a','b'] as String[] | '1234/a/b'
null | '1234' | ['a','b'] as String[] | '1234/a/b'
fs | '1234/c' | [] as String[] | '1234/c'
null | '1234/c' | [] as String[] | '1234/c'
fs | '1234/c' | ['a','b'] as String[] | '1234/c/a/b'
null | '1234/c' | ['a','b'] as String[] | '1234/c/a/b'
fs | '/1234/c' | [] as String[] | '1234/c'
null | '/1234/c' | [] as String[] | '1234/c'
fs | '/1234/c' | ['a','b'] as String[] | '1234/c/a/b'
null | '/1234/c' | ['a','b'] as String[] | '1234/c/a/b'
fs | '../c' | ['a','b'] as String[] | 'c/a/b'
null | '../c' | ['a','b'] as String[] | '../c/a/b'
fs | '../c' | [] as String[] | 'c'
null | '../c' | [] as String[] | '../c'
fs | '..' | [] as String[] | '/'
null | '..' | [] as String[] | '..'
fs | '/..' | [] as String[] | '/'
null | '/..' | [] as String[] | '/'
fs | './1234/c' | ['a','b'] as String[] | '1234/c/a/b'
null | './1234/c' | ['a','b'] as String[] | '1234/c/a/b'
fs | './1234/c' | [] as String[] | '1234/c'
null | './1234/c' | [] as String[] | '1234/c'
fs | '1234' | ['/'] as String[] | '1234'
null | '1234' | ['/'] as String[] | '1234'
null | '../../a/b' | [] as String[] | '../../a/b'
fs | '1234/' | [] as String[] | '1234'
null | '1234/' | [] as String[] | '1234'
}
def 'should get target path' () {
given:
def outputFolder = data.resolve('output')
def outputSubFolder = outputFolder.resolve('some/path')
outputSubFolder.mkdirs()
def outputSubFolderFile = outputSubFolder.resolve('file1.txt')
outputSubFolderFile.text = "this is file1"
def outputFile = data.resolve('file2.txt')
outputFile.text = "this is file2"
def lidFs = new LinFileSystemProvider().newFileSystem(new URI("lid:///"), [enabled: true, store: [location: wdir.toString()]])
wdir.resolve('12345/output1').mkdirs()
wdir.resolve('12345/path/to/file2.txt').mkdirs()
wdir.resolve('12345/.data.json').text = '{"version":"lineage/v1beta1","kind":"TaskRun","spec":{"name":"test"}}'
wdir.resolve('12345/output1/.data.json').text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path": "' + outputFolder.toString() + '"}}'
wdir.resolve('12345/path/to/file2.txt/.data.json').text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path": "' + outputFile.toString() + '"}}'
def time = OffsetDateTime.now()
def wfResultsMetadata = new LinEncoder().withPrettyPrint(true).encode(new WorkflowOutput(time, "lid://1234", [new Parameter( "Path", "a", "lid://1234/a.txt")]))
wdir.resolve('5678/').mkdirs()
wdir.resolve('5678/.data.json').text = wfResultsMetadata
expect: 'Get real path when LinPath is the output data or a subfolder'
new LinPath(lidFs, '12345/output1').getTargetPath() == outputFolder
new LinPath(lidFs,'12345/output1/some/path').getTargetPath() == outputSubFolder
new LinPath(lidFs,'12345/output1/some/path/file1.txt').getTargetPath().text == outputSubFolderFile.text
new LinPath(lidFs, '12345/path/to/file2.txt').getTargetPath().text == outputFile.text
when: 'LinPath fs is null'
new LinPath(null, '12345').getTargetPath()
then:
thrown(IllegalArgumentException)
when: 'LinPath is empty'
new LinPath(lidFs, '/').getTargetPath()
then:
thrown(IllegalArgumentException)
when: 'LinPath is not an output data description'
new LinPath(lidFs, '12345').getTargetPath()
then:
thrown(FileNotFoundException)
when: 'LinPath is not subfolder of an output data description'
new LinPath(lidFs, '12345/path').getTargetPath()
then:
thrown(FileNotFoundException)
when: 'LinPath subfolder of an output data description does not exist'
new LinPath(lidFs, '12345/output1/other/path').getTargetPath()
then:
thrown(FileNotFoundException)
when: 'Lid does not exist'
new LinPath(lidFs, '23456').getTargetPath()
then:
thrown(FileNotFoundException)
when: 'LinPath is not an output data description but is a workflow/task run'
def result = new LinPath(lidFs, '12345').getTargetOrIntermediatePath()
then:
result instanceof LinIntermediatePath
when: 'LinPath is a subpath of a workflow/task run data description'
result = new LinPath(lidFs, '12345/path').getTargetOrIntermediatePath()
then:
result instanceof LinIntermediatePath
when: 'Lid description'
result = new LinPath(lidFs, '5678').getTargetOrMetadataPath()
then:
result instanceof LinMetadataPath
result.text == wfResultsMetadata
when: 'Lid description subobject'
def result2 = new LinPath(lidFs, '5678#output').getTargetOrMetadataPath()
then:
result2 instanceof LinMetadataPath
result2.text == LinUtils.encodeSearchOutputs([new Parameter("Path","a", "lid://1234/a.txt")], true)
when: 'Lid subobject does not exist'
new LinPath(lidFs, '23456#notexists').getTargetOrMetadataPath()
then:
thrown(IllegalArgumentException)
}
def 'should get subobjects as path' (){
given:
def lidFs = new LinFileSystemProvider().newFileSystem(new URI("lid:///"), [enabled: true, store: [location: wdir.toString()]])
def wf = new WorkflowRun(new Workflow([],"repo", "commit"), "sessionId", "runId", [new Parameter("String", "param1", "value1")])
when: 'workflow repo in workflow run'
Path p = LinPath.getMetadataAsTargetPath(wf, lidFs, "123456", "workflow.repository")
then:
p instanceof LinMetadataPath
p.text == '"repo"'
when: 'outputs'
def outputs = new WorkflowOutput(OffsetDateTime.now(), "lid://123456", [new Parameter("Collection", "samples", ["sample1", "sample2"])])
lidFs.store.save("123456#output", outputs)
Path p2 = LinPath.getMetadataAsTargetPath(wf, lidFs, "123456", "output")
then:
p2 instanceof LinMetadataPath
p2.text == LinUtils.encodeSearchOutputs([new Parameter("Collection", "samples", ["sample1", "sample2"])], true)
when: 'child does not exists'
LinPath.getMetadataAsTargetPath(wf, lidFs, "123456", "no-exist")
then:
def exception = thrown(FileNotFoundException)
exception.message == "Target path '123456#no-exist' does not exist"
when: 'outputs does not exists'
LinPath.getMetadataAsTargetPath(wf, lidFs, "6789", "output")
then:
def exception1 = thrown(FileNotFoundException)
exception1.message == "Target path '6789#output' does not exist"
when: 'null object'
LinPath.getMetadataAsTargetPath(null, lidFs, "123456", "no-exist")
then:
def exception2 = thrown(FileNotFoundException)
exception2.message == "Target path '123456' does not exist"
cleanup:
wdir.resolve("123456").deleteDir()
}
def 'should get file name' () {
expect:
new LinPath(fs, PATH).getFileName() == EXPECTED
where:
PATH | EXPECTED
'1234567890/this/file.bam' | new LinPath(null, 'file.bam')
'12345/hola?query#output' | new LinPath("query", "output", "hola", null)
}
def 'should get file parent' () {
when:
def lid1 = new LinPath(fs, '1234567890/this/file.bam')
then:
lid1.getParent() == new LinPath(fs, '1234567890/this')
lid1.getParent().getParent() == new LinPath(fs, '1234567890')
lid1.getParent().getParent().getParent() == new LinPath(fs, "/")
lid1.getParent().getParent().getParent().getParent() == null
}
@Unroll
def 'should get name count' () {
expect:
new LinPath(fs, PATH).getNameCount() == EXPECTED
where:
PATH | EXPECTED
'/' | 0
'123' | 1
'123/a' | 2
'123/a/' | 2
'123/a/b' | 3
'' | 0
}
@Unroll
def 'should get name by index' () {
expect:
new LinPath(fs, PATH).getName(INDEX) == EXPECTED
where:
PATH | INDEX | EXPECTED
'123' | 0 | new LinPath(fs, '123')
'123/a' | 1 | new LinPath(null, 'a')
'123/a/' | 1 | new LinPath(null, 'a')
'123/a/b' | 2 | new LinPath(null, 'b')
'123/a?q#output' | 1 | new LinPath(null, 'a?q#output')
}
@Unroll
def 'should get subpath' () {
expect:
new LinPath(fs, PATH).subpath(BEGIN,END) == EXPECTED
where:
PATH | BEGIN | END | EXPECTED
'123' | 0 | 1 | new LinPath(fs, '123')
'123/a' | 0 | 2 | new LinPath(fs, '123/a')
'123/a/' | 0 | 2 | new LinPath(fs, '123/a')
'123/a' | 1 | 2 | new LinPath(null, 'a')
'123/a/' | 1 | 2 | new LinPath(null, 'a')
'123/a/b' | 2 | 3 | new LinPath(null, 'b')
'123/a/b' | 1 | 3 | new LinPath(null, 'a/b')
}
def 'should normalize a path' () {
expect:
new LinPath(fs, '123').normalize() == new LinPath(fs, '123')
new LinPath(fs, '123/a/b').normalize() == new LinPath(fs, '123/a/b')
new LinPath(fs, '123/./a/b').normalize() == new LinPath(fs, '123/a/b')
new LinPath(fs, '123/a/../a/b').normalize() == new LinPath(fs, '123/a/b')
}
@Unroll
def 'should validate startWith' () {
expect:
new LinPath(fs,PATH).startsWith(OTHER) == EXPECTED
where:
PATH | OTHER | EXPECTED
'12345/a/b' | '12345' | true
'12345/a/b' | '12345/a' | true
'12345/a/b' | '12345/a/b' | true
and:
'12345/a/b' | '12345/b' | false
'12345/a/b' | 'xyz' | false
}
@Unroll
def 'should validate endsWith' () {
expect:
new LinPath(fs,PATH).endsWith(OTHER) == EXPECTED
where:
PATH | OTHER | EXPECTED
'12345/a/b' | 'b' | true
'12345/a/b' | 'a/b' | true
'12345/a/b' | '12345/a/b' | true
and:
'12345/a/b' | '12345/b' | false
'12345/a/b' | 'xyz' | false
}
def 'should validate isAbsolute' () {
expect:
new LinPath(fs,'1234/a/b/c').isAbsolute()
new LinPath(fs,'1234/a/b/c').getRoot().isAbsolute()
new LinPath(fs,'1234/a/b/c').getParent().isAbsolute()
new LinPath(fs,'1234/a/b/c').normalize().isAbsolute()
new LinPath(fs,'1234/a/b/c').getName(0).isAbsolute()
new LinPath(fs,'1234/a/b/c').subpath(0,2).isAbsolute()
and:
!new LinPath(fs,'1234/a/b/c').getFileName().isAbsolute()
!new LinPath(fs,'1234/a/b/c').getName(1).isAbsolute()
!new LinPath(fs,'1234/a/b/c').subpath(1,3).isAbsolute()
}
@Unroll
def 'should get root path' () {
expect:
new LinPath(fs,PATH).getRoot() == new LinPath(fs,EXPECTED)
where:
PATH | EXPECTED
'12345' | '/'
'12345/a' | '/'
}
def 'should relativize path' () {
expect:
BASE_PATH.relativize(PATH) == EXPECTED
where :
BASE_PATH | PATH | EXPECTED
new LinPath(fs, '/') | new LinPath(fs, '123/a/b/c') | new LinPath(null, '123/a/b/c')
new LinPath(fs,'123/a/') | new LinPath(fs, '123/a/b/c') | new LinPath(null, 'b/c')
new LinPath(fs,'123/a/') | new LinPath(fs, '321/a/') | new LinPath(null, '../../321/a')
new LinPath(null,'123/a') | new LinPath(null, '123/a/b/c') | new LinPath(null, 'b/c')
new LinPath(null,'123/a') | new LinPath(null, '321/a') | new LinPath(null, '../../321/a')
new LinPath(fs,'../a/') | new LinPath(fs, '321/a') | new LinPath(null, '../321/a')
new LinPath(fs,'321/a/') | new LinPath(fs, '../a') | new LinPath(null, '../../a')
new LinPath(null,'321/a/') | new LinPath(null, '../a') | new LinPath(null, '../../../a')
}
def 'relativize should throw exception' () {
given:
def lid1 = new LinPath(fs,'123/a/')
def lid2 = new LinPath(null,'123/a/')
def lid3 = new LinPath(null, '../a/b')
when: 'comparing relative with absolute'
lid1.relativize(lid2)
then:
thrown(IllegalArgumentException)
when: 'undefined base path'
lid3.relativize(lid2)
then:
thrown(IllegalArgumentException)
}
def 'should resolve path' () {
when:
def lid1 = new LinPath(fs, '123/a/b/c')
def lid2 = new LinPath(fs, '321/x/y/z')
def rel1 = new LinPath(null, 'foo')
def rel2 = new LinPath(null, 'bar/')
then:
lid1.resolve(lid2) == lid2
lid2.resolve(lid1) == lid1
and:
lid1.resolve(rel1) == new LinPath(fs,'123/a/b/c/foo')
lid1.resolve(rel2) == new LinPath(fs,'123/a/b/c/bar')
and:
rel1.resolve(rel2) == new LinPath(null, 'foo/bar')
rel2.resolve(rel1) == new LinPath(null, 'bar/foo')
}
def 'should resolve path as string' () {
given:
def pr = Mock(LinFileSystemProvider)
def lidfs = Mock(LinFileSystem){
provider() >> pr}
def lid1 = new LinPath(lidfs, '123/a/b/c')
expect:
lid1.resolve('x/y') == new LinPath(lidfs, '123/a/b/c/x/y')
lid1.resolve('/x/y/') == new LinPath(lidfs, '123/a/b/c/x/y')
when:
def result = lid1.resolve('lid://321')
then:
pr.getPath(LinPath.asUri('lid://321')) >> new LinPath(lidfs, '321')
and:
result == new LinPath(lidfs, '321')
}
def 'should throw illegal exception when not correct scheme' (){
when: 'creation'
new LinPath(fs, new URI("http://1234"))
then:
thrown(IllegalArgumentException)
when: 'asUri'
LinPath.asUri("http://1234")
then:
thrown(IllegalArgumentException)
when: 'asUri'
LinPath.asUri("")
then:
thrown(IllegalArgumentException)
}
def 'should throw provider mismatch exception when different path types' () {
given:
def pr = Mock(LinFileSystemProvider)
def fs = Mock(LinFileSystem){
provider() >> pr}
and:
def lid = new LinPath(fs, '123/a/b/c')
when: 'resolve with path'
lid.resolve(Path.of('d'))
then:
thrown(ProviderMismatchException)
when: 'resolve with uri string'
lid.resolve(Path.of('http://1234'))
then:
thrown(ProviderMismatchException)
when: 'relativize'
lid.relativize(Path.of('d'))
then:
thrown(ProviderMismatchException)
}
def 'should throw exception for unsupported methods' () {
given:
def pr = Mock(LinFileSystemProvider)
def fs = Mock(LinFileSystem){
provider() >> pr}
and:
def lid = new LinPath(fs, '123/a/b/c')
when: 'to file'
lid.toFile()
then:
thrown(UnsupportedOperationException)
when: 'register'
lid.register(null, null,null)
then:
thrown(UnsupportedOperationException)
}
def 'should throw exception for incorrect index'() {
when: 'getting name with negative index'
new LinPath(fs, "1234").getName(-1)
then:
thrown(IllegalArgumentException)
when: 'getting name with larger index tha namecount'
new LinPath(fs, "1234").getName(2)
then:
thrown(IllegalArgumentException)
when: 'getting subpath with negative index'
new LinPath(fs, "1234").subpath(-1,1)
then:
thrown(IllegalArgumentException)
when: 'getting subpath with larger index tha namecount'
new LinPath(fs, "1234").subpath(0,2)
then:
thrown(IllegalArgumentException)
}
@Unroll
def 'should get to uri string' () {
expect:
new LinPath(fs, PATH).toUriString() == EXPECTED
where:
PATH | EXPECTED
'/' | 'lid:///'
'1234' | 'lid://1234'
'1234/a/b/c' | 'lid://1234/a/b/c'
'' | 'lid:///'
}
@Unroll
def 'should get string' () {
expect:
new LinPath(fs, PATH).toString() == EXPECTED
where:
PATH | EXPECTED
'/' | '/'
'1234' | '1234'
'1234/a/b/c' | '1234/a/b/c'
'' | '/'
}
@Unroll
def 'should validate asString method'() {
expect:
LinPath.asUriString(FIRST, MORE as String[]) == EXPECTED
where:
FIRST | MORE | EXPECTED
'foo' | [] | 'lid://foo'
'foo/' | [] | 'lid://foo'
'/foo' | [] | 'lid://foo'
and:
'a' | ['/b/'] | 'lid://a/b'
'a' | ['/b','c'] | 'lid://a/b/c'
'a' | ['/b','//c'] | 'lid://a/b/c'
'a' | ['/b/c', 'd'] | 'lid://a/b/c/d'
'/a/' | ['/b/c', 'd'] | 'lid://a/b/c/d'
}
@Unroll
def 'should check is lid uri string' () {
expect:
LinPath.isLidUri(STR) == EXPECTED
where:
STR | EXPECTED
null | false
'' | false
'foo' | false
'/foo' | false
'lid:/foo' | false
'lid:foo' | false
'lid/foo' | false
and:
'lid://' | true
'lid:///' | true
'lid://foo/bar' | true
}
def 'should detect equals'(){
expect:
new LinPath(FS1, PATH1).equals(new LinPath(FS2, PATH2)) == EXPECTED
where:
FS1 | FS2 | PATH1 | PATH2 | EXPECTED
null | fs | "12345/path" | "12345/path" | false
fs | null | "12345/path" | "12345/path" | false
null | null | "12345/" | "12345/path" | false
fs | fs | "12345/" | "12345/path" | false
and:
null | null | "12345/path" | "12345/path" | true
fs | fs | "12345/path" | "12345/path" | true
null | null | "12345/" | "12345" | true
fs | fs | "12345/" | "12345 " | true
}
def 'should validate correct hash'(){
given:
def file = wdir.resolve("file.txt")
file.text = "this is a data file"
def hash = CacheHelper.hasher(file).hash().toString()
def correctData = new FileOutput(file.toString(), new Checksum(hash,"nextflow", "standard"))
when:
def error = LinPath.validateDataOutput(correctData)
then:
!error
cleanup:
file.delete()
}
def 'should warn with incorrect hash'(){
given:
def file = wdir.resolve("file.txt")
file.text = "this is a data file"
def hash = CacheHelper.hasher(file).hash().toString()
def correctData = new FileOutput(file.toString(), new Checksum("abscd","nextflow", "standard"))
when:
def error = LinPath.validateDataOutput(correctData)
then:
error == "Checksum of '$file' does not match with lineage metadata"
cleanup:
file.delete()
}
def 'should warn when hash algorithm is not supported'(){
given:
def file = wdir.resolve("file.txt")
file.text = "this is a data file"
def hash = CacheHelper.hasher(file).hash().toString()
def correctData = new FileOutput(file.toString(), new Checksum(hash,"not-supported", "standard"))
when:
def error = LinPath.validateDataOutput(correctData)
then:
error == "Checksum of '$file' can't be validated - algorithm 'not-supported' is not supported"
cleanup:
file.delete()
}
def 'should throw exception when file not found validating hash'(){
when:
def correctData = new FileOutput("not/existing/file", new Checksum("120741","nextflow", "standard"))
LinPath.validateDataOutput(correctData)
then:
thrown(FileNotFoundException)
}
def 'should validate path' () {
given:
def outputFolder = data.resolve('output')
outputFolder.mkdirs()
def outputFile = outputFolder.resolve('file1.txt')
outputFile.text = "this is file1"
def encoder = new LinEncoder()
def hash = CacheHelper.hasher(outputFile).hash().toString()
def correctData = new FileOutput(outputFile.toString(), new Checksum(hash,"nextflow", "standard"))
def incorrectData = new FileOutput(outputFile.toString(), new Checksum("incorrectHash","nextflow", "standard"))
wdir.resolve('12345/output/file1.txt').mkdirs()
wdir.resolve('12345/output/file2.txt').mkdirs()
wdir.resolve('12345/output/file1.txt/.data.json').text = encoder.encode(correctData)
wdir.resolve('12345/output/file2.txt/.data.json').text = encoder.encode(incorrectData)
def lidFs = new LinFileSystemProvider().newFileSystem(new URI("lid:///"), [enabled: true, store: [location: wdir.toString()]])
when:
def succeed = new LinPath(lidFs, '12345/output/file1.txt').validate()
then:
succeed
when:
succeed = new LinPath(lidFs, '12345/output/file2.txt').validate()
then:
!succeed
when:
succeed = new LinPath(lidFs, '12345/output/file3.txt').validate()
then:
!succeed
cleanup:
outputFile.delete()
}
}

View File

@@ -0,0 +1,58 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.model
import nextflow.lineage.model.v1beta1.Checksum
import nextflow.util.CacheHelper
import spock.lang.Specification
/**
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
class ChecksumTest extends Specification {
def 'should create a checksum'() {
given:
def checksum = new Checksum(algorithm: 'sha1', value: '1234567890abcdef', mode: 'hex')
expect:
checksum.algorithm == 'sha1'
checksum.value == '1234567890abcdef'
checksum.mode == 'hex'
}
def 'should create a checksum with of factory method'() {
given:
def checksum1 = Checksum.of('1234567890abcdef','sha1', CacheHelper.HashMode.DEFAULT())
expect:
checksum1.algorithm == 'sha1'
checksum1.value == '1234567890abcdef'
checksum1.mode == 'standard'
}
def 'should create checksum with ofNextflow factory method'() {
given:
def checksum1 = Checksum.ofNextflow('1234567890abcdef')
expect:
checksum1.algorithm == 'nextflow'
checksum1.value == CacheHelper.hasher('1234567890abcdef').hash().toString()
checksum1.mode == 'standard'
}
}

View File

@@ -0,0 +1,266 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.serde
import java.nio.file.Path
import java.time.OffsetDateTime
import nextflow.lineage.model.v1beta1.Checksum
import nextflow.lineage.model.v1beta1.DataPath
import nextflow.lineage.model.v1beta1.FileOutput
import nextflow.lineage.model.v1beta1.Parameter
import nextflow.lineage.model.v1beta1.TaskOutput
import nextflow.lineage.model.v1beta1.TaskRun
import nextflow.lineage.model.v1beta1.Workflow
import nextflow.lineage.model.v1beta1.WorkflowOutput
import nextflow.lineage.model.v1beta1.WorkflowRun
import spock.lang.Specification
class LinEncoderTest extends Specification{
def 'should encode and decode Outputs'(){
given:
def encoder = new LinEncoder()
and:
def output = new FileOutput("/path/to/file", new Checksum("hash_value", "hash_algorithm", "standard"),
"lid://source", "lid://workflow", "lid://task", 1234)
when:
def encoded = encoder.encode(output)
def object = encoder.decode(encoded)
then:
object instanceof FileOutput
def result = object as FileOutput
result.path == "/path/to/file"
result.checksum instanceof Checksum
result.checksum.value == "hash_value"
result.checksum.algorithm == "hash_algorithm"
result.checksum.mode == "standard"
result.source == "lid://source"
result.size == 1234
}
def 'should encode and decode WorkflowRuns'(){
given:
def encoder = new LinEncoder()
and:
def config = [
process: [
container : "quay.io/nextflow/bash",
executor : "local",
resourceLabels: ["owner": "xxx"],
scratch : false
]
]
def metadata = [
runName : "big_kare",
start : "2025-11-06T13:35:42.049135334Z",
container : "quay.io/nextflow/bash",
commandLine : "nextflow run 'https://github.com/nextflow-io/hello' -name big_kare -with-tower -r master",
nextflow : [version: "25.10.0", enable: [dsl: 2.0]],
containerEngine: "docker",
wave : [enabled: true],
fusion : [enabled: true, version: "2.4"],
platform : [
workflowId: "wf1234",
user : [id: "xxx", userName: "john-smith", email: "john.smith@acme.com", firstName: "John", lastName: "Smith", organization: "acme"],
workspace : [id: "1234", name: "test-workspace", fullName: "Test workspace", organization: "acme"],
computeEnv: [id: "ce3456", name: "test-ce", platform: "aws-cloud"],
pipeline : [id: "pipe294", name: "https://github.com/nextflow-io/hello", revision: "master", commitId: null],
labels : []
],
failOnIgnore : false
]
def uniqueId = UUID.randomUUID()
def mainScript = new DataPath("file://path/to/main.nf", new Checksum("78910", "nextflow", "standard"))
def workflow = new Workflow([mainScript], "https://nextflow.io/nf-test/", "123456")
def wfRun = new WorkflowRun(workflow, uniqueId.toString(), "test_run",
[new Parameter("String", "param1", "value1"), new Parameter("String", "param2", "value2")],
config, metadata
)
when:
def encoded = encoder.encode(wfRun)
def object = encoder.decode(encoded)
then:
object instanceof WorkflowRun
def result = object as WorkflowRun
result.workflow instanceof Workflow
result.workflow.scriptFiles.first instanceof DataPath
result.workflow.scriptFiles.first.path == "file://path/to/main.nf"
result.workflow.scriptFiles.first.checksum instanceof Checksum
result.workflow.scriptFiles.first.checksum.value == "78910"
result.workflow.commitId == "123456"
result.sessionId == uniqueId.toString()
result.name == "test_run"
result.params.size() == 2
result.params.get(0).name == "param1"
result.config.process.container == "quay.io/nextflow/bash"
result.config.process.executor == "local"
result.metadata.platform.workflowId == "wf1234"
}
def 'should decode WorkflowRuns without metadata'(){
given:
def encoder = new LinEncoder()
def wfRunStr = '''
{
"version": "lineage/v1beta1",
"kind": "WorkflowRun",
"spec": {
"workflow": {
"scriptFiles": [
{
"path": "https://github.com/nextflow-io/hello/main.nf",
"checksum": {
"value": "78910",
"algorithm": "nextflow",
"mode": "standard"
}
}
],
"repository": "https://github.com/nextflow-io/hello",
"commitId": "2ce0b0e2943449188092a0e25102f0dadc70cb0a"
},
"sessionId": "4f02559e-9ebd-41d8-8ee2-a8d1e4f09c67",
"name": "test_run",
"params": [],
"config": {
"process": {
"container": "quay.io/nextflow/bash",
"executor": "local",
"resourceLabels": {
"owner": "xxx"
},
"scratch": false
}
}
}
}
'''
when:
def object = encoder.decode(wfRunStr)
then:
object instanceof WorkflowRun
def result = object as WorkflowRun
result.workflow instanceof Workflow
result.workflow.scriptFiles.first instanceof DataPath
result.workflow.scriptFiles.first.path == "https://github.com/nextflow-io/hello/main.nf"
result.workflow.scriptFiles.first.checksum instanceof Checksum
result.workflow.scriptFiles.first.checksum.value == "78910"
result.workflow.commitId == "2ce0b0e2943449188092a0e25102f0dadc70cb0a"
result.sessionId == "4f02559e-9ebd-41d8-8ee2-a8d1e4f09c67"
result.name == "test_run"
result.params.size() == 0
result.config.process.container == "quay.io/nextflow/bash"
result.config.process.executor == "local"
result.metadata == null
}
def 'should encode and decode WorkflowOutputs'(){
given:
def encoder = new LinEncoder()
and:
def time = OffsetDateTime.now()
def wfResults = new WorkflowOutput(time, "lid://1234", [
new Parameter("Collection", "a", [id: 'id', file: 'sample.txt' as Path]),
new Parameter("String", "b", "B")
])
when:
def encoded = encoder.encode(wfResults)
def object = encoder.decode(encoded)
then:
object instanceof WorkflowOutput
def result = object as WorkflowOutput
result.createdAt == time
result.workflowRun == "lid://1234"
result.output == [new Parameter("Collection", "a", [id: 'id', file: 'sample.txt']), new Parameter("String", "b", "B")]
}
def 'should encode and decode TaskRun'() {
given:
def encoder = new LinEncoder()
and:
def uniqueId = UUID.randomUUID()
def taskRun = new TaskRun(
uniqueId.toString(),"name", new Checksum("78910", "nextflow", "standard"), 'this is a script',
[new Parameter("String", "param1", "value1")], "container:version", "conda", "spack", "amd64",
[a: "A", b: "B"], [new DataPath("path/to/file", new Checksum("78910", "nextflow", "standard"))]
)
when:
def encoded = encoder.encode(taskRun)
def object = encoder.decode(encoded)
then:
object instanceof TaskRun
def result = object as TaskRun
result.sessionId == uniqueId.toString()
result.name == "name"
result.codeChecksum.value == "78910"
result.script == "this is a script"
result.input.size() == 1
result.input.get(0).name == "param1"
result.container == "container:version"
result.conda == "conda"
result.spack == "spack"
result.architecture == "amd64"
result.globalVars == [a: "A", b: "B"]
result.binEntries.size() == 1
result.binEntries.get(0).path == "path/to/file"
result.binEntries.get(0).checksum.value == "78910"
}
def 'should encode and decode TaskOutputs'(){
given:
def encoder = new LinEncoder()
and:
def time = OffsetDateTime.now()
def parameter = new Parameter("a","b", "c")
def wfResults = new TaskOutput("lid://1234", "lid://5678", time, [parameter], null)
when:
def encoded = encoder.encode(wfResults)
def object = encoder.decode(encoded)
then:
object instanceof TaskOutput
def result = object as TaskOutput
result.createdAt == time
result.taskRun == "lid://1234"
result.workflowRun == "lid://5678"
result.output.size() == 1
result.output[0] == parameter
}
def 'object with null date attributes' () {
given:
def encoder = new LinEncoder()
and:
def wfResults = new WorkflowOutput(null, "lid://1234")
when:
def encoded = encoder.encode(wfResults)
def object = encoder.decode(encoded)
then:
encoded == '{"version":"lineage/v1beta1","kind":"WorkflowOutput","spec":{"createdAt":null,"workflowRun":"lid://1234","output":null}}'
def result = object as WorkflowOutput
result.createdAt == null
}
}

View File

@@ -0,0 +1,181 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.serde
import nextflow.lineage.model.v1beta1.FileOutput
import nextflow.lineage.model.v1beta1.LinModel
import nextflow.lineage.model.v1beta1.TaskRun
import spock.lang.Specification
class LinTypeAdapterFactoryTest extends Specification {
def 'should read both new and old JSON formats'() {
given:
def encoder = new LinEncoder()
when: 'create old format JSON (without spec wrapper)'
def oldFormatJson = """
{
"version": "${LinModel.VERSION}",
"kind": "FileOutput",
"path": "/path/to/file",
"checksum": {
"value": "hash_value",
"algorithm": "hash_algorithm",
"mode": "standard"
},
"source": "lid://source",
"workflow": "lid://workflow",
"task": "lid://task",
"size": 1234
}
"""
and: 'deserialize old format'
def oldResult = encoder.decode(oldFormatJson)
then: 'should work correctly with backward compatibility'
oldResult instanceof FileOutput
oldResult.path == "/path/to/file"
oldResult.checksum.value == "hash_value"
oldResult.source == "lid://source"
oldResult.size == 1234
when: 'create new format JSON (with spec wrapper)'
def newFormatJson = """
{
"version": "${LinModel.VERSION}",
"kind": "FileOutput",
"spec": {
"path": "/path/to/file",
"checksum": {
"value": "hash_value",
"algorithm": "hash_algorithm",
"mode": "standard"
},
"source": "lid://source",
"workflow": "lid://workflow",
"task": "lid://task",
"size": 1234
}
}
"""
and: 'deserialize new format'
def newResult = encoder.decode(newFormatJson)
then: 'should work correctly'
newResult instanceof FileOutput
newResult.path == "/path/to/file"
newResult.checksum.value == "hash_value"
newResult.source == "lid://source"
newResult.size == 1234
}
def 'should handle TaskRun old format'() {
given:
def encoder = new LinEncoder()
when: 'create old format TaskRun JSON'
def oldFormatJson = """
{
"version": "${LinModel.VERSION}",
"kind": "TaskRun",
"sessionId": "session123",
"name": "testTask",
"codeChecksum": {
"value": "hash123",
"algorithm": "nextflow",
"mode": "standard"
},
"script": "echo hello",
"input": [],
"container": "ubuntu:latest",
"conda": null,
"spack": null,
"architecture": "amd64",
"globalVars": {},
"binEntries": []
}
"""
and: 'deserialize old format'
def result = encoder.decode(oldFormatJson)
then: 'should work correctly'
result instanceof TaskRun
result.sessionId == "session123"
result.name == "testTask"
result.codeChecksum.value == "hash123"
result.script == "echo hello"
result.container == "ubuntu:latest"
result.architecture == "amd64"
}
def 'should reject JSON without version field'() {
given:
def encoder = new LinEncoder()
when: 'try to deserialize JSON without version'
def invalidJson = """
{
"kind": "FileOutput",
"path": "/path/to/file"
}
"""
encoder.decode(invalidJson)
then: 'should throw exception'
thrown(Exception)
}
def 'should reject JSON with wrong version'() {
given:
def encoder = new LinEncoder()
when: 'try to deserialize JSON with wrong version'
def invalidJson = """
{
"version": "wrong/version",
"kind": "FileOutput",
"path": "/path/to/file"
}
"""
encoder.decode(invalidJson)
then: 'should throw exception'
thrown(Exception)
}
def 'should reject JSON without kind field'() {
given:
def encoder = new LinEncoder()
when: 'try to deserialize JSON without kind'
def invalidJson = """
{
"version": "${LinModel.VERSION}",
"path": "/path/to/file"
}
"""
encoder.decode(invalidJson)
then: 'should throw exception'
thrown(Exception)
}
}