add nextflow d30e48d
This commit is contained in:
@@ -0,0 +1,67 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package nextflow.lineage
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.extension.FilesEx
|
||||
|
||||
import static nextflow.lineage.fs.LinPath.*
|
||||
/**
|
||||
* File to store a history of the workflow executions and their corresponding LIDs
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class DefaultLinHistoryLog implements LinHistoryLog {
|
||||
|
||||
Path path
|
||||
|
||||
DefaultLinHistoryLog(Path folder) {
|
||||
this.path = folder
|
||||
if( !path.exists() )
|
||||
Files.createDirectories(path)
|
||||
}
|
||||
|
||||
void write(String name, UUID key, String runLid, Date date = null) {
|
||||
assert name
|
||||
assert key
|
||||
def timestamp = date ?: new Date()
|
||||
final recordFile = path.resolve(runLid.substring(LID_PROT.size()))
|
||||
try {
|
||||
recordFile.text = new LinHistoryRecord(timestamp, name, key, runLid).toString()
|
||||
log.trace("Record for $key written in lineage history log ${FilesEx.toUriString(this.path)}")
|
||||
}catch (Throwable e) {
|
||||
log.warn("Can't write record $key file ${FilesEx.toUriString(recordFile)}", e.message)
|
||||
}
|
||||
}
|
||||
|
||||
List<LinHistoryRecord> getRecords(){
|
||||
List<LinHistoryRecord> list = new LinkedList<LinHistoryRecord>()
|
||||
try {
|
||||
this.path.eachFile { Path file -> list.add(LinHistoryRecord.parse(file.text))}
|
||||
}
|
||||
catch (Throwable e) {
|
||||
log.warn "Exception reading records from lineage history folder: ${FilesEx.toUriString(this.path)}", e.message
|
||||
}
|
||||
return list.sort {it.timestamp }
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.util.stream.Stream
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.file.FileHelper
|
||||
import nextflow.lineage.config.LineageConfig
|
||||
import nextflow.lineage.serde.LinEncoder
|
||||
import nextflow.lineage.serde.LinSerializable
|
||||
/**
|
||||
* Default Implementation for the a lineage store.
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class DefaultLinStore implements LinStore {
|
||||
|
||||
private static String HISTORY_FILE_NAME = ".history"
|
||||
private static final String METADATA_FILE = '.data.json'
|
||||
private static final String DEFAULT_LOCATION = '.lineage'
|
||||
|
||||
private Path location
|
||||
private LinHistoryLog historyLog
|
||||
private LinEncoder encoder
|
||||
|
||||
DefaultLinStore open(LineageConfig config) {
|
||||
location = toLocationPath(config.store.location)
|
||||
encoder = new LinEncoder()
|
||||
if( !Files.exists(location) && !Files.createDirectories(location) ) {
|
||||
throw new AbortOperationException("Unable to create lineage store directory: $location")
|
||||
}
|
||||
historyLog = new DefaultLinHistoryLog(location.resolve(HISTORY_FILE_NAME))
|
||||
return this
|
||||
}
|
||||
|
||||
protected Path toLocationPath(String location) {
|
||||
return location
|
||||
? FileHelper.toCanonicalPath(location)
|
||||
: Path.of('.').toAbsolutePath().normalize().resolve(DEFAULT_LOCATION)
|
||||
}
|
||||
|
||||
@Override
|
||||
void save(String key, LinSerializable value) {
|
||||
final path = location.resolve("$key/$METADATA_FILE")
|
||||
Files.createDirectories(path.parent)
|
||||
log.debug "Save LID file path: $path"
|
||||
path.text = encoder.encode(value)
|
||||
}
|
||||
|
||||
@Override
|
||||
LinSerializable load(String key) {
|
||||
final path = location.resolve("$key/$METADATA_FILE")
|
||||
log.debug("Loading from path $path")
|
||||
if( path.exists() )
|
||||
return encoder.decode(path.text) as LinSerializable
|
||||
log.debug("File for key $key not found")
|
||||
return null
|
||||
}
|
||||
|
||||
Path getLocation() {
|
||||
return location
|
||||
}
|
||||
|
||||
@Override
|
||||
LinHistoryLog getHistoryLog() {
|
||||
return historyLog
|
||||
}
|
||||
|
||||
@Override
|
||||
void close() throws IOException {}
|
||||
|
||||
@Override
|
||||
Stream<String> search(Map<String, List<String>> params) {
|
||||
return Files.walk(location)
|
||||
.filter { Path path ->
|
||||
Files.isRegularFile(path) && path.fileName.toString().startsWith('.data.json')
|
||||
}
|
||||
.map { Path path ->
|
||||
final obj = encoder.decode(path.text)
|
||||
final key = location.relativize(path.parent).toString()
|
||||
return new AbstractMap.SimpleEntry<String, LinSerializable>(key, obj)
|
||||
}
|
||||
.filter { entry ->
|
||||
LinUtils.checkParams(entry.value, params)
|
||||
}
|
||||
.map {it-> it.key }
|
||||
}
|
||||
|
||||
@Override
|
||||
Stream<String> getSubKeys(String parentKey) {
|
||||
final startPath = location.resolve(parentKey)
|
||||
|
||||
return Files.walk(startPath)
|
||||
.filter { Path path ->
|
||||
Files.isRegularFile(path) && path.fileName.toString().startsWith('.data.json') && path.parent != startPath
|
||||
}
|
||||
.map { Path path ->
|
||||
location.relativize(path.parent).toString()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package nextflow.lineage
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.lineage.config.LineageConfig
|
||||
import nextflow.plugin.Priority
|
||||
|
||||
/**
|
||||
* Default Factory for Lineage Store.
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
@CompileStatic
|
||||
@Priority(0)
|
||||
class DefaultLinStoreFactory extends LinStoreFactory {
|
||||
|
||||
private static final Pattern SCHEME = ~/^([a-zA-Z][a-zA-Z\d+\-.]*):/
|
||||
private static final List<String> SUPPORTED_SCHEMES = List.of('file', 's3', 'gs', 'az')
|
||||
|
||||
@Override
|
||||
boolean canOpen(LineageConfig config) {
|
||||
final loc = config.store.location
|
||||
if( !loc ) {
|
||||
return true
|
||||
}
|
||||
final matcher = SCHEME.matcher(loc)
|
||||
return matcher.find() ? matcher.group(1) in SUPPORTED_SCHEMES : true
|
||||
}
|
||||
|
||||
@Override
|
||||
protected LinStore newInstance(LineageConfig config) {
|
||||
return new DefaultLinStore() .open(config)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import groovyx.gpars.dataflow.DataflowWriteChannel
|
||||
import nextflow.Channel
|
||||
import nextflow.Session
|
||||
import nextflow.extension.LinExtension
|
||||
import nextflow.lineage.fs.LinPathFactory
|
||||
import nextflow.lineage.model.v1beta1.FileOutput
|
||||
|
||||
import static nextflow.lineage.fs.LinPath.*
|
||||
|
||||
/**
|
||||
* Lineage channel extensions
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
@CompileStatic
|
||||
@Slf4j
|
||||
class LinExtensionImpl implements LinExtension {
|
||||
|
||||
@Override
|
||||
void fromLineage(Session session, DataflowWriteChannel channel, Map<String,?> opts) {
|
||||
final queryParams = buildQueryParams(opts)
|
||||
log.trace("Querying lineage with params: $queryParams")
|
||||
new LinPropertyValidator().validateQueryParams(queryParams.keySet())
|
||||
final store = getStore(session)
|
||||
try( def stream = store.search(queryParams) ){
|
||||
stream.forEach { channel.bind( LinPathFactory.create( asUriString(it) ) ) }
|
||||
}
|
||||
channel.bind(Channel.STOP)
|
||||
}
|
||||
|
||||
private static Map<String, List<String>> buildQueryParams(Map<String,?> opts) {
|
||||
final queryParams = [type: [FileOutput.class.simpleName] ]
|
||||
if( opts.workflowRun )
|
||||
queryParams['workflowRun'] = [opts.workflowRun as String]
|
||||
if( opts.taskRun )
|
||||
queryParams['taskRun'] = [opts.taskRun as String]
|
||||
if( opts.label ) {
|
||||
if( opts.label instanceof List )
|
||||
queryParams['labels'] = opts.label as List<String>
|
||||
else
|
||||
queryParams['labels'] = [ opts.label.toString() ]
|
||||
}
|
||||
return queryParams
|
||||
}
|
||||
|
||||
protected LinStore getStore(Session session) {
|
||||
final store = LinStoreFactory.getOrCreate(session)
|
||||
if( !store ) {
|
||||
throw new Exception("Lineage store not found - Check Nextflow configuration")
|
||||
}
|
||||
return store
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package nextflow.lineage
|
||||
|
||||
/**
|
||||
* Interface to log workflow executions and their corresponding Lineage IDs
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
interface LinHistoryLog {
|
||||
/**
|
||||
* Write a workflow execution linage history log record.
|
||||
*
|
||||
* @param name Workflow execution name.
|
||||
* @param sessionId Workflow session ID.
|
||||
* @param runLid Workflow run ID.
|
||||
*/
|
||||
void write(String name, UUID sessionId, String runLid)
|
||||
|
||||
/**
|
||||
* Get the store records in the Lineage History Log.
|
||||
*
|
||||
* @return List of stored lineage history records.
|
||||
*/
|
||||
List<LinHistoryRecord> getRecords()
|
||||
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package nextflow.lineage
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
|
||||
import java.text.DateFormat
|
||||
import java.text.SimpleDateFormat
|
||||
|
||||
/**
|
||||
* Record of workflow executions and their corresponding Lineage IDs
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
@CompileStatic
|
||||
@EqualsAndHashCode(includes = 'runName,sessionId')
|
||||
class LinHistoryRecord {
|
||||
|
||||
static final public DateFormat TIMESTAMP_FMT = new SimpleDateFormat('yyyy-MM-dd HH:mm:ss z')
|
||||
|
||||
final Date timestamp
|
||||
final String runName
|
||||
final UUID sessionId
|
||||
final String runLid
|
||||
|
||||
LinHistoryRecord(Date timestamp, String name, UUID sessionId, String runLid) {
|
||||
this.timestamp = timestamp
|
||||
this.runName = name
|
||||
this.sessionId = sessionId
|
||||
this.runLid = runLid
|
||||
}
|
||||
|
||||
protected LinHistoryRecord() {}
|
||||
|
||||
List<String> toList() {
|
||||
return List.of(
|
||||
timestamp ? TIMESTAMP_FMT.format(timestamp) : '-',
|
||||
runName ?: '-',
|
||||
sessionId.toString(),
|
||||
runLid ?: '-',
|
||||
)
|
||||
}
|
||||
|
||||
@Override
|
||||
String toString() {
|
||||
toList().join('\t')
|
||||
}
|
||||
|
||||
static LinHistoryRecord parse(String line) {
|
||||
final cols = line.tokenize('\t')
|
||||
if (cols.size() == 4) {
|
||||
return new LinHistoryRecord(TIMESTAMP_FMT.parse(cols[0]), cols[1], UUID.fromString(cols[2]), cols[3])
|
||||
}
|
||||
throw new IllegalArgumentException("Not a valid history entry: `$line`")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,517 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage
|
||||
|
||||
import nextflow.NextflowMeta
|
||||
import nextflow.extension.FilesEx
|
||||
import nextflow.lineage.exception.OutputRelativePathException
|
||||
|
||||
import static nextflow.lineage.fs.LinPath.*
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.attribute.BasicFileAttributes
|
||||
import java.time.OffsetDateTime
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.Session
|
||||
import nextflow.lineage.model.v1beta1.Checksum
|
||||
import nextflow.lineage.model.v1beta1.FileOutput
|
||||
import nextflow.lineage.model.v1beta1.DataPath
|
||||
import nextflow.lineage.model.v1beta1.Parameter
|
||||
import nextflow.lineage.model.v1beta1.TaskOutput
|
||||
import nextflow.lineage.model.v1beta1.Workflow
|
||||
import nextflow.lineage.model.v1beta1.WorkflowOutput
|
||||
import nextflow.lineage.model.v1beta1.WorkflowRun
|
||||
import nextflow.file.FileHelper
|
||||
import nextflow.file.FileHolder
|
||||
import nextflow.processor.TaskHasher
|
||||
import nextflow.processor.TaskRun
|
||||
import nextflow.script.PlatformMetadata
|
||||
import nextflow.script.ScriptMeta
|
||||
import nextflow.script.params.BaseParam
|
||||
import nextflow.script.params.CmdEvalParam
|
||||
import nextflow.script.params.DefaultInParam
|
||||
import nextflow.script.params.EachInParam
|
||||
import nextflow.script.params.EnvInParam
|
||||
import nextflow.script.params.EnvOutParam
|
||||
import nextflow.script.params.FileInParam
|
||||
import nextflow.script.params.FileOutParam
|
||||
import nextflow.script.params.InParam
|
||||
import nextflow.script.params.OutParam
|
||||
import nextflow.script.params.StdInParam
|
||||
import nextflow.script.params.StdOutParam
|
||||
import nextflow.script.params.ValueInParam
|
||||
import nextflow.script.params.ValueOutParam
|
||||
import nextflow.trace.TraceObserverV2
|
||||
import nextflow.trace.event.FilePublishEvent
|
||||
import nextflow.trace.event.TaskEvent
|
||||
import nextflow.trace.event.WorkflowOutputEvent
|
||||
import nextflow.util.CacheHelper
|
||||
import nextflow.util.PathNormalizer
|
||||
import nextflow.util.SecretHelper
|
||||
import nextflow.util.TestOnly
|
||||
|
||||
/**
|
||||
* Observer to write the generated workflow metadata in a lineage store.
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class LinObserver implements TraceObserverV2 {
|
||||
private static Set<String> workflowMetadataPropertiesToRemove = Set.of(
|
||||
"completed", "duration", "exitStatus", "errorMessage", "errorReport", "stats", "success" // Only existing at the end
|
||||
)
|
||||
private static Map<Class<? extends BaseParam>, String> taskParamToValue = [
|
||||
(StdOutParam) : "stdout",
|
||||
(StdInParam) : "stdin",
|
||||
(FileInParam) : "path",
|
||||
(FileOutParam) : "path",
|
||||
(ValueInParam) : "val",
|
||||
(ValueOutParam): "val",
|
||||
(EnvInParam) : "env",
|
||||
(EnvOutParam) : "env",
|
||||
(CmdEvalParam) : "eval",
|
||||
(EachInParam) : "each"
|
||||
]
|
||||
|
||||
private String executionHash
|
||||
private LinStore store
|
||||
private Session session
|
||||
private WorkflowOutput workflowOutput
|
||||
private Map<String,String> outputsStoreDirLid = new HashMap<String,String>(10)
|
||||
private PathNormalizer normalizer
|
||||
|
||||
LinObserver(Session session, LinStore store){
|
||||
this.session = session
|
||||
this.store = store
|
||||
}
|
||||
|
||||
@TestOnly
|
||||
String getExecutionHash(){ executionHash }
|
||||
|
||||
@TestOnly
|
||||
String setExecutionHash(String hash){ this.executionHash = hash }
|
||||
|
||||
@TestOnly
|
||||
String setNormalizer(PathNormalizer normalizer){ this.normalizer = normalizer }
|
||||
|
||||
@Override
|
||||
void onFlowBegin() {
|
||||
normalizer = new PathNormalizer(session.workflowMetadata)
|
||||
executionHash = storeWorkflowRun(normalizer)
|
||||
final executionUri = asUriString(executionHash)
|
||||
workflowOutput = new WorkflowOutput(
|
||||
OffsetDateTime.now(),
|
||||
executionUri,
|
||||
new LinkedList<Parameter>()
|
||||
)
|
||||
this.store.getHistoryLog().write(session.runName, session.uniqueId, executionUri)
|
||||
}
|
||||
|
||||
@Override
|
||||
void onFlowComplete(){
|
||||
if(workflowOutput?.output ){
|
||||
workflowOutput.createdAt = OffsetDateTime.now()
|
||||
final key = executionHash + '#output'
|
||||
this.store.save(key, workflowOutput)
|
||||
}
|
||||
}
|
||||
|
||||
protected Collection<Path> allScriptFiles() {
|
||||
return ScriptMeta.allScriptNames().values()
|
||||
}
|
||||
|
||||
protected List<DataPath> collectScriptDataPaths(PathNormalizer normalizer) {
|
||||
final allScripts = allScriptFiles().sort()
|
||||
final result = new ArrayList<DataPath>(allScripts.size()+1)
|
||||
// the main script
|
||||
result.add( new DataPath(
|
||||
normalizer.normalizePath(session.workflowMetadata.scriptFile.normalize()),
|
||||
Checksum.of(session.workflowMetadata.scriptId, "nextflow", CacheHelper.HashMode.DEFAULT())
|
||||
) )
|
||||
|
||||
// all other scripts
|
||||
for (Path it: allScripts) {
|
||||
if( it==null || it == session.workflowMetadata.scriptFile )
|
||||
continue
|
||||
final dataPath = new DataPath(normalizer.normalizePath(it.normalize()), Checksum.ofNextflow(it.text))
|
||||
result.add(dataPath)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
protected String storeWorkflowRun(PathNormalizer normalizer) {
|
||||
// create the workflow object holding script files and repo tracking info
|
||||
final workflow = new Workflow(
|
||||
collectScriptDataPaths(normalizer),
|
||||
session.workflowMetadata.repository,
|
||||
session.workflowMetadata.commitId
|
||||
)
|
||||
// create the workflow run main object
|
||||
final value = new WorkflowRun(
|
||||
workflow,
|
||||
session.uniqueId.toString(),
|
||||
session.runName,
|
||||
getNormalizedParams(session.params, normalizer),
|
||||
SecretHelper.hideSecrets(session.config.deepClone()) as Map,
|
||||
collectWorkflowMetadata(normalizer)
|
||||
)
|
||||
final executionHash = CacheHelper.hasher(value).hash().toString()
|
||||
store.save(executionHash, value)
|
||||
return executionHash
|
||||
}
|
||||
|
||||
protected static List<Parameter> getNormalizedParams(Map<String, Object> params, PathNormalizer normalizer){
|
||||
final normalizedParams = new LinkedList<Parameter>()
|
||||
for( Map.Entry<String,Object> entry : params ) {
|
||||
final key = entry.key
|
||||
final val = entry.value
|
||||
normalizedParams.add( new Parameter( getParameterType(val), key, normalizeValue(val, normalizer) ) )
|
||||
}
|
||||
return normalizedParams
|
||||
}
|
||||
|
||||
@Override
|
||||
void onTaskComplete(TaskEvent event) {
|
||||
storeTaskInfo(event.handler.task)
|
||||
}
|
||||
|
||||
protected void storeTaskInfo(TaskRun task) {
|
||||
// store the task run entry
|
||||
storeTaskRun(task, normalizer)
|
||||
// store all task results
|
||||
storeTaskResults(task, normalizer)
|
||||
}
|
||||
|
||||
protected String storeTaskResults(TaskRun task, PathNormalizer normalizer){
|
||||
final outputParams = getNormalizedTaskOutputs(task, normalizer)
|
||||
final value = new TaskOutput( asUriString(task.hash.toString()), asUriString(executionHash), OffsetDateTime.now(), outputParams )
|
||||
final key = task.hash.toString() + '#output'
|
||||
store.save(key,value)
|
||||
return key
|
||||
}
|
||||
|
||||
private List<Parameter> getNormalizedTaskOutputs(TaskRun task, PathNormalizer normalizer){
|
||||
final outputs = task.getOutputs()
|
||||
final outputParams = new LinkedList<Parameter>()
|
||||
for( Map.Entry<OutParam,Object> entry : outputs ) {
|
||||
manageTaskOutputParameter(entry.key, outputParams, entry.value, task, normalizer)
|
||||
}
|
||||
return outputParams
|
||||
}
|
||||
|
||||
private void manageTaskOutputParameter(OutParam key, LinkedList<Parameter> outputParams, value, TaskRun task, PathNormalizer normalizer) {
|
||||
if (key instanceof FileOutParam) {
|
||||
outputParams.add(new Parameter(getParameterType(key), key.name, manageFileOutParam(value, task)))
|
||||
} else {
|
||||
outputParams.add(new Parameter(getParameterType(key), key.name, normalizeValue(value, normalizer)))
|
||||
}
|
||||
}
|
||||
|
||||
private static Object normalizeValue(Object value, PathNormalizer normalizer) {
|
||||
if (value instanceof Path)
|
||||
return normalizer.normalizePath((Path)value)
|
||||
else if (value instanceof CharSequence)
|
||||
return normalizer.normalizePath(value.toString())
|
||||
else
|
||||
return value
|
||||
}
|
||||
|
||||
private Object manageFileOutParam(Object value, TaskRun task) {
|
||||
if (value == null) {
|
||||
log.debug "Unexpected lineage File output value null"
|
||||
return null
|
||||
}
|
||||
if (value instanceof Path) {
|
||||
return asUriString(storeTaskOutput(task, (Path) value))
|
||||
}
|
||||
if (value instanceof Collection<Path>) {
|
||||
final files = new LinkedList<String>()
|
||||
for (Path it : value) {
|
||||
files.add( asUriString(storeTaskOutput(task, (Path)it)) )
|
||||
}
|
||||
return files
|
||||
}
|
||||
// unexpected task output
|
||||
throw new IllegalArgumentException("Unexpected output [${value.getClass().getName()}] '${value}' for task '${task.name}'")
|
||||
}
|
||||
|
||||
protected String storeTaskRun(TaskRun task, PathNormalizer normalizer) {
|
||||
final codeChecksum = Checksum.ofNextflow(session.stubRun ? task.stubSource : task.source)
|
||||
final value = new nextflow.lineage.model.v1beta1.TaskRun(
|
||||
session.uniqueId.toString(),
|
||||
task.getName(),
|
||||
codeChecksum,
|
||||
task.script,
|
||||
task.inputs ? manageTaskInputParameters(task.inputs, normalizer) : null,
|
||||
task.isContainerEnabled() ? task.getContainerFingerprint() : null,
|
||||
normalizer.normalizePath(task.getCondaEnv()),
|
||||
normalizer.normalizePath(task.getSpackEnv()),
|
||||
task.config?.getArchitecture()?.toString(),
|
||||
getTaskGlobalVars(task),
|
||||
getTaskBinEntries(task).collect { Path p -> new DataPath(
|
||||
normalizer.normalizePath(p.normalize()),
|
||||
Checksum.ofNextflow(p) )
|
||||
},
|
||||
asUriString(executionHash)
|
||||
)
|
||||
|
||||
// store in the underlying persistence
|
||||
final key = task.hash.toString()
|
||||
store.save(key, value)
|
||||
return key
|
||||
}
|
||||
|
||||
protected Map<String,Object> getTaskGlobalVars(TaskRun task) {
|
||||
return new TaskHasher(task).getTaskGlobalVars()
|
||||
}
|
||||
|
||||
protected List<Path> getTaskBinEntries(TaskRun task) {
|
||||
return new TaskHasher(task).getTaskBinEntries(task.source)
|
||||
}
|
||||
|
||||
protected String storeTaskOutput(TaskRun task, Path path) {
|
||||
try {
|
||||
final attrs = readAttributes(path)
|
||||
final key = getTaskOutputKey(task, path)
|
||||
final checksum = Checksum.ofNextflow(path)
|
||||
final value = new FileOutput(
|
||||
path.toUriString(),
|
||||
checksum,
|
||||
asUriString(task.hash.toString()),
|
||||
asUriString(executionHash),
|
||||
asUriString(task.hash.toString()),
|
||||
attrs.size(),
|
||||
LinUtils.toDate(attrs?.creationTime()),
|
||||
LinUtils.toDate(attrs?.lastModifiedTime()))
|
||||
store.save(key, value)
|
||||
return key
|
||||
} catch (Throwable e) {
|
||||
log.warn("Unexpected error storing lineage output '${path.toUriString()}' for task '${task.name}'", e)
|
||||
return path.toUriString()
|
||||
}
|
||||
}
|
||||
|
||||
protected String getTaskOutputKey(TaskRun task, Path path) {
|
||||
final rel = getTaskRelative(task, path)
|
||||
return task.hash.toString() + SEPARATOR + rel
|
||||
}
|
||||
|
||||
protected String getWorkflowOutputKey(Path target) {
|
||||
final rel = getWorkflowRelative(target)
|
||||
return executionHash + SEPARATOR + rel
|
||||
}
|
||||
|
||||
protected String getTaskRelative(TaskRun task, Path path){
|
||||
if (path.isAbsolute()) {
|
||||
final rel = getTaskRelative0(task, path)
|
||||
if (rel)
|
||||
return rel
|
||||
throw new IllegalArgumentException("Cannot access the relative path for output '${path.toUriString()}' and task '${task.name}'")
|
||||
}
|
||||
//Check if contains workdir or storeDir
|
||||
final rel = getTaskRelative0(task, path.toAbsolutePath())
|
||||
if (rel) return rel
|
||||
if (path.normalize().getName(0).toString() == "..")
|
||||
throw new IllegalArgumentException("Cannot access the relative path for output '${path.toUriString()}' and task '${task.name}'" )
|
||||
return path.normalize().toString()
|
||||
}
|
||||
|
||||
private String getTaskRelative0(TaskRun task, Path path){
|
||||
final workDirAbsolute = task.workDir.toAbsolutePath()
|
||||
if (path.startsWith(workDirAbsolute)) {
|
||||
return workDirAbsolute.relativize(path).toString()
|
||||
}
|
||||
//If task output is not in the workDir check if output is stored in the task's storeDir
|
||||
final storeDir = task.getConfig().getStoreDir().toAbsolutePath()
|
||||
if( storeDir && path.startsWith(storeDir) ) {
|
||||
final rel = storeDir.relativize(path)
|
||||
//If output stored in storeDir, keep the path in case it is used as workflow output
|
||||
this.outputsStoreDirLid.put(path.toString(), asUriString(task.hash.toString(),rel.toString()))
|
||||
return rel
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
protected BasicFileAttributes readAttributes(Path path) {
|
||||
return Files.readAttributes(path, BasicFileAttributes)
|
||||
}
|
||||
|
||||
@Override
|
||||
void onFilePublish(FilePublishEvent event) {
|
||||
try {
|
||||
final checksum = Checksum.ofNextflow(event.target)
|
||||
final key = getWorkflowOutputKey(event.target)
|
||||
final sourceReference = event.source
|
||||
? getSourceReference(event.source)
|
||||
: asUriString(executionHash)
|
||||
final attrs = readAttributes(event.target)
|
||||
final value = new FileOutput(
|
||||
event.target.toUriString(),
|
||||
checksum,
|
||||
sourceReference,
|
||||
asUriString(executionHash),
|
||||
null,
|
||||
attrs.size(),
|
||||
LinUtils.toDate(attrs?.creationTime()),
|
||||
LinUtils.toDate(attrs?.lastModifiedTime()),
|
||||
event.labels)
|
||||
store.save(key, value)
|
||||
}
|
||||
catch (OutputRelativePathException ignored ){
|
||||
log.warn1("Lineage for workflow output is not supported by publishDir directive")
|
||||
}
|
||||
catch (Throwable e) {
|
||||
log.warn("Unexpected error storing published file '${event.target.toUriString()}' for workflow '${executionHash}'", e)
|
||||
}
|
||||
}
|
||||
|
||||
String getSourceReference(Path source){
|
||||
final hash = FileHelper.getTaskHashFromPath(source, session.workDir)
|
||||
if (hash) {
|
||||
final target = FileHelper.getWorkFolder(session.workDir, hash).relativize(source).toString()
|
||||
return asUriString(hash.toString(), target)
|
||||
}
|
||||
final storeDirReference = outputsStoreDirLid.get(source.toString())
|
||||
return storeDirReference ? asUriString(storeDirReference) : null
|
||||
}
|
||||
|
||||
@Override
|
||||
void onWorkflowOutput(WorkflowOutputEvent event) {
|
||||
final type = getParameterType(event.value)
|
||||
final value = convertPathsToLidReferences(event.index ?: event.value)
|
||||
workflowOutput.output.add(new Parameter(type, event.name, value))
|
||||
}
|
||||
|
||||
protected static String getParameterType(Object param) {
|
||||
if( param instanceof BaseParam )
|
||||
return taskParamToValue.get(param.class)
|
||||
// return generic types
|
||||
if( param instanceof Path )
|
||||
return Path.simpleName
|
||||
if( param instanceof CharSequence )
|
||||
return String.simpleName
|
||||
if( param instanceof Collection )
|
||||
return Collection.simpleName
|
||||
if( param instanceof Map )
|
||||
return Map.simpleName
|
||||
if( param==null ) {
|
||||
log.debug "Unexpected lineage param type null"
|
||||
return null
|
||||
}
|
||||
return param.class.simpleName
|
||||
}
|
||||
|
||||
private Object convertPathsToLidReferences(Object value){
|
||||
if( value instanceof Path ) {
|
||||
try {
|
||||
final key = getWorkflowOutputKey(value)
|
||||
return asUriString(key)
|
||||
} catch (Throwable e){
|
||||
//Workflow output key not found
|
||||
return value
|
||||
}
|
||||
}
|
||||
if( value instanceof Collection ) {
|
||||
return value.collect { el -> convertPathsToLidReferences(el) }
|
||||
}
|
||||
if( value instanceof Map ) {
|
||||
return value
|
||||
.findAll { k, v -> v != null }
|
||||
.collectEntries { k, v -> Map.entry(k, convertPathsToLidReferences(v)) }
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
/**
|
||||
* Relativizes a path from the workflow's output dir.
|
||||
*
|
||||
* @param path Path to relativize
|
||||
* @return Path String with the relative path
|
||||
* @throws IllegalArgumentException
|
||||
*/
|
||||
protected String getWorkflowRelative(Path path) throws IllegalArgumentException{
|
||||
final outputDirAbs = session.outputDir.toAbsolutePath()
|
||||
if (path.isAbsolute()) {
|
||||
if (path.startsWith(outputDirAbs)) {
|
||||
return outputDirAbs.relativize(path).toString()
|
||||
}
|
||||
log.debug("Cannot get relative path for workflow output '${path.toUriString()}'")
|
||||
throw new OutputRelativePathException()
|
||||
}
|
||||
final pathAbs = path.toAbsolutePath()
|
||||
if (pathAbs.startsWith(outputDirAbs)) {
|
||||
return outputDirAbs.relativize(pathAbs).toString()
|
||||
}
|
||||
if (path.normalize().getName(0).toString() == "..") {
|
||||
log.debug("Cannot get relative path for workflow output '${path.toUriString()}'")
|
||||
throw new OutputRelativePathException()
|
||||
}
|
||||
return path.normalize().toString()
|
||||
}
|
||||
|
||||
protected List<Parameter> manageTaskInputParameters(Map<InParam, Object> inputs, PathNormalizer normalizer) {
|
||||
List<Parameter> managedInputs = new LinkedList<Parameter>()
|
||||
inputs.forEach { param, value ->
|
||||
if( param instanceof FileInParam )
|
||||
managedInputs.add( new Parameter( getParameterType(param), param.name, manageFileInParam( (List<FileHolder>)value , normalizer) ) )
|
||||
else if( !(param instanceof DefaultInParam) )
|
||||
managedInputs.add( new Parameter( getParameterType(param), param.name, value) )
|
||||
}
|
||||
return managedInputs
|
||||
}
|
||||
|
||||
private List<Object> manageFileInParam(List<FileHolder> files, PathNormalizer normalizer){
|
||||
final paths = new LinkedList<Object>();
|
||||
for( FileHolder it : files ) {
|
||||
final path = it.sourcePath ?: it.storePath
|
||||
final ref = getSourceReference(path)
|
||||
paths.add(ref ?: new DataPath(
|
||||
normalizer.normalizePath(path),
|
||||
Checksum.ofNextflow(path))
|
||||
)
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
/**
|
||||
* Collects lineage data from workflow metadata applying the following transformations:
|
||||
* - Normalizes paths against the original remote URL, or work directory and convert to URI strings
|
||||
* - Remove transient properties (completed, duration, exitStatus, errorMessage, errorReport, stats, success)
|
||||
* - Convert Nextflow metadata as Json Map
|
||||
* @param normalizer
|
||||
* @return Map with workflow metadata or null when error
|
||||
*/
|
||||
private Map collectWorkflowMetadata(PathNormalizer normalizer) {
|
||||
try {
|
||||
def metadata = session.workflowMetadata.toMap()
|
||||
.collectEntries { it.value instanceof Path ? [it.key, FilesEx.toUriString(it.value as Path) ] : [it.key, it.value] }
|
||||
metadata.removeAll {it.key.toString() in workflowMetadataPropertiesToRemove }
|
||||
if( metadata.containsKey("nextflow") )
|
||||
metadata["nextflow"] = (metadata["nextflow"] as NextflowMeta).toJsonMap()
|
||||
if( metadata.containsKey("configFiles") )
|
||||
metadata["configFiles"] = (metadata["configFiles"] as List<Path>).collect {normalizer.normalizePath(it)}
|
||||
return metadata
|
||||
} catch( Throwable e) {
|
||||
log.debug("Error creating metadata", e)
|
||||
return null
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.Session
|
||||
import nextflow.trace.TraceObserverV2
|
||||
import nextflow.trace.TraceObserverFactoryV2
|
||||
|
||||
/**
|
||||
* Implements factory for {@link LinObserver} object
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class LinObserverFactory implements TraceObserverFactoryV2 {
|
||||
|
||||
@Override
|
||||
Collection<TraceObserverV2> create(Session session) {
|
||||
final result = new ArrayList<TraceObserverV2>(1)
|
||||
final store = LinStoreFactory.getOrCreate(session)
|
||||
if( store )
|
||||
result.add( new LinObserver(session, store) )
|
||||
return result
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.lineage.model.v1beta1.Checksum
|
||||
import nextflow.lineage.model.v1beta1.DataPath
|
||||
import nextflow.lineage.model.v1beta1.FileOutput
|
||||
import nextflow.lineage.model.v1beta1.Parameter
|
||||
import nextflow.lineage.model.v1beta1.TaskOutput
|
||||
import nextflow.lineage.model.v1beta1.TaskRun
|
||||
import nextflow.lineage.model.v1beta1.Workflow
|
||||
import nextflow.lineage.model.v1beta1.WorkflowOutput
|
||||
import nextflow.lineage.model.v1beta1.WorkflowRun
|
||||
|
||||
/**
|
||||
* Class to validate if the string refers to a property in the classes of the Lineage Metadata model.
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
@CompileStatic
|
||||
class LinPropertyValidator {
|
||||
|
||||
private static final List<Class> LIN_MODEL_CLASSES = [
|
||||
Checksum,
|
||||
DataPath,
|
||||
FileOutput,
|
||||
Parameter,
|
||||
TaskOutput,
|
||||
TaskRun,
|
||||
Workflow,
|
||||
WorkflowOutput,
|
||||
WorkflowRun,
|
||||
]
|
||||
|
||||
private Set<String> validProperties
|
||||
|
||||
LinPropertyValidator() {
|
||||
this.validProperties = new HashSet<String>()
|
||||
for( Class clazz : LIN_MODEL_CLASSES ) {
|
||||
for( MetaProperty field : clazz.metaClass.getProperties() ) {
|
||||
validProperties.add( field.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void validate(Collection<String> properties) {
|
||||
for( String property : properties ) {
|
||||
if( property !in this.validProperties ) {
|
||||
def msg = "Property '$property' doesn't exist in the lineage model."
|
||||
final matches = this.validProperties.closest(property)
|
||||
if( matches )
|
||||
msg += " -- Did you mean one of these?" + matches.collect { " $it"}.join(', ')
|
||||
throw new IllegalArgumentException(msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void validateQueryParams(Collection<String> keys) {
|
||||
for( final key : keys ) {
|
||||
validate(key.tokenize('.'))
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage
|
||||
|
||||
import com.google.common.annotations.Beta
|
||||
import groovy.transform.CompileStatic
|
||||
import java.util.stream.Stream
|
||||
import nextflow.lineage.config.LineageConfig
|
||||
import nextflow.lineage.serde.LinSerializable
|
||||
/**
|
||||
* Interface for the lineage store
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Beta
|
||||
@CompileStatic
|
||||
interface LinStore extends Closeable {
|
||||
|
||||
/**
|
||||
* Open the lineage store.
|
||||
* @param config Configuration to open the lineage store.
|
||||
*/
|
||||
LinStore open(LineageConfig config)
|
||||
|
||||
/**
|
||||
* Save a lineage entry in the store for in a given key.
|
||||
*
|
||||
* <p>The key is a hierarchical path-like identifier. Examples:
|
||||
* <ul>
|
||||
* <li>{@code "78ab04"} — a workflow run or task run (hash-based)</li>
|
||||
* <li>{@code "78ab04#output"} — the output metadata for a workflow or task run</li>
|
||||
* <li>{@code "78ab04/samples/file1.txt"} — an individual output file</li>
|
||||
* </ul>
|
||||
*
|
||||
* @param key Entry key.
|
||||
* @param value Entry object.
|
||||
*/
|
||||
void save(String key, LinSerializable value)
|
||||
|
||||
/**
|
||||
* Load an entry for a given Lineage ID key.
|
||||
*
|
||||
* <p>The key is a hierarchical path-like identifier. Examples:
|
||||
* <ul>
|
||||
* <li>{@code "78ab04"} — a workflow run or task run (hash-based)</li>
|
||||
* <li>{@code "78ab04#output"} — the output metadata for a workflow or task run</li>
|
||||
* <li>{@code "78ab04/samples/file1.txt"} — an individual output file</li>
|
||||
* </ul>
|
||||
*
|
||||
* @param key LID key.
|
||||
* @return entry value, or null if key does not exists
|
||||
*/
|
||||
LinSerializable load(String key)
|
||||
|
||||
/**
|
||||
* Get the {@link LinHistoryLog} object associated to the lineage store.
|
||||
* @return {@link LinHistoryLog} object
|
||||
*/
|
||||
LinHistoryLog getHistoryLog()
|
||||
|
||||
/**
|
||||
* Search for lineage entries.
|
||||
* @param params Map of query params
|
||||
* @return Stream with keys fulfilling the query params
|
||||
*/
|
||||
Stream<String> search(Map<String, List<String>> params)
|
||||
|
||||
/**
|
||||
* Search for keys starting with a parent key.
|
||||
* For example, if a LinStore contains the following keys: '123abc', '123abc/samples/file1.txt' and '123abc/summary',
|
||||
* The execution of the function with parentKey='123abc' will return a stream with '123abc/samples/file1.txt' and '123abc/summary'.
|
||||
* Similarly, the execution of the function with parentKey='123abc/samples' will just return '123abc/samples/file1.txt"
|
||||
*
|
||||
* @param parentKey
|
||||
* @return Stream of keys starting with parentKey
|
||||
*/
|
||||
Stream<String> getSubKeys(String parentKey)
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.Session
|
||||
import nextflow.lineage.config.LineageConfig
|
||||
import nextflow.plugin.Plugins
|
||||
import nextflow.util.TestOnly
|
||||
import org.pf4j.ExtensionPoint
|
||||
|
||||
/**
|
||||
* Factory for {@link LinStore} objects
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
abstract class LinStoreFactory implements ExtensionPoint {
|
||||
|
||||
private static LinStore instance
|
||||
|
||||
private static boolean initialized
|
||||
|
||||
protected abstract boolean canOpen(LineageConfig config)
|
||||
|
||||
protected abstract LinStore newInstance(LineageConfig config)
|
||||
|
||||
static LinStore create(LineageConfig config){
|
||||
final factory = Plugins
|
||||
.getPriorityExtensions(LinStoreFactory)
|
||||
.find( f-> f.canOpen(config))
|
||||
if( !factory )
|
||||
throw new IllegalStateException("Unable to find Nextflow Lineage store factory")
|
||||
log.debug "Using Nextflow Lineage store factory: ${factory.getClass().getName()}"
|
||||
return factory.newInstance(config)
|
||||
}
|
||||
|
||||
static LinStore getOrCreate(Session session) {
|
||||
if( instance || initialized )
|
||||
return instance
|
||||
synchronized (LinStoreFactory.class) {
|
||||
if( instance || initialized )
|
||||
return instance
|
||||
initialized = true
|
||||
final config = LineageConfig.create(session)
|
||||
if( !config.enabled )
|
||||
return null
|
||||
return instance = create(config)
|
||||
}
|
||||
}
|
||||
|
||||
@TestOnly
|
||||
static void reset(){
|
||||
synchronized (LinStoreFactory.class) {
|
||||
instance = null
|
||||
initialized = false
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage
|
||||
|
||||
import static nextflow.lineage.fs.LinFileSystemProvider.*
|
||||
import static nextflow.lineage.fs.LinPath.*
|
||||
|
||||
import java.nio.file.attribute.FileTime
|
||||
import java.time.OffsetDateTime
|
||||
import java.time.ZoneId
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.lineage.model.v1beta1.TaskRun
|
||||
import nextflow.lineage.model.v1beta1.WorkflowRun
|
||||
import nextflow.lineage.serde.LinEncoder
|
||||
import nextflow.lineage.serde.LinSerializable
|
||||
import nextflow.lineage.serde.LinTypeAdapterFactory
|
||||
import nextflow.serde.gson.GsonEncoder
|
||||
/**
|
||||
* Utils class for Lineage IDs.
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class LinUtils {
|
||||
|
||||
/**
|
||||
* Get a lineage record or fragment from the Lineage store.
|
||||
*
|
||||
* @param store Lineage store.
|
||||
* @param uri Object or fragment to retrieve in URI-like format.
|
||||
* Format 'lid://<key>[#fragment]' where:
|
||||
* - Key: Metadata Element key
|
||||
* - Fragment: Element fragment to retrieve.
|
||||
* @return Lineage record or fragment.
|
||||
*/
|
||||
static Object getMetadataObject(LinStore store, URI uri) {
|
||||
if( uri.scheme != SCHEME )
|
||||
throw new IllegalArgumentException("Invalid LID URI - scheme is different for $SCHEME")
|
||||
final key = uri.authority ? uri.authority + uri.path : uri.path
|
||||
if( key == SEPARATOR )
|
||||
throw new IllegalArgumentException("Cannot get record from the root LID URI")
|
||||
if ( uri.query )
|
||||
log.warn("Query string is not supported for Lineage URI: `$uri` -- it will be ignored")
|
||||
return getMetadataObject0(store, key, uri.fragment )
|
||||
}
|
||||
|
||||
private static Object getMetadataObject0(LinStore store, String key, String fragment) {
|
||||
final record = store.load(key)
|
||||
if( !record ) {
|
||||
throw new FileNotFoundException("Lineage record $key not found")
|
||||
}
|
||||
if( fragment ) {
|
||||
new LinPropertyValidator().validate(fragment.tokenize('.'))
|
||||
return getSubObject(store, key, record, fragment)
|
||||
}
|
||||
return record
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a lineage sub-record.
|
||||
*
|
||||
* If the requested sub-record is the workflow or task outputs, retrieves the outputs from the outputs description.
|
||||
*
|
||||
* @param store Store to retrieve lineage records.
|
||||
* @param key Parent key.
|
||||
* @param record Parent record.
|
||||
* @param fragment String in indicating the properties to navigate to get the sub-record.
|
||||
* @return Sub-record or null in it does not exist.
|
||||
*/
|
||||
static Object getSubObject(LinStore store, String key, LinSerializable record, String fragment) {
|
||||
if( isSearchingOutputs(record, fragment) ) {
|
||||
// When asking for a Workflow or task output retrieve the outputs description
|
||||
final outputs = store.load("${key}#output")
|
||||
if (!outputs)
|
||||
return []
|
||||
return navigate(outputs, fragment)
|
||||
}
|
||||
return navigate(record, fragment)
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the Lid pseudo path or query is for Task or Workflow outputs.
|
||||
*
|
||||
* @param record Parent lineage record
|
||||
* @param fragment Fragment indicating the properties to navigate to get the sub-record.
|
||||
* @return return 'true' if the parent is a Task/Workflow run and the first element in fragment is 'output'. Otherwise 'false'
|
||||
*/
|
||||
static boolean isSearchingOutputs(LinSerializable record, String fragment) {
|
||||
return (record instanceof WorkflowRun || record instanceof TaskRun) && fragment && fragment.tokenize('.')[0] == 'output'
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluates record or the records in a collection matches a set of parameter-value pairs. It includes in the results collection in case of match.
|
||||
*
|
||||
* @param record Object or collection of records to evaluate
|
||||
* @param params parameter-value pairs to evaluate in each record
|
||||
* @param results results collection to include the matching records
|
||||
*/
|
||||
protected static void treatObject(def record, Map<String, List<String>> params, List<Object> results) {
|
||||
if (params) {
|
||||
if (record instanceof Collection) {
|
||||
(record as Collection).forEach { treatObject(it, params, results) }
|
||||
} else if (checkParams(record, params)) {
|
||||
results.add(record)
|
||||
}
|
||||
} else {
|
||||
results.add(record)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if an record fulfill the parameter-value
|
||||
*
|
||||
* @param record Object to evaluate
|
||||
* @param params parameter-value pairs to evaluate
|
||||
* @return true if all record parameters exist and matches with the value, otherwise false.
|
||||
*/
|
||||
static boolean checkParams(Object record, Map<String, List<String>> params) {
|
||||
for( final entry : params.entrySet() ) {
|
||||
final value = navigate(record, entry.key)
|
||||
if( !checkParam(value, entry.value) ) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
private static boolean checkParam(Object value, List<String> expected) {
|
||||
if( !value )
|
||||
return false
|
||||
|
||||
// If value collection, convert to String and check all expected values are in the value.
|
||||
if( value instanceof Collection ) {
|
||||
final colValue = value as Collection
|
||||
return colValue.collect { it.toString() }.containsAll(expected)
|
||||
}
|
||||
|
||||
//Single record can't be compared with collection with one of more elements
|
||||
if( expected.size() > 1 ) {
|
||||
return false
|
||||
}
|
||||
|
||||
return value.toString() == expected[0]
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves the sub-record or value indicated by a path.
|
||||
*
|
||||
* @param obj Object to navigate
|
||||
* @param path Elements path separated by '.' e.g. field.subfield
|
||||
* @return sub-record / value
|
||||
*/
|
||||
static Object navigate(Object obj, String path) {
|
||||
if (!obj)
|
||||
return null
|
||||
// type has been replaced by class when evaluating LidSerializable records
|
||||
if (obj instanceof LinSerializable && path == 'type')
|
||||
return obj.getClass()?.simpleName
|
||||
try {
|
||||
return path.tokenize('.').inject(obj) { current, key ->
|
||||
getSubPath(current, key)
|
||||
}
|
||||
}
|
||||
catch (Throwable e) {
|
||||
log.debug("Error navigating to $path in record", e)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
private static Object getSubPath(current, String key) {
|
||||
if (current == null) {
|
||||
return null
|
||||
}
|
||||
if (current instanceof Map) {
|
||||
return current[key] // Navigate Map properties
|
||||
}
|
||||
if (current instanceof Collection) {
|
||||
return navigateCollection(current, key)
|
||||
}
|
||||
if (current.metaClass.hasProperty(current, key)) {
|
||||
return current.getAt(key) // Navigate Object properties
|
||||
}
|
||||
log.debug("No property found for $key")
|
||||
return null
|
||||
}
|
||||
|
||||
private static Object navigateCollection(Collection collection, String key) {
|
||||
final results = []
|
||||
for (Object record : collection) {
|
||||
final res = getSubPath(record, key)
|
||||
if (res)
|
||||
results.add(res)
|
||||
}
|
||||
if (results.isEmpty() ) {
|
||||
log.trace("No property found for $key")
|
||||
return null
|
||||
}
|
||||
// Return a single record if only ine results is found.
|
||||
return results.size() == 1 ? results[0] : results
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper function to convert from FileTime to ISO 8601 with offset
|
||||
* of current timezone.
|
||||
*
|
||||
* @param time File time to convert
|
||||
* @return The {@link OffsetDateTime} for the corresponding file time or null in case of not available (null)
|
||||
*/
|
||||
static OffsetDateTime toDate(FileTime time) {
|
||||
return time != null
|
||||
? time.toInstant().atZone(ZoneId.systemDefault()).toOffsetDateTime()
|
||||
: null
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper function to convert from String ISO 8601 to FileTime.
|
||||
*
|
||||
* @param date ISO formated time
|
||||
* @return Converted FileTime or null if date is not available (null or 'N/A')
|
||||
*/
|
||||
static FileTime toFileTime(OffsetDateTime date) {
|
||||
if (!date)
|
||||
return null
|
||||
return FileTime.from(date.toInstant())
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper function to unify the encoding of outputs when querying and navigating the lineage pseudoFS.
|
||||
* Outputs can include LinSerializable records, collections or parts of these records.
|
||||
* LinSerializable records can be encoded with the LinEncoder, but collections or parts of
|
||||
* these records require to extend the GsonEncoder.
|
||||
*
|
||||
* @param output Output to encode
|
||||
* @return Output encoded as a JSON string
|
||||
*/
|
||||
static String encodeSearchOutputs(Object output, boolean prettyPrint = false) {
|
||||
if (output instanceof LinSerializable) {
|
||||
return new LinEncoder().withPrettyPrint(prettyPrint).encode(output)
|
||||
} else {
|
||||
return new GsonEncoder<Object>() {}
|
||||
.withPrettyPrint(prettyPrint)
|
||||
.withSerializeNulls(true)
|
||||
.withTypeAdapterFactory(new LinTypeAdapterFactory())
|
||||
.encode(output)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.cli
|
||||
|
||||
import static nextflow.lineage.fs.LinPath.*
|
||||
|
||||
import java.nio.charset.StandardCharsets
|
||||
import java.nio.file.Path
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.Session
|
||||
import nextflow.cli.CmdLineage
|
||||
import nextflow.config.ConfigMap
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.lineage.LinHistoryRecord
|
||||
import nextflow.lineage.LinPropertyValidator
|
||||
import nextflow.lineage.LinStore
|
||||
import nextflow.lineage.LinStoreFactory
|
||||
import nextflow.lineage.LinUtils
|
||||
import nextflow.lineage.fs.LinPathFactory
|
||||
import nextflow.lineage.serde.LinEncoder
|
||||
import nextflow.ui.TableBuilder
|
||||
import org.eclipse.jgit.diff.DiffAlgorithm
|
||||
import org.eclipse.jgit.diff.DiffFormatter
|
||||
import org.eclipse.jgit.diff.RawText
|
||||
import org.eclipse.jgit.diff.RawTextComparator
|
||||
/**
|
||||
* Implements lineage command line operations
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class LinCommandImpl implements CmdLineage.LinCommand {
|
||||
|
||||
private static final Path DEFAULT_HTML_FILE = Path.of("lineage.html")
|
||||
|
||||
private static final String ERR_NOT_LOADED = 'Error lineage store not loaded - Check Nextflow configuration'
|
||||
|
||||
@Override
|
||||
void list(ConfigMap config) {
|
||||
final session = new Session(config)
|
||||
final store = LinStoreFactory.getOrCreate(session)
|
||||
if (store) {
|
||||
printHistory(store)
|
||||
} else {
|
||||
println ERR_NOT_LOADED
|
||||
}
|
||||
}
|
||||
|
||||
private void printHistory(LinStore store) {
|
||||
final records = store.historyLog?.records
|
||||
if( !records ) {
|
||||
println("No workflow runs found in lineage history log")
|
||||
return
|
||||
}
|
||||
def table = new TableBuilder(cellSeparator: '\t')
|
||||
.head('TIMESTAMP')
|
||||
.head('RUN NAME')
|
||||
.head('SESSION ID')
|
||||
.head('LINEAGE ID')
|
||||
for (LinHistoryRecord record : records) {
|
||||
table.append(record.toList())
|
||||
}
|
||||
println table.toString()
|
||||
}
|
||||
|
||||
@Override
|
||||
void view(ConfigMap config, List<String> args) {
|
||||
if( !isLidUri(args[0]) )
|
||||
throw new Exception("Identifier is not a lineage URL")
|
||||
final store = LinStoreFactory.getOrCreate(new Session(config))
|
||||
if ( !store ) {
|
||||
println ERR_NOT_LOADED
|
||||
return
|
||||
}
|
||||
try {
|
||||
def entry = LinUtils.getMetadataObject(store, new URI(args[0]))
|
||||
if( entry == null ) {
|
||||
println "No entry found for ${args[0]}"
|
||||
return
|
||||
}
|
||||
println LinUtils.encodeSearchOutputs(entry, true)
|
||||
} catch (Throwable e) {
|
||||
println "Error loading ${args[0]} - ${e.message}"
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
void render(ConfigMap config, List<String> args) {
|
||||
final store = LinStoreFactory.getOrCreate(new Session(config))
|
||||
if( !store ) {
|
||||
println ERR_NOT_LOADED
|
||||
return
|
||||
}
|
||||
try {
|
||||
final renderFile = args.size() > 1 ? Path.of(args[1]) : DEFAULT_HTML_FILE
|
||||
new LinDagRenderer(store).render(args[0], renderFile)
|
||||
println("Rendered lineage graph for ${args[0]} to $renderFile")
|
||||
} catch (Throwable e) {
|
||||
println("ERROR: rendering lineage graph - ${e.message}")
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
void diff(ConfigMap config, List<String> args) {
|
||||
if (!isLidUri(args[0]) || !isLidUri(args[1]))
|
||||
throw new Exception("Identifier is not a lineage URL")
|
||||
|
||||
final store = LinStoreFactory.getOrCreate(new Session(config))
|
||||
if (!store) {
|
||||
println ERR_NOT_LOADED
|
||||
return
|
||||
}
|
||||
try {
|
||||
final key1 = args[0].substring(LID_PROT.size())
|
||||
final entry1 = store.load(key1)
|
||||
if (!entry1) {
|
||||
println "No entry found for ${args[0]}."
|
||||
return
|
||||
}
|
||||
final key2 = args[1].substring(LID_PROT.size())
|
||||
final entry2 = store.load(key2)
|
||||
if (!entry2) {
|
||||
println "No entry found for ${args[1]}."
|
||||
return
|
||||
}
|
||||
final encoder = new LinEncoder().withPrettyPrint(true)
|
||||
generateDiff(encoder.encode(entry1), key1, encoder.encode(entry2), key2)
|
||||
} catch (Throwable e) {
|
||||
println "Error generating diff between ${args[0]}: $e.message"
|
||||
}
|
||||
}
|
||||
|
||||
private static void generateDiff(String entry1, String key1, String entry2, String key2) {
|
||||
// Convert strings to JGit RawText format
|
||||
final text1 = new RawText(entry1.getBytes(StandardCharsets.UTF_8))
|
||||
final text2 = new RawText(entry2.getBytes(StandardCharsets.UTF_8))
|
||||
|
||||
// Set up the diff algorithm (Git-style diff)
|
||||
final diffAlgorithm = DiffAlgorithm.getAlgorithm(DiffAlgorithm.SupportedAlgorithm.MYERS)
|
||||
final diffComparator = RawTextComparator.DEFAULT
|
||||
|
||||
// Compute the differences
|
||||
final editList = diffAlgorithm.diff(diffComparator, text1, text2)
|
||||
|
||||
final output = new StringBuilder()
|
||||
// Add header
|
||||
output.append("diff --git ${key1} ${key2}\n")
|
||||
output.append("--- ${key1}\n")
|
||||
output.append("+++ ${key2}\n")
|
||||
|
||||
// Use DiffFormatter to display results in Git-style format
|
||||
final outputStream = new ByteArrayOutputStream()
|
||||
final diffFormatter = new DiffFormatter(outputStream)
|
||||
diffFormatter.setOldPrefix(key1)
|
||||
diffFormatter.setNewPrefix(key2)
|
||||
diffFormatter.format(editList, text1, text2)
|
||||
output.append(outputStream.toString(StandardCharsets.UTF_8))
|
||||
|
||||
println output.toString()
|
||||
}
|
||||
|
||||
@Override
|
||||
void find(ConfigMap config, List<String> args) {
|
||||
final store = LinStoreFactory.getOrCreate(new Session(config))
|
||||
if (!store) {
|
||||
println ERR_NOT_LOADED
|
||||
return
|
||||
}
|
||||
try {
|
||||
final params = parseFindArgs(args)
|
||||
new LinPropertyValidator().validateQueryParams(params.keySet())
|
||||
try (def stream = store.search(params) ) {
|
||||
stream.forEach { println asUriString(it) }
|
||||
}
|
||||
} catch (Throwable e){
|
||||
println "Error searching for ${args[0]}. ${e.message}"
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
void check(ConfigMap config, List<String> args) {
|
||||
final store = LinStoreFactory.getOrCreate(new Session(config))
|
||||
if (!store) {
|
||||
println ERR_NOT_LOADED
|
||||
return
|
||||
}
|
||||
final valid = LinPathFactory.create(args[0]).validate()
|
||||
if( !valid )
|
||||
throw new AbortOperationException(valid.error)
|
||||
else
|
||||
println("Checksum validation succeed")
|
||||
}
|
||||
|
||||
private Map<String, List<String>> parseFindArgs(List<String> args){
|
||||
Map<String, List<String>> params = [:].withDefault { [] }
|
||||
|
||||
args.collectEntries { pair ->
|
||||
final idx = pair.indexOf('=')
|
||||
if( idx < 0 )
|
||||
throw new IllegalArgumentException("Parameter $pair doesn't contain '=' separator")
|
||||
def key = URLDecoder.decode(pair[0..<idx], 'UTF-8')
|
||||
final value = URLDecoder.decode(pair[(idx + 1)..<pair.length()], 'UTF-8')
|
||||
|
||||
// Convert 'label' key in the CLI to 'labels' property in the model
|
||||
if( key == 'label' )
|
||||
key = 'labels'
|
||||
|
||||
params[key] << value
|
||||
}
|
||||
return params
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.cli
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
import groovy.transform.Canonical
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.dag.MermaidHtmlRenderer
|
||||
import nextflow.lineage.LinStore
|
||||
import nextflow.lineage.model.v1beta1.FileOutput
|
||||
import nextflow.lineage.model.v1beta1.TaskRun
|
||||
import nextflow.lineage.model.v1beta1.WorkflowRun
|
||||
|
||||
import static nextflow.lineage.fs.LinPath.LID_PROT
|
||||
import static nextflow.lineage.fs.LinPath.isLidUri
|
||||
/**
|
||||
* Renderer for the lineage graph.
|
||||
*
|
||||
* @author Ben Sherman <bentshermann@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class LinDagRenderer {
|
||||
|
||||
private final LinStore store
|
||||
|
||||
private Queue<String> queue
|
||||
|
||||
private List<Node> nodes
|
||||
|
||||
private List<Edge> edges
|
||||
|
||||
LinDagRenderer(LinStore store) {
|
||||
this.store = store
|
||||
}
|
||||
|
||||
private void enqueueLid(String lid) {
|
||||
queue.add(lid)
|
||||
}
|
||||
|
||||
private void addNode(String id, String label, NodeType type) {
|
||||
nodes.add(new Node(id, label, type))
|
||||
}
|
||||
|
||||
private void addEdge(String source, String target) {
|
||||
edges.add(new Edge(source, target))
|
||||
}
|
||||
|
||||
void render(String lid, Path file) {
|
||||
// visit nodes
|
||||
this.queue = new LinkedList<String>()
|
||||
this.nodes = new LinkedList<Node>()
|
||||
this.edges = new LinkedList<Edge>()
|
||||
|
||||
enqueueLid(lid)
|
||||
while( !queue.isEmpty() )
|
||||
visitLid(queue.remove())
|
||||
|
||||
// render Mermaid diagram
|
||||
final lines = new LinkedList<String>()
|
||||
lines << "flowchart TB"
|
||||
|
||||
for( final node : nodes ) {
|
||||
if( node.type == NodeType.FILE )
|
||||
lines << " ${node.id}[\"${node.label}\"]".toString()
|
||||
if( node.type == NodeType.TASK )
|
||||
lines << " ${node.id}([\"${node.label}\"])".toString()
|
||||
}
|
||||
|
||||
for( final edge : edges )
|
||||
lines << " ${edge.source} --> ${edge.target}".toString()
|
||||
|
||||
final template = MermaidHtmlRenderer.readTemplate()
|
||||
file.text = template.replace('REPLACE_WITH_NETWORK_DATA', lines.join('\n'))
|
||||
}
|
||||
|
||||
private void visitLid(String lid) {
|
||||
if( !isLidUri(lid) )
|
||||
throw new Exception("Identifier is not a lineage URL: ${lid}")
|
||||
final record = store.load(rawLid(lid))
|
||||
if( !record ) {
|
||||
log.warn "Lineage record references an LID that does not exist: ${lid}"
|
||||
return
|
||||
}
|
||||
if( record instanceof FileOutput )
|
||||
visitFileOutput(lid, record)
|
||||
else if( record instanceof TaskRun )
|
||||
visitTaskRun(lid, record)
|
||||
else if( record instanceof WorkflowRun )
|
||||
visitWorkflowRun(lid, record)
|
||||
else
|
||||
throw new Exception("Cannot render lineage for type ${record.getClass().getSimpleName()} -- must be a FileOutput, TaskRun, or WorkflowRun")
|
||||
}
|
||||
|
||||
private void visitFileOutput(String lid, FileOutput fileOutput) {
|
||||
addNode(lid, lid, NodeType.FILE)
|
||||
final source = fileOutput.source
|
||||
if( !source )
|
||||
return
|
||||
if( isLidUri(source) ) {
|
||||
enqueueLid(source)
|
||||
addEdge(source, lid)
|
||||
}
|
||||
else {
|
||||
final id = safeId(source)
|
||||
final label = safeLabel(source)
|
||||
addNode(id, label, NodeType.FILE)
|
||||
addEdge(id, lid)
|
||||
}
|
||||
}
|
||||
|
||||
private void visitTaskRun(String lid, TaskRun taskRun) {
|
||||
addNode(lid, "${taskRun.name} [${lid}]", NodeType.TASK)
|
||||
for( final param : taskRun.input ) {
|
||||
visitParameter(lid, param.value)
|
||||
}
|
||||
}
|
||||
|
||||
private void visitWorkflowRun(String lid, WorkflowRun workflowRun) {
|
||||
addNode(lid, "${workflowRun.name} [${lid}]", NodeType.TASK)
|
||||
for( final param : workflowRun.params ) {
|
||||
visitParameter0(lid, param.value.toString())
|
||||
}
|
||||
}
|
||||
|
||||
private void visitParameter(String lid, Object value) {
|
||||
if( value instanceof Collection ) {
|
||||
for( final el : value )
|
||||
visitParameter(lid, el)
|
||||
}
|
||||
else if( value instanceof CharSequence ) {
|
||||
final source = value.toString()
|
||||
if( isLidUri(source) ) {
|
||||
enqueueLid(source)
|
||||
addEdge(source, lid)
|
||||
}
|
||||
else {
|
||||
visitParameter0(lid, source)
|
||||
}
|
||||
}
|
||||
else if( value instanceof Map && value.path ) {
|
||||
final path = value.path.toString()
|
||||
if( isLidUri(path) ) {
|
||||
enqueueLid(path)
|
||||
addEdge(path, lid)
|
||||
}
|
||||
else {
|
||||
visitParameter0(lid, path)
|
||||
}
|
||||
}
|
||||
else {
|
||||
visitParameter0(lid, value.toString())
|
||||
}
|
||||
}
|
||||
|
||||
private void visitParameter0(String lid, String value) {
|
||||
final id = safeId(value)
|
||||
final label = safeLabel(value)
|
||||
addNode(id, label, NodeType.FILE)
|
||||
addEdge(id, lid)
|
||||
}
|
||||
|
||||
private static String rawLid(String lid) {
|
||||
return lid.substring(LID_PROT.size())
|
||||
}
|
||||
|
||||
private static String safeId(String rawId) {
|
||||
return rawId.replaceAll(/[^a-zA-Z0-9_.:\/\-]/, '_')
|
||||
}
|
||||
|
||||
private static String safeLabel(String label) {
|
||||
return label.replace('http', 'h\u200Ettp')
|
||||
}
|
||||
|
||||
@Canonical
|
||||
private static class Node {
|
||||
String id
|
||||
String label
|
||||
NodeType type
|
||||
}
|
||||
|
||||
@Canonical
|
||||
private static class Edge {
|
||||
String source
|
||||
String target
|
||||
}
|
||||
|
||||
private static enum NodeType {
|
||||
FILE,
|
||||
TASK,
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.config
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.ToString
|
||||
import nextflow.Global
|
||||
import nextflow.Session
|
||||
import nextflow.config.spec.ConfigOption
|
||||
import nextflow.config.spec.ConfigScope
|
||||
import nextflow.config.spec.ScopeName
|
||||
import nextflow.script.dsl.Description
|
||||
|
||||
/**
|
||||
* Model workflow data lineage config
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@ScopeName("lineage")
|
||||
@Description("""
|
||||
The `lineage` scope controls the generation of lineage metadata.
|
||||
""")
|
||||
@ToString
|
||||
@CompileStatic
|
||||
class LineageConfig implements ConfigScope {
|
||||
|
||||
final LineageStoreOpts store
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Enable generation of lineage metadata (default: `false`).
|
||||
""")
|
||||
final boolean enabled
|
||||
|
||||
/* required by extension point -- do not remove */
|
||||
LineageConfig() {}
|
||||
|
||||
LineageConfig(Map opts) {
|
||||
this.store = new LineageStoreOpts(opts.store as Map ?: Collections.emptyMap())
|
||||
this.enabled = opts.enabled as boolean
|
||||
}
|
||||
|
||||
static Map<String,Object> asMap() {
|
||||
final result = session?.config?.navigate('lineage') as Map
|
||||
return result != null ? result : new HashMap<String,Object>()
|
||||
}
|
||||
|
||||
static LineageConfig create(Map config) {
|
||||
new LineageConfig(config.lineage as Map ?: Collections.emptyMap())
|
||||
}
|
||||
|
||||
static LineageConfig create(Session session) {
|
||||
if( session ) {
|
||||
return LineageConfig.create(session.config)
|
||||
}
|
||||
else
|
||||
throw new IllegalStateException("Missing Nextflow session")
|
||||
}
|
||||
|
||||
static LineageConfig create() {
|
||||
create(getSession())
|
||||
}
|
||||
|
||||
static private Session getSession() {
|
||||
return Global.session as Session
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.config
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.ToString
|
||||
import nextflow.config.spec.ConfigOption
|
||||
import nextflow.config.spec.ConfigScope
|
||||
import nextflow.script.dsl.Description
|
||||
|
||||
/**
|
||||
* Model data store options
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@ToString
|
||||
@CompileStatic
|
||||
class LineageStoreOpts implements ConfigScope {
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The location of the lineage metadata store (default: `./.lineage`).
|
||||
""")
|
||||
final String location
|
||||
|
||||
LineageStoreOpts(Map opts) {
|
||||
this.location = opts.location as String
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package nextflow.lineage.exception
|
||||
|
||||
/**
|
||||
* Exception to indicate the an output path is not relative to the output dir.
|
||||
* It is used to detect the cases where publishDir is used with Data Lineage.
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
class OutputRelativePathException extends Exception {
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.fs
|
||||
|
||||
import com.google.common.collect.ImmutableSet
|
||||
import nextflow.lineage.LinStore
|
||||
import nextflow.lineage.LinStoreFactory
|
||||
|
||||
import java.nio.file.FileStore
|
||||
import java.nio.file.FileSystem
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.PathMatcher
|
||||
import java.nio.file.WatchService
|
||||
import java.nio.file.attribute.UserPrincipalLookupService
|
||||
import java.nio.file.spi.FileSystemProvider
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.lineage.LinStore
|
||||
import nextflow.lineage.LinStoreFactory
|
||||
import nextflow.lineage.config.LineageConfig
|
||||
|
||||
/**
|
||||
* File system for LID Paths
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
@CompileStatic
|
||||
class LinFileSystem extends FileSystem {
|
||||
|
||||
private LinFileSystemProvider provider
|
||||
|
||||
private LinStore store
|
||||
|
||||
/*
|
||||
* Only needed to prevent serialization issues - see https://github.com/nextflow-io/nextflow/issues/5208
|
||||
*/
|
||||
protected LinFileSystem(){}
|
||||
|
||||
LinFileSystem(LinFileSystemProvider provider, LineageConfig config) {
|
||||
this.provider = provider
|
||||
this.store = LinStoreFactory.create(config)
|
||||
}
|
||||
|
||||
LinStore getStore() {
|
||||
return store
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean equals( Object other ) {
|
||||
if( this.class != other.class ) return false
|
||||
final that = (LinFileSystem)other
|
||||
this.provider == that.provider && this.store == that.store
|
||||
}
|
||||
|
||||
@Override
|
||||
int hashCode() {
|
||||
Objects.hash(provider,store)
|
||||
}
|
||||
|
||||
@Override
|
||||
FileSystemProvider provider() {
|
||||
return provider
|
||||
}
|
||||
|
||||
@Override
|
||||
void close() throws IOException {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean isOpen() {
|
||||
return false
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean isReadOnly() {
|
||||
return true
|
||||
}
|
||||
|
||||
@Override
|
||||
String getSeparator() {
|
||||
return LinPath.SEPARATOR
|
||||
}
|
||||
|
||||
@Override
|
||||
Iterable<Path> getRootDirectories() {
|
||||
return null
|
||||
}
|
||||
|
||||
@Override
|
||||
Iterable<FileStore> getFileStores() {
|
||||
return null
|
||||
}
|
||||
|
||||
@Override
|
||||
Set<String> supportedFileAttributeViews() {
|
||||
return ImmutableSet.of("basic")
|
||||
}
|
||||
|
||||
@Override
|
||||
Path getPath(String first, String... more) {
|
||||
final path = more ? LinPath.SEPARATOR + more.join(LinPath. SEPARATOR) : ''
|
||||
return getPath(LinPath.asUri(LinPath.LID_PROT + first + path))
|
||||
}
|
||||
|
||||
Path getPath(URI uri){
|
||||
return new LinPath(this, uri)
|
||||
}
|
||||
|
||||
@Override
|
||||
PathMatcher getPathMatcher(String syntaxAndPattern) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
UserPrincipalLookupService getUserPrincipalLookupService() {
|
||||
throw new UnsupportedOperationException('User Principal Lookup Service not supported')
|
||||
}
|
||||
|
||||
@Override
|
||||
WatchService newWatchService() throws IOException {
|
||||
throw new UnsupportedOperationException('Watch Service not supported')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.fs
|
||||
|
||||
import java.nio.ByteBuffer
|
||||
import java.nio.channels.NonWritableChannelException
|
||||
import java.nio.channels.SeekableByteChannel
|
||||
import java.nio.file.AccessDeniedException
|
||||
import java.nio.file.AccessMode
|
||||
import java.nio.file.CopyOption
|
||||
import java.nio.file.DirectoryStream
|
||||
import java.nio.file.FileStore
|
||||
import java.nio.file.FileSystem
|
||||
import java.nio.file.FileSystemNotFoundException
|
||||
import java.nio.file.LinkOption
|
||||
import java.nio.file.OpenOption
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.ProviderMismatchException
|
||||
import java.nio.file.StandardOpenOption
|
||||
import java.nio.file.attribute.BasicFileAttributes
|
||||
import java.nio.file.attribute.FileAttribute
|
||||
import java.nio.file.attribute.FileAttributeView
|
||||
import java.nio.file.spi.FileSystemProvider
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.lineage.config.LineageConfig
|
||||
import nextflow.util.TestOnly
|
||||
/**
|
||||
* File System Provider for LID Paths
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
@CompileStatic
|
||||
class LinFileSystemProvider extends FileSystemProvider {
|
||||
|
||||
public static final String SCHEME = "lid"
|
||||
|
||||
private LinFileSystem fileSystem
|
||||
|
||||
@Override
|
||||
String getScheme() {
|
||||
return SCHEME
|
||||
}
|
||||
|
||||
protected LinPath toLinPath(Path path) {
|
||||
if (path !instanceof LinPath)
|
||||
throw new ProviderMismatchException()
|
||||
if (path instanceof LinMetadataPath)
|
||||
return (LinMetadataPath) path
|
||||
return (LinPath) path
|
||||
}
|
||||
|
||||
private void checkScheme(URI uri) {
|
||||
final scheme = uri.scheme.toLowerCase()
|
||||
if (scheme != getScheme())
|
||||
throw new IllegalArgumentException("Not a valid ${getScheme().toUpperCase()} scheme: $scheme")
|
||||
}
|
||||
|
||||
@Override
|
||||
synchronized FileSystem newFileSystem(URI uri, Map<String, ?> config) throws IOException {
|
||||
checkScheme(uri)
|
||||
if (fileSystem) {
|
||||
return fileSystem
|
||||
}
|
||||
//Overwrite default values with provided configuration
|
||||
final defaultConfig = LineageConfig.asMap()
|
||||
if (config) {
|
||||
for (Map.Entry<String, ?> e : config.entrySet()) {
|
||||
defaultConfig.put(e.key, e.value)
|
||||
}
|
||||
}
|
||||
return fileSystem = new LinFileSystem(this, new LineageConfig(defaultConfig))
|
||||
}
|
||||
|
||||
@Override
|
||||
FileSystem getFileSystem(URI uri) throws FileSystemNotFoundException {
|
||||
if (!fileSystem)
|
||||
throw new FileSystemNotFoundException()
|
||||
return fileSystem
|
||||
}
|
||||
|
||||
synchronized FileSystem getFileSystemOrCreate(URI uri) {
|
||||
checkScheme(uri)
|
||||
if (!fileSystem) {
|
||||
fileSystem = (LinFileSystem) newFileSystem(uri, LineageConfig.asMap())
|
||||
}
|
||||
return fileSystem
|
||||
}
|
||||
|
||||
@Override
|
||||
LinPath getPath(URI uri) {
|
||||
return (LinPath) ((LinFileSystem) getFileSystemOrCreate(uri)).getPath(uri)
|
||||
}
|
||||
|
||||
@Override
|
||||
OutputStream newOutputStream(Path path, OpenOption... options) throws IOException {
|
||||
throw new UnsupportedOperationException("Write not supported by ${getScheme().toUpperCase()} file system provider")
|
||||
}
|
||||
|
||||
@Override
|
||||
InputStream newInputStream(Path path, OpenOption... options) throws IOException {
|
||||
final lid = toLinPath(path)
|
||||
if (lid instanceof LinMetadataPath)
|
||||
return (lid as LinMetadataPath).newInputStream()
|
||||
return newInputStream0(lid, options)
|
||||
}
|
||||
|
||||
private static InputStream newInputStream0(LinPath lid, OpenOption... options) throws IOException {
|
||||
final realPath = lid.getTargetOrMetadataPath()
|
||||
if (realPath instanceof LinMetadataPath)
|
||||
return (realPath as LinMetadataPath).newInputStream()
|
||||
return realPath.fileSystem.provider().newInputStream(realPath, options)
|
||||
}
|
||||
|
||||
@Override
|
||||
SeekableByteChannel newByteChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException {
|
||||
final lid = toLinPath(path)
|
||||
validateOptions(options)
|
||||
return newByteChannel0(lid, options, attrs)
|
||||
}
|
||||
|
||||
@CompileStatic
|
||||
private class LinPathSeekableByteChannel implements SeekableByteChannel {
|
||||
SeekableByteChannel channel
|
||||
|
||||
LinPathSeekableByteChannel(SeekableByteChannel channel) {
|
||||
this.channel = channel
|
||||
}
|
||||
|
||||
@Override
|
||||
int read(ByteBuffer dst) throws IOException {
|
||||
return channel.read(dst)
|
||||
}
|
||||
|
||||
@Override
|
||||
int write(ByteBuffer src) throws IOException {
|
||||
throw new NonWritableChannelException(){}
|
||||
}
|
||||
|
||||
@Override
|
||||
long position() throws IOException {
|
||||
return channel.position()
|
||||
}
|
||||
|
||||
@Override
|
||||
SeekableByteChannel position(long newPosition) throws IOException {
|
||||
channel.position(newPosition)
|
||||
return this
|
||||
}
|
||||
|
||||
@Override
|
||||
long size() throws IOException {
|
||||
return channel.size()
|
||||
}
|
||||
|
||||
@Override
|
||||
SeekableByteChannel truncate(long unused) throws IOException {
|
||||
throw new NonWritableChannelException()
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean isOpen() {
|
||||
return channel.isOpen()
|
||||
}
|
||||
|
||||
@Override
|
||||
void close() throws IOException {
|
||||
channel.close()
|
||||
}
|
||||
}
|
||||
|
||||
private static void validateOptions(Set<? extends OpenOption> options) {
|
||||
if (!options || options.empty)
|
||||
return
|
||||
for (OpenOption opt : options) {
|
||||
// All OpenOption values except for APPEND and WRITE are allowed
|
||||
if (opt == StandardOpenOption.APPEND || opt == StandardOpenOption.WRITE)
|
||||
throw new UnsupportedOperationException("'$opt' not allowed");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private SeekableByteChannel newByteChannel0(LinPath lid, Set<? extends OpenOption> options, FileAttribute<?>... attrs) {
|
||||
if (lid instanceof LinMetadataPath) {
|
||||
return (lid as LinMetadataPath).newSeekableByteChannel()
|
||||
}
|
||||
final realPath = lid.getTargetOrMetadataPath()
|
||||
if (realPath instanceof LinMetadataPath) {
|
||||
return (realPath as LinMetadataPath).newSeekableByteChannel()
|
||||
} else {
|
||||
SeekableByteChannel channel = realPath.fileSystem.provider().newByteChannel(realPath, options, attrs)
|
||||
return new LinPathSeekableByteChannel(channel)
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
DirectoryStream<Path> newDirectoryStream(Path path, DirectoryStream.Filter<? super Path> filter) throws IOException {
|
||||
final lid = toLinPath(path)
|
||||
final real = lid.getTargetOrIntermediatePath()
|
||||
if (real instanceof LinIntermediatePath)
|
||||
return getDirectoryStreamFromSubPath(lid)
|
||||
return getDirectoryStreamFromRealPath(real, lid)
|
||||
}
|
||||
|
||||
private static DirectoryStream<Path> getDirectoryStreamFromSubPath(LinPath lid){
|
||||
final paths = lid.getSubPaths()
|
||||
if( !paths )
|
||||
throw new FileNotFoundException("Sub paths for '$lid' do not exist")
|
||||
return new DirectoryStream<Path>() {
|
||||
Iterator<Path> iterator() {
|
||||
return paths.iterator()
|
||||
}
|
||||
|
||||
void close() {
|
||||
paths.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
private DirectoryStream<Path> getDirectoryStreamFromRealPath(Path real, LinPath lid) {
|
||||
final stream = real
|
||||
.getFileSystem()
|
||||
.provider()
|
||||
.newDirectoryStream(real, new LidFilter(fileSystem))
|
||||
|
||||
return new DirectoryStream<Path>() {
|
||||
|
||||
@Override
|
||||
Iterator<Path> iterator() {
|
||||
return new LidIterator(fileSystem, stream.iterator(), lid, real)
|
||||
}
|
||||
|
||||
@Override
|
||||
void close() throws IOException {
|
||||
stream.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@CompileStatic
|
||||
private class LidFilter implements DirectoryStream.Filter<Path> {
|
||||
|
||||
private final LinFileSystem fs
|
||||
|
||||
LidFilter(LinFileSystem fs) {
|
||||
this.fs = fs
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean accept(Path entry) throws IOException {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
private static LinPath fromRealToLinPath(Path toConvert, Path realBase, LinPath lidBase) {
|
||||
if (toConvert.isAbsolute()) {
|
||||
if (toConvert.class != realBase.class) {
|
||||
throw new ProviderMismatchException()
|
||||
}
|
||||
final relative = realBase.relativize(toConvert)
|
||||
return (LinPath) lidBase.resolve(relative.toString())
|
||||
} else {
|
||||
return (LinPath) lidBase.resolve(toConvert.toString())
|
||||
}
|
||||
}
|
||||
|
||||
private static class LidIterator implements Iterator<Path> {
|
||||
|
||||
private final LinFileSystem fs
|
||||
private final Iterator<Path> target
|
||||
private final LinPath parent
|
||||
private final Path parentReal
|
||||
|
||||
LidIterator(LinFileSystem fs, Iterator<Path> itr, LinPath parent, Path real) {
|
||||
this.fs = fs
|
||||
this.target = itr
|
||||
this.parent = parent
|
||||
this.parentReal = real
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean hasNext() {
|
||||
return target.hasNext()
|
||||
}
|
||||
|
||||
@Override
|
||||
LinPath next() {
|
||||
final path = target.next()
|
||||
return path ? fromRealToLinPath(path, parentReal, parent) : null
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
void createDirectory(Path dir, FileAttribute<?>... attrs) throws IOException {
|
||||
throw new UnsupportedOperationException("Create directory not supported by ${getScheme().toUpperCase()} file system provider")
|
||||
}
|
||||
|
||||
@Override
|
||||
void delete(Path path) throws IOException {
|
||||
throw new UnsupportedOperationException("Delete not supported by ${getScheme().toUpperCase()} file system provider")
|
||||
}
|
||||
|
||||
@Override
|
||||
void copy(Path source, Path target, CopyOption... options) throws IOException {
|
||||
throw new UnsupportedOperationException("Copy not supported by ${getScheme().toUpperCase()} file system provider")
|
||||
}
|
||||
|
||||
@Override
|
||||
void move(Path source, Path target, CopyOption... options) throws IOException {
|
||||
throw new UnsupportedOperationException("Move not supported by ${getScheme().toUpperCase()} file system provider")
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean isSameFile(Path path, Path path2) throws IOException {
|
||||
return path == path2
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean isHidden(Path path) throws IOException {
|
||||
return toLinPath(path).getTargetOrMetadataPath().isHidden()
|
||||
}
|
||||
|
||||
@Override
|
||||
FileStore getFileStore(Path path) throws IOException {
|
||||
throw new UnsupportedOperationException("File store not supported by ${getScheme().toUpperCase()} file system provider")
|
||||
}
|
||||
|
||||
@Override
|
||||
void checkAccess(Path path, AccessMode... modes) throws IOException {
|
||||
validateAccessModes(modes)
|
||||
final lid = toLinPath(path)
|
||||
if (lid instanceof LinMetadataPath)
|
||||
return
|
||||
checkAccess0(lid, modes)
|
||||
}
|
||||
|
||||
private void checkAccess0(LinPath lid, AccessMode... modes) {
|
||||
final real = lid.getTargetOrMetadataPath()
|
||||
if (real instanceof LinMetadataPath)
|
||||
return
|
||||
real.fileSystem.provider().checkAccess(real, modes)
|
||||
}
|
||||
|
||||
private void validateAccessModes(AccessMode... modes) {
|
||||
for (AccessMode m : modes) {
|
||||
if (m == AccessMode.WRITE)
|
||||
throw new AccessDeniedException("Write mode not supported")
|
||||
if (m == AccessMode.EXECUTE)
|
||||
throw new AccessDeniedException("Execute mode not supported")
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
<V extends FileAttributeView> V getFileAttributeView(Path path, Class<V> type, LinkOption... options) {
|
||||
return null
|
||||
}
|
||||
|
||||
@Override
|
||||
<A extends BasicFileAttributes> A readAttributes(Path path, Class<A> type, LinkOption... options) throws IOException {
|
||||
final lid = toLinPath(path)
|
||||
if (lid instanceof LinMetadataPath)
|
||||
return (lid as LinMetadataPath).readAttributes(type)
|
||||
return readAttributes0(lid, type, options)
|
||||
}
|
||||
|
||||
private <A extends BasicFileAttributes> A readAttributes0(LinPath lid, Class<A> type, LinkOption... options) throws IOException {
|
||||
final target = lid.getTargetOrIntermediatePath()
|
||||
if (target instanceof LinIntermediatePath)
|
||||
return (target as LinIntermediatePath).readAttributes(type)
|
||||
return target.fileSystem.provider().readAttributes(target, type, options)
|
||||
}
|
||||
|
||||
@Override
|
||||
Map<String, Object> readAttributes(Path path, String attributes, LinkOption... options) throws IOException {
|
||||
throw new UnsupportedOperationException("Read file attributes not supported by ${getScheme().toUpperCase()} file system provider")
|
||||
}
|
||||
|
||||
@Override
|
||||
void setAttribute(Path path, String attribute, Object value, LinkOption... options) throws IOException {
|
||||
throw new UnsupportedOperationException("Set file attributes not supported by ${getScheme().toUpperCase()} file system provider")
|
||||
}
|
||||
|
||||
@TestOnly
|
||||
void reset() {
|
||||
fileSystem=null
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.fs
|
||||
|
||||
import java.nio.channels.SeekableByteChannel
|
||||
import java.nio.file.attribute.BasicFileAttributes
|
||||
import java.nio.file.attribute.FileTime
|
||||
import java.time.Instant
|
||||
|
||||
class LinIntermediatePath extends LinMetadataPath{
|
||||
|
||||
LinIntermediatePath(LinFileSystem fs, String path) {
|
||||
super("", FileTime.from(Instant.now()), fs, path, null)
|
||||
}
|
||||
|
||||
@Override
|
||||
InputStream newInputStream() {
|
||||
throw new UnsupportedOperationException()
|
||||
}
|
||||
@Override
|
||||
SeekableByteChannel newSeekableByteChannel(){
|
||||
throw new UnsupportedOperationException()
|
||||
}
|
||||
@Override
|
||||
<A extends BasicFileAttributes> A readAttributes(Class<A> type){
|
||||
return (A) new BasicFileAttributes() {
|
||||
@Override
|
||||
long size() { return 0 }
|
||||
|
||||
@Override
|
||||
FileTime lastModifiedTime() { FileTime.from(Instant.now()) }
|
||||
|
||||
@Override
|
||||
FileTime lastAccessTime() { FileTime.from(Instant.now()) }
|
||||
|
||||
@Override
|
||||
FileTime creationTime() { FileTime.from(Instant.now()) }
|
||||
|
||||
@Override
|
||||
boolean isRegularFile() { return false }
|
||||
|
||||
@Override
|
||||
boolean isDirectory() { return true }
|
||||
|
||||
@Override
|
||||
boolean isSymbolicLink() { return false }
|
||||
|
||||
@Override
|
||||
boolean isOther() { return false }
|
||||
|
||||
@Override
|
||||
Object fileKey() { return null }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.fs
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
|
||||
import java.nio.channels.SeekableByteChannel
|
||||
import java.nio.file.attribute.BasicFileAttributes
|
||||
import java.nio.file.attribute.FileTime
|
||||
|
||||
/**
|
||||
* Class to model the metadata descriptions as a file.
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
@CompileStatic
|
||||
class LinMetadataPath extends LinPath {
|
||||
private byte[] results
|
||||
private FileTime creationTime
|
||||
|
||||
LinMetadataPath(String resultsObject, FileTime creationTime, LinFileSystem fs, String path, String fragment) {
|
||||
super(fs, "${path}${fragment ? '#'+ fragment : ''}")
|
||||
this.results = resultsObject.getBytes("UTF-8")
|
||||
this.creationTime = creationTime
|
||||
}
|
||||
|
||||
InputStream newInputStream() {
|
||||
return new ByteArrayInputStream(results)
|
||||
}
|
||||
|
||||
SeekableByteChannel newSeekableByteChannel(){
|
||||
return new LinMetadataSeekableByteChannel(results)
|
||||
}
|
||||
|
||||
<A extends BasicFileAttributes> A readAttributes(Class<A> type){
|
||||
return (A) new BasicFileAttributes() {
|
||||
@Override
|
||||
long size() { return results.length }
|
||||
|
||||
@Override
|
||||
FileTime lastModifiedTime() { return creationTime }
|
||||
|
||||
@Override
|
||||
FileTime lastAccessTime() { return creationTime }
|
||||
|
||||
@Override
|
||||
FileTime creationTime() { return creationTime }
|
||||
|
||||
@Override
|
||||
boolean isRegularFile() { return true }
|
||||
|
||||
@Override
|
||||
boolean isDirectory() { return false }
|
||||
|
||||
@Override
|
||||
boolean isSymbolicLink() { return false }
|
||||
|
||||
@Override
|
||||
boolean isOther() { return false }
|
||||
|
||||
@Override
|
||||
Object fileKey() { return null }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.fs
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
|
||||
import java.nio.ByteBuffer
|
||||
import java.nio.channels.ClosedChannelException
|
||||
import java.nio.channels.NonWritableChannelException
|
||||
import java.nio.channels.SeekableByteChannel
|
||||
|
||||
/**
|
||||
* SeekableByteChannel for metadata results description as a file.
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
@CompileStatic
|
||||
class LinMetadataSeekableByteChannel implements SeekableByteChannel {
|
||||
private final ByteBuffer buffer
|
||||
private boolean open
|
||||
|
||||
LinMetadataSeekableByteChannel(byte[] bytes){
|
||||
this.open = true
|
||||
this.buffer = ByteBuffer.wrap(bytes)
|
||||
}
|
||||
|
||||
@Override
|
||||
int read(ByteBuffer dst) {
|
||||
if (!open) throw new ClosedChannelException()
|
||||
if (!buffer.hasRemaining()) return -1
|
||||
int remaining = Math.min(dst.remaining(), buffer.remaining())
|
||||
byte[] temp = new byte[remaining]
|
||||
buffer.get(temp)
|
||||
dst.put(temp)
|
||||
return remaining
|
||||
}
|
||||
|
||||
@Override
|
||||
int write(ByteBuffer src) { throw new NonWritableChannelException() }
|
||||
|
||||
@Override
|
||||
long position() { return buffer.position() }
|
||||
|
||||
@Override
|
||||
SeekableByteChannel position(long newPosition) {
|
||||
if (newPosition < 0 || newPosition > buffer.limit()) throw new IllegalArgumentException()
|
||||
buffer.position((int) newPosition)
|
||||
return this
|
||||
}
|
||||
|
||||
@Override
|
||||
long size() { return buffer.limit() }
|
||||
|
||||
@Override
|
||||
SeekableByteChannel truncate(long size) { throw new NonWritableChannelException() }
|
||||
|
||||
@Override
|
||||
boolean isOpen() { return open }
|
||||
|
||||
@Override
|
||||
void close() { open = false }
|
||||
}
|
||||
@@ -0,0 +1,626 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.fs
|
||||
|
||||
import static nextflow.lineage.LinUtils.*
|
||||
import static nextflow.lineage.fs.LinFileSystemProvider.*
|
||||
|
||||
import java.nio.file.FileSystem
|
||||
import java.nio.file.LinkOption
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.ProviderMismatchException
|
||||
import java.nio.file.WatchEvent
|
||||
import java.nio.file.WatchKey
|
||||
import java.nio.file.WatchService
|
||||
import java.time.OffsetDateTime
|
||||
import java.util.stream.Stream
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import groovy.transform.Memoized
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.file.FileHelper
|
||||
import nextflow.file.LogicalDataPath
|
||||
import nextflow.lineage.LinPropertyValidator
|
||||
import nextflow.lineage.LinStore
|
||||
import nextflow.lineage.model.v1beta1.Checksum
|
||||
import nextflow.lineage.model.v1beta1.FileOutput
|
||||
import nextflow.lineage.model.v1beta1.TaskRun
|
||||
import nextflow.lineage.model.v1beta1.WorkflowRun
|
||||
import nextflow.lineage.serde.LinSerializable
|
||||
import nextflow.util.CacheHelper
|
||||
import nextflow.util.TestOnly
|
||||
/**
|
||||
* LID file system path
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class LinPath implements Path, LogicalDataPath {
|
||||
|
||||
static public final List<String> SUPPORTED_CHECKSUM_ALGORITHMS = ["nextflow"]
|
||||
static public final String SEPARATOR = '/'
|
||||
public static final String LID_PROT = "${SCHEME}://"
|
||||
|
||||
static private final String[] EMPTY = new String[]{}
|
||||
|
||||
private LinFileSystem fileSystem
|
||||
|
||||
// String with the lineage file path
|
||||
private String filePath
|
||||
|
||||
private String query
|
||||
|
||||
private String fragment
|
||||
|
||||
/*
|
||||
* Only needed to prevent serialization issues - see https://github.com/nextflow-io/nextflow/issues/5208
|
||||
*/
|
||||
protected LinPath(){}
|
||||
|
||||
LinPath(LinFileSystem fs, URI uri) {
|
||||
if( uri.scheme != SCHEME ) {
|
||||
throw new IllegalArgumentException("Invalid LID URI - scheme is different for $SCHEME")
|
||||
}
|
||||
this.fileSystem = fs
|
||||
setFieldsFormURI(uri)
|
||||
// Check if query and fragment are with filePath
|
||||
if( query == null && fragment == null )
|
||||
setFieldsFormURI(new URI(toUriString()))
|
||||
// Warn if query is specified
|
||||
if( query )
|
||||
log.warn("Query string is not supported for Lineage URI: `$uri` -- it will be ignored")
|
||||
// Validate fragment
|
||||
if( fragment )
|
||||
new LinPropertyValidator().validate(fragment.tokenize('.'))
|
||||
}
|
||||
|
||||
private void setFieldsFormURI(URI uri){
|
||||
this.query = uri.query
|
||||
this.fragment = uri.fragment
|
||||
this.filePath = resolve0(fileSystem, norm0("${uri.authority?:''}${uri.path}") )
|
||||
}
|
||||
|
||||
protected LinPath(String query, String fragment, String filepath, LinFileSystem fs) {
|
||||
this.fileSystem = fs
|
||||
this.query = query
|
||||
this.fragment = fragment
|
||||
this.filePath = filepath
|
||||
}
|
||||
|
||||
LinPath(LinFileSystem fs, String path) {
|
||||
this( fs, asUri( LID_PROT + norm0(path)) )
|
||||
}
|
||||
|
||||
LinPath(LinFileSystem fs, String first, String[] more) {
|
||||
this( fs, asUri( LID_PROT + buildPath(first, more) ) )
|
||||
}
|
||||
|
||||
static String asUriString(String first, String... more) {
|
||||
return LID_PROT + buildPath(first, more)
|
||||
}
|
||||
|
||||
static boolean isLidUri(String path) {
|
||||
return path && path.startsWith(LID_PROT)
|
||||
}
|
||||
|
||||
private static String buildPath(String first, String[] more) {
|
||||
first = norm0(first)
|
||||
if( more ) {
|
||||
final morePath = norm0(more).join(SEPARATOR)
|
||||
return first.isEmpty() ? morePath : first + SEPARATOR + morePath
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
@Memoized
|
||||
protected static String validateDataOutput(FileOutput lidObject) {
|
||||
final hashedPath = FileHelper.toCanonicalPath(lidObject.path as String)
|
||||
if( !hashedPath.exists() )
|
||||
throw new FileNotFoundException("Target path $lidObject.path does not exist")
|
||||
return validateChecksum(lidObject.checksum, hashedPath)
|
||||
}
|
||||
|
||||
protected static String validateChecksum(Checksum checksum, Path hashedPath) {
|
||||
if( !checksum )
|
||||
return null
|
||||
if( !isAlgorithmSupported(checksum.algorithm) ) {
|
||||
return "Checksum of '$hashedPath' can't be validated - algorithm '${checksum.algorithm}' is not supported"
|
||||
}
|
||||
final hash = checksum.mode
|
||||
? CacheHelper.hasher(hashedPath, CacheHelper.HashMode.of(checksum.mode.toString().toLowerCase())).hash().toString()
|
||||
: CacheHelper.hasher(hashedPath).hash().toString()
|
||||
return hash != checksum.value
|
||||
? "Checksum of '$hashedPath' does not match with lineage metadata"
|
||||
: null
|
||||
}
|
||||
|
||||
protected static isAlgorithmSupported(String algorithm) {
|
||||
return algorithm && algorithm in SUPPORTED_CHECKSUM_ALGORITHMS
|
||||
}
|
||||
|
||||
@TestOnly
|
||||
protected String getFilePath() { this.filePath }
|
||||
|
||||
protected Stream<Path> getSubPaths(){
|
||||
if( !fileSystem )
|
||||
throw new IllegalArgumentException("Cannot get sub-paths for a relative lineage path")
|
||||
if( filePath.isEmpty() || filePath == SEPARATOR )
|
||||
throw new IllegalArgumentException("Cannot get sub-paths for an empty lineage path (lid:///)")
|
||||
final store = fileSystem.getStore()
|
||||
if( !store )
|
||||
throw new Exception("Lineage store not found - Check Nextflow configuration")
|
||||
return store.getSubKeys(filePath).map {new LinPath(fileSystem as LinFileSystem, it) as Path }
|
||||
}
|
||||
|
||||
/**
|
||||
* Finds the target path of a LinPath.
|
||||
* This method return the different types depending on the type of metadata pointing:
|
||||
* - When the LinPath point to FileOutput metadata or a subpath, it returns the real path.
|
||||
* - When it points other lineage metadata or a fragment of a lineage metadata and asMetadata is true, it returns a LinMetadataPath
|
||||
* which contains in memory the lineage metadata description or the requested fragment of this description.
|
||||
* - When it points to a WorkflowRun or TaskRun metadata or subpath and asIntermediate is set to true. LinIntermediatePath, which is representing a directory
|
||||
* In other cases it will return a FileNotFoundException
|
||||
*
|
||||
* @param fs LinFileSystem associated to the LinPath to find
|
||||
* @param filePath Path to look for the target path
|
||||
* @param fragment String with path to sub-object inside the description
|
||||
* @param asMetadata Flag to indicate if other metadata descriptions must be returned as LinMetadataPath.
|
||||
* @param asIntermediate Flag to indicate if WorkflowRun and TaskRun subpaths must be returned as LinIntermediatePath.
|
||||
* @param subpath subpath associated to the target path to find. Used when looking for a parent path
|
||||
* @return Real Path, LinMetadataPath or LinIntermediatePath path associated to the LinPath
|
||||
* @throws Exception
|
||||
* IllegalArgumentException if the filepath, filesystem or its LinStore are null.
|
||||
* FileNotFoundException if the filePath, subpath and fragment is not found.
|
||||
*/
|
||||
protected static Path findTarget(LinFileSystem fs, String filePath, String fragment, boolean asMetadata, boolean asIntermediate) throws Exception {
|
||||
if( !fs )
|
||||
throw new IllegalArgumentException("Cannot get target path for a relative lineage path")
|
||||
if( filePath.isEmpty() || filePath == SEPARATOR )
|
||||
throw new IllegalArgumentException("Cannot get target path for an empty lineage path (lid:///)")
|
||||
final store = fs.getStore()
|
||||
if( !store )
|
||||
throw new Exception("Lineage store not found - Check Nextflow configuration")
|
||||
findTarget0(fs, store, filePath, fragment, asMetadata, asIntermediate, [])
|
||||
}
|
||||
|
||||
private static Path findTarget0(LinFileSystem fs, LinStore store, String filePath, String fragment, boolean asMetadata, boolean asIntermediate, List<String> subpath) {
|
||||
final object = store.load(filePath)
|
||||
if( object ) {
|
||||
return getTargetPathFromObject(object, fs, filePath, fragment, asMetadata, asIntermediate, subpath)
|
||||
} else {
|
||||
if( fragment ) {
|
||||
// If object doesn't exit, it's not possible to get fragment.
|
||||
throw new FileNotFoundException("Target path '$filePath#$fragment' does not exist")
|
||||
}
|
||||
return findTargetFromParent(fs, store, filePath, asIntermediate, subpath)
|
||||
}
|
||||
}
|
||||
|
||||
private static Path findTargetFromParent(LinFileSystem fs, LinStore store, String filePath, boolean asIntermediate, List<String> subpath) {
|
||||
final currentPath = Path.of(filePath)
|
||||
final parent = Path.of(filePath).getParent()
|
||||
if( !parent ) {
|
||||
throw new FileNotFoundException("Target path '$filePath/${subpath.join('/')} does not exist")
|
||||
}
|
||||
ArrayList<String> newChildren = new ArrayList<String>()
|
||||
newChildren.add(currentPath.getFileName().toString())
|
||||
newChildren.addAll(subpath)
|
||||
//As Metadata set as false because parent path only inspected for FileOutput or intermediate.
|
||||
return findTarget0(fs, store, parent.toString(), null, false, asIntermediate, newChildren)
|
||||
}
|
||||
|
||||
private static Path getTargetPathFromObject(LinSerializable object, LinFileSystem fs, String filePath, String fragment, boolean asMetadataPath, boolean asIntermediatePath,List<String> subpath) {
|
||||
// It's not possible to get a target path with both fragment and subpath
|
||||
if( fragment && subpath ) {
|
||||
throw new FileNotFoundException("Unable to get a target path for '$filePath' with fragments and subpath")
|
||||
}
|
||||
// If metadata flag is active and looks for a fragment returns the metadata despite the type of object
|
||||
if( asMetadataPath && fragment ){
|
||||
return getMetadataAsTargetPath(object, fs, filePath, fragment)
|
||||
}
|
||||
// Return real files when FileOutput sub-path
|
||||
if( object instanceof FileOutput ) {
|
||||
return getTargetPathFromOutput(object, subpath)
|
||||
}
|
||||
// Intermediate run case
|
||||
if( asIntermediatePath && (object instanceof WorkflowRun || object instanceof TaskRun) ) {
|
||||
return new LinIntermediatePath(fs, "$filePath/${subpath.join('/')}")
|
||||
}
|
||||
|
||||
// It is not possible to get a metadata path with subpath. For other cases return metadata path if activated or throw exception
|
||||
if( asMetadataPath && !subpath)
|
||||
return getMetadataAsTargetPath(object, fs, filePath, fragment)
|
||||
else
|
||||
throw new FileNotFoundException("Target path '${filePath}/${subpath ? '/' + subpath.join('/') : ''}${fragment ? '#' + fragment : ''}' does not exist")
|
||||
}
|
||||
|
||||
protected static Path getMetadataAsTargetPath(LinSerializable results, LinFileSystem fs, String filePath, String fragment) {
|
||||
if( !results ) {
|
||||
throw new FileNotFoundException("Target path '$filePath' does not exist")
|
||||
}
|
||||
if( fragment ) {
|
||||
return getSubObjectAsPath(fs, filePath, results, fragment)
|
||||
} else {
|
||||
return generateLinMetadataPath(fs, filePath, results, fragment)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a metadata sub-object as LinMetadataPath.
|
||||
* If the requested sub-object is the workflow or task outputs, retrieves the outputs from the outputs description.
|
||||
*
|
||||
* @param fs LinFilesystem for the te.
|
||||
* @param key Parent metadata key.
|
||||
* @param object Parent object.
|
||||
* @param children Array of string in indicating the properties to navigate to get the sub-object.
|
||||
* @return LinMetadataPath or null in it does not exist
|
||||
*/
|
||||
static LinMetadataPath getSubObjectAsPath(LinFileSystem fs, String key, LinSerializable object, String fragment) {
|
||||
if( isSearchingOutputs(object, fragment) ) {
|
||||
// When asking for a Workflow or task output retrieve the outputs description
|
||||
final outputs = fs.store.load("${key}#output")
|
||||
if( !outputs ) {
|
||||
throw new FileNotFoundException("Target path '$key#output' does not exist")
|
||||
}
|
||||
return generateLinMetadataPath(fs, key, outputs, fragment)
|
||||
} else {
|
||||
return generateLinMetadataPath(fs, key, object, fragment)
|
||||
}
|
||||
}
|
||||
|
||||
private static LinMetadataPath generateLinMetadataPath(LinFileSystem fs, String key, Object object, String fragment) {
|
||||
def creationTime = toFileTime(navigate(object, 'createdAt') as OffsetDateTime ?: OffsetDateTime.now())
|
||||
final output = fragment ? navigate(object, fragment) : object
|
||||
if( !output ) {
|
||||
throw new FileNotFoundException("Target path '$key#${fragment}' does not exist")
|
||||
}
|
||||
return new LinMetadataPath(encodeSearchOutputs(output, true), creationTime, fs, key, fragment)
|
||||
}
|
||||
|
||||
private static Path getTargetPathFromOutput(FileOutput object, List<String> children) {
|
||||
final lidObject = object as FileOutput
|
||||
// verify checksum validation
|
||||
final violation = validateDataOutput(lidObject)
|
||||
if( violation )
|
||||
log.warn1(violation)
|
||||
// return the real path stored in the metadata
|
||||
def realPath = FileHelper.toCanonicalPath(lidObject.path as String)
|
||||
if( children && children.size() > 0 )
|
||||
realPath = realPath.resolve(children.join(SEPARATOR))
|
||||
if( !realPath.exists() )
|
||||
throw new FileNotFoundException("Target path '$realPath' does not exist")
|
||||
return realPath
|
||||
}
|
||||
|
||||
private static boolean isEmptyBase(LinFileSystem fs, String base) {
|
||||
return !base || base == SEPARATOR || (fs && base == "..")
|
||||
}
|
||||
|
||||
private static String resolve0(LinFileSystem fs, String base, String[] more) {
|
||||
if( isEmptyBase(fs, base) ) {
|
||||
return resolveEmptyPathCase(fs, more as List)
|
||||
}
|
||||
if( base.contains(SEPARATOR) ) {
|
||||
final parts = base.tokenize(SEPARATOR)
|
||||
final remain = parts[1..-1] + more.toList()
|
||||
return resolve0(fs, parts[0], remain as String[])
|
||||
}
|
||||
final result = Path.of(base)
|
||||
return more ? result.resolve(more.join(SEPARATOR)).toString() : result.toString()
|
||||
}
|
||||
|
||||
private static String resolveEmptyPathCase(LinFileSystem fs, List<String> more) {
|
||||
switch( more.size() ) {
|
||||
case 0:
|
||||
return "/"
|
||||
case 1:
|
||||
return resolve0(fs, more[0], EMPTY)
|
||||
default:
|
||||
return resolve0(fs, more[0], more[1..-1] as String[])
|
||||
}
|
||||
}
|
||||
|
||||
static private String norm0(String path) {
|
||||
if( !path || path == SEPARATOR )
|
||||
return ""
|
||||
//Remove repeated elements
|
||||
path = Path.of(path.trim()).normalize().toString()
|
||||
//Remove initial and final separators
|
||||
if( path.startsWith(SEPARATOR) )
|
||||
path = path.substring(1)
|
||||
if( path.endsWith(SEPARATOR) )
|
||||
path = path.substring(0, path.size() - 1)
|
||||
return path
|
||||
}
|
||||
|
||||
static private String[] norm0(String... path) {
|
||||
for( int i = 0; i < path.length; i++ ) {
|
||||
path[i] = norm0(path[i])
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
@Override
|
||||
FileSystem getFileSystem() {
|
||||
return fileSystem
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean isAbsolute() {
|
||||
return fileSystem != null
|
||||
}
|
||||
|
||||
@Override
|
||||
Path getRoot() {
|
||||
return new LinPath(fileSystem, SEPARATOR)
|
||||
}
|
||||
|
||||
@Override
|
||||
Path getFileName() {
|
||||
final result = Path.of(filePath).getFileName()?.toString()
|
||||
return result ? new LinPath(query, fragment, result, null) : null
|
||||
}
|
||||
|
||||
@Override
|
||||
Path getParent() {
|
||||
final c = getNameCount()
|
||||
if( c > 1 )
|
||||
return subpath(0, c - 1)
|
||||
if( c == 1 )
|
||||
return new LinPath(fileSystem, SEPARATOR)
|
||||
return null
|
||||
}
|
||||
|
||||
@Override
|
||||
int getNameCount() {
|
||||
return Path.of(filePath).nameCount
|
||||
}
|
||||
|
||||
@Override
|
||||
Path getName(int index) {
|
||||
if( index < 0 )
|
||||
throw new IllegalArgumentException("Path name index cannot be less than zero - offending value: $index")
|
||||
final path = Path.of(filePath)
|
||||
if( index == path.nameCount - 1 ) {
|
||||
return new LinPath( query, fragment, path.getName(index).toString(), null)
|
||||
}
|
||||
return new LinPath(index == 0 ? fileSystem : null, path.getName(index).toString())
|
||||
}
|
||||
|
||||
@Override
|
||||
Path subpath(int beginIndex, int endIndex) {
|
||||
if( beginIndex < 0 )
|
||||
throw new IllegalArgumentException("subpath begin index cannot be less than zero - offending value: $beginIndex")
|
||||
final path = Path.of(filePath)
|
||||
return new LinPath(beginIndex == 0 ? fileSystem : null, path.subpath(beginIndex, endIndex).toString())
|
||||
}
|
||||
|
||||
@Override
|
||||
Path normalize() {
|
||||
return new LinPath(fileSystem, Path.of(filePath).normalize().toString())
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean startsWith(Path other) {
|
||||
return startsWith(other.toString())
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean startsWith(String other) {
|
||||
return filePath.startsWith(other)
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean endsWith(Path other) {
|
||||
return endsWith(other.toString())
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean endsWith(String other) {
|
||||
return filePath.endsWith(other)
|
||||
}
|
||||
|
||||
@Override
|
||||
Path resolve(Path other) {
|
||||
if( LinPath.class != other.class )
|
||||
throw new ProviderMismatchException()
|
||||
|
||||
final that = (LinPath) other
|
||||
|
||||
if( that.fileSystem && this.fileSystem != that.fileSystem )
|
||||
return other
|
||||
if( that.isAbsolute() ) {
|
||||
return that
|
||||
} else {
|
||||
final newPath = Path.of(filePath).resolve(that.toString())
|
||||
return new LinPath(that.query, that.fragment, newPath.toString(), fileSystem)
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
Path resolve(String path) {
|
||||
if( !path )
|
||||
return this
|
||||
final scheme = FileHelper.getUrlProtocol(path)
|
||||
if( !scheme ) {
|
||||
// consider the path as a lid relative path
|
||||
return resolve(new LinPath(null, path))
|
||||
}
|
||||
if( scheme != SCHEME ) {
|
||||
throw new ProviderMismatchException()
|
||||
}
|
||||
final that = fileSystem.provider().getPath(asUri(path))
|
||||
return resolve(that)
|
||||
}
|
||||
|
||||
@Override
|
||||
Path relativize(Path other) {
|
||||
if( LinPath.class != other.class ) {
|
||||
throw new ProviderMismatchException()
|
||||
}
|
||||
LinPath lidOther = other as LinPath
|
||||
if( this.isAbsolute() != lidOther.isAbsolute() )
|
||||
throw new IllegalArgumentException("Cannot compare absolute with relative paths");
|
||||
def path
|
||||
if( this.isAbsolute() ) {
|
||||
// Compare 'filePath' as absolute paths adding the root separator
|
||||
path = Path.of(SEPARATOR + filePath).relativize(Path.of(SEPARATOR + lidOther.filePath))
|
||||
} else {
|
||||
// Compare 'filePath' as relative paths
|
||||
path = Path.of(filePath).relativize(Path.of(lidOther.filePath))
|
||||
}
|
||||
return new LinPath(lidOther.query, lidOther.fragment, path.getNameCount() > 0 ? path.toString() : SEPARATOR, null)
|
||||
}
|
||||
|
||||
@Override
|
||||
URI toUri() {
|
||||
return asUri("${SCHEME}://${filePath}${query ? '?' + query : ''}${fragment ? '#' + fragment : ''}")
|
||||
}
|
||||
|
||||
String toUriString() {
|
||||
return toUri().toString()
|
||||
}
|
||||
|
||||
@Override
|
||||
Path toAbsolutePath() {
|
||||
return this
|
||||
}
|
||||
|
||||
@Override
|
||||
Path toRealPath(LinkOption... options) throws IOException {
|
||||
return this.getTargetOrMetadataPath()
|
||||
}
|
||||
|
||||
Path toTargetPath() {
|
||||
return getTargetOrMetadataPath()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the path associated with a FileOutput record.
|
||||
*
|
||||
* @return Path associated with a FileOutput record
|
||||
* @throws FileNotFoundException if the record does not exist or its type is not a FileOutput.
|
||||
*/
|
||||
protected Path getTargetPath() {
|
||||
return findTarget(fileSystem, filePath, fragment, false, false)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the path associated with a FileOutput record or an intermediate subpath.
|
||||
*
|
||||
* @return Path associated with a FileOutput record or a LinIntermediatePath if LinPath points to a workflow and task run subpath.
|
||||
* @throws FileNotFoundException if the record does not exist or its type is not a FileOutput or a intermediate directory
|
||||
*/
|
||||
protected Path getTargetOrIntermediatePath() {
|
||||
return findTarget(fileSystem, filePath, fragment, false, true)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the path associated with a lineage record.
|
||||
*
|
||||
* @return Path associated with a FileOutput record or a LinMetadataFile with the lineage record for other types, or a intermediate directory
|
||||
* @throws FileNotFoundException if the record does not exist
|
||||
*/
|
||||
protected Path getTargetOrMetadataPath() {
|
||||
return findTarget(fileSystem, filePath, fragment,true, false)
|
||||
}
|
||||
|
||||
@Override
|
||||
File toFile() throws IOException {
|
||||
throw new UnsupportedOperationException("toFile not supported by LinPath")
|
||||
}
|
||||
|
||||
@Override
|
||||
WatchKey register(WatchService watcher, WatchEvent.Kind<?>[] events, WatchEvent.Modifier... modifiers) throws IOException {
|
||||
throw new UnsupportedOperationException("Register not supported by LinPath")
|
||||
}
|
||||
|
||||
@Override
|
||||
int compareTo(Path other) {
|
||||
return toString().compareTo(other.toString());
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean equals(Object other) {
|
||||
if( LinPath.class != other.class ) {
|
||||
return false
|
||||
}
|
||||
final that = (LinPath) other
|
||||
return this.fileSystem == that.fileSystem && this.filePath.equals(that.filePath)
|
||||
}
|
||||
|
||||
/**
|
||||
* @return The unique hash code for this path
|
||||
*/
|
||||
@Override
|
||||
int hashCode() {
|
||||
return Objects.hash(fileSystem, filePath)
|
||||
}
|
||||
|
||||
static URI asUri(String path) {
|
||||
if( !path )
|
||||
throw new IllegalArgumentException("Missing 'path' argument")
|
||||
if( !path.startsWith(LID_PROT) )
|
||||
throw new IllegalArgumentException("Invalid LID file system path URI - it must start with '${LID_PROT}' prefix - offendinf value: $path")
|
||||
if( path.startsWith(LID_PROT + SEPARATOR) && path.length() > 7 )
|
||||
throw new IllegalArgumentException("Invalid LID file system path URI - make sure the schema prefix does not container more than two slash characters or a query in the root '/' - offending value: $path")
|
||||
if( path == LID_PROT ) //Empty path case
|
||||
return new URI("lid:///")
|
||||
return new URI(path)
|
||||
}
|
||||
|
||||
@Override
|
||||
String toString() {
|
||||
return "$filePath${query ? '?' + query : ''}${fragment ? '#' + fragment : ''}".toString()
|
||||
}
|
||||
/**
|
||||
* Validates the integrity of the LinPath. If there is a problem with the validation an exception is thrown.
|
||||
* To validate just try to get the find target target path. It checks if lid exists, it is a FileOutput,
|
||||
* the target path exists and the checksum is the same as the stored in the metadata.
|
||||
*/
|
||||
FileCheck validate() throws Exception{
|
||||
final obj = fileSystem.store.load(filePath)
|
||||
if( !obj )
|
||||
return new FileCheck("File cannot be found")
|
||||
if( obj instanceof FileOutput ) {
|
||||
final res = validateDataOutput(obj as FileOutput)
|
||||
return new FileCheck(res, obj)
|
||||
}
|
||||
return new FileCheck("Unexpected lineage object type: ${obj.getClass().getName()}")
|
||||
}
|
||||
|
||||
@EqualsAndHashCode
|
||||
static class FileCheck {
|
||||
final String error
|
||||
final FileOutput file
|
||||
|
||||
FileCheck(String error, FileOutput out=null) {
|
||||
this.error = error
|
||||
this.file = out
|
||||
}
|
||||
|
||||
/**
|
||||
* Implements groovy truth
|
||||
*/
|
||||
boolean asBoolean() {
|
||||
return error==null && file!=null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.fs
|
||||
|
||||
import static LinPath.*
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.lineage.config.LineageConfig
|
||||
import nextflow.file.FileHelper
|
||||
import nextflow.file.FileSystemPathFactory
|
||||
/**
|
||||
* Implements a {@link FileSystemPathFactory} for LID file system
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
@CompileStatic
|
||||
class LinPathFactory extends FileSystemPathFactory {
|
||||
|
||||
@Override
|
||||
protected Path parseUri(String uri) {
|
||||
return isLidUri(uri) ? create(uri) : null
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String toUriString(Path path) {
|
||||
return path instanceof LinPath ? ((LinPath)path).toUriString() : null
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String getBashLib(Path target) {
|
||||
return null
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String getUploadCmd(String source, Path target) {
|
||||
return null
|
||||
}
|
||||
|
||||
static LinPath create(String path) {
|
||||
final uri = LinPath.asUri(path)
|
||||
return (LinPath) FileHelper.getOrCreateFileSystemFor(uri, LineageConfig.asMap()).provider().getPath(uri)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.model.v1beta1
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
import groovy.transform.Canonical
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.util.CacheHelper
|
||||
/**
|
||||
* Models a checksum including the value as well as the algortihm and mode used to compute it.
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io
|
||||
*/
|
||||
@Canonical
|
||||
@CompileStatic
|
||||
class Checksum {
|
||||
String value
|
||||
String algorithm
|
||||
String mode
|
||||
|
||||
static Checksum of(String value, String algorithm, CacheHelper.HashMode mode) {
|
||||
new Checksum(value, algorithm, mode.toString().toLowerCase())
|
||||
}
|
||||
|
||||
static Checksum ofNextflow(String value) {
|
||||
new Checksum(CacheHelper.hasher(value).hash().toString(), 'nextflow', CacheHelper.HashMode.DEFAULT().toString().toLowerCase())
|
||||
}
|
||||
|
||||
static Checksum ofNextflow(Path path) {
|
||||
new Checksum(CacheHelper.hasher(path).hash().toString(), 'nextflow', CacheHelper.HashMode.DEFAULT().toString().toLowerCase())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.model.v1beta1
|
||||
|
||||
import groovy.transform.Canonical
|
||||
import groovy.transform.CompileStatic
|
||||
|
||||
/**
|
||||
* Models a data path which includes the path and a checksum to validate the content of the path.
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io
|
||||
*/
|
||||
@Canonical
|
||||
@CompileStatic
|
||||
class DataPath {
|
||||
/**
|
||||
* Real path of the output data.
|
||||
*/
|
||||
String path
|
||||
/**
|
||||
* Checksum of the output data.
|
||||
*/
|
||||
Checksum checksum
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.model.v1beta1
|
||||
|
||||
import groovy.transform.Canonical
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.lineage.serde.LinSerializable
|
||||
|
||||
import java.time.OffsetDateTime
|
||||
|
||||
/**
|
||||
* Model a base class for workflow and task outputs
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Canonical
|
||||
@CompileStatic
|
||||
class FileOutput implements LinSerializable {
|
||||
/**
|
||||
* Real path of the output data.
|
||||
*/
|
||||
String path
|
||||
/**
|
||||
* Checksum of the output data.
|
||||
*/
|
||||
Checksum checksum
|
||||
/**
|
||||
* Entity that generated the data. Possible entities are:
|
||||
* - a FileOutput if the workflow published from a task data.
|
||||
* - a TaskRun if the data is a task output.
|
||||
* - a WorkflowRun if the data is generated by the workflow (e.g., an index file).
|
||||
*/
|
||||
String source
|
||||
/**
|
||||
* Reference to the WorkflowRun that generated the data.
|
||||
*/
|
||||
String workflowRun
|
||||
/**
|
||||
* Reference to the task that generated the data.
|
||||
*/
|
||||
String taskRun
|
||||
/**
|
||||
* Size of the data.
|
||||
*/
|
||||
long size
|
||||
/**
|
||||
* Data creation date.
|
||||
*/
|
||||
OffsetDateTime createdAt
|
||||
/**
|
||||
* Data last modified date.
|
||||
*/
|
||||
OffsetDateTime modifiedAt
|
||||
/**
|
||||
* Labels attached to the data
|
||||
*/
|
||||
List<String> labels
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.model.v1beta1
|
||||
|
||||
/**
|
||||
* Marker interface holding lineage model common definitions
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
interface LinModel {
|
||||
static final public String VERSION = 'lineage/v1beta1'
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.model.v1beta1
|
||||
|
||||
import groovy.transform.Canonical
|
||||
import groovy.transform.CompileStatic
|
||||
|
||||
/**
|
||||
* Model Workflow and Task Parameters.
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io
|
||||
*/
|
||||
@Canonical
|
||||
@CompileStatic
|
||||
class Parameter {
|
||||
String type
|
||||
String name
|
||||
Object value
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.model.v1beta1
|
||||
|
||||
import groovy.transform.Canonical
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.lineage.serde.LinSerializable
|
||||
|
||||
import java.time.OffsetDateTime
|
||||
|
||||
/**
|
||||
* Models task results.
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
@Canonical
|
||||
@CompileStatic
|
||||
class TaskOutput implements LinSerializable {
|
||||
/**
|
||||
* Reference to the task that generated the output.
|
||||
*/
|
||||
String taskRun
|
||||
/**
|
||||
* Reference to the WorkflowRun that generated the output.
|
||||
*/
|
||||
String workflowRun
|
||||
/**
|
||||
* Creation date of this task output description
|
||||
*/
|
||||
OffsetDateTime createdAt
|
||||
/**
|
||||
* Output of the task
|
||||
*/
|
||||
List<Parameter> output
|
||||
/**
|
||||
* Labels attached to the task output
|
||||
*/
|
||||
List<String> labels
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.model.v1beta1
|
||||
|
||||
import groovy.transform.Canonical
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.lineage.serde.LinSerializable
|
||||
|
||||
/**
|
||||
* Models a task execution.
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Canonical
|
||||
@CompileStatic
|
||||
class TaskRun implements LinSerializable {
|
||||
/**
|
||||
* Execution session identifier
|
||||
*/
|
||||
String sessionId
|
||||
/**
|
||||
* Task name
|
||||
*/
|
||||
String name
|
||||
/**
|
||||
* Checksum of the task source code
|
||||
*/
|
||||
Checksum codeChecksum
|
||||
/**
|
||||
* Resolved task script
|
||||
*/
|
||||
String script
|
||||
/**
|
||||
* Task run input
|
||||
*/
|
||||
List<Parameter> input
|
||||
/**
|
||||
* Container used for the task run
|
||||
*/
|
||||
String container
|
||||
/**
|
||||
* Conda environment used for the task run
|
||||
*/
|
||||
String conda
|
||||
/**
|
||||
* Spack environment used for the task run
|
||||
*/
|
||||
String spack
|
||||
/**
|
||||
* Architecture defined in the Spack environment used for the task run
|
||||
*/
|
||||
String architecture
|
||||
/**
|
||||
* Global variables defined in the task run
|
||||
*/
|
||||
Map globalVars
|
||||
/**
|
||||
* Binaries used in the task run
|
||||
*/
|
||||
List<DataPath> binEntries
|
||||
/**
|
||||
* Workflow run associated to the task run
|
||||
*/
|
||||
String workflowRun
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.model.v1beta1
|
||||
|
||||
import groovy.transform.Canonical
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.lineage.serde.LinSerializable
|
||||
|
||||
|
||||
/**
|
||||
* Models a workflow definition.
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io
|
||||
*/
|
||||
@Canonical
|
||||
@CompileStatic
|
||||
class Workflow implements LinSerializable {
|
||||
/**
|
||||
* List of script files used by a workflow, starting with the main script
|
||||
*/
|
||||
List<DataPath> scriptFiles
|
||||
/**
|
||||
* Workflow repository
|
||||
*/
|
||||
String repository
|
||||
/**
|
||||
* Workflow commit identifier
|
||||
*/
|
||||
String commitId
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.model.v1beta1
|
||||
|
||||
import groovy.transform.Canonical
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.lineage.serde.LinSerializable
|
||||
|
||||
import java.time.OffsetDateTime
|
||||
|
||||
/**
|
||||
* Models the results of a workflow execution.
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io
|
||||
*/
|
||||
@Canonical
|
||||
@CompileStatic
|
||||
class WorkflowOutput implements LinSerializable {
|
||||
/**
|
||||
* Creation date of the workflow output
|
||||
*/
|
||||
OffsetDateTime createdAt
|
||||
/**
|
||||
* Workflow run that generated the output
|
||||
*/
|
||||
String workflowRun
|
||||
/**
|
||||
* Workflow output
|
||||
*/
|
||||
List<Parameter> output
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.model.v1beta1
|
||||
|
||||
import groovy.transform.Canonical
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.lineage.serde.LinSerializable
|
||||
|
||||
/**
|
||||
* Models a Workflow Execution
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io
|
||||
*/
|
||||
@Canonical
|
||||
@CompileStatic
|
||||
class WorkflowRun implements LinSerializable {
|
||||
/**
|
||||
* Description of the workflow associated with the workflow run.
|
||||
*/
|
||||
Workflow workflow
|
||||
/**
|
||||
* Session identifier used in the workflow run
|
||||
*/
|
||||
String sessionId
|
||||
/**
|
||||
* Workflow run name
|
||||
*/
|
||||
String name
|
||||
/**
|
||||
* Workflow parameters
|
||||
*/
|
||||
List<Parameter> params
|
||||
/**
|
||||
* Resolved Configuration
|
||||
*/
|
||||
Map config
|
||||
/**
|
||||
* Raw metadata
|
||||
*/
|
||||
Map metadata
|
||||
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.serde
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.serde.gson.GsonEncoder
|
||||
|
||||
/**
|
||||
* Implements a JSON encoder for lineage model objects
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class LinEncoder extends GsonEncoder<LinSerializable> {
|
||||
|
||||
LinEncoder() {
|
||||
withTypeAdapterFactory(new LinTypeAdapterFactory())
|
||||
// enable rendering of null values
|
||||
withSerializeNulls(true)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.serde
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.serde.JsonSerializable
|
||||
/**
|
||||
* Marker interface for lineage serializable objects
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
interface LinSerializable extends JsonSerializable {
|
||||
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package nextflow.lineage.serde
|
||||
|
||||
import com.google.gson.Gson
|
||||
import com.google.gson.JsonObject
|
||||
import com.google.gson.JsonParseException
|
||||
import com.google.gson.JsonParser
|
||||
import com.google.gson.TypeAdapter
|
||||
import com.google.gson.reflect.TypeToken
|
||||
import com.google.gson.stream.JsonReader
|
||||
import com.google.gson.stream.JsonWriter
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.lineage.model.v1beta1.FileOutput
|
||||
import nextflow.lineage.model.v1beta1.LinModel
|
||||
import nextflow.lineage.model.v1beta1.TaskOutput
|
||||
import nextflow.lineage.model.v1beta1.TaskRun
|
||||
import nextflow.lineage.model.v1beta1.Workflow
|
||||
import nextflow.lineage.model.v1beta1.WorkflowOutput
|
||||
import nextflow.lineage.model.v1beta1.WorkflowRun
|
||||
import nextflow.serde.gson.RuntimeTypeAdapterFactory
|
||||
/**
|
||||
* Class to serialize LiSerializable objects including the Lineage model version.
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
@CompileStatic
|
||||
class LinTypeAdapterFactory<T> extends RuntimeTypeAdapterFactory<T> {
|
||||
|
||||
public static final String VERSION_FIELD = 'version'
|
||||
public static final String SPEC_FIELD = 'spec'
|
||||
public static final String CURRENT_VERSION = LinModel.VERSION
|
||||
|
||||
LinTypeAdapterFactory() {
|
||||
super(LinSerializable.class, "kind", false)
|
||||
this.registerSubtype(WorkflowRun, WorkflowRun.simpleName)
|
||||
.registerSubtype(WorkflowOutput, WorkflowOutput.simpleName)
|
||||
.registerSubtype(Workflow, Workflow.simpleName)
|
||||
.registerSubtype(TaskRun, TaskRun.simpleName)
|
||||
.registerSubtype(TaskOutput, TaskOutput.simpleName)
|
||||
.registerSubtype(FileOutput, FileOutput.simpleName)
|
||||
}
|
||||
|
||||
@Override
|
||||
<R> TypeAdapter<R> create(Gson gson, TypeToken<R> type) {
|
||||
if (!LinSerializable.class.isAssignableFrom(type.rawType)) {
|
||||
return null
|
||||
}
|
||||
|
||||
def delegate = super.create(gson, type as TypeToken<T>)
|
||||
if (delegate == null) {
|
||||
return null
|
||||
}
|
||||
|
||||
return new TypeAdapter<R>() {
|
||||
@Override
|
||||
void write(JsonWriter out, R value) throws IOException {
|
||||
final object = new JsonObject()
|
||||
object.addProperty(VERSION_FIELD, CURRENT_VERSION)
|
||||
String label = getLabelFromSubtype(value.class)
|
||||
if (!label)
|
||||
throw new JsonParseException("Not registered class ${value.class}")
|
||||
object.addProperty(getTypeFieldName(), label)
|
||||
def json = gson.toJsonTree(value)
|
||||
object.add(SPEC_FIELD, json)
|
||||
gson.toJson(object, out)
|
||||
}
|
||||
|
||||
@Override
|
||||
R read(JsonReader reader) throws IOException {
|
||||
final obj = JsonParser.parseReader(reader)?.getAsJsonObject()
|
||||
if( obj==null )
|
||||
throw new JsonParseException("Parsed JSON object is null")
|
||||
final versionEl = obj.get(VERSION_FIELD)
|
||||
if (versionEl == null || versionEl.asString != CURRENT_VERSION) {
|
||||
throw new JsonParseException("Invalid or missing '${VERSION_FIELD}' JSON property")
|
||||
}
|
||||
final typeEl = obj.get(getTypeFieldName())
|
||||
if( typeEl==null )
|
||||
throw new JsonParseException("JSON property '${getTypeFieldName()}' not found")
|
||||
|
||||
// Check if this is the new format (has 'spec' field) or old format (data at root level)
|
||||
final specEl = obj.get(SPEC_FIELD)?.asJsonObject
|
||||
if ( specEl != null ) {
|
||||
// New format: data is wrapped in 'spec' field
|
||||
specEl.add(getTypeFieldName(), typeEl)
|
||||
return (R) delegate.fromJsonTree(specEl)
|
||||
} else {
|
||||
// Old format: data is at root level, just remove version field
|
||||
obj.remove(VERSION_FIELD)
|
||||
return (R) delegate.fromJsonTree(obj)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
#
|
||||
# Copyright 2013-2025, Seqera Labs
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
nextflow.lineage.DefaultLinStoreFactory
|
||||
nextflow.lineage.LinExtensionImpl
|
||||
nextflow.lineage.LinObserverFactory
|
||||
nextflow.lineage.cli.LinCommandImpl
|
||||
nextflow.lineage.config.LineageConfig
|
||||
@@ -0,0 +1,17 @@
|
||||
#
|
||||
# Copyright 2013-2026, Seqera Labs
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
nextflow.lineage.fs.LinFileSystemProvider
|
||||
@@ -0,0 +1,79 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package nextflow.lineage
|
||||
|
||||
import spock.lang.Specification
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
|
||||
/**
|
||||
* Lineage History file tests
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
class DefaultLinHistoryLogTest extends Specification {
|
||||
|
||||
Path tempDir
|
||||
Path historyFile
|
||||
DefaultLinHistoryLog linHistoryLog
|
||||
|
||||
def setup() {
|
||||
tempDir = Files.createTempDirectory("wdir")
|
||||
historyFile = tempDir.resolve("lin-history")
|
||||
linHistoryLog = new DefaultLinHistoryLog(historyFile)
|
||||
}
|
||||
|
||||
def cleanup(){
|
||||
tempDir?.deleteDir()
|
||||
}
|
||||
|
||||
def "write should add a new file to the history folder"() {
|
||||
given:
|
||||
UUID sessionId = UUID.randomUUID()
|
||||
String runName = "TestRun"
|
||||
String runLid = "lid://123"
|
||||
|
||||
when:
|
||||
linHistoryLog.write(runName, sessionId, runLid)
|
||||
|
||||
then:
|
||||
def files = historyFile.listFiles()
|
||||
files.size() == 1
|
||||
def parsedRecord = LinHistoryRecord.parse(files[0].text)
|
||||
parsedRecord.sessionId == sessionId
|
||||
parsedRecord.runName == runName
|
||||
parsedRecord.runLid == runLid
|
||||
}
|
||||
|
||||
def 'should get records' () {
|
||||
given:
|
||||
UUID sessionId = UUID.randomUUID()
|
||||
String runName = "Run1"
|
||||
String runLid = "lid://123"
|
||||
and:
|
||||
linHistoryLog.write(runName, sessionId, runLid)
|
||||
|
||||
when:
|
||||
def records = linHistoryLog.getRecords()
|
||||
then:
|
||||
records.size() == 1
|
||||
records[0].sessionId == sessionId
|
||||
records[0].runName == runName
|
||||
records[0].runLid == runLid
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage
|
||||
|
||||
import nextflow.lineage.config.LineageConfig
|
||||
import spock.lang.Specification
|
||||
import spock.lang.Unroll
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class DefaultLinStoreFactoryTest extends Specification {
|
||||
|
||||
@Unroll
|
||||
def 'should validate can open' () {
|
||||
given:
|
||||
def factory = new DefaultLinStoreFactory()
|
||||
def config = new LineageConfig(CONFIG)
|
||||
|
||||
expect:
|
||||
factory.canOpen(config) == EXPECTED
|
||||
|
||||
where:
|
||||
EXPECTED | CONFIG
|
||||
true | [:]
|
||||
true | [store:[location:'/some/path']]
|
||||
true | [store:[location:'some/rel/path']]
|
||||
true | [store:[location:'file:/this/that']]
|
||||
true | [store:[location:'s3://some/path']]
|
||||
false | [store:[location:'http://some/path']]
|
||||
false | [store:[location:'jdbc:foo']]
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.time.Instant
|
||||
import java.time.OffsetDateTime
|
||||
import java.time.ZoneOffset
|
||||
|
||||
import nextflow.lineage.model.v1beta1.Checksum
|
||||
import nextflow.lineage.model.v1beta1.DataPath
|
||||
import nextflow.lineage.model.v1beta1.FileOutput
|
||||
import nextflow.lineage.model.v1beta1.Parameter
|
||||
import nextflow.lineage.model.v1beta1.Workflow
|
||||
import nextflow.lineage.model.v1beta1.WorkflowRun
|
||||
import nextflow.lineage.serde.LinEncoder
|
||||
import nextflow.lineage.config.LineageConfig
|
||||
import spock.lang.Specification
|
||||
import spock.lang.TempDir
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
class DefaultLinStoreTest extends Specification {
|
||||
|
||||
@TempDir
|
||||
Path tempDir
|
||||
|
||||
Path storeLocation
|
||||
LineageConfig config
|
||||
|
||||
def setup() {
|
||||
storeLocation = tempDir.resolve("store")
|
||||
def configMap = [enabled: true, store: [location: storeLocation.toString(), logLocation: storeLocation.resolve(".log").toString()]]
|
||||
config = new LineageConfig(configMap)
|
||||
}
|
||||
|
||||
def 'should open store'() {
|
||||
given:
|
||||
def store = new DefaultLinStore()
|
||||
when:
|
||||
store.open(config)
|
||||
def historyLog = store.getHistoryLog()
|
||||
then:
|
||||
store.getLocation() == storeLocation
|
||||
historyLog != null
|
||||
historyLog instanceof DefaultLinHistoryLog
|
||||
}
|
||||
|
||||
def "save should store value in the correct file location"() {
|
||||
given:
|
||||
def key = "testKey"
|
||||
def value = new FileOutput("/path/to/file", new Checksum("hash_value", "hash_algorithm", "standard"), "lid://source", "lid://workflow", "lid://task", 1234)
|
||||
def lidStore = new DefaultLinStore()
|
||||
lidStore.open(config)
|
||||
|
||||
when:
|
||||
lidStore.save(key, value)
|
||||
|
||||
then:
|
||||
def filePath = storeLocation.resolve("$key/.data.json")
|
||||
Files.exists(filePath)
|
||||
filePath.text == new LinEncoder().encode(value)
|
||||
}
|
||||
|
||||
def "load should retrieve stored value correctly"() {
|
||||
given:
|
||||
def key = "testKey"
|
||||
def value = new FileOutput("/path/to/file", new Checksum("hash_value", "hash_algorithm", "standard"), "lid://source", "lid://workflow", "lid://task", 1234)
|
||||
def lidStore = new DefaultLinStore()
|
||||
lidStore.open(config)
|
||||
lidStore.save(key, value)
|
||||
|
||||
expect:
|
||||
lidStore.load(key).toString() == value.toString()
|
||||
}
|
||||
|
||||
def "load should return null if key does not exist"() {
|
||||
given:
|
||||
def lidStore = new DefaultLinStore()
|
||||
lidStore.open(config)
|
||||
|
||||
expect:
|
||||
lidStore.load("nonexistentKey") == null
|
||||
}
|
||||
|
||||
def 'should query' () {
|
||||
given:
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(1234567), ZoneOffset.UTC)
|
||||
def mainScript = new DataPath("file://path/to/main.nf", new Checksum("78910", "nextflow", "standard"))
|
||||
def workflow = new Workflow([mainScript],"https://nextflow.io/nf-test/", "123456" )
|
||||
def key = "testKey"
|
||||
def value1 = new WorkflowRun(workflow, uniqueId.toString(), "test_run", [ new Parameter("String", "param1", "value1"), new Parameter("String", "param2", "value2")] )
|
||||
def key2 = "testKey2"
|
||||
def value2 = new FileOutput("/path/tp/file1", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", null, 1234, time, time, ["value1", "value2"])
|
||||
def key3 = "testKey3"
|
||||
def value3 = new FileOutput("/path/tp/file2", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", null, 1234, time, time, ["value2", "value3"])
|
||||
def key4 = "testKey4"
|
||||
def value4 = new FileOutput("/path/tp/file", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", null, 1234, time, time, ["value4", "value3"])
|
||||
|
||||
def lidStore = new DefaultLinStore()
|
||||
lidStore.open(config)
|
||||
lidStore.save(key, value1)
|
||||
lidStore.save(key2, value2)
|
||||
lidStore.save(key3, value3)
|
||||
lidStore.save(key4, value4)
|
||||
|
||||
when:
|
||||
def results = lidStore.search( [type:['FileOutput'], labels:['value2']]).toList()
|
||||
then:
|
||||
results.size() == 2
|
||||
results.containsAll([key2,key3])
|
||||
}
|
||||
|
||||
def 'should search subkeys' () {
|
||||
given:
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(1234567), ZoneOffset.UTC)
|
||||
def mainScript = new DataPath("file://path/to/main.nf", new Checksum("78910", "nextflow", "standard"))
|
||||
def workflow = new Workflow([mainScript], "https://nextflow.io/nf-test/", "123456")
|
||||
def key = "testKey"
|
||||
def value1 = new WorkflowRun(workflow, uniqueId.toString(), "test_run", [new Parameter("String", "param1", "value1"), new Parameter("String", "param2", "value2")])
|
||||
def key2 = "testKey/file1"
|
||||
def value2 = new FileOutput("/path/file1", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", null, 1234, time, time, ["value1", "value2"])
|
||||
def key3 = "testKey/subfolder/file3"
|
||||
def value3 = new FileOutput("/path//file2", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", null, 1234, time, time, ["value2", "value3"])
|
||||
def key4 = "testKey2/file2"
|
||||
def value4 = new FileOutput("/path/tp/file", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", null, 1234, time, time, ["value4", "value3"])
|
||||
|
||||
def lidStore = new DefaultLinStore()
|
||||
lidStore.open(config)
|
||||
lidStore.save(key, value1)
|
||||
lidStore.save(key2, value2)
|
||||
lidStore.save(key3, value3)
|
||||
lidStore.save(key4, value4)
|
||||
|
||||
when:
|
||||
def results = lidStore.getSubKeys("testKey").toList()
|
||||
then:
|
||||
results.size() == 2
|
||||
results.containsAll([key2, key3])
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage
|
||||
|
||||
import java.nio.file.Path
|
||||
import java.time.Instant
|
||||
import java.time.OffsetDateTime
|
||||
import java.time.ZoneOffset
|
||||
|
||||
import groovyx.gpars.dataflow.DataflowQueue
|
||||
import nextflow.Channel
|
||||
import nextflow.Session
|
||||
import nextflow.lineage.config.LineageConfig
|
||||
import nextflow.lineage.fs.LinPathFactory
|
||||
import nextflow.lineage.model.v1beta1.Checksum
|
||||
import nextflow.lineage.model.v1beta1.DataPath
|
||||
import nextflow.lineage.model.v1beta1.FileOutput
|
||||
import nextflow.lineage.model.v1beta1.Parameter
|
||||
import nextflow.lineage.model.v1beta1.Workflow
|
||||
import nextflow.lineage.model.v1beta1.WorkflowRun
|
||||
import spock.lang.Specification
|
||||
import spock.lang.TempDir
|
||||
|
||||
import static nextflow.lineage.fs.LinPath.*
|
||||
import static test.ScriptHelper.runDataflow
|
||||
|
||||
/**
|
||||
* Lineage channel extensions tests
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
class LinExtensionImplTest extends Specification {
|
||||
|
||||
@TempDir
|
||||
Path tempDir
|
||||
|
||||
Path storeLocation
|
||||
Map configMap
|
||||
|
||||
def setup() {
|
||||
storeLocation = tempDir.resolve("store")
|
||||
configMap = [linage: [enabled: true, store: [location: storeLocation.toString()]]]
|
||||
}
|
||||
|
||||
def 'should return global query results' () {
|
||||
given:
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(1234567), ZoneOffset.UTC)
|
||||
def mainScript = new DataPath("file://path/to/main.nf", new Checksum("78910", "nextflow", "standard"))
|
||||
def workflow = new Workflow([mainScript],"https://nextflow.io/nf-test/", "123456" )
|
||||
def key = "testKey"
|
||||
def value1 = new WorkflowRun(workflow, uniqueId.toString(), "test_run", [ new Parameter("String", "param1", "value1"), new Parameter("String", "param2", "value2")] )
|
||||
def key2 = "testKey2"
|
||||
def value2 = new FileOutput("/path/tp/file1", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", "taskid", 1234, time, time, ["value1","value2"])
|
||||
def key3 = "testKey3"
|
||||
def value3 = new FileOutput("/path/tp/file2", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", null, 1234, time, time, ["value2", "value3"])
|
||||
def key4 = "testKey4"
|
||||
def value4 = new FileOutput("/path/tp/file", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", "taskid", 1234, time, time, ["value4","value3"])
|
||||
def lidStore = new DefaultLinStore()
|
||||
def session = Mock(Session) {
|
||||
getConfig() >> configMap
|
||||
}
|
||||
lidStore.open(LineageConfig.create(session))
|
||||
lidStore.save(key, value1)
|
||||
lidStore.save(key2, value2)
|
||||
lidStore.save(key3, value3)
|
||||
lidStore.save(key4, value4)
|
||||
def linExt = Spy(new LinExtensionImpl())
|
||||
when:
|
||||
def results = new DataflowQueue()
|
||||
linExt.fromLineage(session, results, [label: ["value2", "value3"]])
|
||||
then:
|
||||
linExt.getStore(session) >> lidStore
|
||||
and:
|
||||
results.val == LinPathFactory.create( asUriString(key3) )
|
||||
results.val == Channel.STOP
|
||||
|
||||
when:
|
||||
results = new DataflowQueue()
|
||||
linExt.fromLineage(session, results, [taskRun: "taskid", label: ["value4"]])
|
||||
then:
|
||||
linExt.getStore(session) >> lidStore
|
||||
and:
|
||||
results.val == LinPathFactory.create( asUriString(key4) )
|
||||
results.val == Channel.STOP
|
||||
|
||||
when:
|
||||
results = new DataflowQueue()
|
||||
linExt.fromLineage(session, results, [workflowRun: "testkey", taskRun: "taskid", label: "value2"])
|
||||
then:
|
||||
linExt.getStore(session) >> lidStore
|
||||
and:
|
||||
results.val == LinPathFactory.create( asUriString(key2) )
|
||||
results.val == Channel.STOP
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package nextflow.lineage
|
||||
|
||||
import spock.lang.Specification
|
||||
|
||||
/**
|
||||
* Lineage History Record tests
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
class LinHistoryRecordTest extends Specification {
|
||||
def "LinHistoryRecord parse should throw for invalid record"() {
|
||||
when:
|
||||
LinHistoryRecord.parse("invalid-record")
|
||||
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
}
|
||||
|
||||
def "LinHistoryRecord parse should handle 4-column record"() {
|
||||
given:
|
||||
def timestamp = new Date()
|
||||
def formattedTimestamp = LinHistoryRecord.TIMESTAMP_FMT.format(timestamp)
|
||||
def line = "${formattedTimestamp}\trun-1\t${UUID.randomUUID()}\tlid://123"
|
||||
|
||||
when:
|
||||
def record = LinHistoryRecord.parse(line)
|
||||
|
||||
then:
|
||||
record.timestamp != null
|
||||
record.runName == "run-1"
|
||||
record.runLid == "lid://123"
|
||||
}
|
||||
|
||||
def "LinHistoryRecord toString should produce tab-separated format"() {
|
||||
given:
|
||||
UUID sessionId = UUID.randomUUID()
|
||||
def record = new LinHistoryRecord(new Date(), "TestRun", sessionId, "lid://123")
|
||||
|
||||
when:
|
||||
def line = record.toString()
|
||||
|
||||
then:
|
||||
line.contains("\t")
|
||||
line.split("\t").size() == 4
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,983 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage
|
||||
|
||||
import nextflow.extension.FilesEx
|
||||
import nextflow.lineage.exception.OutputRelativePathException
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.attribute.BasicFileAttributes
|
||||
|
||||
import com.google.common.hash.HashCode
|
||||
import nextflow.NextflowMeta
|
||||
import nextflow.Session
|
||||
import nextflow.file.FileHolder
|
||||
import nextflow.lineage.model.v1beta1.Checksum
|
||||
import nextflow.lineage.model.v1beta1.FileOutput
|
||||
import nextflow.lineage.model.v1beta1.DataPath
|
||||
import nextflow.lineage.model.v1beta1.Parameter
|
||||
import nextflow.lineage.model.v1beta1.TaskOutput
|
||||
import nextflow.lineage.model.v1beta1.Workflow
|
||||
import nextflow.lineage.model.v1beta1.WorkflowOutput
|
||||
import nextflow.lineage.model.v1beta1.WorkflowRun
|
||||
import nextflow.lineage.serde.LinEncoder
|
||||
import nextflow.lineage.config.LineageConfig
|
||||
import nextflow.processor.TaskConfig
|
||||
import nextflow.processor.TaskHandler
|
||||
import nextflow.processor.TaskId
|
||||
import nextflow.processor.TaskRun
|
||||
import nextflow.script.ScriptBinding
|
||||
import nextflow.script.PlatformMetadata
|
||||
import nextflow.script.ScriptMeta
|
||||
import nextflow.script.TokenVar
|
||||
import nextflow.script.WorkflowMetadata
|
||||
import nextflow.script.params.EnvOutParam
|
||||
import nextflow.script.params.FileInParam
|
||||
import nextflow.script.params.FileOutParam
|
||||
import nextflow.script.params.InParam
|
||||
import nextflow.script.params.OutParam
|
||||
import nextflow.script.params.StdInParam
|
||||
import nextflow.script.params.StdOutParam
|
||||
import nextflow.script.params.ValueInParam
|
||||
import nextflow.script.params.ValueOutParam
|
||||
import nextflow.trace.event.FilePublishEvent
|
||||
import nextflow.trace.event.TaskEvent
|
||||
import nextflow.trace.event.WorkflowOutputEvent
|
||||
import nextflow.util.CacheHelper
|
||||
import nextflow.util.PathNormalizer
|
||||
import spock.lang.Shared
|
||||
import spock.lang.Specification
|
||||
import spock.lang.Unroll
|
||||
|
||||
import static nextflow.lineage.fs.LinPath.*
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class LinObserverTest extends Specification {
|
||||
@Shared
|
||||
Path lidFolder = Files.createTempDirectory("lid")
|
||||
def cleanupSpec(){
|
||||
lidFolder.deleteDir()
|
||||
}
|
||||
|
||||
def 'should normalize paths' (){
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def workDir = folder.resolve("workDir")
|
||||
def projectDir = folder.resolve("projectDir")
|
||||
def metadata = Mock(WorkflowMetadata){
|
||||
getRepository() >> "https://nextflow.io/nf-test/"
|
||||
getCommitId() >> "123456"
|
||||
getScriptId() >> "78910"
|
||||
getProjectDir() >> projectDir
|
||||
getWorkDir() >> workDir
|
||||
}
|
||||
def params = [path: workDir.resolve("path/file.txt"), sequence: projectDir.resolve("file2.txt").toString(), value: 12]
|
||||
when:
|
||||
def results = LinObserver.getNormalizedParams(params, new PathNormalizer(metadata))
|
||||
then:
|
||||
results.size() == 3
|
||||
results.get(0).name == "path"
|
||||
results.get(0).type == Path.simpleName
|
||||
results.get(0).value == "work/path/file.txt"
|
||||
results.get(1).name == "sequence"
|
||||
results.get(1).type == "String"
|
||||
results.get(1).value == projectDir.resolve("file2.txt").toString()
|
||||
results.get(2).name == "value"
|
||||
results.get(2).type == "Integer"
|
||||
results.get(2).value == 12
|
||||
|
||||
cleanup:
|
||||
ScriptMeta.reset()
|
||||
folder?.deleteDir()
|
||||
}
|
||||
def 'should collect script files' () {
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
and:
|
||||
def config = [workflow:[lineage:[enabled: true, store:[location:folder.toString()]]]]
|
||||
def store = new DefaultLinStore();
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def scriptFile = folder.resolve("main.nf")
|
||||
def module1 = folder.resolve("a_script1.nf"); module1.text = 'hola'
|
||||
def module2 = folder.resolve("b_script2.nf"); module2.text = 'world'
|
||||
and:
|
||||
|
||||
def metadata = Mock(WorkflowMetadata){
|
||||
getRepository() >> "https://nextflow.io/nf-test/"
|
||||
getCommitId() >> "123456"
|
||||
getScriptId() >> "78910"
|
||||
getScriptFile() >> scriptFile
|
||||
getProjectDir() >> folder.resolve("projectDir")
|
||||
getWorkDir() >> folder.resolve("workDir")
|
||||
}
|
||||
def session = Mock(Session) {
|
||||
getConfig() >> config
|
||||
getUniqueId() >> uniqueId
|
||||
getRunName() >> "test_run"
|
||||
getWorkflowMetadata() >> metadata
|
||||
getParams() >> new ScriptBinding.ParamsMap()
|
||||
}
|
||||
store.open(LineageConfig.create(session))
|
||||
def observer = Spy(new LinObserver(session, store))
|
||||
|
||||
when:
|
||||
def files = observer.collectScriptDataPaths(new PathNormalizer(metadata))
|
||||
then:
|
||||
observer.allScriptFiles() >> [ module2, scriptFile, module1 ]
|
||||
and:
|
||||
files.size() == 3
|
||||
and:
|
||||
files[0].path == "file://${scriptFile.toString()}"
|
||||
files[0].checksum == new Checksum("78910", "nextflow", "standard")
|
||||
and:
|
||||
files[1].path == "file://$module1"
|
||||
files[1].checksum == Checksum.ofNextflow(module1.text)
|
||||
and:
|
||||
files[2].path == "file://$module2"
|
||||
files[2].checksum == Checksum.ofNextflow(module2.text)
|
||||
|
||||
cleanup:
|
||||
ScriptMeta.reset()
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
def 'should save workflow' (){
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
|
||||
def store = new DefaultLinStore();
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def scriptFile = folder.resolve("main.nf")
|
||||
def map = [
|
||||
repository: "https://nextflow.io/nf-test/",
|
||||
commitId: "123456",
|
||||
scriptId: "78910",
|
||||
scriptFile: scriptFile,
|
||||
projectDir: folder.resolve("projectDir"),
|
||||
revision: "main",
|
||||
projectName: "nextflow.io/nf-test",
|
||||
workDir: folder.resolve("workDir")
|
||||
]
|
||||
def metadata = Mock(WorkflowMetadata){
|
||||
getRepository() >> map.repository
|
||||
getCommitId() >> map.commitId
|
||||
getScriptId() >> map.scriptId
|
||||
getScriptFile() >> map.scriptFile
|
||||
getProjectDir() >> map.projectDir
|
||||
getRevision() >> map.revision
|
||||
getProjectName() >> map.projectName
|
||||
getWorkDir() >> map.workDir
|
||||
toMap() >> map
|
||||
}
|
||||
def session = Mock(Session) {
|
||||
getConfig() >> config
|
||||
getUniqueId() >> uniqueId
|
||||
getRunName() >> "test_run"
|
||||
getWorkflowMetadata() >> metadata
|
||||
getParams() >> new ScriptBinding.ParamsMap()
|
||||
}
|
||||
store.open(LineageConfig.create(session))
|
||||
def observer = new LinObserver(session, store)
|
||||
def mainScript = new DataPath("file://${scriptFile.toString()}", new Checksum("78910", "nextflow", "standard"))
|
||||
def workflow = new Workflow([mainScript], map.repository, map.commitId)
|
||||
def workflowRun = new WorkflowRun(workflow, uniqueId.toString(), "test_run", [], config, map)
|
||||
when:
|
||||
observer.onFlowCreate(session)
|
||||
observer.onFlowBegin()
|
||||
then:
|
||||
folder.resolve("${observer.executionHash}/.data.json").text == new LinEncoder().encode(workflowRun)
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
def 'should strip sensitive user data from platform metadata in lineage' () {
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
|
||||
def store = new DefaultLinStore()
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def scriptFile = folder.resolve("main.nf")
|
||||
and:
|
||||
def platformMeta = new PlatformMetadata()
|
||||
platformMeta.user = new PlatformMetadata.User([
|
||||
id: 'u1234', userName: 'john-smith', email: 'john.smith@acme.com',
|
||||
firstName: 'John', lastName: 'Smith', organization: 'ACME'
|
||||
])
|
||||
platformMeta.workspace = new PlatformMetadata.Workspace([
|
||||
workspaceId: '1234', workspaceName: 'my-ws', workspaceFullName: 'My WS', orgName: 'ACME'
|
||||
])
|
||||
def map = [
|
||||
repository: "https://nextflow.io/nf-test/",
|
||||
commitId: "123456",
|
||||
scriptId: "78910",
|
||||
scriptFile: scriptFile,
|
||||
projectDir: folder.resolve("projectDir"),
|
||||
revision: "main",
|
||||
projectName: "nextflow.io/nf-test",
|
||||
workDir: folder.resolve("workDir"),
|
||||
platform: platformMeta
|
||||
]
|
||||
def metadata = Mock(WorkflowMetadata) {
|
||||
getRepository() >> map.repository
|
||||
getCommitId() >> map.commitId
|
||||
getScriptId() >> map.scriptId
|
||||
getScriptFile() >> map.scriptFile
|
||||
getProjectDir() >> map.projectDir
|
||||
getWorkDir() >> map.workDir
|
||||
toMap() >> map
|
||||
}
|
||||
def session = Mock(Session) {
|
||||
getConfig() >> config
|
||||
getUniqueId() >> uniqueId
|
||||
getRunName() >> "test_run"
|
||||
getWorkflowMetadata() >> metadata
|
||||
getParams() >> new ScriptBinding.ParamsMap()
|
||||
}
|
||||
store.open(LineageConfig.create(session))
|
||||
def observer = new LinObserver(session, store)
|
||||
|
||||
when:
|
||||
observer.onFlowCreate(session)
|
||||
observer.onFlowBegin()
|
||||
def stored = store.load(observer.executionHash) as WorkflowRun
|
||||
|
||||
then:
|
||||
def storedPlatform = stored.metadata.platform as Map
|
||||
storedPlatform.user.id == 'u1234'
|
||||
storedPlatform.user.userName == 'john-smith'
|
||||
storedPlatform.user.organization == 'ACME'
|
||||
!storedPlatform.user.containsKey('email')
|
||||
!storedPlatform.user.containsKey('firstName')
|
||||
!storedPlatform.user.containsKey('lastName')
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
def 'should convert Path values to URI strings in workflow metadata' () {
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
|
||||
def store = new DefaultLinStore()
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def scriptFile = folder.resolve("main.nf")
|
||||
def workDir = folder.resolve("workDir")
|
||||
def somePath = folder.resolve("data/input.txt")
|
||||
def map = [
|
||||
repository : "https://nextflow.io/nf-test/",
|
||||
commitId : "abc123",
|
||||
scriptId : "78910",
|
||||
scriptFile : scriptFile,
|
||||
projectDir : folder.resolve("projectDir"),
|
||||
workDir : workDir,
|
||||
somePathKey : somePath,
|
||||
someStringKey: "hello"
|
||||
]
|
||||
def metadata = Mock(WorkflowMetadata) {
|
||||
getRepository() >> map.repository
|
||||
getCommitId() >> map.commitId
|
||||
getScriptId() >> map.scriptId
|
||||
getScriptFile() >> map.scriptFile
|
||||
getProjectDir() >> map.projectDir
|
||||
getWorkDir() >> map.workDir
|
||||
toMap() >> map
|
||||
}
|
||||
def session = Mock(Session) {
|
||||
getConfig() >> config
|
||||
getUniqueId() >> uniqueId
|
||||
getRunName() >> "test_run"
|
||||
getWorkflowMetadata() >> metadata
|
||||
getParams() >> new ScriptBinding.ParamsMap()
|
||||
}
|
||||
store.open(LineageConfig.create(session))
|
||||
def observer = new LinObserver(session, store)
|
||||
|
||||
when:
|
||||
observer.onFlowCreate(session)
|
||||
observer.onFlowBegin()
|
||||
def stored = store.load(observer.executionHash) as WorkflowRun
|
||||
|
||||
then:
|
||||
stored.metadata.somePathKey == FilesEx.toUriString(somePath)
|
||||
stored.metadata.someStringKey == "hello"
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
def 'should remove transient properties from workflow metadata' () {
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
|
||||
def store = new DefaultLinStore()
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def scriptFile = folder.resolve("main.nf")
|
||||
def map = [
|
||||
repository : "https://nextflow.io/nf-test/",
|
||||
commitId : "abc123",
|
||||
scriptId : "78910",
|
||||
scriptFile : scriptFile,
|
||||
projectDir : folder.resolve("projectDir"),
|
||||
workDir : folder.resolve("workDir"),
|
||||
// transient properties that must be removed
|
||||
completed : new Date(),
|
||||
duration : 1000L,
|
||||
exitStatus : 0,
|
||||
errorMessage: "none",
|
||||
errorReport : "none",
|
||||
stats : [:],
|
||||
success : true
|
||||
]
|
||||
def metadata = Mock(WorkflowMetadata) {
|
||||
getRepository() >> map.repository
|
||||
getCommitId() >> map.commitId
|
||||
getScriptId() >> map.scriptId
|
||||
getScriptFile() >> map.scriptFile
|
||||
getProjectDir() >> map.projectDir
|
||||
getWorkDir() >> map.workDir
|
||||
toMap() >> map
|
||||
}
|
||||
def session = Mock(Session) {
|
||||
getConfig() >> config
|
||||
getUniqueId() >> uniqueId
|
||||
getRunName() >> "test_run"
|
||||
getWorkflowMetadata() >> metadata
|
||||
getParams() >> new ScriptBinding.ParamsMap()
|
||||
}
|
||||
store.open(LineageConfig.create(session))
|
||||
def observer = new LinObserver(session, store)
|
||||
|
||||
when:
|
||||
observer.onFlowCreate(session)
|
||||
observer.onFlowBegin()
|
||||
def stored = store.load(observer.executionHash) as WorkflowRun
|
||||
|
||||
then:
|
||||
!stored.metadata.containsKey('completed')
|
||||
!stored.metadata.containsKey('duration')
|
||||
!stored.metadata.containsKey('exitStatus')
|
||||
!stored.metadata.containsKey('errorMessage')
|
||||
!stored.metadata.containsKey('errorReport')
|
||||
!stored.metadata.containsKey('stats')
|
||||
!stored.metadata.containsKey('success')
|
||||
stored.metadata.containsKey('commitId')
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
def 'should convert NextflowMeta to JSON map in workflow metadata' () {
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
|
||||
def store = new DefaultLinStore()
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def scriptFile = folder.resolve("main.nf")
|
||||
def nfMeta = new NextflowMeta("24.10.0", 9999, "01-01-2024 00:00 UTC")
|
||||
def map = [
|
||||
repository : "https://nextflow.io/nf-test/",
|
||||
commitId : "abc123",
|
||||
scriptId : "78910",
|
||||
scriptFile : scriptFile,
|
||||
projectDir : folder.resolve("projectDir"),
|
||||
workDir : folder.resolve("workDir"),
|
||||
nextflow : nfMeta
|
||||
]
|
||||
def metadata = Mock(WorkflowMetadata) {
|
||||
getRepository() >> map.repository
|
||||
getCommitId() >> map.commitId
|
||||
getScriptId() >> map.scriptId
|
||||
getScriptFile() >> map.scriptFile
|
||||
getProjectDir() >> map.projectDir
|
||||
getWorkDir() >> map.workDir
|
||||
toMap() >> map
|
||||
}
|
||||
def session = Mock(Session) {
|
||||
getConfig() >> config
|
||||
getUniqueId() >> uniqueId
|
||||
getRunName() >> "test_run"
|
||||
getWorkflowMetadata() >> metadata
|
||||
getParams() >> new ScriptBinding.ParamsMap()
|
||||
}
|
||||
store.open(LineageConfig.create(session))
|
||||
def observer = new LinObserver(session, store)
|
||||
|
||||
when:
|
||||
observer.onFlowCreate(session)
|
||||
observer.onFlowBegin()
|
||||
def stored = store.load(observer.executionHash) as WorkflowRun
|
||||
|
||||
then:
|
||||
stored.metadata.nextflow instanceof Map
|
||||
(stored.metadata.nextflow as Map).version == "24.10.0"
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
def 'should normalize configFiles in workflow metadata' () {
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
|
||||
def store = new DefaultLinStore()
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def scriptFile = folder.resolve("main.nf")
|
||||
def workDir = folder.resolve("workDir")
|
||||
def projectDir = folder.resolve("projectDir")
|
||||
def configFile = projectDir.resolve("nextflow.config")
|
||||
def map = [
|
||||
repository : "https://nextflow.io/nf-test/",
|
||||
commitId : "abc123",
|
||||
scriptId : "78910",
|
||||
scriptFile : scriptFile,
|
||||
projectDir : projectDir,
|
||||
workDir : workDir,
|
||||
configFiles : [configFile]
|
||||
]
|
||||
def metadata = Mock(WorkflowMetadata) {
|
||||
getRepository() >> map.repository
|
||||
getCommitId() >> map.commitId
|
||||
getScriptId() >> map.scriptId
|
||||
getScriptFile() >> map.scriptFile
|
||||
getProjectDir() >> map.projectDir
|
||||
getWorkDir() >> map.workDir
|
||||
toMap() >> map
|
||||
}
|
||||
def session = Mock(Session) {
|
||||
getConfig() >> config
|
||||
getUniqueId() >> uniqueId
|
||||
getRunName() >> "test_run"
|
||||
getWorkflowMetadata() >> metadata
|
||||
getParams() >> new ScriptBinding.ParamsMap()
|
||||
}
|
||||
store.open(LineageConfig.create(session))
|
||||
def observer = new LinObserver(session, store)
|
||||
|
||||
when:
|
||||
observer.onFlowCreate(session)
|
||||
observer.onFlowBegin()
|
||||
def stored = store.load(observer.executionHash) as WorkflowRun
|
||||
|
||||
then:
|
||||
def normalizer = new PathNormalizer(metadata)
|
||||
(stored.metadata.configFiles as List) == [normalizer.normalizePath(configFile)]
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
def 'should handle null user in platform metadata' () {
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
|
||||
def store = new DefaultLinStore()
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def scriptFile = folder.resolve("main.nf")
|
||||
def platformMeta = new PlatformMetadata()
|
||||
platformMeta.workflowId = 'wf-999'
|
||||
platformMeta.workflowUrl = 'https://tower.nf/orgs/org/workspaces/ws/watch/wf-999'
|
||||
// user is intentionally left null
|
||||
def map = [
|
||||
repository : "https://nextflow.io/nf-test/",
|
||||
commitId : "abc123",
|
||||
scriptId : "78910",
|
||||
scriptFile : scriptFile,
|
||||
projectDir : folder.resolve("projectDir"),
|
||||
workDir : folder.resolve("workDir"),
|
||||
platform : platformMeta
|
||||
]
|
||||
def metadata = Mock(WorkflowMetadata) {
|
||||
getRepository() >> map.repository
|
||||
getCommitId() >> map.commitId
|
||||
getScriptId() >> map.scriptId
|
||||
getScriptFile() >> map.scriptFile
|
||||
getProjectDir() >> map.projectDir
|
||||
getWorkDir() >> map.workDir
|
||||
toMap() >> map
|
||||
}
|
||||
def session = Mock(Session) {
|
||||
getConfig() >> config
|
||||
getUniqueId() >> uniqueId
|
||||
getRunName() >> "test_run"
|
||||
getWorkflowMetadata() >> metadata
|
||||
getParams() >> new ScriptBinding.ParamsMap()
|
||||
}
|
||||
store.open(LineageConfig.create(session))
|
||||
def observer = new LinObserver(session, store)
|
||||
|
||||
when:
|
||||
observer.onFlowCreate(session)
|
||||
observer.onFlowBegin()
|
||||
def stored = store.load(observer.executionHash) as WorkflowRun
|
||||
|
||||
then:
|
||||
def storedPlatform = stored.metadata.platform as Map
|
||||
storedPlatform.workflowId == 'wf-999'
|
||||
storedPlatform.workflowUrl == 'https://tower.nf/orgs/org/workspaces/ws/watch/wf-999'
|
||||
storedPlatform.user == null
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
def 'should return empty metadata when toMap throws' () {
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
|
||||
def store = new DefaultLinStore()
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def scriptFile = folder.resolve("main.nf")
|
||||
def metadata = Mock(WorkflowMetadata) {
|
||||
getRepository() >> "https://nextflow.io/nf-test/"
|
||||
getCommitId() >> "abc123"
|
||||
getScriptId() >> "78910"
|
||||
getScriptFile() >> scriptFile
|
||||
getProjectDir() >> folder.resolve("projectDir")
|
||||
getWorkDir() >> folder.resolve("workDir")
|
||||
toMap() >> { throw new RuntimeException("metadata error") }
|
||||
}
|
||||
def session = Mock(Session) {
|
||||
getConfig() >> config
|
||||
getUniqueId() >> uniqueId
|
||||
getRunName() >> "test_run"
|
||||
getWorkflowMetadata() >> metadata
|
||||
getParams() >> new ScriptBinding.ParamsMap()
|
||||
}
|
||||
store.open(LineageConfig.create(session))
|
||||
def observer = new LinObserver(session, store)
|
||||
|
||||
when:
|
||||
observer.onFlowCreate(session)
|
||||
observer.onFlowBegin()
|
||||
def stored = store.load(observer.executionHash) as WorkflowRun
|
||||
|
||||
then:
|
||||
stored.metadata == null
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should get parameter type' () {
|
||||
expect:
|
||||
LinObserver.getParameterType(PARAM) == STRING
|
||||
where:
|
||||
PARAM | STRING
|
||||
null | null
|
||||
new FileInParam(null, []) | "path"
|
||||
new ValueOutParam(null, []) | "val"
|
||||
new EnvOutParam(null, []) | "env"
|
||||
new StdInParam(null, []) | "stdin"
|
||||
new StdOutParam(null, []) | "stdout"
|
||||
Path.of("test") | "Path"
|
||||
["test"] | "Collection"
|
||||
[key:"value"] | "Map"
|
||||
}
|
||||
|
||||
def 'should save task run' () {
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test').toRealPath()
|
||||
def config = [workflow:[lineage:[enabled: true, store:[location:folder.toString()]]]]
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def workDir = folder.resolve("work")
|
||||
def session = Mock(Session) {
|
||||
getConfig()>>config
|
||||
getUniqueId()>>uniqueId
|
||||
getRunName()>>"test_run"
|
||||
getWorkDir() >> workDir
|
||||
}
|
||||
def metadata = Mock(WorkflowMetadata){
|
||||
getRepository() >> "https://nextflow.io/nf-test/"
|
||||
getCommitId() >> "123456"
|
||||
getScriptId() >> "78910"
|
||||
getProjectDir() >> folder.resolve("projectDir")
|
||||
getWorkDir() >> workDir
|
||||
}
|
||||
and:
|
||||
def store = new DefaultLinStore();
|
||||
store.open(LineageConfig.create(session))
|
||||
and:
|
||||
def observer = Spy(new LinObserver(session, store))
|
||||
def normalizer = new PathNormalizer(metadata)
|
||||
observer.executionHash = "hash"
|
||||
observer.normalizer = normalizer
|
||||
and:
|
||||
def hash = HashCode.fromString("1234567890")
|
||||
def taskWd = workDir.resolve('12/34567890')
|
||||
Files.createDirectories(taskWd)
|
||||
and:
|
||||
observer.getTaskGlobalVars(_) >> [:]
|
||||
observer.getTaskBinEntries(_) >> []
|
||||
|
||||
and: 'Task Inputs'
|
||||
def inputs = new LinkedHashMap<InParam, Object>()
|
||||
// File from task
|
||||
inputs.put(new FileInParam(null, []).bind("file1"), [new FileHolder(workDir.resolve('78/567890/file1.txt'))])
|
||||
// Normal file
|
||||
def file = folder.resolve("file2.txt")
|
||||
file.text = "this is a test file"
|
||||
def fileHash = CacheHelper.hasher(file).hash().toString()
|
||||
inputs.put(new FileInParam(null, []).bind("file2"), [new FileHolder(file)])
|
||||
//Value input
|
||||
inputs.put(new ValueInParam(null, []).bind("id"), "value")
|
||||
|
||||
and: 'Task Outputs'
|
||||
def outputs = new LinkedHashMap<OutParam, Object>()
|
||||
// Single Path output
|
||||
def outFile1 = taskWd.resolve('fileOut1.txt')
|
||||
outFile1.text = 'some data'
|
||||
def fileHash1 = CacheHelper.hasher(outFile1).hash().toString()
|
||||
def attrs1 = Files.readAttributes(outFile1, BasicFileAttributes)
|
||||
outputs.put(new FileOutParam(null, []).bind(new TokenVar("file1")), outFile1)
|
||||
// Collection Path output
|
||||
def outFile2 = taskWd.resolve('fileOut2.txt')
|
||||
outFile2.text = 'some other data'
|
||||
def fileHash2 = CacheHelper.hasher(outFile2).hash().toString()
|
||||
def attrs2 = Files.readAttributes(outFile2, BasicFileAttributes)
|
||||
outputs.put(new FileOutParam(null, []).bind(new TokenVar("file2")), [outFile2])
|
||||
outputs.put(new ValueOutParam(null, []).bind(new TokenVar("id")), "value")
|
||||
|
||||
and: 'Task description'
|
||||
def task = Mock(TaskRun) {
|
||||
getId() >> TaskId.of(100)
|
||||
getName() >> 'foo'
|
||||
getHash() >> hash
|
||||
getSource() >> 'echo task source'
|
||||
getScript() >> 'this is the script'
|
||||
getInputs() >> inputs
|
||||
getOutputs() >> outputs
|
||||
getWorkDir() >> taskWd
|
||||
}
|
||||
def handler = Mock(TaskHandler){
|
||||
getTask() >> task
|
||||
}
|
||||
|
||||
and: 'Expected LID objects'
|
||||
def sourceHash = CacheHelper.hasher('echo task source').hash().toString()
|
||||
def script = 'this is the script'
|
||||
def taskDescription = new nextflow.lineage.model.v1beta1.TaskRun(uniqueId.toString(), "foo",
|
||||
new Checksum(sourceHash, "nextflow", "standard"),
|
||||
script,
|
||||
[
|
||||
new Parameter("path", "file1", ['lid://78567890/file1.txt']),
|
||||
new Parameter("path", "file2", [[path: normalizer.normalizePath(file), checksum: [value:fileHash, algorithm: "nextflow", mode: "standard"]]]),
|
||||
new Parameter("val", "id", "value")
|
||||
], null, null, null, null, [:], [], "lid://hash")
|
||||
def dataOutput1 = new FileOutput(outFile1.toString(), new Checksum(fileHash1, "nextflow", "standard"),
|
||||
"lid://1234567890", "lid://hash", "lid://1234567890", attrs1.size(), LinUtils.toDate(attrs1.creationTime()), LinUtils.toDate(attrs1.lastModifiedTime()) )
|
||||
def dataOutput2 = new FileOutput(outFile2.toString(), new Checksum(fileHash2, "nextflow", "standard"),
|
||||
"lid://1234567890", "lid://hash", "lid://1234567890", attrs2.size(), LinUtils.toDate(attrs2.creationTime()), LinUtils.toDate(attrs2.lastModifiedTime()) )
|
||||
|
||||
when:
|
||||
observer.onTaskComplete(new TaskEvent(handler, null))
|
||||
def taskRunResult = store.load("${hash.toString()}")
|
||||
def dataOutputResult1 = store.load("${hash}/fileOut1.txt") as FileOutput
|
||||
def dataOutputResult2 = store.load("${hash}/fileOut2.txt") as FileOutput
|
||||
def taskOutputsResult = store.load("${hash}#output") as TaskOutput
|
||||
then:
|
||||
taskRunResult == taskDescription
|
||||
dataOutputResult1 == dataOutput1
|
||||
dataOutputResult2 == dataOutput2
|
||||
taskOutputsResult.taskRun == "lid://1234567890"
|
||||
taskOutputsResult.workflowRun == "lid://hash"
|
||||
taskOutputsResult.output.size() == 3
|
||||
taskOutputsResult.output.get(0).type == "path"
|
||||
taskOutputsResult.output.get(0).name == "file1"
|
||||
taskOutputsResult.output.get(0).value == "lid://1234567890/fileOut1.txt"
|
||||
taskOutputsResult.output.get(1).type == "path"
|
||||
taskOutputsResult.output.get(1).name == "file2"
|
||||
taskOutputsResult.output.get(1).value == ["lid://1234567890/fileOut2.txt"]
|
||||
taskOutputsResult.output.get(2).type == "val"
|
||||
taskOutputsResult.output.get(2).name == "id"
|
||||
taskOutputsResult.output.get(2).value == "value"
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
def 'should save task data output' () {
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
|
||||
def store = new DefaultLinStore();
|
||||
def session = Mock(Session) {
|
||||
getConfig()>>config
|
||||
}
|
||||
store.open(LineageConfig.create(session))
|
||||
def observer = Spy(new LinObserver(session, store))
|
||||
observer.executionHash = "hash"
|
||||
and:
|
||||
def workDir = folder.resolve('12/34567890')
|
||||
Files.createDirectories(workDir)
|
||||
and:
|
||||
def outFile = workDir.resolve('foo/bar/file.bam')
|
||||
Files.createDirectories(outFile.parent)
|
||||
outFile.text = 'some data'
|
||||
def fileHash = CacheHelper.hasher(outFile).hash().toString()
|
||||
and:
|
||||
def hash = HashCode.fromInt(123456789)
|
||||
and:
|
||||
def task = Mock(TaskRun) {
|
||||
getId() >> TaskId.of(100)
|
||||
getName() >> 'foo'
|
||||
getHash() >> hash
|
||||
getWorkDir() >> workDir
|
||||
}
|
||||
and:
|
||||
def attrs = Files.readAttributes(outFile, BasicFileAttributes)
|
||||
def output = new FileOutput(outFile.toString(), new Checksum(fileHash, "nextflow", "standard"),
|
||||
"lid://15cd5b07", "lid://hash", "lid://15cd5b07", attrs.size(), LinUtils.toDate(attrs.creationTime()), LinUtils.toDate(attrs.lastModifiedTime()) )
|
||||
and:
|
||||
observer.readAttributes(outFile) >> attrs
|
||||
|
||||
when:
|
||||
observer.storeTaskOutput(task, outFile)
|
||||
then:
|
||||
folder.resolve("${hash}/foo/bar/file.bam/.data.json").text == new LinEncoder().encode(output)
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
def 'should relativise task output dirs' (){
|
||||
when:
|
||||
def config = [workflow:[lineage:[enabled: true, store:[location:lidFolder.toString()]]]]
|
||||
def store = new DefaultLinStore();
|
||||
def session = Mock(Session) {
|
||||
getConfig()>>config
|
||||
}
|
||||
def hash = HashCode.fromInt(123456789)
|
||||
def taskConfig = Mock(TaskConfig){
|
||||
getStoreDir() >> STORE_DIR
|
||||
}
|
||||
def task = Mock(TaskRun) {
|
||||
getId() >> TaskId.of(100)
|
||||
getName() >> 'foo'
|
||||
getHash() >> hash
|
||||
getWorkDir() >> WORK_DIR
|
||||
getConfig() >> taskConfig
|
||||
}
|
||||
store.open(LineageConfig.create(session))
|
||||
def observer = new LinObserver(session, store)
|
||||
then:
|
||||
observer.getTaskRelative(task, PATH) == EXPECTED
|
||||
where:
|
||||
WORK_DIR | STORE_DIR | PATH | EXPECTED
|
||||
Path.of('/path/to/work/12/3456789') | Path.of('/path/to/storeDir') | Path.of('/path/to/work/12/3456789/relative') | "relative"
|
||||
Path.of('/path/to/work/12/3456789') | Path.of('/path/to/storeDir') | Path.of('/path/to/storeDir/relative') | "relative"
|
||||
Path.of('work/12/3456789') | Path.of('storeDir') | Path.of('work/12/3456789/relative') | "relative"
|
||||
Path.of('work/12/3456789') | Path.of('storeDir') | Path.of('storeDir/relative') | "relative"
|
||||
Path.of('work/12/3456789') | Path.of('storeDir') | Path.of('results/relative') | "results/relative"
|
||||
Path.of('/path/to/work/12/3456789') | Path.of('storeDir') | Path.of('./relative') | "relative"
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should return exception when relativize task output dirs'() {
|
||||
when:
|
||||
def config = [workflow:[lineage:[enabled: true, store:[location:lidFolder.toString()]]]]
|
||||
def store = new DefaultLinStore();
|
||||
def session = Mock(Session) {
|
||||
getConfig()>>config
|
||||
}
|
||||
def hash = HashCode.fromInt(123456789)
|
||||
def taskConfig = Mock(TaskConfig){
|
||||
getStoreDir() >> STORE_DIR
|
||||
}
|
||||
def task = Mock(TaskRun) {
|
||||
getId() >> TaskId.of(100)
|
||||
getName() >> 'foo'
|
||||
getHash() >> hash
|
||||
getWorkDir() >> WORK_DIR
|
||||
getConfig() >> taskConfig
|
||||
}
|
||||
store.open(LineageConfig.create(session))
|
||||
def observer = new LinObserver(session, store)
|
||||
observer.getTaskRelative(task, PATH)
|
||||
then:
|
||||
def e = thrown(IllegalArgumentException)
|
||||
e.message == "Cannot access the relative path for output '$PATH' and task '${task.name}'".toString()
|
||||
|
||||
where:
|
||||
WORK_DIR | STORE_DIR | PATH
|
||||
Path.of('/path/to/work/12/3456789') | Path.of('/path/to/storeDir') | Path.of('/another/path/relative')
|
||||
Path.of('/path/to/work/12/3456789') | Path.of('/path/to/storeDir') | Path.of('../path/to/storeDir/relative')
|
||||
}
|
||||
|
||||
def 'should relativize workflow output dirs' (){
|
||||
when:
|
||||
def config = [workflow:[lineage:[enabled: true, store:[location:lidFolder.toString()]]]]
|
||||
def store = new DefaultLinStore();
|
||||
def session = Mock(Session) {
|
||||
getOutputDir()>>OUTPUT_DIR
|
||||
getConfig()>>config
|
||||
}
|
||||
store.open(LineageConfig.create(session))
|
||||
def observer = new LinObserver(session, store)
|
||||
then:
|
||||
observer.getWorkflowRelative(PATH) == EXPECTED
|
||||
where:
|
||||
OUTPUT_DIR | PATH | EXPECTED
|
||||
Path.of('/path/to/outDir') | Path.of('/path/to/outDir/relative') | "relative"
|
||||
Path.of('outDir') | Path.of('outDir/relative') | "relative"
|
||||
Path.of('/path/to/outDir') | Path.of('results/relative') | "results/relative"
|
||||
Path.of('/path/to/outDir') | Path.of('./relative') | "relative"
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should return exception when relativize workflow output dirs' (){
|
||||
when:
|
||||
def config = [workflow:[lineage:[enabled: true, store:[location:lidFolder.toString()]]]]
|
||||
def store = new DefaultLinStore();
|
||||
def session = Mock(Session) {
|
||||
getOutputDir()>>OUTPUT_DIR
|
||||
getConfig()>>config
|
||||
}
|
||||
def observer = new LinObserver(session, store)
|
||||
observer.getWorkflowRelative(PATH)
|
||||
then:
|
||||
thrown(OutputRelativePathException)
|
||||
where:
|
||||
OUTPUT_DIR | PATH | EXPECTED
|
||||
Path.of('/path/to/outDir') | Path.of('/another/path/') | "relative"
|
||||
Path.of('/path/to/outDir') | Path.of('../relative') | "relative"
|
||||
}
|
||||
|
||||
def 'should save workflow output'() {
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
|
||||
def store = new DefaultLinStore();
|
||||
def outputDir = folder.resolve('results')
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def scriptFile = folder.resolve("main.nf")
|
||||
def workDir= folder.resolve("work")
|
||||
def metadata = Mock(WorkflowMetadata){
|
||||
getRepository() >> "https://nextflow.io/nf-test/"
|
||||
getCommitId() >> "123456"
|
||||
getScriptId() >> "78910"
|
||||
getScriptFile() >> scriptFile
|
||||
getProjectDir() >> folder.resolve("projectDir")
|
||||
getWorkDir() >> workDir
|
||||
}
|
||||
def session = Mock(Session) {
|
||||
getConfig()>>config
|
||||
getOutputDir()>>outputDir
|
||||
getWorkDir() >> workDir
|
||||
getWorkflowMetadata()>>metadata
|
||||
getUniqueId()>>uniqueId
|
||||
getRunName()>>"test_run"
|
||||
getParams() >> new ScriptBinding.ParamsMap()
|
||||
}
|
||||
store.open(LineageConfig.create(session))
|
||||
def observer = new LinObserver(session, store)
|
||||
def encoder = new LinEncoder()
|
||||
|
||||
when: 'Starting workflow'
|
||||
observer.onFlowCreate(session)
|
||||
observer.onFlowBegin()
|
||||
then: 'History file should contain execution hash'
|
||||
def lid = LinHistoryRecord.parse(folder.resolve(".history/${observer.executionHash}").text)
|
||||
lid.runLid == asUriString(observer.executionHash)
|
||||
lid.sessionId == uniqueId
|
||||
lid.runName == "test_run"
|
||||
|
||||
when: ' publish output with source file'
|
||||
def outFile1 = outputDir.resolve('foo/file.bam')
|
||||
Files.createDirectories(outFile1.parent)
|
||||
outFile1.text = 'some data1'
|
||||
def sourceFile1 = workDir.resolve('12/3987/file.bam')
|
||||
Files.createDirectories(sourceFile1.parent)
|
||||
sourceFile1.text = 'some data1'
|
||||
observer.onFilePublish(new FilePublishEvent(sourceFile1, outFile1))
|
||||
observer.onWorkflowOutput(new WorkflowOutputEvent("a", outFile1))
|
||||
|
||||
then: 'check file 1 output metadata in lid store'
|
||||
def attrs1 = Files.readAttributes(outFile1, BasicFileAttributes)
|
||||
def fileHash1 = CacheHelper.hasher(outFile1).hash().toString()
|
||||
def output1 = new FileOutput(outFile1.toString(), new Checksum(fileHash1, "nextflow", "standard"),
|
||||
"lid://123987/file.bam", "$LID_PROT${observer.executionHash}", null,
|
||||
attrs1.size(), LinUtils.toDate(attrs1.creationTime()), LinUtils.toDate(attrs1.lastModifiedTime()) )
|
||||
folder.resolve("${observer.executionHash}/foo/file.bam/.data.json").text == encoder.encode(output1)
|
||||
|
||||
when: 'publish without source path'
|
||||
def outFile2 = outputDir.resolve('foo/file2.bam')
|
||||
Files.createDirectories(outFile2.parent)
|
||||
outFile2.text = 'some data2'
|
||||
def attrs2 = Files.readAttributes(outFile2, BasicFileAttributes)
|
||||
def fileHash2 = CacheHelper.hasher(outFile2).hash().toString()
|
||||
observer.onFilePublish(new FilePublishEvent(null, outFile2))
|
||||
observer.onWorkflowOutput(new WorkflowOutputEvent("b", outFile2))
|
||||
then: 'Check outFile2 metadata in lid store'
|
||||
def output2 = new FileOutput(outFile2.toString(), new Checksum(fileHash2, "nextflow", "standard"),
|
||||
"lid://${observer.executionHash}" , "lid://${observer.executionHash}", null,
|
||||
attrs2.size(), LinUtils.toDate(attrs2.creationTime()), LinUtils.toDate(attrs2.lastModifiedTime()) )
|
||||
folder.resolve("${observer.executionHash}/foo/file2.bam/.data.json").text == encoder.encode(output2)
|
||||
|
||||
when: 'Workflow complete'
|
||||
observer.onFlowComplete()
|
||||
then: 'Check WorkflowOutput is written in the lid store'
|
||||
def resultsRetrieved = store.load("${observer.executionHash}#output") as WorkflowOutput
|
||||
resultsRetrieved.output == [new Parameter(Path.simpleName, "a", "lid://${observer.executionHash}/foo/file.bam"), new Parameter(Path.simpleName, "b", "lid://${observer.executionHash}/foo/file2.bam")]
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
def 'should not save workflow output entry when no outputs'() {
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def config = [lineage: [enabled: true, store: [location: folder.toString()]]]
|
||||
def store = new DefaultLinStore();
|
||||
def outputDir = folder.resolve('results')
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def scriptFile = folder.resolve("main.nf")
|
||||
def workDir = folder.resolve("work")
|
||||
def metadata = Mock(WorkflowMetadata) {
|
||||
getRepository() >> "https://nextflow.io/nf-test/"
|
||||
getCommitId() >> "123456"
|
||||
getScriptId() >> "78910"
|
||||
getScriptFile() >> scriptFile
|
||||
getProjectDir() >> folder.resolve("projectDir")
|
||||
getWorkDir() >> workDir
|
||||
}
|
||||
def session = Mock(Session) {
|
||||
getConfig() >> config
|
||||
getOutputDir() >> outputDir
|
||||
getWorkDir() >> workDir
|
||||
getWorkflowMetadata() >> metadata
|
||||
getUniqueId() >> uniqueId
|
||||
getRunName() >> "test_run"
|
||||
getParams() >> new ScriptBinding.ParamsMap()
|
||||
}
|
||||
store.open(LineageConfig.create(session))
|
||||
def observer = new LinObserver(session, store)
|
||||
|
||||
when:
|
||||
observer.onFlowCreate(session)
|
||||
observer.onFlowBegin()
|
||||
observer.onFlowComplete()
|
||||
def resultFile = folder.resolve("${observer.executionHash}#output")
|
||||
then:
|
||||
!resultFile.exists()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package nextflow.lineage
|
||||
|
||||
import spock.lang.Specification
|
||||
|
||||
/**
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
class LinPropertyValidationTest extends Specification{
|
||||
|
||||
def 'should throw exception when property does not exist'(){
|
||||
when:
|
||||
new LinPropertyValidator().validate(['value', 'not_existing'])
|
||||
then:
|
||||
def e = thrown(IllegalArgumentException)
|
||||
e.message.startsWith( "Property 'not_existing' doesn't exist in the lineage model")
|
||||
}
|
||||
|
||||
def 'should not throw exception when property exist'(){
|
||||
when:
|
||||
new LinPropertyValidator().validate(['value', 'output'])
|
||||
then:
|
||||
noExceptionThrown()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage
|
||||
|
||||
import java.time.ZoneId
|
||||
|
||||
import nextflow.lineage.model.v1beta1.Checksum
|
||||
import nextflow.lineage.model.v1beta1.DataPath
|
||||
import nextflow.lineage.model.v1beta1.Parameter
|
||||
import nextflow.lineage.model.v1beta1.Workflow
|
||||
import nextflow.lineage.model.v1beta1.WorkflowOutput
|
||||
import nextflow.lineage.model.v1beta1.WorkflowRun
|
||||
import nextflow.lineage.config.LineageConfig
|
||||
import spock.lang.Specification
|
||||
import spock.lang.TempDir
|
||||
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.attribute.FileTime
|
||||
import java.time.Instant
|
||||
import java.time.OffsetDateTime
|
||||
import java.time.ZoneOffset
|
||||
|
||||
class LinUtilsTest extends Specification{
|
||||
|
||||
@TempDir
|
||||
Path tempDir
|
||||
|
||||
Path storeLocation
|
||||
LineageConfig config
|
||||
|
||||
def setup() {
|
||||
storeLocation = tempDir.resolve("store")
|
||||
def configMap = [enabled: true, store: [location: storeLocation.toString()]]
|
||||
config = new LineageConfig(configMap)
|
||||
}
|
||||
|
||||
def 'should convert to Date'(){
|
||||
expect:
|
||||
LinUtils.toDate(FILE_TIME) == DATE
|
||||
where:
|
||||
FILE_TIME | DATE
|
||||
null | null
|
||||
FileTime.fromMillis(1234) | Instant.ofEpochMilli(1234).atZone(ZoneId.systemDefault())?.toOffsetDateTime()
|
||||
}
|
||||
|
||||
def 'should convert to FileTime'(){
|
||||
expect:
|
||||
LinUtils.toFileTime(DATE) == FILE_TIME
|
||||
where:
|
||||
FILE_TIME | DATE
|
||||
null | null
|
||||
FileTime.fromMillis(1234) | OffsetDateTime.ofInstant(Instant.ofEpochMilli(1234), ZoneOffset.UTC)
|
||||
}
|
||||
|
||||
|
||||
def 'should get lineage record'() {
|
||||
given:
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def mainScript = new DataPath("file://path/to/main.nf", new Checksum("78910", "nextflow", "standard"))
|
||||
def workflow = new Workflow([mainScript], "https://nextflow.io/nf-test/", "123456")
|
||||
def key1 = "testKey"
|
||||
def value1 = new WorkflowRun(workflow, uniqueId.toString(), "test_run", [new Parameter("String", "param1", "value1"), new Parameter("String", "param2", "value2")])
|
||||
def outputs1 = new WorkflowOutput(OffsetDateTime.now(), "lid://testKey", [new Parameter( "String", "output", "name")] )
|
||||
def key2 = "testKey2"
|
||||
def value2 = new WorkflowRun(workflow, uniqueId.toString(), "test_run2", [new Parameter("String", "param1", "value1"), new Parameter("String", "param2", "value2")])
|
||||
def lidStore = new DefaultLinStore()
|
||||
lidStore.open(config)
|
||||
lidStore.save(key1, value1)
|
||||
lidStore.save("$key1#output", outputs1)
|
||||
lidStore.save(key2, value2)
|
||||
|
||||
when:
|
||||
def params = LinUtils.getMetadataObject(lidStore, new URI('lid://testKey#params'))
|
||||
then:
|
||||
params instanceof List<Parameter>
|
||||
(params as List<Parameter>).size() == 2
|
||||
|
||||
when:
|
||||
def outputs = LinUtils.getMetadataObject(lidStore, new URI('lid://testKey#output'))
|
||||
then:
|
||||
outputs instanceof List<Parameter>
|
||||
def param = (outputs as List)[0] as Parameter
|
||||
param.name == "output"
|
||||
|
||||
when:
|
||||
outputs = LinUtils.getMetadataObject(lidStore, new URI('lid://testKey2#output'))
|
||||
then:
|
||||
outputs instanceof List
|
||||
(outputs as List).isEmpty()
|
||||
|
||||
when:
|
||||
LinUtils.getMetadataObject(lidStore, new URI('lid://testKey#no-exist'))
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
|
||||
when:
|
||||
LinUtils.getMetadataObject(lidStore, new URI('lid://testKey#outputs.no-exist'))
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
|
||||
when:
|
||||
LinUtils.getMetadataObject(lidStore, new URI('lid://no-exist#something'))
|
||||
then:
|
||||
thrown(FileNotFoundException)
|
||||
}
|
||||
|
||||
|
||||
def "should check params in an object"() {
|
||||
given:
|
||||
def obj = [ "type": "value", "workflow": ["repository": "subvalue"], "output" : [ ["path":"/to/file"],["path":"file2"] ], "labels": ["a","b"] ]
|
||||
|
||||
expect:
|
||||
LinUtils.checkParams(obj, PARAMS) == EXPECTED
|
||||
|
||||
where:
|
||||
PARAMS | EXPECTED
|
||||
["type": ["value"]] | true
|
||||
["type": ["wrong"]] | false
|
||||
["workflow.repository": ["subvalue"]] | true
|
||||
["workflow.repository": ["wrong"]] | false
|
||||
["output.path": ["wrong"]] | false
|
||||
["output.path": ["/to/file"]] | true
|
||||
["output.path": ["file2"]] | true
|
||||
["labels": ["a"]] | true
|
||||
["labels": ["c"]] | false
|
||||
["labels": ["a","b"]] | true
|
||||
["labels": ["a","b","c"]] | false
|
||||
["type" : ["value", "value2"]] | false
|
||||
|
||||
}
|
||||
|
||||
def "should navigate in object params"() {
|
||||
given:
|
||||
def obj = [
|
||||
"key1": "value1",
|
||||
"nested": [
|
||||
"subkey": "subvalue"
|
||||
]
|
||||
]
|
||||
|
||||
expect:
|
||||
LinUtils.navigate(obj, PATH) == EXPECTED
|
||||
|
||||
where:
|
||||
PATH | EXPECTED
|
||||
"key1" | "value1"
|
||||
"nested.subkey" | "subvalue"
|
||||
"wrongKey" | null
|
||||
}
|
||||
|
||||
def "should add objects matching parameters"() {
|
||||
given:
|
||||
def results = []
|
||||
|
||||
when:
|
||||
LinUtils.treatObject(OBJECT, PARAMS, results)
|
||||
|
||||
then:
|
||||
results == EXPECTED
|
||||
|
||||
where:
|
||||
OBJECT | PARAMS | EXPECTED
|
||||
["field": "value"] | ["field": ["value"]] | [["field": "value"]]
|
||||
["field": "wrong"] | ["field": ["value"]] | []
|
||||
[["field": "value"], ["field": "x"]] | ["field": ["value"]] | [["field": "value"]]
|
||||
"string" | [:] | ["string"]
|
||||
["nested": ["subfield": "match"]] | ["nested.subfield": ["match"]] | [["nested": ["subfield": "match"]]]
|
||||
["nested": ["subfield": "nomatch"]] | ["nested.subfield": ["match"]] | []
|
||||
[["nested": ["subfield": "match"]], ["nested": ["subfield": "other"]]] | ["nested.subfield": ["match"]] | [["nested": ["subfield": "match"]]]
|
||||
}
|
||||
|
||||
def 'should navigate' (){
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def mainScript = new DataPath("file://path/to/main.nf", new Checksum("78910", "nextflow", "standard"))
|
||||
def workflow = new Workflow([mainScript], "https://nextflow.io/nf-test/", "123456")
|
||||
def wfRun = new WorkflowRun(workflow, uniqueId.toString(), "test_run", [new Parameter("String", "param1", [key: "value1"]), new Parameter("String", "param2", "value2")])
|
||||
|
||||
expect:
|
||||
LinUtils.navigate(wfRun, "workflow.commitId") == "123456"
|
||||
LinUtils.navigate(wfRun, "params.name") == ["param1", "param2"]
|
||||
LinUtils.navigate(wfRun, "params.value.key") == "value1"
|
||||
LinUtils.navigate(wfRun, "params.value.no-exist") == null
|
||||
LinUtils.navigate(wfRun, "params.no-exist") == null
|
||||
LinUtils.navigate(wfRun, "no-exist") == null
|
||||
LinUtils.navigate(null, "something") == null
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,478 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.cli
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.time.Instant
|
||||
import java.time.OffsetDateTime
|
||||
import java.time.ZoneOffset
|
||||
|
||||
import nextflow.SysEnv
|
||||
import nextflow.config.ConfigMap
|
||||
import nextflow.dag.MermaidHtmlRenderer
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.file.FileHelper
|
||||
import nextflow.lineage.DefaultLinHistoryLog
|
||||
import nextflow.lineage.LinHistoryRecord
|
||||
import nextflow.lineage.LinStoreFactory
|
||||
import nextflow.lineage.fs.LinFileSystemProvider
|
||||
import nextflow.lineage.model.v1beta1.Checksum
|
||||
import nextflow.lineage.model.v1beta1.DataPath
|
||||
import nextflow.lineage.model.v1beta1.FileOutput
|
||||
import nextflow.lineage.model.v1beta1.Parameter
|
||||
import nextflow.lineage.model.v1beta1.TaskRun
|
||||
import nextflow.lineage.model.v1beta1.Workflow
|
||||
import nextflow.lineage.model.v1beta1.WorkflowRun
|
||||
import nextflow.lineage.serde.LinEncoder
|
||||
import nextflow.plugin.Plugins
|
||||
import nextflow.util.CacheHelper
|
||||
import org.junit.Rule
|
||||
import spock.lang.Shared
|
||||
import spock.lang.Specification
|
||||
import test.OutputCapture
|
||||
|
||||
import static test.TestHelper.filterLogNoise
|
||||
|
||||
class LinCommandImplTest extends Specification{
|
||||
|
||||
@Shared
|
||||
Path tmpDir
|
||||
|
||||
@Shared
|
||||
Path storeLocation
|
||||
|
||||
@Shared
|
||||
ConfigMap configMap
|
||||
|
||||
def reset() {
|
||||
def provider = FileHelper.getProviderFor('lid') as LinFileSystemProvider
|
||||
provider?.reset()
|
||||
LinStoreFactory.reset()
|
||||
}
|
||||
|
||||
def setup() {
|
||||
reset()
|
||||
// clear the environment to avoid the local env pollute the test env
|
||||
SysEnv.push([:])
|
||||
tmpDir = Files.createTempDirectory('tmp')
|
||||
storeLocation = tmpDir.resolve("store")
|
||||
configMap = new ConfigMap([lineage: [enabled: true, store: [location: storeLocation.toString(), logLocation: storeLocation.resolve(".log").toString()]]])
|
||||
}
|
||||
|
||||
def cleanup() {
|
||||
Plugins.stop()
|
||||
LinStoreFactory.reset()
|
||||
SysEnv.pop()
|
||||
tmpDir?.deleteDir()
|
||||
}
|
||||
|
||||
def setupSpec() {
|
||||
reset()
|
||||
}
|
||||
|
||||
/*
|
||||
* Read more http://mrhaki.blogspot.com.es/2015/02/spocklight-capture-and-assert-system.html
|
||||
*/
|
||||
@Rule
|
||||
OutputCapture capture = new OutputCapture()
|
||||
|
||||
def 'should print executions lids' (){
|
||||
given:
|
||||
def historyFile = storeLocation.resolve(".history")
|
||||
def lidLog = new DefaultLinHistoryLog(historyFile)
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def date = new Date();
|
||||
def recordEntry = "${LinHistoryRecord.TIMESTAMP_FMT.format(date)}\trun_name\t${uniqueId}\tlid://123456".toString()
|
||||
lidLog.write("run_name", uniqueId, "lid://123456", date)
|
||||
when:
|
||||
new LinCommandImpl().list(configMap)
|
||||
def stdout = filterLogNoise(capture)
|
||||
|
||||
then:
|
||||
stdout.size() == 2
|
||||
stdout[1] == recordEntry
|
||||
}
|
||||
|
||||
def 'should print no history' (){
|
||||
given:
|
||||
def historyFile = storeLocation.resolve(".meta/.history")
|
||||
Files.createDirectories(historyFile.parent)
|
||||
|
||||
when:
|
||||
new LinCommandImpl().list(configMap)
|
||||
def stdout = filterLogNoise(capture, true)
|
||||
|
||||
then:
|
||||
stdout.size() == 1
|
||||
stdout[0] == "No workflow runs found in lineage history log"
|
||||
}
|
||||
|
||||
def 'should show lid content' (){
|
||||
given:
|
||||
def lidFile = storeLocation.resolve("12345/.data.json")
|
||||
Files.createDirectories(lidFile.parent)
|
||||
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
|
||||
def encoder = new LinEncoder().withPrettyPrint(true)
|
||||
def entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
|
||||
"lid://123987/file.bam","lid://123987/", null, 1234, time, time, null)
|
||||
def jsonSer = encoder.encode(entry)
|
||||
def expectedOutput = jsonSer
|
||||
lidFile.text = jsonSer
|
||||
when:
|
||||
new LinCommandImpl().view(configMap, ["lid://12345"])
|
||||
def stdout = filterLogNoise(capture)
|
||||
|
||||
then:
|
||||
stdout.size() == expectedOutput.readLines().size()
|
||||
stdout.join('\n') == expectedOutput
|
||||
}
|
||||
|
||||
def 'should show empty lists content' (){
|
||||
given:
|
||||
def lidFile = storeLocation.resolve("12345/.data.json")
|
||||
Files.createDirectories(lidFile.parent)
|
||||
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
|
||||
def encoder = new LinEncoder().withPrettyPrint(true)
|
||||
def entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
|
||||
"lid://123987/file.bam","lid://123987/", null, 1234, time, time, [])
|
||||
def jsonSer = encoder.encode(entry)
|
||||
def expectedOutput = '[]'
|
||||
lidFile.text = jsonSer
|
||||
when:
|
||||
new LinCommandImpl().view(configMap, ["lid://12345#labels"])
|
||||
def stdout = filterLogNoise(capture)
|
||||
|
||||
then:
|
||||
stdout.size() == expectedOutput.readLines().size()
|
||||
stdout.join('\n') == expectedOutput
|
||||
}
|
||||
|
||||
def 'should show empty lists when no outputs' () {
|
||||
def lidFile = storeLocation.resolve("12345/.data.json")
|
||||
Files.createDirectories(lidFile.parent)
|
||||
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
|
||||
def encoder = new LinEncoder().withPrettyPrint(true)
|
||||
def expectedOutput = '[]'
|
||||
def wf = new Workflow([new DataPath("/path/to/main.nf)")], "hello-nf", "aasdklk")
|
||||
def entry = new WorkflowRun(wf, "sessionId", "run_name",
|
||||
[new Parameter("String", "sample_id", "ggal_gut"),
|
||||
new Parameter("Integer", "reads", 2)])
|
||||
lidFile.text = encoder.encode(entry)
|
||||
|
||||
when:
|
||||
new LinCommandImpl().view(configMap, ["lid://12345#output"])
|
||||
def stdout = filterLogNoise(capture)
|
||||
|
||||
then:
|
||||
stdout.size() == expectedOutput.readLines().size()
|
||||
stdout.join('\n') == expectedOutput
|
||||
}
|
||||
|
||||
def 'should warn if no lid content' (){
|
||||
given:
|
||||
|
||||
when:
|
||||
new LinCommandImpl().view(configMap, ["lid://12345"])
|
||||
def stdout = filterLogNoise(capture)
|
||||
|
||||
then:
|
||||
stdout.size() == 1
|
||||
stdout[0] == "Error loading lid://12345 - Lineage record 12345 not found"
|
||||
}
|
||||
|
||||
def 'should get lineage lid content' (){
|
||||
given:
|
||||
|
||||
def outputHtml = tmpDir.resolve('lineage.html')
|
||||
|
||||
def lidFile = storeLocation.resolve("12345/file.bam/.data.json")
|
||||
def lidFile2 = storeLocation.resolve("123987/file.bam/.data.json")
|
||||
def lidFile3 = storeLocation.resolve("123987/.data.json")
|
||||
def lidFile4 = storeLocation.resolve("45678/output.txt/.data.json")
|
||||
def lidFile5 = storeLocation.resolve("45678/.data.json")
|
||||
Files.createDirectories(lidFile.parent)
|
||||
Files.createDirectories(lidFile2.parent)
|
||||
Files.createDirectories(lidFile3.parent)
|
||||
Files.createDirectories(lidFile4.parent)
|
||||
Files.createDirectories(lidFile5.parent)
|
||||
def encoder = new LinEncoder()
|
||||
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
|
||||
def entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
|
||||
"lid://123987/file.bam", "lid://45678", null, 1234, time, time, null)
|
||||
lidFile.text = encoder.encode(entry)
|
||||
entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
|
||||
"lid://123987", "lid://45678", "lid://123987", 1234, time, time, null)
|
||||
lidFile2.text = encoder.encode(entry)
|
||||
entry = new TaskRun("u345-2346-1stw2", "foo",
|
||||
new Checksum("abcde2345","nextflow","standard"),
|
||||
'this is a script',
|
||||
[new Parameter( "val", "sample_id","ggal_gut"),
|
||||
new Parameter("path","reads", ["lid://45678/output.txt"] ),
|
||||
new Parameter("path","input", [new DataPath("path/to/file",new Checksum("45372qe","nextflow","standard"))])
|
||||
],
|
||||
null, null, null, null, [:],[])
|
||||
lidFile3.text = encoder.encode(entry)
|
||||
entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
|
||||
"lid://45678", "lid://45678", null, 1234, time, time, null)
|
||||
lidFile4.text = encoder.encode(entry)
|
||||
entry = new TaskRun("u345-2346-1stw2", "bar",
|
||||
new Checksum("abfs2556","nextflow","standard"),
|
||||
'this is a script',
|
||||
null,null, null, null, null, [:],[])
|
||||
lidFile5.text = encoder.encode(entry)
|
||||
final network = """\
|
||||
flowchart TB
|
||||
lid://12345/file.bam["lid://12345/file.bam"]
|
||||
lid://123987/file.bam["lid://123987/file.bam"]
|
||||
lid://123987(["foo [lid://123987]"])
|
||||
ggal_gut["ggal_gut"]
|
||||
path/to/file["path/to/file"]
|
||||
lid://45678/output.txt["lid://45678/output.txt"]
|
||||
lid://45678(["bar [lid://45678]"])
|
||||
lid://123987/file.bam --> lid://12345/file.bam
|
||||
lid://123987 --> lid://123987/file.bam
|
||||
ggal_gut --> lid://123987
|
||||
lid://45678/output.txt --> lid://123987
|
||||
path/to/file --> lid://123987
|
||||
lid://45678 --> lid://45678/output.txt""".stripIndent()
|
||||
final template = MermaidHtmlRenderer.readTemplate()
|
||||
def expectedOutput = template.replace('REPLACE_WITH_NETWORK_DATA', network)
|
||||
|
||||
when:
|
||||
new LinCommandImpl().render(configMap, ["lid://12345/file.bam", outputHtml.toString()])
|
||||
def stdout = filterLogNoise(capture)
|
||||
|
||||
then:
|
||||
stdout.size() == 1
|
||||
stdout[0] == "Rendered lineage graph for lid://12345/file.bam to ${outputHtml}"
|
||||
outputHtml.exists()
|
||||
outputHtml.text == expectedOutput
|
||||
}
|
||||
|
||||
def 'should get lineage from workflow lid content' (){
|
||||
given:
|
||||
|
||||
def outputHtml = tmpDir.resolve('lineage.html')
|
||||
|
||||
def lidFile = storeLocation.resolve("12345/file.bam/.data.json")
|
||||
def lidFile3 = storeLocation.resolve("12345/.data.json")
|
||||
Files.createDirectories(lidFile.parent)
|
||||
Files.createDirectories(lidFile3.parent)
|
||||
def encoder = new LinEncoder()
|
||||
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
|
||||
def entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
|
||||
"lid://12345", "lid://12345", null, 1234, time, time, null)
|
||||
lidFile.text = encoder.encode(entry)
|
||||
def wf = new Workflow([new DataPath("/path/to/main.nf)")], "hello-nf", "aasdklk")
|
||||
entry = new WorkflowRun(wf,"sessionId","run_name",
|
||||
[new Parameter( "String", "sample_id","ggal_gut"),
|
||||
new Parameter("Integer","reads",2)])
|
||||
lidFile3.text = encoder.encode(entry)
|
||||
final network = """\
|
||||
flowchart TB
|
||||
lid://12345/file.bam["lid://12345/file.bam"]
|
||||
lid://12345(["run_name [lid://12345]"])
|
||||
ggal_gut["ggal_gut"]
|
||||
2.0["2.0"]
|
||||
lid://12345 --> lid://12345/file.bam
|
||||
ggal_gut --> lid://12345
|
||||
2.0 --> lid://12345""".stripIndent()
|
||||
final template = MermaidHtmlRenderer.readTemplate()
|
||||
def expectedOutput = template.replace('REPLACE_WITH_NETWORK_DATA', network)
|
||||
|
||||
when:
|
||||
new LinCommandImpl().render(configMap, ["lid://12345/file.bam", outputHtml.toString()])
|
||||
def stdout = filterLogNoise(capture)
|
||||
|
||||
then:
|
||||
stdout.size() == 1
|
||||
stdout[0] == "Rendered lineage graph for lid://12345/file.bam to ${outputHtml}"
|
||||
outputHtml.exists()
|
||||
outputHtml.text == expectedOutput
|
||||
}
|
||||
|
||||
def 'should show an error if trying to do a query'(){
|
||||
given:
|
||||
def lidFile = storeLocation.resolve("12345/.data.json")
|
||||
Files.createDirectories(lidFile.parent)
|
||||
def encoder = new LinEncoder().withPrettyPrint(true)
|
||||
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
|
||||
def entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
|
||||
"lid://123987/file.bam", "lid://123987/", null, 1234, time, time, null)
|
||||
def jsonSer = encoder.encode(entry)
|
||||
def expectedOutput = "Error loading lid:///?type=FileOutput - Cannot get record from the root LID URI"
|
||||
lidFile.text = jsonSer
|
||||
when:
|
||||
new LinCommandImpl().view(configMap, ["lid:///?type=FileOutput"])
|
||||
def stdout = filterLogNoise(capture)
|
||||
|
||||
then:
|
||||
stdout.size() == expectedOutput.readLines().size()
|
||||
stdout.join('\n') == expectedOutput
|
||||
}
|
||||
|
||||
def 'should diff'(){
|
||||
given:
|
||||
def lidFile = storeLocation.resolve("12345/.data.json")
|
||||
Files.createDirectories(lidFile.parent)
|
||||
def lidFile2 = storeLocation.resolve("67890/.data.json")
|
||||
Files.createDirectories(lidFile2.parent)
|
||||
def encoder = new LinEncoder().withPrettyPrint(true)
|
||||
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
|
||||
def entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
|
||||
"lid://123987/file.bam", "lid://123987/", null, 1234, time, time, null)
|
||||
def entry2 = new FileOutput("path/to/file2",new Checksum("42472qet","nextflow","standard"),
|
||||
"lid://123987/file2.bam", "lid://123987/", null, 1235, time, time, null)
|
||||
lidFile.text = encoder.encode(entry)
|
||||
lidFile2.text = encoder.encode(entry2)
|
||||
def expectedOutput = '''diff --git 12345 67890
|
||||
--- 12345
|
||||
+++ 67890
|
||||
@@ -2,16 +2,16 @@
|
||||
"version": "lineage/v1beta1",
|
||||
"kind": "FileOutput",
|
||||
"spec": {
|
||||
- "path": "path/to/file",
|
||||
+ "path": "path/to/file2",
|
||||
"checksum": {
|
||||
- "value": "45372qe",
|
||||
+ "value": "42472qet",
|
||||
"algorithm": "nextflow",
|
||||
"mode": "standard"
|
||||
},
|
||||
- "source": "lid://123987/file.bam",
|
||||
+ "source": "lid://123987/file2.bam",
|
||||
"workflowRun": "lid://123987/",
|
||||
"taskRun": null,
|
||||
- "size": 1234,
|
||||
+ "size": 1235,
|
||||
"createdAt": "1970-01-02T10:17:36.789Z",
|
||||
"modifiedAt": "1970-01-02T10:17:36.789Z",
|
||||
"labels": null
|
||||
'''
|
||||
|
||||
when:
|
||||
new LinCommandImpl().diff(configMap, ["lid://12345", "lid://67890"])
|
||||
def stdout = filterLogNoise(capture)
|
||||
|
||||
then:
|
||||
stdout.join('\n') == expectedOutput
|
||||
}
|
||||
|
||||
def 'should print error if no entry found diff'(){
|
||||
given:
|
||||
def lidFile = storeLocation.resolve("12345/.data.json")
|
||||
Files.createDirectories(lidFile.parent)
|
||||
def encoder = new LinEncoder().withPrettyPrint(true)
|
||||
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
|
||||
def entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
|
||||
"lid://123987/file.bam", "lid://123987/", null, 1234, time, time, null)
|
||||
lidFile.text = encoder.encode(entry)
|
||||
|
||||
when:
|
||||
new LinCommandImpl().diff(configMap, ["lid://89012", "lid://12345"])
|
||||
new LinCommandImpl().diff(configMap, ["lid://12345", "lid://67890"])
|
||||
def stdout = filterLogNoise(capture)
|
||||
|
||||
then:
|
||||
stdout.size() == 2
|
||||
stdout[0] == "No entry found for lid://89012."
|
||||
stdout[1] == "No entry found for lid://67890."
|
||||
}
|
||||
|
||||
def 'should print error store is not found in diff'(){
|
||||
when:
|
||||
def config = new ConfigMap()
|
||||
new LinCommandImpl().list(config)
|
||||
new LinCommandImpl().view(config, ["lid:///12345"])
|
||||
new LinCommandImpl().render(config, ["lid://12345", "output.html"])
|
||||
new LinCommandImpl().diff(config, ["lid://89012", "lid://12345"])
|
||||
|
||||
def stdout = filterLogNoise(capture)
|
||||
def expectedOutput = "Error lineage store not loaded - Check Nextflow configuration"
|
||||
then:
|
||||
stdout.size() == 4
|
||||
stdout[0] == expectedOutput
|
||||
stdout[1] == expectedOutput
|
||||
stdout[2] == expectedOutput
|
||||
stdout[3] == expectedOutput
|
||||
}
|
||||
|
||||
def 'should find metadata descriptions'(){
|
||||
given:
|
||||
def lidFile = storeLocation.resolve("123987/file.bam/.data.json")
|
||||
Files.createDirectories(lidFile.parent)
|
||||
def lidFile2 = storeLocation.resolve("123987/file2.bam/.data.json")
|
||||
Files.createDirectories(lidFile2.parent)
|
||||
def lidFile3 = storeLocation.resolve(".meta/123987/file3.bam/.data.json")
|
||||
Files.createDirectories(lidFile3.parent)
|
||||
def encoder = new LinEncoder().withPrettyPrint(true)
|
||||
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
|
||||
def entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
|
||||
"lid://123987/file.bam", "lid://123987/", null, 1234, time, time, ["experiment=test"])
|
||||
def entry2 = new FileOutput("path/to/file2",new Checksum("42472qet","nextflow","standard"),
|
||||
"lid://123987/file2.bam", "lid://123987/", null, 1235, time, time, ["experiment=test"])
|
||||
def entry3 = new FileOutput("path/to/file3",new Checksum("42472qet","nextflow","standard"),
|
||||
"lid://123987/file2.bam", "lid://123987/", null, 1235, time, time, null)
|
||||
def expectedOutput1 = 'lid://123987/file.bam\nlid://123987/file2.bam'
|
||||
def expectedOutput2 = 'lid://123987/file2.bam\nlid://123987/file.bam'
|
||||
lidFile.text = encoder.encode(entry)
|
||||
lidFile2.text = encoder.encode(entry2)
|
||||
lidFile3.text = encoder.encode(entry3)
|
||||
when:
|
||||
new LinCommandImpl().find(configMap, ["type=FileOutput", "label=experiment=test"])
|
||||
def stdout = filterLogNoise(capture)
|
||||
|
||||
then:
|
||||
stdout.join('\n') == expectedOutput1 || stdout.join('\n') == expectedOutput2
|
||||
}
|
||||
|
||||
def 'should print correct validate path' () {
|
||||
given:
|
||||
def outputFolder = tmpDir.resolve('output')
|
||||
Files.createDirectories(outputFolder)
|
||||
def outputFile = outputFolder.resolve('file1.txt')
|
||||
outputFile.text = "this is file1 == "
|
||||
|
||||
and:
|
||||
def encoder = new LinEncoder().withPrettyPrint(true)
|
||||
def hash = CacheHelper.hasher(outputFile).hash().toString()
|
||||
def correctData = new FileOutput(outputFile.toString(), new Checksum(hash,"nextflow", "standard"))
|
||||
def incorrectData = new FileOutput(outputFile.toString(), new Checksum("incorrectHash","nextflow", "standard"))
|
||||
def lid1 = storeLocation.resolve('12345/output/file1.txt/.data.json')
|
||||
Files.createDirectories(lid1.parent)
|
||||
lid1.text = encoder.encode(correctData)
|
||||
def lid2 = storeLocation.resolve('12345/output/file2.txt/.data.json')
|
||||
Files.createDirectories(lid2.parent)
|
||||
lid2.text = encoder.encode(incorrectData)
|
||||
when:
|
||||
new LinCommandImpl().check(configMap, ["lid://12345/output/file1.txt"])
|
||||
def stdout = filterLogNoise(capture)
|
||||
def expectedOutput1 = "Checksum validation succeed"
|
||||
then:
|
||||
stdout.size() == 1
|
||||
stdout[0] == expectedOutput1
|
||||
|
||||
when:
|
||||
new LinCommandImpl().check(configMap, ["lid://12345/output/file2.txt"])
|
||||
then:
|
||||
def err = thrown(AbortOperationException)
|
||||
err.message == "Checksum of '${outputFile}' does not match with lineage metadata"
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.config
|
||||
|
||||
|
||||
import spock.lang.Specification
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class LineageConfigTest extends Specification {
|
||||
|
||||
def 'should create default config' () {
|
||||
when:
|
||||
def config = new LineageConfig(Map.of())
|
||||
then:
|
||||
!config.enabled
|
||||
!config.store.location
|
||||
}
|
||||
|
||||
def 'should create default with enable' () {
|
||||
when:
|
||||
def config = new LineageConfig([enabled: true])
|
||||
then:
|
||||
config.enabled
|
||||
!config.store.location
|
||||
}
|
||||
|
||||
def 'should create data config with location' () {
|
||||
when:
|
||||
def config = new LineageConfig(enabled: true, store: [location: "/some/data/store"])
|
||||
then:
|
||||
config.enabled
|
||||
config.store.location == '/some/data/store'
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,496 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.fs
|
||||
|
||||
import java.nio.ByteBuffer
|
||||
import java.nio.channels.NonWritableChannelException
|
||||
import java.nio.file.AccessDeniedException
|
||||
import java.nio.file.AccessMode
|
||||
import java.nio.file.FileSystemNotFoundException
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.ProviderMismatchException
|
||||
import java.nio.file.StandardOpenOption
|
||||
import java.nio.file.attribute.BasicFileAttributes
|
||||
|
||||
import nextflow.Global
|
||||
import nextflow.Session
|
||||
import nextflow.lineage.DefaultLinStore
|
||||
import spock.lang.Shared
|
||||
import spock.lang.Specification
|
||||
/**
|
||||
* LID File system provider tests
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
class LinFileSystemProviderTest extends Specification {
|
||||
|
||||
@Shared def wdir = Files.createTempDirectory('wdir')
|
||||
@Shared def data = Files.createTempDirectory('work')
|
||||
|
||||
|
||||
def cleanupSpec(){
|
||||
wdir.deleteDir()
|
||||
data.deleteDir()
|
||||
}
|
||||
|
||||
def 'should return lid scheme' () {
|
||||
given:
|
||||
def provider = new LinFileSystemProvider()
|
||||
expect:
|
||||
provider.getScheme() == 'lid'
|
||||
}
|
||||
|
||||
def 'should get lid path' () {
|
||||
given:
|
||||
def lid = Mock(LinPath)
|
||||
and:
|
||||
def provider = new LinFileSystemProvider()
|
||||
expect:
|
||||
provider.toLinPath(lid) == lid
|
||||
|
||||
when:
|
||||
provider.toLinPath(Path.of('foo'))
|
||||
then:
|
||||
thrown(ProviderMismatchException)
|
||||
}
|
||||
|
||||
def 'should create new file system' () {
|
||||
given:
|
||||
def provider = new LinFileSystemProvider()
|
||||
def config = [store:[location:data.toString()]]
|
||||
def lid = LinPath.asUri('lid://12345')
|
||||
when:
|
||||
def fs = provider.newFileSystem(lid, config) as LinFileSystem
|
||||
then:
|
||||
(fs.store as DefaultLinStore).location == data
|
||||
}
|
||||
|
||||
def 'should get a file system' () {
|
||||
given:
|
||||
def provider = new LinFileSystemProvider()
|
||||
def config = [store:[location: data.toString()]]
|
||||
def uri = LinPath.asUri('lid://12345')
|
||||
when:
|
||||
provider.getFileSystem(uri)
|
||||
then:
|
||||
thrown(FileSystemNotFoundException)
|
||||
|
||||
when:
|
||||
provider.newFileSystem(uri, config) as LinFileSystem
|
||||
and:
|
||||
def fs = provider.getFileSystem(uri) as LinFileSystem
|
||||
then:
|
||||
(fs.store as DefaultLinStore).location == data
|
||||
}
|
||||
|
||||
def 'should get or create a file system' () {
|
||||
given:
|
||||
def config = [lineage:[store:[location: data.toString()]]]
|
||||
Global.session = Mock(Session) { getConfig()>>config }
|
||||
and:
|
||||
def uri = LinPath.asUri('lid://12345')
|
||||
def provider = new LinFileSystemProvider()
|
||||
|
||||
when:
|
||||
def fs = provider.getFileSystemOrCreate(uri) as LinFileSystem
|
||||
then:
|
||||
(fs.store as DefaultLinStore).location == data
|
||||
|
||||
when:
|
||||
def fs2 = provider.getFileSystemOrCreate(uri) as LinFileSystem
|
||||
then:
|
||||
fs2.is(fs)
|
||||
}
|
||||
|
||||
def 'should create new byte channel' () {
|
||||
given:
|
||||
def config = [lineage:[store:[location:wdir.toString()]]]
|
||||
def outputMeta = wdir.resolve("12345/output.txt")
|
||||
def output = data.resolve("output.txt")
|
||||
output.text = "Hello, World!"
|
||||
outputMeta.mkdirs()
|
||||
outputMeta.resolve(".data.json").text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path":"'+output.toString()+'"}}'
|
||||
|
||||
Global.session = Mock(Session) { getConfig()>>config }
|
||||
and:
|
||||
def provider = new LinFileSystemProvider()
|
||||
def lid = provider.getPath(LinPath.asUri('lid://12345/output.txt'))
|
||||
def opts = Set.of(StandardOpenOption.READ)
|
||||
when:
|
||||
def channel = provider.newByteChannel(lid, opts)
|
||||
then:
|
||||
channel.isOpen()
|
||||
channel.position() == 0
|
||||
channel.size() == "Hello, World!".getBytes().size()
|
||||
when:
|
||||
channel.truncate(25)
|
||||
then:
|
||||
thrown(NonWritableChannelException)
|
||||
|
||||
when:
|
||||
def buffer = ByteBuffer.allocate(1000);
|
||||
def read = channel.read(buffer)
|
||||
def bytes = new byte[read]
|
||||
buffer.get(0,bytes)
|
||||
then:
|
||||
bytes == "Hello, World!".getBytes()
|
||||
when:
|
||||
channel.position(2)
|
||||
then:
|
||||
channel.position() == 2
|
||||
|
||||
when:
|
||||
channel.write(buffer)
|
||||
then:
|
||||
thrown(NonWritableChannelException)
|
||||
|
||||
when:
|
||||
provider.newByteChannel(lid, Set.of(StandardOpenOption.WRITE))
|
||||
then:
|
||||
thrown(UnsupportedOperationException)
|
||||
|
||||
when:
|
||||
provider.newByteChannel(lid, Set.of(StandardOpenOption.APPEND))
|
||||
then:
|
||||
thrown(UnsupportedOperationException)
|
||||
|
||||
cleanup:
|
||||
channel.close()
|
||||
outputMeta.deleteDir()
|
||||
output.delete()
|
||||
}
|
||||
|
||||
def 'should create new byte channel for LinMetadata' () {
|
||||
given:
|
||||
def config = [lineage:[store:[location:wdir.toString()]]]
|
||||
def outputMeta = wdir.resolve("12345")
|
||||
outputMeta.mkdirs()
|
||||
outputMeta.resolve(".data.json").text = '{"version":"lineage/v1beta1","kind":"WorkflowRun","spec":{"sessionId":"session","name":"run_name","params":[{"type":"String","name":"param1","value":"value1"}]}}'
|
||||
|
||||
Global.session = Mock(Session) { getConfig()>>config }
|
||||
and:
|
||||
def provider = new LinFileSystemProvider()
|
||||
def lid = provider.getPath(LinPath.asUri('lid://12345#name'))
|
||||
|
||||
when:
|
||||
def channel = provider.newByteChannel(lid, Set.of(StandardOpenOption.READ))
|
||||
then:
|
||||
channel.isOpen()
|
||||
channel.position() == 0
|
||||
channel.size() == '"run_name"'.getBytes().size()
|
||||
|
||||
when:
|
||||
channel.truncate(25)
|
||||
then:
|
||||
thrown(NonWritableChannelException)
|
||||
|
||||
when:
|
||||
def buffer = ByteBuffer.allocate(1000);
|
||||
def read = channel.read(buffer)
|
||||
def bytes = new byte[read]
|
||||
buffer.get(0,bytes)
|
||||
then:
|
||||
bytes =='"run_name"'.getBytes()
|
||||
|
||||
when:
|
||||
channel.position(2)
|
||||
then:
|
||||
channel.position() == 2
|
||||
|
||||
when:
|
||||
channel.write(buffer)
|
||||
then:
|
||||
thrown(NonWritableChannelException)
|
||||
|
||||
when:
|
||||
provider.newByteChannel(lid, Set.of(StandardOpenOption.WRITE))
|
||||
then:
|
||||
thrown(UnsupportedOperationException)
|
||||
|
||||
when:
|
||||
provider.newByteChannel(lid, Set.of(StandardOpenOption.APPEND))
|
||||
then:
|
||||
thrown(UnsupportedOperationException)
|
||||
|
||||
cleanup:
|
||||
channel.close()
|
||||
outputMeta.deleteDir()
|
||||
}
|
||||
|
||||
def 'should read lid' () {
|
||||
given:
|
||||
def config = [lineage:[store:[location:wdir.toString()]]]
|
||||
def outputMeta = wdir.resolve("12345/output.txt")
|
||||
def output = data.resolve("output.txt")
|
||||
output.text = "Hello, World!"
|
||||
outputMeta.mkdirs()
|
||||
outputMeta.resolve(".data.json").text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path":"'+output.toString()+'"}}'
|
||||
|
||||
Global.session = Mock(Session) { getConfig()>>config }
|
||||
and:
|
||||
def provider = new LinFileSystemProvider()
|
||||
def lid = provider.getPath(LinPath.asUri('lid://12345/output.txt'))
|
||||
def opts = Set.of(StandardOpenOption.READ)
|
||||
|
||||
expect:
|
||||
lid.text == "Hello, World!"
|
||||
|
||||
cleanup:
|
||||
outputMeta.deleteDir()
|
||||
output.delete()
|
||||
}
|
||||
|
||||
def 'should not create a directory' () {
|
||||
given:
|
||||
def config = [lineage:[store:[location:wdir.toString()]]]
|
||||
Global.session = Mock(Session) { getConfig()>>config }
|
||||
and:
|
||||
def provider = new LinFileSystemProvider()
|
||||
def lid = provider.getPath(LinPath.asUri('lid://12345'))
|
||||
|
||||
when:
|
||||
provider.createDirectory(lid)
|
||||
then:
|
||||
thrown(UnsupportedOperationException)
|
||||
|
||||
}
|
||||
|
||||
def 'should create directory stream' () {
|
||||
given:
|
||||
def output1 = data.resolve('path')
|
||||
output1.mkdir()
|
||||
output1.resolve('file1.txt').text = 'file1'
|
||||
output1.resolve('file2.txt').text = 'file2'
|
||||
output1.resolve('file3.txt').text = 'file3'
|
||||
wdir.resolve('12345/output1').mkdirs()
|
||||
wdir.resolve('12345/output2').mkdirs()
|
||||
wdir.resolve('12345/.data.json').text = '{"version":"lineage/v1beta1","kind":"TaskRun","spec":{"name":"dummy"}}'
|
||||
wdir.resolve('12345/output1/.data.json').text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path": "' + output1.toString() + '"}}'
|
||||
|
||||
and:
|
||||
def config = [lineage:[store:[location:wdir.toString()]]]
|
||||
Global.session = Mock(Session) { getConfig()>>config }
|
||||
and:
|
||||
def provider = new LinFileSystemProvider()
|
||||
def lid = provider.getPath(LinPath.asUri('lid://12345/output1'))
|
||||
def lid2 = provider.getPath(LinPath.asUri('lid://12345'))
|
||||
def lid3 = provider.getPath(LinPath.asUri('lid://678'))
|
||||
|
||||
expect:
|
||||
Files.exists(lid)
|
||||
Files.exists(lid.resolve('file1.txt'))
|
||||
Files.exists(lid.resolve('file2.txt'))
|
||||
Files.exists(lid.resolve('file3.txt'))
|
||||
|
||||
when:
|
||||
provider.newDirectoryStream(lid3, (p) -> true)
|
||||
then:
|
||||
thrown(FileNotFoundException)
|
||||
|
||||
when:
|
||||
def stream = provider.newDirectoryStream(lid, (p) -> true)
|
||||
and:
|
||||
def result = stream.toList()
|
||||
then:
|
||||
result.toSet() == [
|
||||
lid.resolve('file1.txt'),
|
||||
lid.resolve('file2.txt'),
|
||||
lid.resolve('file3.txt')
|
||||
] as Set
|
||||
|
||||
when:
|
||||
stream = provider.newDirectoryStream(lid2, (p) -> true)
|
||||
and:
|
||||
result = stream.toList()
|
||||
then:
|
||||
result.size() == 1
|
||||
result[0] == lid2.resolve('output1')
|
||||
|
||||
cleanup:
|
||||
wdir.resolve('12345').deleteDir()
|
||||
output1.deleteDir()
|
||||
|
||||
}
|
||||
|
||||
def 'should not delete a file' () {
|
||||
given:
|
||||
def config = [lineage:[store:[location:wdir.toString()]]]
|
||||
Global.session = Mock(Session) { getConfig()>>config }
|
||||
and:
|
||||
def provider = new LinFileSystemProvider()
|
||||
def lid = provider.getPath(LinPath.asUri('lid://12345'))
|
||||
|
||||
when:
|
||||
provider.delete(lid)
|
||||
then:
|
||||
thrown(UnsupportedOperationException)
|
||||
|
||||
}
|
||||
|
||||
def 'should not copy a file' () {
|
||||
given:
|
||||
def config = [lineage:[store:[location:wdir.toString()]]]
|
||||
Global.session = Mock(Session) { getConfig()>>config }
|
||||
and:
|
||||
def provider = new LinFileSystemProvider()
|
||||
def lid1 = provider.getPath(LinPath.asUri('lid://12345/abc'))
|
||||
def lid2 = provider.getPath(LinPath.asUri('lid://54321/foo'))
|
||||
|
||||
when:
|
||||
provider.copy(lid1, lid2)
|
||||
then:
|
||||
thrown(UnsupportedOperationException)
|
||||
}
|
||||
|
||||
def 'should not move a file' () {
|
||||
given:
|
||||
def config = [lineage:[store:[location:wdir.toString()]]]
|
||||
Global.session = Mock(Session) { getConfig()>>config }
|
||||
and:
|
||||
def provider = new LinFileSystemProvider()
|
||||
def lid1 = provider.getPath(LinPath.asUri('lid://12345/abc'))
|
||||
def lid2 = provider.getPath(LinPath.asUri('lid://54321/foo'))
|
||||
|
||||
when:
|
||||
provider.move(lid1, lid2)
|
||||
then:
|
||||
thrown(UnsupportedOperationException)
|
||||
}
|
||||
|
||||
def 'should check is same file' () {
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def config = [lineage:[store:[location:folder.toString()]]]
|
||||
Global.session = Mock(Session) { getConfig()>>config }
|
||||
and:
|
||||
def provider = new LinFileSystemProvider()
|
||||
def lid1 = provider.getPath(LinPath.asUri('lid://12345/abc'))
|
||||
def lid2 = provider.getPath(LinPath.asUri('lid://54321/foo'))
|
||||
def lid3 = provider.getPath(LinPath.asUri('lid://54321/foo'))
|
||||
|
||||
expect:
|
||||
!provider.isSameFile(lid1, lid2)
|
||||
!provider.isSameFile(lid1, lid3)
|
||||
and:
|
||||
provider.isSameFile(lid2, lid3)
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
def 'should check is hidden file' () {
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def config = [lineage:[store:[location:wdir.toString()]]]
|
||||
Global.session = Mock(Session) { getConfig()>>config }
|
||||
and:
|
||||
def output = folder.resolve('path')
|
||||
output.mkdir()
|
||||
output.resolve('abc').text = 'file1'
|
||||
output.resolve('.foo').text = 'file2'
|
||||
wdir.resolve('12345/output').mkdirs()
|
||||
wdir.resolve('12345/output/.data.json').text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path": "' + output.toString() + '"}}'
|
||||
and:
|
||||
def provider = new LinFileSystemProvider()
|
||||
def lid1 = provider.getPath(LinPath.asUri('lid://12345/output/abc'))
|
||||
def lid2 = provider.getPath(LinPath.asUri('lid://12345/output/.foo'))
|
||||
|
||||
expect:
|
||||
!provider.isHidden(lid1)
|
||||
provider.isHidden(lid2)
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
def 'should read file attributes' () {
|
||||
given:
|
||||
def config = [lineage:[store:[location:wdir.toString()]]]
|
||||
def file = data.resolve('abc')
|
||||
file.text = 'Hello'
|
||||
wdir.resolve('12345/abc').mkdirs()
|
||||
wdir.resolve('12345/abc/.data.json').text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path":"' + file.toString() + '"}}'
|
||||
and:
|
||||
Global.session = Mock(Session) { getConfig()>>config }
|
||||
and:
|
||||
def provider = new LinFileSystemProvider()
|
||||
def lid1 = provider.getPath(LinPath.asUri('lid://12345/abc'))
|
||||
|
||||
when:
|
||||
def attr1 = provider.readAttributes(lid1, BasicFileAttributes)
|
||||
def real1= Files.readAttributes(file,BasicFileAttributes)
|
||||
then:
|
||||
!attr1.directory
|
||||
attr1.isRegularFile()
|
||||
attr1.size() == real1.size()
|
||||
attr1.creationTime() == real1.creationTime()
|
||||
attr1.lastModifiedTime() == real1.lastModifiedTime()
|
||||
attr1.lastAccessTime() == real1.lastAccessTime()
|
||||
|
||||
cleanup:
|
||||
file?.delete()
|
||||
wdir.resolve('12345').deleteDir()
|
||||
}
|
||||
|
||||
def 'should throw exception in unsupported methods'() {
|
||||
given:
|
||||
def config = [lineage:[store:[location:wdir.toString()]]]
|
||||
Global.session = Mock(Session) { getConfig()>>config }
|
||||
def provider = new LinFileSystemProvider()
|
||||
|
||||
when:
|
||||
provider.newOutputStream(null)
|
||||
then:
|
||||
thrown(UnsupportedOperationException)
|
||||
|
||||
when:
|
||||
provider.getFileStore(null)
|
||||
then:
|
||||
thrown(UnsupportedOperationException)
|
||||
|
||||
when:
|
||||
provider.readAttributes(null, "attrib")
|
||||
then:
|
||||
thrown(UnsupportedOperationException)
|
||||
|
||||
when:
|
||||
provider.setAttribute(null, "attrib", null)
|
||||
then:
|
||||
thrown(UnsupportedOperationException)
|
||||
}
|
||||
|
||||
def 'should throw exception when checking access mode'(){
|
||||
given:
|
||||
def config = [lineage:[store:[location:wdir.toString()]]]
|
||||
Global.session = Mock(Session) { getConfig()>>config }
|
||||
def provider = new LinFileSystemProvider()
|
||||
def lid1 = provider.getPath(LinPath.asUri('lid://12345/abc'))
|
||||
|
||||
when:
|
||||
provider.checkAccess(lid1, AccessMode.WRITE)
|
||||
then:
|
||||
def ex1 = thrown(AccessDeniedException)
|
||||
ex1.message == "Write mode not supported"
|
||||
|
||||
when:
|
||||
provider.checkAccess(lid1, AccessMode.EXECUTE)
|
||||
then:
|
||||
def ex2 = thrown(AccessDeniedException)
|
||||
ex2.message == "Execute mode not supported"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.fs
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
|
||||
import nextflow.Global
|
||||
import nextflow.Session
|
||||
import spock.lang.Specification
|
||||
import spock.lang.Unroll
|
||||
|
||||
/**
|
||||
* LID Path Factory tests.
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
class LinPathFactoryTest extends Specification {
|
||||
|
||||
Path tmp
|
||||
|
||||
def setup() {
|
||||
tmp = Files.createTempDirectory("data")
|
||||
Global.session = Mock(Session) { getConfig()>> [workflow:[lineage:[store:[location: tmp.toString()]]]] }
|
||||
}
|
||||
|
||||
def cleanup() {
|
||||
Global.session = null
|
||||
tmp.deleteDir()
|
||||
}
|
||||
|
||||
def 'should create lin path' () {
|
||||
given:
|
||||
def factory = new LinPathFactory()
|
||||
|
||||
expect:
|
||||
factory.parseUri('foo') == null
|
||||
|
||||
when:
|
||||
def p1 = factory.parseUri('lid://12345')
|
||||
then:
|
||||
p1.toUriString() == 'lid://12345'
|
||||
|
||||
when:
|
||||
def p2 = factory.parseUri('lid://12345/x/y/z')
|
||||
then:
|
||||
p2.toUriString() == 'lid://12345/x/y/z'
|
||||
|
||||
when:
|
||||
def p3 = factory.parseUri('lid://12345//x///y/z//')
|
||||
then:
|
||||
p3.toUriString() == 'lid://12345/x/y/z'
|
||||
|
||||
when:
|
||||
factory.parseUri('lid:///12345')
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should convert get lid uri string' () {
|
||||
given:
|
||||
def factory = new LinPathFactory()
|
||||
|
||||
when:
|
||||
def lid = LinPathFactory.create(EXPECTED)
|
||||
then:
|
||||
factory.toUriString(lid) == EXPECTED
|
||||
|
||||
where:
|
||||
_ | EXPECTED
|
||||
_ | 'lid://123'
|
||||
_ | 'lid://123/a/b/c'
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,718 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.fs
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.ProviderMismatchException
|
||||
import java.time.OffsetDateTime
|
||||
|
||||
import nextflow.lineage.LinUtils
|
||||
import nextflow.lineage.model.v1beta1.Checksum
|
||||
import nextflow.lineage.model.v1beta1.FileOutput
|
||||
import nextflow.lineage.model.v1beta1.Parameter
|
||||
import nextflow.lineage.model.v1beta1.Workflow
|
||||
import nextflow.lineage.model.v1beta1.WorkflowOutput
|
||||
import nextflow.lineage.model.v1beta1.WorkflowRun
|
||||
import nextflow.lineage.serde.LinEncoder
|
||||
import nextflow.util.CacheHelper
|
||||
import org.junit.Rule
|
||||
import spock.lang.Shared
|
||||
import spock.lang.Specification
|
||||
import spock.lang.Unroll
|
||||
import test.OutputCapture
|
||||
|
||||
import static test.TestHelper.filterLogNoise
|
||||
/**
|
||||
* LID Path Tests
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
class LinPathTest extends Specification {
|
||||
|
||||
@Shared
|
||||
Path wdir
|
||||
@Shared
|
||||
Path data
|
||||
@Shared
|
||||
def fs = Mock(LinFileSystem)
|
||||
|
||||
def setupSpec(){
|
||||
wdir = Files.createTempDirectory("wdir")
|
||||
data = wdir.resolve('work')
|
||||
}
|
||||
|
||||
def cleanupSpec(){
|
||||
wdir.deleteDir()
|
||||
}
|
||||
|
||||
@Rule
|
||||
OutputCapture capture = new OutputCapture()
|
||||
|
||||
def 'should create from URI' () {
|
||||
when:
|
||||
def path = new LinPath(fs, new URI( URI_STRING ))
|
||||
then:
|
||||
path.filePath == PATH
|
||||
path.fragment == FRAGMENT
|
||||
path.query == QUERY
|
||||
|
||||
where:
|
||||
URI_STRING | PATH | QUERY | FRAGMENT
|
||||
"lid://1234/hola" | "1234/hola" | null | null
|
||||
"lid://1234/hola#workflow.repository" | "1234/hola" | null | "workflow.repository"
|
||||
"lid://1234/#workflow.repository" | "1234" | null | "workflow.repository"
|
||||
"lid://1234/?q=a&b=c" | "1234" | "q=a&b=c" | null
|
||||
"lid://1234/?q=a&b=c#workflow.repository" | "1234" | "q=a&b=c" | "workflow.repository"
|
||||
"lid:///" | "/" | null | null
|
||||
}
|
||||
|
||||
def 'should throw exception if fragment contains an unknown property'() {
|
||||
when:
|
||||
new LinPath(fs, new URI ("lid://1234/hola#no-exist"))
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
|
||||
}
|
||||
|
||||
def 'should warn if query is specified'() {
|
||||
when:
|
||||
new LinPath(fs, new URI("lid://1234/hola?query"))
|
||||
def stdout = filterLogNoise(capture)
|
||||
|
||||
then:
|
||||
stdout.size() == 1
|
||||
stdout[0].endsWith("Query string is not supported for Lineage URI: `lid://1234/hola?query` -- it will be ignored")
|
||||
}
|
||||
|
||||
def 'should create correct lid Path' () {
|
||||
when:
|
||||
def lid = new LinPath(FS, PATH, MORE)
|
||||
then:
|
||||
lid.filePath == EXPECTED_FILE
|
||||
where:
|
||||
FS | PATH | MORE | EXPECTED_FILE
|
||||
fs | '/' | [] as String[] | '/'
|
||||
null | '/' | [] as String[] | '/'
|
||||
fs | '/' | ['a','b'] as String[] | 'a/b'
|
||||
null | '/' | ['a','b'] as String[] | 'a/b'
|
||||
fs | '' | [] as String[] | '/'
|
||||
null | '' | [] as String[] | '/'
|
||||
fs | '' | ['a','b'] as String[] | 'a/b'
|
||||
null | '' | ['a','b'] as String[] | 'a/b'
|
||||
fs | '1234' | [] as String[] | '1234'
|
||||
null | '1234' | [] as String[] | '1234'
|
||||
fs | '1234' | ['a','b'] as String[] | '1234/a/b'
|
||||
null | '1234' | ['a','b'] as String[] | '1234/a/b'
|
||||
fs | '1234/c' | [] as String[] | '1234/c'
|
||||
null | '1234/c' | [] as String[] | '1234/c'
|
||||
fs | '1234/c' | ['a','b'] as String[] | '1234/c/a/b'
|
||||
null | '1234/c' | ['a','b'] as String[] | '1234/c/a/b'
|
||||
fs | '/1234/c' | [] as String[] | '1234/c'
|
||||
null | '/1234/c' | [] as String[] | '1234/c'
|
||||
fs | '/1234/c' | ['a','b'] as String[] | '1234/c/a/b'
|
||||
null | '/1234/c' | ['a','b'] as String[] | '1234/c/a/b'
|
||||
fs | '../c' | ['a','b'] as String[] | 'c/a/b'
|
||||
null | '../c' | ['a','b'] as String[] | '../c/a/b'
|
||||
fs | '../c' | [] as String[] | 'c'
|
||||
null | '../c' | [] as String[] | '../c'
|
||||
fs | '..' | [] as String[] | '/'
|
||||
null | '..' | [] as String[] | '..'
|
||||
fs | '/..' | [] as String[] | '/'
|
||||
null | '/..' | [] as String[] | '/'
|
||||
fs | './1234/c' | ['a','b'] as String[] | '1234/c/a/b'
|
||||
null | './1234/c' | ['a','b'] as String[] | '1234/c/a/b'
|
||||
fs | './1234/c' | [] as String[] | '1234/c'
|
||||
null | './1234/c' | [] as String[] | '1234/c'
|
||||
fs | '1234' | ['/'] as String[] | '1234'
|
||||
null | '1234' | ['/'] as String[] | '1234'
|
||||
null | '../../a/b' | [] as String[] | '../../a/b'
|
||||
fs | '1234/' | [] as String[] | '1234'
|
||||
null | '1234/' | [] as String[] | '1234'
|
||||
}
|
||||
|
||||
def 'should get target path' () {
|
||||
given:
|
||||
def outputFolder = data.resolve('output')
|
||||
def outputSubFolder = outputFolder.resolve('some/path')
|
||||
outputSubFolder.mkdirs()
|
||||
def outputSubFolderFile = outputSubFolder.resolve('file1.txt')
|
||||
outputSubFolderFile.text = "this is file1"
|
||||
def outputFile = data.resolve('file2.txt')
|
||||
outputFile.text = "this is file2"
|
||||
|
||||
def lidFs = new LinFileSystemProvider().newFileSystem(new URI("lid:///"), [enabled: true, store: [location: wdir.toString()]])
|
||||
|
||||
wdir.resolve('12345/output1').mkdirs()
|
||||
wdir.resolve('12345/path/to/file2.txt').mkdirs()
|
||||
wdir.resolve('12345/.data.json').text = '{"version":"lineage/v1beta1","kind":"TaskRun","spec":{"name":"test"}}'
|
||||
wdir.resolve('12345/output1/.data.json').text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path": "' + outputFolder.toString() + '"}}'
|
||||
wdir.resolve('12345/path/to/file2.txt/.data.json').text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path": "' + outputFile.toString() + '"}}'
|
||||
def time = OffsetDateTime.now()
|
||||
def wfResultsMetadata = new LinEncoder().withPrettyPrint(true).encode(new WorkflowOutput(time, "lid://1234", [new Parameter( "Path", "a", "lid://1234/a.txt")]))
|
||||
wdir.resolve('5678/').mkdirs()
|
||||
wdir.resolve('5678/.data.json').text = wfResultsMetadata
|
||||
|
||||
expect: 'Get real path when LinPath is the output data or a subfolder'
|
||||
new LinPath(lidFs, '12345/output1').getTargetPath() == outputFolder
|
||||
new LinPath(lidFs,'12345/output1/some/path').getTargetPath() == outputSubFolder
|
||||
new LinPath(lidFs,'12345/output1/some/path/file1.txt').getTargetPath().text == outputSubFolderFile.text
|
||||
new LinPath(lidFs, '12345/path/to/file2.txt').getTargetPath().text == outputFile.text
|
||||
|
||||
when: 'LinPath fs is null'
|
||||
new LinPath(null, '12345').getTargetPath()
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
|
||||
when: 'LinPath is empty'
|
||||
new LinPath(lidFs, '/').getTargetPath()
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
|
||||
when: 'LinPath is not an output data description'
|
||||
new LinPath(lidFs, '12345').getTargetPath()
|
||||
then:
|
||||
thrown(FileNotFoundException)
|
||||
|
||||
when: 'LinPath is not subfolder of an output data description'
|
||||
new LinPath(lidFs, '12345/path').getTargetPath()
|
||||
then:
|
||||
thrown(FileNotFoundException)
|
||||
|
||||
when: 'LinPath subfolder of an output data description does not exist'
|
||||
new LinPath(lidFs, '12345/output1/other/path').getTargetPath()
|
||||
then:
|
||||
thrown(FileNotFoundException)
|
||||
|
||||
when: 'Lid does not exist'
|
||||
new LinPath(lidFs, '23456').getTargetPath()
|
||||
then:
|
||||
thrown(FileNotFoundException)
|
||||
|
||||
when: 'LinPath is not an output data description but is a workflow/task run'
|
||||
def result = new LinPath(lidFs, '12345').getTargetOrIntermediatePath()
|
||||
then:
|
||||
result instanceof LinIntermediatePath
|
||||
|
||||
when: 'LinPath is a subpath of a workflow/task run data description'
|
||||
result = new LinPath(lidFs, '12345/path').getTargetOrIntermediatePath()
|
||||
then:
|
||||
result instanceof LinIntermediatePath
|
||||
|
||||
when: 'Lid description'
|
||||
result = new LinPath(lidFs, '5678').getTargetOrMetadataPath()
|
||||
then:
|
||||
result instanceof LinMetadataPath
|
||||
result.text == wfResultsMetadata
|
||||
|
||||
when: 'Lid description subobject'
|
||||
def result2 = new LinPath(lidFs, '5678#output').getTargetOrMetadataPath()
|
||||
then:
|
||||
result2 instanceof LinMetadataPath
|
||||
result2.text == LinUtils.encodeSearchOutputs([new Parameter("Path","a", "lid://1234/a.txt")], true)
|
||||
|
||||
when: 'Lid subobject does not exist'
|
||||
new LinPath(lidFs, '23456#notexists').getTargetOrMetadataPath()
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
}
|
||||
|
||||
def 'should get subobjects as path' (){
|
||||
given:
|
||||
def lidFs = new LinFileSystemProvider().newFileSystem(new URI("lid:///"), [enabled: true, store: [location: wdir.toString()]])
|
||||
def wf = new WorkflowRun(new Workflow([],"repo", "commit"), "sessionId", "runId", [new Parameter("String", "param1", "value1")])
|
||||
|
||||
when: 'workflow repo in workflow run'
|
||||
Path p = LinPath.getMetadataAsTargetPath(wf, lidFs, "123456", "workflow.repository")
|
||||
then:
|
||||
p instanceof LinMetadataPath
|
||||
p.text == '"repo"'
|
||||
|
||||
when: 'outputs'
|
||||
def outputs = new WorkflowOutput(OffsetDateTime.now(), "lid://123456", [new Parameter("Collection", "samples", ["sample1", "sample2"])])
|
||||
lidFs.store.save("123456#output", outputs)
|
||||
Path p2 = LinPath.getMetadataAsTargetPath(wf, lidFs, "123456", "output")
|
||||
then:
|
||||
p2 instanceof LinMetadataPath
|
||||
p2.text == LinUtils.encodeSearchOutputs([new Parameter("Collection", "samples", ["sample1", "sample2"])], true)
|
||||
|
||||
when: 'child does not exists'
|
||||
LinPath.getMetadataAsTargetPath(wf, lidFs, "123456", "no-exist")
|
||||
then:
|
||||
def exception = thrown(FileNotFoundException)
|
||||
exception.message == "Target path '123456#no-exist' does not exist"
|
||||
|
||||
when: 'outputs does not exists'
|
||||
LinPath.getMetadataAsTargetPath(wf, lidFs, "6789", "output")
|
||||
then:
|
||||
def exception1 = thrown(FileNotFoundException)
|
||||
exception1.message == "Target path '6789#output' does not exist"
|
||||
|
||||
when: 'null object'
|
||||
LinPath.getMetadataAsTargetPath(null, lidFs, "123456", "no-exist")
|
||||
then:
|
||||
def exception2 = thrown(FileNotFoundException)
|
||||
exception2.message == "Target path '123456' does not exist"
|
||||
|
||||
cleanup:
|
||||
wdir.resolve("123456").deleteDir()
|
||||
}
|
||||
|
||||
def 'should get file name' () {
|
||||
expect:
|
||||
new LinPath(fs, PATH).getFileName() == EXPECTED
|
||||
where:
|
||||
PATH | EXPECTED
|
||||
'1234567890/this/file.bam' | new LinPath(null, 'file.bam')
|
||||
'12345/hola?query#output' | new LinPath("query", "output", "hola", null)
|
||||
|
||||
}
|
||||
|
||||
def 'should get file parent' () {
|
||||
when:
|
||||
def lid1 = new LinPath(fs, '1234567890/this/file.bam')
|
||||
then:
|
||||
lid1.getParent() == new LinPath(fs, '1234567890/this')
|
||||
lid1.getParent().getParent() == new LinPath(fs, '1234567890')
|
||||
lid1.getParent().getParent().getParent() == new LinPath(fs, "/")
|
||||
lid1.getParent().getParent().getParent().getParent() == null
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should get name count' () {
|
||||
expect:
|
||||
new LinPath(fs, PATH).getNameCount() == EXPECTED
|
||||
where:
|
||||
PATH | EXPECTED
|
||||
'/' | 0
|
||||
'123' | 1
|
||||
'123/a' | 2
|
||||
'123/a/' | 2
|
||||
'123/a/b' | 3
|
||||
'' | 0
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should get name by index' () {
|
||||
expect:
|
||||
new LinPath(fs, PATH).getName(INDEX) == EXPECTED
|
||||
where:
|
||||
PATH | INDEX | EXPECTED
|
||||
'123' | 0 | new LinPath(fs, '123')
|
||||
'123/a' | 1 | new LinPath(null, 'a')
|
||||
'123/a/' | 1 | new LinPath(null, 'a')
|
||||
'123/a/b' | 2 | new LinPath(null, 'b')
|
||||
'123/a?q#output' | 1 | new LinPath(null, 'a?q#output')
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should get subpath' () {
|
||||
expect:
|
||||
new LinPath(fs, PATH).subpath(BEGIN,END) == EXPECTED
|
||||
where:
|
||||
PATH | BEGIN | END | EXPECTED
|
||||
'123' | 0 | 1 | new LinPath(fs, '123')
|
||||
'123/a' | 0 | 2 | new LinPath(fs, '123/a')
|
||||
'123/a/' | 0 | 2 | new LinPath(fs, '123/a')
|
||||
'123/a' | 1 | 2 | new LinPath(null, 'a')
|
||||
'123/a/' | 1 | 2 | new LinPath(null, 'a')
|
||||
'123/a/b' | 2 | 3 | new LinPath(null, 'b')
|
||||
'123/a/b' | 1 | 3 | new LinPath(null, 'a/b')
|
||||
}
|
||||
|
||||
def 'should normalize a path' () {
|
||||
expect:
|
||||
new LinPath(fs, '123').normalize() == new LinPath(fs, '123')
|
||||
new LinPath(fs, '123/a/b').normalize() == new LinPath(fs, '123/a/b')
|
||||
new LinPath(fs, '123/./a/b').normalize() == new LinPath(fs, '123/a/b')
|
||||
new LinPath(fs, '123/a/../a/b').normalize() == new LinPath(fs, '123/a/b')
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should validate startWith' () {
|
||||
expect:
|
||||
new LinPath(fs,PATH).startsWith(OTHER) == EXPECTED
|
||||
where:
|
||||
PATH | OTHER | EXPECTED
|
||||
'12345/a/b' | '12345' | true
|
||||
'12345/a/b' | '12345/a' | true
|
||||
'12345/a/b' | '12345/a/b' | true
|
||||
and:
|
||||
'12345/a/b' | '12345/b' | false
|
||||
'12345/a/b' | 'xyz' | false
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should validate endsWith' () {
|
||||
expect:
|
||||
new LinPath(fs,PATH).endsWith(OTHER) == EXPECTED
|
||||
where:
|
||||
PATH | OTHER | EXPECTED
|
||||
'12345/a/b' | 'b' | true
|
||||
'12345/a/b' | 'a/b' | true
|
||||
'12345/a/b' | '12345/a/b' | true
|
||||
and:
|
||||
'12345/a/b' | '12345/b' | false
|
||||
'12345/a/b' | 'xyz' | false
|
||||
}
|
||||
|
||||
def 'should validate isAbsolute' () {
|
||||
expect:
|
||||
new LinPath(fs,'1234/a/b/c').isAbsolute()
|
||||
new LinPath(fs,'1234/a/b/c').getRoot().isAbsolute()
|
||||
new LinPath(fs,'1234/a/b/c').getParent().isAbsolute()
|
||||
new LinPath(fs,'1234/a/b/c').normalize().isAbsolute()
|
||||
new LinPath(fs,'1234/a/b/c').getName(0).isAbsolute()
|
||||
new LinPath(fs,'1234/a/b/c').subpath(0,2).isAbsolute()
|
||||
and:
|
||||
!new LinPath(fs,'1234/a/b/c').getFileName().isAbsolute()
|
||||
!new LinPath(fs,'1234/a/b/c').getName(1).isAbsolute()
|
||||
!new LinPath(fs,'1234/a/b/c').subpath(1,3).isAbsolute()
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should get root path' () {
|
||||
expect:
|
||||
new LinPath(fs,PATH).getRoot() == new LinPath(fs,EXPECTED)
|
||||
where:
|
||||
PATH | EXPECTED
|
||||
'12345' | '/'
|
||||
'12345/a' | '/'
|
||||
}
|
||||
|
||||
def 'should relativize path' () {
|
||||
expect:
|
||||
BASE_PATH.relativize(PATH) == EXPECTED
|
||||
where :
|
||||
BASE_PATH | PATH | EXPECTED
|
||||
new LinPath(fs, '/') | new LinPath(fs, '123/a/b/c') | new LinPath(null, '123/a/b/c')
|
||||
new LinPath(fs,'123/a/') | new LinPath(fs, '123/a/b/c') | new LinPath(null, 'b/c')
|
||||
new LinPath(fs,'123/a/') | new LinPath(fs, '321/a/') | new LinPath(null, '../../321/a')
|
||||
new LinPath(null,'123/a') | new LinPath(null, '123/a/b/c') | new LinPath(null, 'b/c')
|
||||
new LinPath(null,'123/a') | new LinPath(null, '321/a') | new LinPath(null, '../../321/a')
|
||||
new LinPath(fs,'../a/') | new LinPath(fs, '321/a') | new LinPath(null, '../321/a')
|
||||
new LinPath(fs,'321/a/') | new LinPath(fs, '../a') | new LinPath(null, '../../a')
|
||||
new LinPath(null,'321/a/') | new LinPath(null, '../a') | new LinPath(null, '../../../a')
|
||||
}
|
||||
|
||||
def 'relativize should throw exception' () {
|
||||
given:
|
||||
def lid1 = new LinPath(fs,'123/a/')
|
||||
def lid2 = new LinPath(null,'123/a/')
|
||||
def lid3 = new LinPath(null, '../a/b')
|
||||
when: 'comparing relative with absolute'
|
||||
lid1.relativize(lid2)
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
|
||||
when: 'undefined base path'
|
||||
lid3.relativize(lid2)
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
}
|
||||
|
||||
def 'should resolve path' () {
|
||||
when:
|
||||
def lid1 = new LinPath(fs, '123/a/b/c')
|
||||
def lid2 = new LinPath(fs, '321/x/y/z')
|
||||
def rel1 = new LinPath(null, 'foo')
|
||||
def rel2 = new LinPath(null, 'bar/')
|
||||
|
||||
then:
|
||||
lid1.resolve(lid2) == lid2
|
||||
lid2.resolve(lid1) == lid1
|
||||
and:
|
||||
lid1.resolve(rel1) == new LinPath(fs,'123/a/b/c/foo')
|
||||
lid1.resolve(rel2) == new LinPath(fs,'123/a/b/c/bar')
|
||||
and:
|
||||
rel1.resolve(rel2) == new LinPath(null, 'foo/bar')
|
||||
rel2.resolve(rel1) == new LinPath(null, 'bar/foo')
|
||||
}
|
||||
|
||||
def 'should resolve path as string' () {
|
||||
given:
|
||||
def pr = Mock(LinFileSystemProvider)
|
||||
def lidfs = Mock(LinFileSystem){
|
||||
provider() >> pr}
|
||||
|
||||
|
||||
def lid1 = new LinPath(lidfs, '123/a/b/c')
|
||||
|
||||
expect:
|
||||
lid1.resolve('x/y') == new LinPath(lidfs, '123/a/b/c/x/y')
|
||||
lid1.resolve('/x/y/') == new LinPath(lidfs, '123/a/b/c/x/y')
|
||||
|
||||
when:
|
||||
def result = lid1.resolve('lid://321')
|
||||
then:
|
||||
pr.getPath(LinPath.asUri('lid://321')) >> new LinPath(lidfs, '321')
|
||||
and:
|
||||
result == new LinPath(lidfs, '321')
|
||||
}
|
||||
|
||||
def 'should throw illegal exception when not correct scheme' (){
|
||||
when: 'creation'
|
||||
new LinPath(fs, new URI("http://1234"))
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
|
||||
when: 'asUri'
|
||||
LinPath.asUri("http://1234")
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
|
||||
when: 'asUri'
|
||||
LinPath.asUri("")
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
|
||||
}
|
||||
|
||||
def 'should throw provider mismatch exception when different path types' () {
|
||||
given:
|
||||
def pr = Mock(LinFileSystemProvider)
|
||||
def fs = Mock(LinFileSystem){
|
||||
provider() >> pr}
|
||||
and:
|
||||
def lid = new LinPath(fs, '123/a/b/c')
|
||||
|
||||
when: 'resolve with path'
|
||||
lid.resolve(Path.of('d'))
|
||||
then:
|
||||
thrown(ProviderMismatchException)
|
||||
|
||||
when: 'resolve with uri string'
|
||||
lid.resolve(Path.of('http://1234'))
|
||||
then:
|
||||
thrown(ProviderMismatchException)
|
||||
|
||||
when: 'relativize'
|
||||
lid.relativize(Path.of('d'))
|
||||
then:
|
||||
thrown(ProviderMismatchException)
|
||||
}
|
||||
|
||||
def 'should throw exception for unsupported methods' () {
|
||||
given:
|
||||
def pr = Mock(LinFileSystemProvider)
|
||||
def fs = Mock(LinFileSystem){
|
||||
provider() >> pr}
|
||||
and:
|
||||
def lid = new LinPath(fs, '123/a/b/c')
|
||||
|
||||
when: 'to file'
|
||||
lid.toFile()
|
||||
then:
|
||||
thrown(UnsupportedOperationException)
|
||||
|
||||
when: 'register'
|
||||
lid.register(null, null,null)
|
||||
then:
|
||||
thrown(UnsupportedOperationException)
|
||||
}
|
||||
|
||||
def 'should throw exception for incorrect index'() {
|
||||
when: 'getting name with negative index'
|
||||
new LinPath(fs, "1234").getName(-1)
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
|
||||
when: 'getting name with larger index tha namecount'
|
||||
new LinPath(fs, "1234").getName(2)
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
|
||||
when: 'getting subpath with negative index'
|
||||
new LinPath(fs, "1234").subpath(-1,1)
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
|
||||
when: 'getting subpath with larger index tha namecount'
|
||||
new LinPath(fs, "1234").subpath(0,2)
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should get to uri string' () {
|
||||
expect:
|
||||
new LinPath(fs, PATH).toUriString() == EXPECTED
|
||||
where:
|
||||
PATH | EXPECTED
|
||||
'/' | 'lid:///'
|
||||
'1234' | 'lid://1234'
|
||||
'1234/a/b/c' | 'lid://1234/a/b/c'
|
||||
'' | 'lid:///'
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should get string' () {
|
||||
expect:
|
||||
new LinPath(fs, PATH).toString() == EXPECTED
|
||||
where:
|
||||
PATH | EXPECTED
|
||||
'/' | '/'
|
||||
'1234' | '1234'
|
||||
'1234/a/b/c' | '1234/a/b/c'
|
||||
'' | '/'
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should validate asString method'() {
|
||||
expect:
|
||||
LinPath.asUriString(FIRST, MORE as String[]) == EXPECTED
|
||||
|
||||
where:
|
||||
FIRST | MORE | EXPECTED
|
||||
'foo' | [] | 'lid://foo'
|
||||
'foo/' | [] | 'lid://foo'
|
||||
'/foo' | [] | 'lid://foo'
|
||||
and:
|
||||
'a' | ['/b/'] | 'lid://a/b'
|
||||
'a' | ['/b','c'] | 'lid://a/b/c'
|
||||
'a' | ['/b','//c'] | 'lid://a/b/c'
|
||||
'a' | ['/b/c', 'd'] | 'lid://a/b/c/d'
|
||||
'/a/' | ['/b/c', 'd'] | 'lid://a/b/c/d'
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should check is lid uri string' () {
|
||||
expect:
|
||||
LinPath.isLidUri(STR) == EXPECTED
|
||||
|
||||
where:
|
||||
STR | EXPECTED
|
||||
null | false
|
||||
'' | false
|
||||
'foo' | false
|
||||
'/foo' | false
|
||||
'lid:/foo' | false
|
||||
'lid:foo' | false
|
||||
'lid/foo' | false
|
||||
and:
|
||||
'lid://' | true
|
||||
'lid:///' | true
|
||||
'lid://foo/bar' | true
|
||||
}
|
||||
|
||||
def 'should detect equals'(){
|
||||
expect:
|
||||
new LinPath(FS1, PATH1).equals(new LinPath(FS2, PATH2)) == EXPECTED
|
||||
where:
|
||||
FS1 | FS2 | PATH1 | PATH2 | EXPECTED
|
||||
null | fs | "12345/path" | "12345/path" | false
|
||||
fs | null | "12345/path" | "12345/path" | false
|
||||
null | null | "12345/" | "12345/path" | false
|
||||
fs | fs | "12345/" | "12345/path" | false
|
||||
and:
|
||||
null | null | "12345/path" | "12345/path" | true
|
||||
fs | fs | "12345/path" | "12345/path" | true
|
||||
null | null | "12345/" | "12345" | true
|
||||
fs | fs | "12345/" | "12345 " | true
|
||||
}
|
||||
|
||||
def 'should validate correct hash'(){
|
||||
given:
|
||||
def file = wdir.resolve("file.txt")
|
||||
file.text = "this is a data file"
|
||||
def hash = CacheHelper.hasher(file).hash().toString()
|
||||
def correctData = new FileOutput(file.toString(), new Checksum(hash,"nextflow", "standard"))
|
||||
when:
|
||||
def error = LinPath.validateDataOutput(correctData)
|
||||
then:
|
||||
!error
|
||||
|
||||
cleanup:
|
||||
file.delete()
|
||||
}
|
||||
|
||||
def 'should warn with incorrect hash'(){
|
||||
given:
|
||||
def file = wdir.resolve("file.txt")
|
||||
file.text = "this is a data file"
|
||||
def hash = CacheHelper.hasher(file).hash().toString()
|
||||
def correctData = new FileOutput(file.toString(), new Checksum("abscd","nextflow", "standard"))
|
||||
when:
|
||||
def error = LinPath.validateDataOutput(correctData)
|
||||
then:
|
||||
error == "Checksum of '$file' does not match with lineage metadata"
|
||||
|
||||
cleanup:
|
||||
file.delete()
|
||||
}
|
||||
|
||||
def 'should warn when hash algorithm is not supported'(){
|
||||
given:
|
||||
def file = wdir.resolve("file.txt")
|
||||
file.text = "this is a data file"
|
||||
def hash = CacheHelper.hasher(file).hash().toString()
|
||||
def correctData = new FileOutput(file.toString(), new Checksum(hash,"not-supported", "standard"))
|
||||
when:
|
||||
def error = LinPath.validateDataOutput(correctData)
|
||||
then:
|
||||
error == "Checksum of '$file' can't be validated - algorithm 'not-supported' is not supported"
|
||||
|
||||
cleanup:
|
||||
file.delete()
|
||||
}
|
||||
|
||||
def 'should throw exception when file not found validating hash'(){
|
||||
when:
|
||||
def correctData = new FileOutput("not/existing/file", new Checksum("120741","nextflow", "standard"))
|
||||
LinPath.validateDataOutput(correctData)
|
||||
|
||||
then:
|
||||
thrown(FileNotFoundException)
|
||||
}
|
||||
|
||||
def 'should validate path' () {
|
||||
given:
|
||||
def outputFolder = data.resolve('output')
|
||||
outputFolder.mkdirs()
|
||||
def outputFile = outputFolder.resolve('file1.txt')
|
||||
outputFile.text = "this is file1"
|
||||
def encoder = new LinEncoder()
|
||||
def hash = CacheHelper.hasher(outputFile).hash().toString()
|
||||
def correctData = new FileOutput(outputFile.toString(), new Checksum(hash,"nextflow", "standard"))
|
||||
def incorrectData = new FileOutput(outputFile.toString(), new Checksum("incorrectHash","nextflow", "standard"))
|
||||
wdir.resolve('12345/output/file1.txt').mkdirs()
|
||||
wdir.resolve('12345/output/file2.txt').mkdirs()
|
||||
wdir.resolve('12345/output/file1.txt/.data.json').text = encoder.encode(correctData)
|
||||
wdir.resolve('12345/output/file2.txt/.data.json').text = encoder.encode(incorrectData)
|
||||
def lidFs = new LinFileSystemProvider().newFileSystem(new URI("lid:///"), [enabled: true, store: [location: wdir.toString()]])
|
||||
|
||||
when:
|
||||
def succeed = new LinPath(lidFs, '12345/output/file1.txt').validate()
|
||||
then:
|
||||
succeed
|
||||
|
||||
when:
|
||||
succeed = new LinPath(lidFs, '12345/output/file2.txt').validate()
|
||||
then:
|
||||
!succeed
|
||||
|
||||
when:
|
||||
succeed = new LinPath(lidFs, '12345/output/file3.txt').validate()
|
||||
then:
|
||||
!succeed
|
||||
|
||||
cleanup:
|
||||
outputFile.delete()
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.model
|
||||
|
||||
import nextflow.lineage.model.v1beta1.Checksum
|
||||
import nextflow.util.CacheHelper
|
||||
import spock.lang.Specification
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class ChecksumTest extends Specification {
|
||||
|
||||
def 'should create a checksum'() {
|
||||
given:
|
||||
def checksum = new Checksum(algorithm: 'sha1', value: '1234567890abcdef', mode: 'hex')
|
||||
|
||||
expect:
|
||||
checksum.algorithm == 'sha1'
|
||||
checksum.value == '1234567890abcdef'
|
||||
checksum.mode == 'hex'
|
||||
}
|
||||
|
||||
def 'should create a checksum with of factory method'() {
|
||||
given:
|
||||
def checksum1 = Checksum.of('1234567890abcdef','sha1', CacheHelper.HashMode.DEFAULT())
|
||||
|
||||
expect:
|
||||
checksum1.algorithm == 'sha1'
|
||||
checksum1.value == '1234567890abcdef'
|
||||
checksum1.mode == 'standard'
|
||||
}
|
||||
|
||||
def 'should create checksum with ofNextflow factory method'() {
|
||||
given:
|
||||
def checksum1 = Checksum.ofNextflow('1234567890abcdef')
|
||||
|
||||
expect:
|
||||
checksum1.algorithm == 'nextflow'
|
||||
checksum1.value == CacheHelper.hasher('1234567890abcdef').hash().toString()
|
||||
checksum1.mode == 'standard'
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,266 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.serde
|
||||
|
||||
import java.nio.file.Path
|
||||
import java.time.OffsetDateTime
|
||||
|
||||
import nextflow.lineage.model.v1beta1.Checksum
|
||||
import nextflow.lineage.model.v1beta1.DataPath
|
||||
import nextflow.lineage.model.v1beta1.FileOutput
|
||||
import nextflow.lineage.model.v1beta1.Parameter
|
||||
import nextflow.lineage.model.v1beta1.TaskOutput
|
||||
import nextflow.lineage.model.v1beta1.TaskRun
|
||||
import nextflow.lineage.model.v1beta1.Workflow
|
||||
import nextflow.lineage.model.v1beta1.WorkflowOutput
|
||||
import nextflow.lineage.model.v1beta1.WorkflowRun
|
||||
import spock.lang.Specification
|
||||
|
||||
class LinEncoderTest extends Specification{
|
||||
|
||||
def 'should encode and decode Outputs'(){
|
||||
given:
|
||||
def encoder = new LinEncoder()
|
||||
and:
|
||||
def output = new FileOutput("/path/to/file", new Checksum("hash_value", "hash_algorithm", "standard"),
|
||||
"lid://source", "lid://workflow", "lid://task", 1234)
|
||||
|
||||
when:
|
||||
def encoded = encoder.encode(output)
|
||||
def object = encoder.decode(encoded)
|
||||
|
||||
then:
|
||||
object instanceof FileOutput
|
||||
def result = object as FileOutput
|
||||
result.path == "/path/to/file"
|
||||
result.checksum instanceof Checksum
|
||||
result.checksum.value == "hash_value"
|
||||
result.checksum.algorithm == "hash_algorithm"
|
||||
result.checksum.mode == "standard"
|
||||
result.source == "lid://source"
|
||||
result.size == 1234
|
||||
|
||||
}
|
||||
|
||||
def 'should encode and decode WorkflowRuns'(){
|
||||
given:
|
||||
def encoder = new LinEncoder()
|
||||
and:
|
||||
def config = [
|
||||
process: [
|
||||
container : "quay.io/nextflow/bash",
|
||||
executor : "local",
|
||||
resourceLabels: ["owner": "xxx"],
|
||||
scratch : false
|
||||
]
|
||||
]
|
||||
def metadata = [
|
||||
runName : "big_kare",
|
||||
start : "2025-11-06T13:35:42.049135334Z",
|
||||
container : "quay.io/nextflow/bash",
|
||||
commandLine : "nextflow run 'https://github.com/nextflow-io/hello' -name big_kare -with-tower -r master",
|
||||
nextflow : [version: "25.10.0", enable: [dsl: 2.0]],
|
||||
containerEngine: "docker",
|
||||
wave : [enabled: true],
|
||||
fusion : [enabled: true, version: "2.4"],
|
||||
platform : [
|
||||
workflowId: "wf1234",
|
||||
user : [id: "xxx", userName: "john-smith", email: "john.smith@acme.com", firstName: "John", lastName: "Smith", organization: "acme"],
|
||||
workspace : [id: "1234", name: "test-workspace", fullName: "Test workspace", organization: "acme"],
|
||||
computeEnv: [id: "ce3456", name: "test-ce", platform: "aws-cloud"],
|
||||
pipeline : [id: "pipe294", name: "https://github.com/nextflow-io/hello", revision: "master", commitId: null],
|
||||
labels : []
|
||||
],
|
||||
failOnIgnore : false
|
||||
]
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def mainScript = new DataPath("file://path/to/main.nf", new Checksum("78910", "nextflow", "standard"))
|
||||
def workflow = new Workflow([mainScript], "https://nextflow.io/nf-test/", "123456")
|
||||
def wfRun = new WorkflowRun(workflow, uniqueId.toString(), "test_run",
|
||||
[new Parameter("String", "param1", "value1"), new Parameter("String", "param2", "value2")],
|
||||
config, metadata
|
||||
)
|
||||
|
||||
when:
|
||||
def encoded = encoder.encode(wfRun)
|
||||
def object = encoder.decode(encoded)
|
||||
|
||||
then:
|
||||
object instanceof WorkflowRun
|
||||
def result = object as WorkflowRun
|
||||
result.workflow instanceof Workflow
|
||||
result.workflow.scriptFiles.first instanceof DataPath
|
||||
result.workflow.scriptFiles.first.path == "file://path/to/main.nf"
|
||||
result.workflow.scriptFiles.first.checksum instanceof Checksum
|
||||
result.workflow.scriptFiles.first.checksum.value == "78910"
|
||||
result.workflow.commitId == "123456"
|
||||
result.sessionId == uniqueId.toString()
|
||||
result.name == "test_run"
|
||||
result.params.size() == 2
|
||||
result.params.get(0).name == "param1"
|
||||
result.config.process.container == "quay.io/nextflow/bash"
|
||||
result.config.process.executor == "local"
|
||||
result.metadata.platform.workflowId == "wf1234"
|
||||
}
|
||||
|
||||
def 'should decode WorkflowRuns without metadata'(){
|
||||
given:
|
||||
def encoder = new LinEncoder()
|
||||
def wfRunStr = '''
|
||||
{
|
||||
"version": "lineage/v1beta1",
|
||||
"kind": "WorkflowRun",
|
||||
"spec": {
|
||||
"workflow": {
|
||||
"scriptFiles": [
|
||||
{
|
||||
"path": "https://github.com/nextflow-io/hello/main.nf",
|
||||
"checksum": {
|
||||
"value": "78910",
|
||||
"algorithm": "nextflow",
|
||||
"mode": "standard"
|
||||
}
|
||||
}
|
||||
],
|
||||
"repository": "https://github.com/nextflow-io/hello",
|
||||
"commitId": "2ce0b0e2943449188092a0e25102f0dadc70cb0a"
|
||||
},
|
||||
"sessionId": "4f02559e-9ebd-41d8-8ee2-a8d1e4f09c67",
|
||||
"name": "test_run",
|
||||
"params": [],
|
||||
"config": {
|
||||
"process": {
|
||||
"container": "quay.io/nextflow/bash",
|
||||
"executor": "local",
|
||||
"resourceLabels": {
|
||||
"owner": "xxx"
|
||||
},
|
||||
"scratch": false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
'''
|
||||
when:
|
||||
def object = encoder.decode(wfRunStr)
|
||||
|
||||
then:
|
||||
object instanceof WorkflowRun
|
||||
def result = object as WorkflowRun
|
||||
result.workflow instanceof Workflow
|
||||
result.workflow.scriptFiles.first instanceof DataPath
|
||||
result.workflow.scriptFiles.first.path == "https://github.com/nextflow-io/hello/main.nf"
|
||||
result.workflow.scriptFiles.first.checksum instanceof Checksum
|
||||
result.workflow.scriptFiles.first.checksum.value == "78910"
|
||||
result.workflow.commitId == "2ce0b0e2943449188092a0e25102f0dadc70cb0a"
|
||||
result.sessionId == "4f02559e-9ebd-41d8-8ee2-a8d1e4f09c67"
|
||||
result.name == "test_run"
|
||||
result.params.size() == 0
|
||||
result.config.process.container == "quay.io/nextflow/bash"
|
||||
result.config.process.executor == "local"
|
||||
result.metadata == null
|
||||
}
|
||||
|
||||
def 'should encode and decode WorkflowOutputs'(){
|
||||
given:
|
||||
def encoder = new LinEncoder()
|
||||
and:
|
||||
def time = OffsetDateTime.now()
|
||||
def wfResults = new WorkflowOutput(time, "lid://1234", [
|
||||
new Parameter("Collection", "a", [id: 'id', file: 'sample.txt' as Path]),
|
||||
new Parameter("String", "b", "B")
|
||||
])
|
||||
|
||||
when:
|
||||
def encoded = encoder.encode(wfResults)
|
||||
def object = encoder.decode(encoded)
|
||||
then:
|
||||
object instanceof WorkflowOutput
|
||||
def result = object as WorkflowOutput
|
||||
result.createdAt == time
|
||||
result.workflowRun == "lid://1234"
|
||||
result.output == [new Parameter("Collection", "a", [id: 'id', file: 'sample.txt']), new Parameter("String", "b", "B")]
|
||||
}
|
||||
|
||||
def 'should encode and decode TaskRun'() {
|
||||
given:
|
||||
def encoder = new LinEncoder()
|
||||
and:
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def taskRun = new TaskRun(
|
||||
uniqueId.toString(),"name", new Checksum("78910", "nextflow", "standard"), 'this is a script',
|
||||
[new Parameter("String", "param1", "value1")], "container:version", "conda", "spack", "amd64",
|
||||
[a: "A", b: "B"], [new DataPath("path/to/file", new Checksum("78910", "nextflow", "standard"))]
|
||||
)
|
||||
when:
|
||||
def encoded = encoder.encode(taskRun)
|
||||
def object = encoder.decode(encoded)
|
||||
then:
|
||||
object instanceof TaskRun
|
||||
def result = object as TaskRun
|
||||
result.sessionId == uniqueId.toString()
|
||||
result.name == "name"
|
||||
result.codeChecksum.value == "78910"
|
||||
result.script == "this is a script"
|
||||
result.input.size() == 1
|
||||
result.input.get(0).name == "param1"
|
||||
result.container == "container:version"
|
||||
result.conda == "conda"
|
||||
result.spack == "spack"
|
||||
result.architecture == "amd64"
|
||||
result.globalVars == [a: "A", b: "B"]
|
||||
result.binEntries.size() == 1
|
||||
result.binEntries.get(0).path == "path/to/file"
|
||||
result.binEntries.get(0).checksum.value == "78910"
|
||||
}
|
||||
|
||||
def 'should encode and decode TaskOutputs'(){
|
||||
given:
|
||||
def encoder = new LinEncoder()
|
||||
and:
|
||||
def time = OffsetDateTime.now()
|
||||
def parameter = new Parameter("a","b", "c")
|
||||
def wfResults = new TaskOutput("lid://1234", "lid://5678", time, [parameter], null)
|
||||
when:
|
||||
def encoded = encoder.encode(wfResults)
|
||||
def object = encoder.decode(encoded)
|
||||
|
||||
then:
|
||||
object instanceof TaskOutput
|
||||
def result = object as TaskOutput
|
||||
result.createdAt == time
|
||||
result.taskRun == "lid://1234"
|
||||
result.workflowRun == "lid://5678"
|
||||
result.output.size() == 1
|
||||
result.output[0] == parameter
|
||||
}
|
||||
|
||||
def 'object with null date attributes' () {
|
||||
given:
|
||||
def encoder = new LinEncoder()
|
||||
and:
|
||||
def wfResults = new WorkflowOutput(null, "lid://1234")
|
||||
when:
|
||||
def encoded = encoder.encode(wfResults)
|
||||
def object = encoder.decode(encoded)
|
||||
then:
|
||||
encoded == '{"version":"lineage/v1beta1","kind":"WorkflowOutput","spec":{"createdAt":null,"workflowRun":"lid://1234","output":null}}'
|
||||
def result = object as WorkflowOutput
|
||||
result.createdAt == null
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.serde
|
||||
|
||||
|
||||
import nextflow.lineage.model.v1beta1.FileOutput
|
||||
import nextflow.lineage.model.v1beta1.LinModel
|
||||
import nextflow.lineage.model.v1beta1.TaskRun
|
||||
import spock.lang.Specification
|
||||
|
||||
class LinTypeAdapterFactoryTest extends Specification {
|
||||
|
||||
def 'should read both new and old JSON formats'() {
|
||||
given:
|
||||
def encoder = new LinEncoder()
|
||||
|
||||
when: 'create old format JSON (without spec wrapper)'
|
||||
def oldFormatJson = """
|
||||
{
|
||||
"version": "${LinModel.VERSION}",
|
||||
"kind": "FileOutput",
|
||||
"path": "/path/to/file",
|
||||
"checksum": {
|
||||
"value": "hash_value",
|
||||
"algorithm": "hash_algorithm",
|
||||
"mode": "standard"
|
||||
},
|
||||
"source": "lid://source",
|
||||
"workflow": "lid://workflow",
|
||||
"task": "lid://task",
|
||||
"size": 1234
|
||||
}
|
||||
"""
|
||||
|
||||
and: 'deserialize old format'
|
||||
def oldResult = encoder.decode(oldFormatJson)
|
||||
|
||||
then: 'should work correctly with backward compatibility'
|
||||
oldResult instanceof FileOutput
|
||||
oldResult.path == "/path/to/file"
|
||||
oldResult.checksum.value == "hash_value"
|
||||
oldResult.source == "lid://source"
|
||||
oldResult.size == 1234
|
||||
|
||||
when: 'create new format JSON (with spec wrapper)'
|
||||
def newFormatJson = """
|
||||
{
|
||||
"version": "${LinModel.VERSION}",
|
||||
"kind": "FileOutput",
|
||||
"spec": {
|
||||
"path": "/path/to/file",
|
||||
"checksum": {
|
||||
"value": "hash_value",
|
||||
"algorithm": "hash_algorithm",
|
||||
"mode": "standard"
|
||||
},
|
||||
"source": "lid://source",
|
||||
"workflow": "lid://workflow",
|
||||
"task": "lid://task",
|
||||
"size": 1234
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
and: 'deserialize new format'
|
||||
def newResult = encoder.decode(newFormatJson)
|
||||
|
||||
then: 'should work correctly'
|
||||
newResult instanceof FileOutput
|
||||
newResult.path == "/path/to/file"
|
||||
newResult.checksum.value == "hash_value"
|
||||
newResult.source == "lid://source"
|
||||
newResult.size == 1234
|
||||
}
|
||||
|
||||
def 'should handle TaskRun old format'() {
|
||||
given:
|
||||
def encoder = new LinEncoder()
|
||||
|
||||
when: 'create old format TaskRun JSON'
|
||||
def oldFormatJson = """
|
||||
{
|
||||
"version": "${LinModel.VERSION}",
|
||||
"kind": "TaskRun",
|
||||
"sessionId": "session123",
|
||||
"name": "testTask",
|
||||
"codeChecksum": {
|
||||
"value": "hash123",
|
||||
"algorithm": "nextflow",
|
||||
"mode": "standard"
|
||||
},
|
||||
"script": "echo hello",
|
||||
"input": [],
|
||||
"container": "ubuntu:latest",
|
||||
"conda": null,
|
||||
"spack": null,
|
||||
"architecture": "amd64",
|
||||
"globalVars": {},
|
||||
"binEntries": []
|
||||
}
|
||||
"""
|
||||
|
||||
and: 'deserialize old format'
|
||||
def result = encoder.decode(oldFormatJson)
|
||||
|
||||
then: 'should work correctly'
|
||||
result instanceof TaskRun
|
||||
result.sessionId == "session123"
|
||||
result.name == "testTask"
|
||||
result.codeChecksum.value == "hash123"
|
||||
result.script == "echo hello"
|
||||
result.container == "ubuntu:latest"
|
||||
result.architecture == "amd64"
|
||||
}
|
||||
|
||||
def 'should reject JSON without version field'() {
|
||||
given:
|
||||
def encoder = new LinEncoder()
|
||||
|
||||
when: 'try to deserialize JSON without version'
|
||||
def invalidJson = """
|
||||
{
|
||||
"kind": "FileOutput",
|
||||
"path": "/path/to/file"
|
||||
}
|
||||
"""
|
||||
encoder.decode(invalidJson)
|
||||
|
||||
then: 'should throw exception'
|
||||
thrown(Exception)
|
||||
}
|
||||
|
||||
def 'should reject JSON with wrong version'() {
|
||||
given:
|
||||
def encoder = new LinEncoder()
|
||||
|
||||
when: 'try to deserialize JSON with wrong version'
|
||||
def invalidJson = """
|
||||
{
|
||||
"version": "wrong/version",
|
||||
"kind": "FileOutput",
|
||||
"path": "/path/to/file"
|
||||
}
|
||||
"""
|
||||
encoder.decode(invalidJson)
|
||||
|
||||
then: 'should throw exception'
|
||||
thrown(Exception)
|
||||
}
|
||||
|
||||
def 'should reject JSON without kind field'() {
|
||||
given:
|
||||
def encoder = new LinEncoder()
|
||||
|
||||
when: 'try to deserialize JSON without kind'
|
||||
def invalidJson = """
|
||||
{
|
||||
"version": "${LinModel.VERSION}",
|
||||
"path": "/path/to/file"
|
||||
}
|
||||
"""
|
||||
encoder.decode(invalidJson)
|
||||
|
||||
then: 'should throw exception'
|
||||
thrown(Exception)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user