add nextflow d30e48d

This commit is contained in:
2026-04-29 23:01:54 +02:00
parent d0b12d668d
commit 97cc9058d3
2840 changed files with 730250 additions and 0 deletions

View File

@@ -0,0 +1,79 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage
import spock.lang.Specification
import java.nio.file.Files
import java.nio.file.Path
/**
* Lineage History file tests
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
class DefaultLinHistoryLogTest extends Specification {
Path tempDir
Path historyFile
DefaultLinHistoryLog linHistoryLog
def setup() {
tempDir = Files.createTempDirectory("wdir")
historyFile = tempDir.resolve("lin-history")
linHistoryLog = new DefaultLinHistoryLog(historyFile)
}
def cleanup(){
tempDir?.deleteDir()
}
def "write should add a new file to the history folder"() {
given:
UUID sessionId = UUID.randomUUID()
String runName = "TestRun"
String runLid = "lid://123"
when:
linHistoryLog.write(runName, sessionId, runLid)
then:
def files = historyFile.listFiles()
files.size() == 1
def parsedRecord = LinHistoryRecord.parse(files[0].text)
parsedRecord.sessionId == sessionId
parsedRecord.runName == runName
parsedRecord.runLid == runLid
}
def 'should get records' () {
given:
UUID sessionId = UUID.randomUUID()
String runName = "Run1"
String runLid = "lid://123"
and:
linHistoryLog.write(runName, sessionId, runLid)
when:
def records = linHistoryLog.getRecords()
then:
records.size() == 1
records[0].sessionId == sessionId
records[0].runName == runName
records[0].runLid == runLid
}
}

View File

@@ -0,0 +1,49 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage
import nextflow.lineage.config.LineageConfig
import spock.lang.Specification
import spock.lang.Unroll
/**
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
class DefaultLinStoreFactoryTest extends Specification {
@Unroll
def 'should validate can open' () {
given:
def factory = new DefaultLinStoreFactory()
def config = new LineageConfig(CONFIG)
expect:
factory.canOpen(config) == EXPECTED
where:
EXPECTED | CONFIG
true | [:]
true | [store:[location:'/some/path']]
true | [store:[location:'some/rel/path']]
true | [store:[location:'file:/this/that']]
true | [store:[location:'s3://some/path']]
false | [store:[location:'http://some/path']]
false | [store:[location:'jdbc:foo']]
}
}

View File

@@ -0,0 +1,161 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage
import java.nio.file.Files
import java.nio.file.Path
import java.time.Instant
import java.time.OffsetDateTime
import java.time.ZoneOffset
import nextflow.lineage.model.v1beta1.Checksum
import nextflow.lineage.model.v1beta1.DataPath
import nextflow.lineage.model.v1beta1.FileOutput
import nextflow.lineage.model.v1beta1.Parameter
import nextflow.lineage.model.v1beta1.Workflow
import nextflow.lineage.model.v1beta1.WorkflowRun
import nextflow.lineage.serde.LinEncoder
import nextflow.lineage.config.LineageConfig
import spock.lang.Specification
import spock.lang.TempDir
/**
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
class DefaultLinStoreTest extends Specification {
@TempDir
Path tempDir
Path storeLocation
LineageConfig config
def setup() {
storeLocation = tempDir.resolve("store")
def configMap = [enabled: true, store: [location: storeLocation.toString(), logLocation: storeLocation.resolve(".log").toString()]]
config = new LineageConfig(configMap)
}
def 'should open store'() {
given:
def store = new DefaultLinStore()
when:
store.open(config)
def historyLog = store.getHistoryLog()
then:
store.getLocation() == storeLocation
historyLog != null
historyLog instanceof DefaultLinHistoryLog
}
def "save should store value in the correct file location"() {
given:
def key = "testKey"
def value = new FileOutput("/path/to/file", new Checksum("hash_value", "hash_algorithm", "standard"), "lid://source", "lid://workflow", "lid://task", 1234)
def lidStore = new DefaultLinStore()
lidStore.open(config)
when:
lidStore.save(key, value)
then:
def filePath = storeLocation.resolve("$key/.data.json")
Files.exists(filePath)
filePath.text == new LinEncoder().encode(value)
}
def "load should retrieve stored value correctly"() {
given:
def key = "testKey"
def value = new FileOutput("/path/to/file", new Checksum("hash_value", "hash_algorithm", "standard"), "lid://source", "lid://workflow", "lid://task", 1234)
def lidStore = new DefaultLinStore()
lidStore.open(config)
lidStore.save(key, value)
expect:
lidStore.load(key).toString() == value.toString()
}
def "load should return null if key does not exist"() {
given:
def lidStore = new DefaultLinStore()
lidStore.open(config)
expect:
lidStore.load("nonexistentKey") == null
}
def 'should query' () {
given:
def uniqueId = UUID.randomUUID()
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(1234567), ZoneOffset.UTC)
def mainScript = new DataPath("file://path/to/main.nf", new Checksum("78910", "nextflow", "standard"))
def workflow = new Workflow([mainScript],"https://nextflow.io/nf-test/", "123456" )
def key = "testKey"
def value1 = new WorkflowRun(workflow, uniqueId.toString(), "test_run", [ new Parameter("String", "param1", "value1"), new Parameter("String", "param2", "value2")] )
def key2 = "testKey2"
def value2 = new FileOutput("/path/tp/file1", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", null, 1234, time, time, ["value1", "value2"])
def key3 = "testKey3"
def value3 = new FileOutput("/path/tp/file2", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", null, 1234, time, time, ["value2", "value3"])
def key4 = "testKey4"
def value4 = new FileOutput("/path/tp/file", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", null, 1234, time, time, ["value4", "value3"])
def lidStore = new DefaultLinStore()
lidStore.open(config)
lidStore.save(key, value1)
lidStore.save(key2, value2)
lidStore.save(key3, value3)
lidStore.save(key4, value4)
when:
def results = lidStore.search( [type:['FileOutput'], labels:['value2']]).toList()
then:
results.size() == 2
results.containsAll([key2,key3])
}
def 'should search subkeys' () {
given:
def uniqueId = UUID.randomUUID()
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(1234567), ZoneOffset.UTC)
def mainScript = new DataPath("file://path/to/main.nf", new Checksum("78910", "nextflow", "standard"))
def workflow = new Workflow([mainScript], "https://nextflow.io/nf-test/", "123456")
def key = "testKey"
def value1 = new WorkflowRun(workflow, uniqueId.toString(), "test_run", [new Parameter("String", "param1", "value1"), new Parameter("String", "param2", "value2")])
def key2 = "testKey/file1"
def value2 = new FileOutput("/path/file1", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", null, 1234, time, time, ["value1", "value2"])
def key3 = "testKey/subfolder/file3"
def value3 = new FileOutput("/path//file2", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", null, 1234, time, time, ["value2", "value3"])
def key4 = "testKey2/file2"
def value4 = new FileOutput("/path/tp/file", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", null, 1234, time, time, ["value4", "value3"])
def lidStore = new DefaultLinStore()
lidStore.open(config)
lidStore.save(key, value1)
lidStore.save(key2, value2)
lidStore.save(key3, value3)
lidStore.save(key4, value4)
when:
def results = lidStore.getSubKeys("testKey").toList()
then:
results.size() == 2
results.containsAll([key2, key3])
}
}

View File

@@ -0,0 +1,112 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage
import java.nio.file.Path
import java.time.Instant
import java.time.OffsetDateTime
import java.time.ZoneOffset
import groovyx.gpars.dataflow.DataflowQueue
import nextflow.Channel
import nextflow.Session
import nextflow.lineage.config.LineageConfig
import nextflow.lineage.fs.LinPathFactory
import nextflow.lineage.model.v1beta1.Checksum
import nextflow.lineage.model.v1beta1.DataPath
import nextflow.lineage.model.v1beta1.FileOutput
import nextflow.lineage.model.v1beta1.Parameter
import nextflow.lineage.model.v1beta1.Workflow
import nextflow.lineage.model.v1beta1.WorkflowRun
import spock.lang.Specification
import spock.lang.TempDir
import static nextflow.lineage.fs.LinPath.*
import static test.ScriptHelper.runDataflow
/**
* Lineage channel extensions tests
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
class LinExtensionImplTest extends Specification {
@TempDir
Path tempDir
Path storeLocation
Map configMap
def setup() {
storeLocation = tempDir.resolve("store")
configMap = [linage: [enabled: true, store: [location: storeLocation.toString()]]]
}
def 'should return global query results' () {
given:
def uniqueId = UUID.randomUUID()
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(1234567), ZoneOffset.UTC)
def mainScript = new DataPath("file://path/to/main.nf", new Checksum("78910", "nextflow", "standard"))
def workflow = new Workflow([mainScript],"https://nextflow.io/nf-test/", "123456" )
def key = "testKey"
def value1 = new WorkflowRun(workflow, uniqueId.toString(), "test_run", [ new Parameter("String", "param1", "value1"), new Parameter("String", "param2", "value2")] )
def key2 = "testKey2"
def value2 = new FileOutput("/path/tp/file1", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", "taskid", 1234, time, time, ["value1","value2"])
def key3 = "testKey3"
def value3 = new FileOutput("/path/tp/file2", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", null, 1234, time, time, ["value2", "value3"])
def key4 = "testKey4"
def value4 = new FileOutput("/path/tp/file", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", "taskid", 1234, time, time, ["value4","value3"])
def lidStore = new DefaultLinStore()
def session = Mock(Session) {
getConfig() >> configMap
}
lidStore.open(LineageConfig.create(session))
lidStore.save(key, value1)
lidStore.save(key2, value2)
lidStore.save(key3, value3)
lidStore.save(key4, value4)
def linExt = Spy(new LinExtensionImpl())
when:
def results = new DataflowQueue()
linExt.fromLineage(session, results, [label: ["value2", "value3"]])
then:
linExt.getStore(session) >> lidStore
and:
results.val == LinPathFactory.create( asUriString(key3) )
results.val == Channel.STOP
when:
results = new DataflowQueue()
linExt.fromLineage(session, results, [taskRun: "taskid", label: ["value4"]])
then:
linExt.getStore(session) >> lidStore
and:
results.val == LinPathFactory.create( asUriString(key4) )
results.val == Channel.STOP
when:
results = new DataflowQueue()
linExt.fromLineage(session, results, [workflowRun: "testkey", taskRun: "taskid", label: "value2"])
then:
linExt.getStore(session) >> lidStore
and:
results.val == LinPathFactory.create( asUriString(key2) )
results.val == Channel.STOP
}
}

View File

@@ -0,0 +1,61 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage
import spock.lang.Specification
/**
* Lineage History Record tests
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
class LinHistoryRecordTest extends Specification {
def "LinHistoryRecord parse should throw for invalid record"() {
when:
LinHistoryRecord.parse("invalid-record")
then:
thrown(IllegalArgumentException)
}
def "LinHistoryRecord parse should handle 4-column record"() {
given:
def timestamp = new Date()
def formattedTimestamp = LinHistoryRecord.TIMESTAMP_FMT.format(timestamp)
def line = "${formattedTimestamp}\trun-1\t${UUID.randomUUID()}\tlid://123"
when:
def record = LinHistoryRecord.parse(line)
then:
record.timestamp != null
record.runName == "run-1"
record.runLid == "lid://123"
}
def "LinHistoryRecord toString should produce tab-separated format"() {
given:
UUID sessionId = UUID.randomUUID()
def record = new LinHistoryRecord(new Date(), "TestRun", sessionId, "lid://123")
when:
def line = record.toString()
then:
line.contains("\t")
line.split("\t").size() == 4
}
}

View File

@@ -0,0 +1,983 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage
import nextflow.extension.FilesEx
import nextflow.lineage.exception.OutputRelativePathException
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.attribute.BasicFileAttributes
import com.google.common.hash.HashCode
import nextflow.NextflowMeta
import nextflow.Session
import nextflow.file.FileHolder
import nextflow.lineage.model.v1beta1.Checksum
import nextflow.lineage.model.v1beta1.FileOutput
import nextflow.lineage.model.v1beta1.DataPath
import nextflow.lineage.model.v1beta1.Parameter
import nextflow.lineage.model.v1beta1.TaskOutput
import nextflow.lineage.model.v1beta1.Workflow
import nextflow.lineage.model.v1beta1.WorkflowOutput
import nextflow.lineage.model.v1beta1.WorkflowRun
import nextflow.lineage.serde.LinEncoder
import nextflow.lineage.config.LineageConfig
import nextflow.processor.TaskConfig
import nextflow.processor.TaskHandler
import nextflow.processor.TaskId
import nextflow.processor.TaskRun
import nextflow.script.ScriptBinding
import nextflow.script.PlatformMetadata
import nextflow.script.ScriptMeta
import nextflow.script.TokenVar
import nextflow.script.WorkflowMetadata
import nextflow.script.params.EnvOutParam
import nextflow.script.params.FileInParam
import nextflow.script.params.FileOutParam
import nextflow.script.params.InParam
import nextflow.script.params.OutParam
import nextflow.script.params.StdInParam
import nextflow.script.params.StdOutParam
import nextflow.script.params.ValueInParam
import nextflow.script.params.ValueOutParam
import nextflow.trace.event.FilePublishEvent
import nextflow.trace.event.TaskEvent
import nextflow.trace.event.WorkflowOutputEvent
import nextflow.util.CacheHelper
import nextflow.util.PathNormalizer
import spock.lang.Shared
import spock.lang.Specification
import spock.lang.Unroll
import static nextflow.lineage.fs.LinPath.*
/**
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
class LinObserverTest extends Specification {
@Shared
Path lidFolder = Files.createTempDirectory("lid")
def cleanupSpec(){
lidFolder.deleteDir()
}
def 'should normalize paths' (){
given:
def folder = Files.createTempDirectory('test')
def workDir = folder.resolve("workDir")
def projectDir = folder.resolve("projectDir")
def metadata = Mock(WorkflowMetadata){
getRepository() >> "https://nextflow.io/nf-test/"
getCommitId() >> "123456"
getScriptId() >> "78910"
getProjectDir() >> projectDir
getWorkDir() >> workDir
}
def params = [path: workDir.resolve("path/file.txt"), sequence: projectDir.resolve("file2.txt").toString(), value: 12]
when:
def results = LinObserver.getNormalizedParams(params, new PathNormalizer(metadata))
then:
results.size() == 3
results.get(0).name == "path"
results.get(0).type == Path.simpleName
results.get(0).value == "work/path/file.txt"
results.get(1).name == "sequence"
results.get(1).type == "String"
results.get(1).value == projectDir.resolve("file2.txt").toString()
results.get(2).name == "value"
results.get(2).type == "Integer"
results.get(2).value == 12
cleanup:
ScriptMeta.reset()
folder?.deleteDir()
}
def 'should collect script files' () {
given:
def folder = Files.createTempDirectory('test')
and:
def config = [workflow:[lineage:[enabled: true, store:[location:folder.toString()]]]]
def store = new DefaultLinStore();
def uniqueId = UUID.randomUUID()
def scriptFile = folder.resolve("main.nf")
def module1 = folder.resolve("a_script1.nf"); module1.text = 'hola'
def module2 = folder.resolve("b_script2.nf"); module2.text = 'world'
and:
def metadata = Mock(WorkflowMetadata){
getRepository() >> "https://nextflow.io/nf-test/"
getCommitId() >> "123456"
getScriptId() >> "78910"
getScriptFile() >> scriptFile
getProjectDir() >> folder.resolve("projectDir")
getWorkDir() >> folder.resolve("workDir")
}
def session = Mock(Session) {
getConfig() >> config
getUniqueId() >> uniqueId
getRunName() >> "test_run"
getWorkflowMetadata() >> metadata
getParams() >> new ScriptBinding.ParamsMap()
}
store.open(LineageConfig.create(session))
def observer = Spy(new LinObserver(session, store))
when:
def files = observer.collectScriptDataPaths(new PathNormalizer(metadata))
then:
observer.allScriptFiles() >> [ module2, scriptFile, module1 ]
and:
files.size() == 3
and:
files[0].path == "file://${scriptFile.toString()}"
files[0].checksum == new Checksum("78910", "nextflow", "standard")
and:
files[1].path == "file://$module1"
files[1].checksum == Checksum.ofNextflow(module1.text)
and:
files[2].path == "file://$module2"
files[2].checksum == Checksum.ofNextflow(module2.text)
cleanup:
ScriptMeta.reset()
folder?.deleteDir()
}
def 'should save workflow' (){
given:
def folder = Files.createTempDirectory('test')
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
def store = new DefaultLinStore();
def uniqueId = UUID.randomUUID()
def scriptFile = folder.resolve("main.nf")
def map = [
repository: "https://nextflow.io/nf-test/",
commitId: "123456",
scriptId: "78910",
scriptFile: scriptFile,
projectDir: folder.resolve("projectDir"),
revision: "main",
projectName: "nextflow.io/nf-test",
workDir: folder.resolve("workDir")
]
def metadata = Mock(WorkflowMetadata){
getRepository() >> map.repository
getCommitId() >> map.commitId
getScriptId() >> map.scriptId
getScriptFile() >> map.scriptFile
getProjectDir() >> map.projectDir
getRevision() >> map.revision
getProjectName() >> map.projectName
getWorkDir() >> map.workDir
toMap() >> map
}
def session = Mock(Session) {
getConfig() >> config
getUniqueId() >> uniqueId
getRunName() >> "test_run"
getWorkflowMetadata() >> metadata
getParams() >> new ScriptBinding.ParamsMap()
}
store.open(LineageConfig.create(session))
def observer = new LinObserver(session, store)
def mainScript = new DataPath("file://${scriptFile.toString()}", new Checksum("78910", "nextflow", "standard"))
def workflow = new Workflow([mainScript], map.repository, map.commitId)
def workflowRun = new WorkflowRun(workflow, uniqueId.toString(), "test_run", [], config, map)
when:
observer.onFlowCreate(session)
observer.onFlowBegin()
then:
folder.resolve("${observer.executionHash}/.data.json").text == new LinEncoder().encode(workflowRun)
cleanup:
folder?.deleteDir()
}
def 'should strip sensitive user data from platform metadata in lineage' () {
given:
def folder = Files.createTempDirectory('test')
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
def store = new DefaultLinStore()
def uniqueId = UUID.randomUUID()
def scriptFile = folder.resolve("main.nf")
and:
def platformMeta = new PlatformMetadata()
platformMeta.user = new PlatformMetadata.User([
id: 'u1234', userName: 'john-smith', email: 'john.smith@acme.com',
firstName: 'John', lastName: 'Smith', organization: 'ACME'
])
platformMeta.workspace = new PlatformMetadata.Workspace([
workspaceId: '1234', workspaceName: 'my-ws', workspaceFullName: 'My WS', orgName: 'ACME'
])
def map = [
repository: "https://nextflow.io/nf-test/",
commitId: "123456",
scriptId: "78910",
scriptFile: scriptFile,
projectDir: folder.resolve("projectDir"),
revision: "main",
projectName: "nextflow.io/nf-test",
workDir: folder.resolve("workDir"),
platform: platformMeta
]
def metadata = Mock(WorkflowMetadata) {
getRepository() >> map.repository
getCommitId() >> map.commitId
getScriptId() >> map.scriptId
getScriptFile() >> map.scriptFile
getProjectDir() >> map.projectDir
getWorkDir() >> map.workDir
toMap() >> map
}
def session = Mock(Session) {
getConfig() >> config
getUniqueId() >> uniqueId
getRunName() >> "test_run"
getWorkflowMetadata() >> metadata
getParams() >> new ScriptBinding.ParamsMap()
}
store.open(LineageConfig.create(session))
def observer = new LinObserver(session, store)
when:
observer.onFlowCreate(session)
observer.onFlowBegin()
def stored = store.load(observer.executionHash) as WorkflowRun
then:
def storedPlatform = stored.metadata.platform as Map
storedPlatform.user.id == 'u1234'
storedPlatform.user.userName == 'john-smith'
storedPlatform.user.organization == 'ACME'
!storedPlatform.user.containsKey('email')
!storedPlatform.user.containsKey('firstName')
!storedPlatform.user.containsKey('lastName')
cleanup:
folder?.deleteDir()
}
def 'should convert Path values to URI strings in workflow metadata' () {
given:
def folder = Files.createTempDirectory('test')
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
def store = new DefaultLinStore()
def uniqueId = UUID.randomUUID()
def scriptFile = folder.resolve("main.nf")
def workDir = folder.resolve("workDir")
def somePath = folder.resolve("data/input.txt")
def map = [
repository : "https://nextflow.io/nf-test/",
commitId : "abc123",
scriptId : "78910",
scriptFile : scriptFile,
projectDir : folder.resolve("projectDir"),
workDir : workDir,
somePathKey : somePath,
someStringKey: "hello"
]
def metadata = Mock(WorkflowMetadata) {
getRepository() >> map.repository
getCommitId() >> map.commitId
getScriptId() >> map.scriptId
getScriptFile() >> map.scriptFile
getProjectDir() >> map.projectDir
getWorkDir() >> map.workDir
toMap() >> map
}
def session = Mock(Session) {
getConfig() >> config
getUniqueId() >> uniqueId
getRunName() >> "test_run"
getWorkflowMetadata() >> metadata
getParams() >> new ScriptBinding.ParamsMap()
}
store.open(LineageConfig.create(session))
def observer = new LinObserver(session, store)
when:
observer.onFlowCreate(session)
observer.onFlowBegin()
def stored = store.load(observer.executionHash) as WorkflowRun
then:
stored.metadata.somePathKey == FilesEx.toUriString(somePath)
stored.metadata.someStringKey == "hello"
cleanup:
folder?.deleteDir()
}
def 'should remove transient properties from workflow metadata' () {
given:
def folder = Files.createTempDirectory('test')
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
def store = new DefaultLinStore()
def uniqueId = UUID.randomUUID()
def scriptFile = folder.resolve("main.nf")
def map = [
repository : "https://nextflow.io/nf-test/",
commitId : "abc123",
scriptId : "78910",
scriptFile : scriptFile,
projectDir : folder.resolve("projectDir"),
workDir : folder.resolve("workDir"),
// transient properties that must be removed
completed : new Date(),
duration : 1000L,
exitStatus : 0,
errorMessage: "none",
errorReport : "none",
stats : [:],
success : true
]
def metadata = Mock(WorkflowMetadata) {
getRepository() >> map.repository
getCommitId() >> map.commitId
getScriptId() >> map.scriptId
getScriptFile() >> map.scriptFile
getProjectDir() >> map.projectDir
getWorkDir() >> map.workDir
toMap() >> map
}
def session = Mock(Session) {
getConfig() >> config
getUniqueId() >> uniqueId
getRunName() >> "test_run"
getWorkflowMetadata() >> metadata
getParams() >> new ScriptBinding.ParamsMap()
}
store.open(LineageConfig.create(session))
def observer = new LinObserver(session, store)
when:
observer.onFlowCreate(session)
observer.onFlowBegin()
def stored = store.load(observer.executionHash) as WorkflowRun
then:
!stored.metadata.containsKey('completed')
!stored.metadata.containsKey('duration')
!stored.metadata.containsKey('exitStatus')
!stored.metadata.containsKey('errorMessage')
!stored.metadata.containsKey('errorReport')
!stored.metadata.containsKey('stats')
!stored.metadata.containsKey('success')
stored.metadata.containsKey('commitId')
cleanup:
folder?.deleteDir()
}
def 'should convert NextflowMeta to JSON map in workflow metadata' () {
given:
def folder = Files.createTempDirectory('test')
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
def store = new DefaultLinStore()
def uniqueId = UUID.randomUUID()
def scriptFile = folder.resolve("main.nf")
def nfMeta = new NextflowMeta("24.10.0", 9999, "01-01-2024 00:00 UTC")
def map = [
repository : "https://nextflow.io/nf-test/",
commitId : "abc123",
scriptId : "78910",
scriptFile : scriptFile,
projectDir : folder.resolve("projectDir"),
workDir : folder.resolve("workDir"),
nextflow : nfMeta
]
def metadata = Mock(WorkflowMetadata) {
getRepository() >> map.repository
getCommitId() >> map.commitId
getScriptId() >> map.scriptId
getScriptFile() >> map.scriptFile
getProjectDir() >> map.projectDir
getWorkDir() >> map.workDir
toMap() >> map
}
def session = Mock(Session) {
getConfig() >> config
getUniqueId() >> uniqueId
getRunName() >> "test_run"
getWorkflowMetadata() >> metadata
getParams() >> new ScriptBinding.ParamsMap()
}
store.open(LineageConfig.create(session))
def observer = new LinObserver(session, store)
when:
observer.onFlowCreate(session)
observer.onFlowBegin()
def stored = store.load(observer.executionHash) as WorkflowRun
then:
stored.metadata.nextflow instanceof Map
(stored.metadata.nextflow as Map).version == "24.10.0"
cleanup:
folder?.deleteDir()
}
def 'should normalize configFiles in workflow metadata' () {
given:
def folder = Files.createTempDirectory('test')
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
def store = new DefaultLinStore()
def uniqueId = UUID.randomUUID()
def scriptFile = folder.resolve("main.nf")
def workDir = folder.resolve("workDir")
def projectDir = folder.resolve("projectDir")
def configFile = projectDir.resolve("nextflow.config")
def map = [
repository : "https://nextflow.io/nf-test/",
commitId : "abc123",
scriptId : "78910",
scriptFile : scriptFile,
projectDir : projectDir,
workDir : workDir,
configFiles : [configFile]
]
def metadata = Mock(WorkflowMetadata) {
getRepository() >> map.repository
getCommitId() >> map.commitId
getScriptId() >> map.scriptId
getScriptFile() >> map.scriptFile
getProjectDir() >> map.projectDir
getWorkDir() >> map.workDir
toMap() >> map
}
def session = Mock(Session) {
getConfig() >> config
getUniqueId() >> uniqueId
getRunName() >> "test_run"
getWorkflowMetadata() >> metadata
getParams() >> new ScriptBinding.ParamsMap()
}
store.open(LineageConfig.create(session))
def observer = new LinObserver(session, store)
when:
observer.onFlowCreate(session)
observer.onFlowBegin()
def stored = store.load(observer.executionHash) as WorkflowRun
then:
def normalizer = new PathNormalizer(metadata)
(stored.metadata.configFiles as List) == [normalizer.normalizePath(configFile)]
cleanup:
folder?.deleteDir()
}
def 'should handle null user in platform metadata' () {
given:
def folder = Files.createTempDirectory('test')
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
def store = new DefaultLinStore()
def uniqueId = UUID.randomUUID()
def scriptFile = folder.resolve("main.nf")
def platformMeta = new PlatformMetadata()
platformMeta.workflowId = 'wf-999'
platformMeta.workflowUrl = 'https://tower.nf/orgs/org/workspaces/ws/watch/wf-999'
// user is intentionally left null
def map = [
repository : "https://nextflow.io/nf-test/",
commitId : "abc123",
scriptId : "78910",
scriptFile : scriptFile,
projectDir : folder.resolve("projectDir"),
workDir : folder.resolve("workDir"),
platform : platformMeta
]
def metadata = Mock(WorkflowMetadata) {
getRepository() >> map.repository
getCommitId() >> map.commitId
getScriptId() >> map.scriptId
getScriptFile() >> map.scriptFile
getProjectDir() >> map.projectDir
getWorkDir() >> map.workDir
toMap() >> map
}
def session = Mock(Session) {
getConfig() >> config
getUniqueId() >> uniqueId
getRunName() >> "test_run"
getWorkflowMetadata() >> metadata
getParams() >> new ScriptBinding.ParamsMap()
}
store.open(LineageConfig.create(session))
def observer = new LinObserver(session, store)
when:
observer.onFlowCreate(session)
observer.onFlowBegin()
def stored = store.load(observer.executionHash) as WorkflowRun
then:
def storedPlatform = stored.metadata.platform as Map
storedPlatform.workflowId == 'wf-999'
storedPlatform.workflowUrl == 'https://tower.nf/orgs/org/workspaces/ws/watch/wf-999'
storedPlatform.user == null
cleanup:
folder?.deleteDir()
}
def 'should return empty metadata when toMap throws' () {
given:
def folder = Files.createTempDirectory('test')
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
def store = new DefaultLinStore()
def uniqueId = UUID.randomUUID()
def scriptFile = folder.resolve("main.nf")
def metadata = Mock(WorkflowMetadata) {
getRepository() >> "https://nextflow.io/nf-test/"
getCommitId() >> "abc123"
getScriptId() >> "78910"
getScriptFile() >> scriptFile
getProjectDir() >> folder.resolve("projectDir")
getWorkDir() >> folder.resolve("workDir")
toMap() >> { throw new RuntimeException("metadata error") }
}
def session = Mock(Session) {
getConfig() >> config
getUniqueId() >> uniqueId
getRunName() >> "test_run"
getWorkflowMetadata() >> metadata
getParams() >> new ScriptBinding.ParamsMap()
}
store.open(LineageConfig.create(session))
def observer = new LinObserver(session, store)
when:
observer.onFlowCreate(session)
observer.onFlowBegin()
def stored = store.load(observer.executionHash) as WorkflowRun
then:
stored.metadata == null
cleanup:
folder?.deleteDir()
}
@Unroll
def 'should get parameter type' () {
expect:
LinObserver.getParameterType(PARAM) == STRING
where:
PARAM | STRING
null | null
new FileInParam(null, []) | "path"
new ValueOutParam(null, []) | "val"
new EnvOutParam(null, []) | "env"
new StdInParam(null, []) | "stdin"
new StdOutParam(null, []) | "stdout"
Path.of("test") | "Path"
["test"] | "Collection"
[key:"value"] | "Map"
}
def 'should save task run' () {
given:
def folder = Files.createTempDirectory('test').toRealPath()
def config = [workflow:[lineage:[enabled: true, store:[location:folder.toString()]]]]
def uniqueId = UUID.randomUUID()
def workDir = folder.resolve("work")
def session = Mock(Session) {
getConfig()>>config
getUniqueId()>>uniqueId
getRunName()>>"test_run"
getWorkDir() >> workDir
}
def metadata = Mock(WorkflowMetadata){
getRepository() >> "https://nextflow.io/nf-test/"
getCommitId() >> "123456"
getScriptId() >> "78910"
getProjectDir() >> folder.resolve("projectDir")
getWorkDir() >> workDir
}
and:
def store = new DefaultLinStore();
store.open(LineageConfig.create(session))
and:
def observer = Spy(new LinObserver(session, store))
def normalizer = new PathNormalizer(metadata)
observer.executionHash = "hash"
observer.normalizer = normalizer
and:
def hash = HashCode.fromString("1234567890")
def taskWd = workDir.resolve('12/34567890')
Files.createDirectories(taskWd)
and:
observer.getTaskGlobalVars(_) >> [:]
observer.getTaskBinEntries(_) >> []
and: 'Task Inputs'
def inputs = new LinkedHashMap<InParam, Object>()
// File from task
inputs.put(new FileInParam(null, []).bind("file1"), [new FileHolder(workDir.resolve('78/567890/file1.txt'))])
// Normal file
def file = folder.resolve("file2.txt")
file.text = "this is a test file"
def fileHash = CacheHelper.hasher(file).hash().toString()
inputs.put(new FileInParam(null, []).bind("file2"), [new FileHolder(file)])
//Value input
inputs.put(new ValueInParam(null, []).bind("id"), "value")
and: 'Task Outputs'
def outputs = new LinkedHashMap<OutParam, Object>()
// Single Path output
def outFile1 = taskWd.resolve('fileOut1.txt')
outFile1.text = 'some data'
def fileHash1 = CacheHelper.hasher(outFile1).hash().toString()
def attrs1 = Files.readAttributes(outFile1, BasicFileAttributes)
outputs.put(new FileOutParam(null, []).bind(new TokenVar("file1")), outFile1)
// Collection Path output
def outFile2 = taskWd.resolve('fileOut2.txt')
outFile2.text = 'some other data'
def fileHash2 = CacheHelper.hasher(outFile2).hash().toString()
def attrs2 = Files.readAttributes(outFile2, BasicFileAttributes)
outputs.put(new FileOutParam(null, []).bind(new TokenVar("file2")), [outFile2])
outputs.put(new ValueOutParam(null, []).bind(new TokenVar("id")), "value")
and: 'Task description'
def task = Mock(TaskRun) {
getId() >> TaskId.of(100)
getName() >> 'foo'
getHash() >> hash
getSource() >> 'echo task source'
getScript() >> 'this is the script'
getInputs() >> inputs
getOutputs() >> outputs
getWorkDir() >> taskWd
}
def handler = Mock(TaskHandler){
getTask() >> task
}
and: 'Expected LID objects'
def sourceHash = CacheHelper.hasher('echo task source').hash().toString()
def script = 'this is the script'
def taskDescription = new nextflow.lineage.model.v1beta1.TaskRun(uniqueId.toString(), "foo",
new Checksum(sourceHash, "nextflow", "standard"),
script,
[
new Parameter("path", "file1", ['lid://78567890/file1.txt']),
new Parameter("path", "file2", [[path: normalizer.normalizePath(file), checksum: [value:fileHash, algorithm: "nextflow", mode: "standard"]]]),
new Parameter("val", "id", "value")
], null, null, null, null, [:], [], "lid://hash")
def dataOutput1 = new FileOutput(outFile1.toString(), new Checksum(fileHash1, "nextflow", "standard"),
"lid://1234567890", "lid://hash", "lid://1234567890", attrs1.size(), LinUtils.toDate(attrs1.creationTime()), LinUtils.toDate(attrs1.lastModifiedTime()) )
def dataOutput2 = new FileOutput(outFile2.toString(), new Checksum(fileHash2, "nextflow", "standard"),
"lid://1234567890", "lid://hash", "lid://1234567890", attrs2.size(), LinUtils.toDate(attrs2.creationTime()), LinUtils.toDate(attrs2.lastModifiedTime()) )
when:
observer.onTaskComplete(new TaskEvent(handler, null))
def taskRunResult = store.load("${hash.toString()}")
def dataOutputResult1 = store.load("${hash}/fileOut1.txt") as FileOutput
def dataOutputResult2 = store.load("${hash}/fileOut2.txt") as FileOutput
def taskOutputsResult = store.load("${hash}#output") as TaskOutput
then:
taskRunResult == taskDescription
dataOutputResult1 == dataOutput1
dataOutputResult2 == dataOutput2
taskOutputsResult.taskRun == "lid://1234567890"
taskOutputsResult.workflowRun == "lid://hash"
taskOutputsResult.output.size() == 3
taskOutputsResult.output.get(0).type == "path"
taskOutputsResult.output.get(0).name == "file1"
taskOutputsResult.output.get(0).value == "lid://1234567890/fileOut1.txt"
taskOutputsResult.output.get(1).type == "path"
taskOutputsResult.output.get(1).name == "file2"
taskOutputsResult.output.get(1).value == ["lid://1234567890/fileOut2.txt"]
taskOutputsResult.output.get(2).type == "val"
taskOutputsResult.output.get(2).name == "id"
taskOutputsResult.output.get(2).value == "value"
cleanup:
folder?.deleteDir()
}
def 'should save task data output' () {
given:
def folder = Files.createTempDirectory('test')
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
def store = new DefaultLinStore();
def session = Mock(Session) {
getConfig()>>config
}
store.open(LineageConfig.create(session))
def observer = Spy(new LinObserver(session, store))
observer.executionHash = "hash"
and:
def workDir = folder.resolve('12/34567890')
Files.createDirectories(workDir)
and:
def outFile = workDir.resolve('foo/bar/file.bam')
Files.createDirectories(outFile.parent)
outFile.text = 'some data'
def fileHash = CacheHelper.hasher(outFile).hash().toString()
and:
def hash = HashCode.fromInt(123456789)
and:
def task = Mock(TaskRun) {
getId() >> TaskId.of(100)
getName() >> 'foo'
getHash() >> hash
getWorkDir() >> workDir
}
and:
def attrs = Files.readAttributes(outFile, BasicFileAttributes)
def output = new FileOutput(outFile.toString(), new Checksum(fileHash, "nextflow", "standard"),
"lid://15cd5b07", "lid://hash", "lid://15cd5b07", attrs.size(), LinUtils.toDate(attrs.creationTime()), LinUtils.toDate(attrs.lastModifiedTime()) )
and:
observer.readAttributes(outFile) >> attrs
when:
observer.storeTaskOutput(task, outFile)
then:
folder.resolve("${hash}/foo/bar/file.bam/.data.json").text == new LinEncoder().encode(output)
cleanup:
folder?.deleteDir()
}
def 'should relativise task output dirs' (){
when:
def config = [workflow:[lineage:[enabled: true, store:[location:lidFolder.toString()]]]]
def store = new DefaultLinStore();
def session = Mock(Session) {
getConfig()>>config
}
def hash = HashCode.fromInt(123456789)
def taskConfig = Mock(TaskConfig){
getStoreDir() >> STORE_DIR
}
def task = Mock(TaskRun) {
getId() >> TaskId.of(100)
getName() >> 'foo'
getHash() >> hash
getWorkDir() >> WORK_DIR
getConfig() >> taskConfig
}
store.open(LineageConfig.create(session))
def observer = new LinObserver(session, store)
then:
observer.getTaskRelative(task, PATH) == EXPECTED
where:
WORK_DIR | STORE_DIR | PATH | EXPECTED
Path.of('/path/to/work/12/3456789') | Path.of('/path/to/storeDir') | Path.of('/path/to/work/12/3456789/relative') | "relative"
Path.of('/path/to/work/12/3456789') | Path.of('/path/to/storeDir') | Path.of('/path/to/storeDir/relative') | "relative"
Path.of('work/12/3456789') | Path.of('storeDir') | Path.of('work/12/3456789/relative') | "relative"
Path.of('work/12/3456789') | Path.of('storeDir') | Path.of('storeDir/relative') | "relative"
Path.of('work/12/3456789') | Path.of('storeDir') | Path.of('results/relative') | "results/relative"
Path.of('/path/to/work/12/3456789') | Path.of('storeDir') | Path.of('./relative') | "relative"
}
@Unroll
def 'should return exception when relativize task output dirs'() {
when:
def config = [workflow:[lineage:[enabled: true, store:[location:lidFolder.toString()]]]]
def store = new DefaultLinStore();
def session = Mock(Session) {
getConfig()>>config
}
def hash = HashCode.fromInt(123456789)
def taskConfig = Mock(TaskConfig){
getStoreDir() >> STORE_DIR
}
def task = Mock(TaskRun) {
getId() >> TaskId.of(100)
getName() >> 'foo'
getHash() >> hash
getWorkDir() >> WORK_DIR
getConfig() >> taskConfig
}
store.open(LineageConfig.create(session))
def observer = new LinObserver(session, store)
observer.getTaskRelative(task, PATH)
then:
def e = thrown(IllegalArgumentException)
e.message == "Cannot access the relative path for output '$PATH' and task '${task.name}'".toString()
where:
WORK_DIR | STORE_DIR | PATH
Path.of('/path/to/work/12/3456789') | Path.of('/path/to/storeDir') | Path.of('/another/path/relative')
Path.of('/path/to/work/12/3456789') | Path.of('/path/to/storeDir') | Path.of('../path/to/storeDir/relative')
}
def 'should relativize workflow output dirs' (){
when:
def config = [workflow:[lineage:[enabled: true, store:[location:lidFolder.toString()]]]]
def store = new DefaultLinStore();
def session = Mock(Session) {
getOutputDir()>>OUTPUT_DIR
getConfig()>>config
}
store.open(LineageConfig.create(session))
def observer = new LinObserver(session, store)
then:
observer.getWorkflowRelative(PATH) == EXPECTED
where:
OUTPUT_DIR | PATH | EXPECTED
Path.of('/path/to/outDir') | Path.of('/path/to/outDir/relative') | "relative"
Path.of('outDir') | Path.of('outDir/relative') | "relative"
Path.of('/path/to/outDir') | Path.of('results/relative') | "results/relative"
Path.of('/path/to/outDir') | Path.of('./relative') | "relative"
}
@Unroll
def 'should return exception when relativize workflow output dirs' (){
when:
def config = [workflow:[lineage:[enabled: true, store:[location:lidFolder.toString()]]]]
def store = new DefaultLinStore();
def session = Mock(Session) {
getOutputDir()>>OUTPUT_DIR
getConfig()>>config
}
def observer = new LinObserver(session, store)
observer.getWorkflowRelative(PATH)
then:
thrown(OutputRelativePathException)
where:
OUTPUT_DIR | PATH | EXPECTED
Path.of('/path/to/outDir') | Path.of('/another/path/') | "relative"
Path.of('/path/to/outDir') | Path.of('../relative') | "relative"
}
def 'should save workflow output'() {
given:
def folder = Files.createTempDirectory('test')
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
def store = new DefaultLinStore();
def outputDir = folder.resolve('results')
def uniqueId = UUID.randomUUID()
def scriptFile = folder.resolve("main.nf")
def workDir= folder.resolve("work")
def metadata = Mock(WorkflowMetadata){
getRepository() >> "https://nextflow.io/nf-test/"
getCommitId() >> "123456"
getScriptId() >> "78910"
getScriptFile() >> scriptFile
getProjectDir() >> folder.resolve("projectDir")
getWorkDir() >> workDir
}
def session = Mock(Session) {
getConfig()>>config
getOutputDir()>>outputDir
getWorkDir() >> workDir
getWorkflowMetadata()>>metadata
getUniqueId()>>uniqueId
getRunName()>>"test_run"
getParams() >> new ScriptBinding.ParamsMap()
}
store.open(LineageConfig.create(session))
def observer = new LinObserver(session, store)
def encoder = new LinEncoder()
when: 'Starting workflow'
observer.onFlowCreate(session)
observer.onFlowBegin()
then: 'History file should contain execution hash'
def lid = LinHistoryRecord.parse(folder.resolve(".history/${observer.executionHash}").text)
lid.runLid == asUriString(observer.executionHash)
lid.sessionId == uniqueId
lid.runName == "test_run"
when: ' publish output with source file'
def outFile1 = outputDir.resolve('foo/file.bam')
Files.createDirectories(outFile1.parent)
outFile1.text = 'some data1'
def sourceFile1 = workDir.resolve('12/3987/file.bam')
Files.createDirectories(sourceFile1.parent)
sourceFile1.text = 'some data1'
observer.onFilePublish(new FilePublishEvent(sourceFile1, outFile1))
observer.onWorkflowOutput(new WorkflowOutputEvent("a", outFile1))
then: 'check file 1 output metadata in lid store'
def attrs1 = Files.readAttributes(outFile1, BasicFileAttributes)
def fileHash1 = CacheHelper.hasher(outFile1).hash().toString()
def output1 = new FileOutput(outFile1.toString(), new Checksum(fileHash1, "nextflow", "standard"),
"lid://123987/file.bam", "$LID_PROT${observer.executionHash}", null,
attrs1.size(), LinUtils.toDate(attrs1.creationTime()), LinUtils.toDate(attrs1.lastModifiedTime()) )
folder.resolve("${observer.executionHash}/foo/file.bam/.data.json").text == encoder.encode(output1)
when: 'publish without source path'
def outFile2 = outputDir.resolve('foo/file2.bam')
Files.createDirectories(outFile2.parent)
outFile2.text = 'some data2'
def attrs2 = Files.readAttributes(outFile2, BasicFileAttributes)
def fileHash2 = CacheHelper.hasher(outFile2).hash().toString()
observer.onFilePublish(new FilePublishEvent(null, outFile2))
observer.onWorkflowOutput(new WorkflowOutputEvent("b", outFile2))
then: 'Check outFile2 metadata in lid store'
def output2 = new FileOutput(outFile2.toString(), new Checksum(fileHash2, "nextflow", "standard"),
"lid://${observer.executionHash}" , "lid://${observer.executionHash}", null,
attrs2.size(), LinUtils.toDate(attrs2.creationTime()), LinUtils.toDate(attrs2.lastModifiedTime()) )
folder.resolve("${observer.executionHash}/foo/file2.bam/.data.json").text == encoder.encode(output2)
when: 'Workflow complete'
observer.onFlowComplete()
then: 'Check WorkflowOutput is written in the lid store'
def resultsRetrieved = store.load("${observer.executionHash}#output") as WorkflowOutput
resultsRetrieved.output == [new Parameter(Path.simpleName, "a", "lid://${observer.executionHash}/foo/file.bam"), new Parameter(Path.simpleName, "b", "lid://${observer.executionHash}/foo/file2.bam")]
cleanup:
folder?.deleteDir()
}
def 'should not save workflow output entry when no outputs'() {
given:
def folder = Files.createTempDirectory('test')
def config = [lineage: [enabled: true, store: [location: folder.toString()]]]
def store = new DefaultLinStore();
def outputDir = folder.resolve('results')
def uniqueId = UUID.randomUUID()
def scriptFile = folder.resolve("main.nf")
def workDir = folder.resolve("work")
def metadata = Mock(WorkflowMetadata) {
getRepository() >> "https://nextflow.io/nf-test/"
getCommitId() >> "123456"
getScriptId() >> "78910"
getScriptFile() >> scriptFile
getProjectDir() >> folder.resolve("projectDir")
getWorkDir() >> workDir
}
def session = Mock(Session) {
getConfig() >> config
getOutputDir() >> outputDir
getWorkDir() >> workDir
getWorkflowMetadata() >> metadata
getUniqueId() >> uniqueId
getRunName() >> "test_run"
getParams() >> new ScriptBinding.ParamsMap()
}
store.open(LineageConfig.create(session))
def observer = new LinObserver(session, store)
when:
observer.onFlowCreate(session)
observer.onFlowBegin()
observer.onFlowComplete()
def resultFile = folder.resolve("${observer.executionHash}#output")
then:
!resultFile.exists()
}
}

View File

@@ -0,0 +1,39 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage
import spock.lang.Specification
/**
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
class LinPropertyValidationTest extends Specification{
def 'should throw exception when property does not exist'(){
when:
new LinPropertyValidator().validate(['value', 'not_existing'])
then:
def e = thrown(IllegalArgumentException)
e.message.startsWith( "Property 'not_existing' doesn't exist in the lineage model")
}
def 'should not throw exception when property exist'(){
when:
new LinPropertyValidator().validate(['value', 'output'])
then:
noExceptionThrown()
}
}

View File

@@ -0,0 +1,202 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage
import java.time.ZoneId
import nextflow.lineage.model.v1beta1.Checksum
import nextflow.lineage.model.v1beta1.DataPath
import nextflow.lineage.model.v1beta1.Parameter
import nextflow.lineage.model.v1beta1.Workflow
import nextflow.lineage.model.v1beta1.WorkflowOutput
import nextflow.lineage.model.v1beta1.WorkflowRun
import nextflow.lineage.config.LineageConfig
import spock.lang.Specification
import spock.lang.TempDir
import java.nio.file.Path
import java.nio.file.attribute.FileTime
import java.time.Instant
import java.time.OffsetDateTime
import java.time.ZoneOffset
class LinUtilsTest extends Specification{
@TempDir
Path tempDir
Path storeLocation
LineageConfig config
def setup() {
storeLocation = tempDir.resolve("store")
def configMap = [enabled: true, store: [location: storeLocation.toString()]]
config = new LineageConfig(configMap)
}
def 'should convert to Date'(){
expect:
LinUtils.toDate(FILE_TIME) == DATE
where:
FILE_TIME | DATE
null | null
FileTime.fromMillis(1234) | Instant.ofEpochMilli(1234).atZone(ZoneId.systemDefault())?.toOffsetDateTime()
}
def 'should convert to FileTime'(){
expect:
LinUtils.toFileTime(DATE) == FILE_TIME
where:
FILE_TIME | DATE
null | null
FileTime.fromMillis(1234) | OffsetDateTime.ofInstant(Instant.ofEpochMilli(1234), ZoneOffset.UTC)
}
def 'should get lineage record'() {
given:
def uniqueId = UUID.randomUUID()
def mainScript = new DataPath("file://path/to/main.nf", new Checksum("78910", "nextflow", "standard"))
def workflow = new Workflow([mainScript], "https://nextflow.io/nf-test/", "123456")
def key1 = "testKey"
def value1 = new WorkflowRun(workflow, uniqueId.toString(), "test_run", [new Parameter("String", "param1", "value1"), new Parameter("String", "param2", "value2")])
def outputs1 = new WorkflowOutput(OffsetDateTime.now(), "lid://testKey", [new Parameter( "String", "output", "name")] )
def key2 = "testKey2"
def value2 = new WorkflowRun(workflow, uniqueId.toString(), "test_run2", [new Parameter("String", "param1", "value1"), new Parameter("String", "param2", "value2")])
def lidStore = new DefaultLinStore()
lidStore.open(config)
lidStore.save(key1, value1)
lidStore.save("$key1#output", outputs1)
lidStore.save(key2, value2)
when:
def params = LinUtils.getMetadataObject(lidStore, new URI('lid://testKey#params'))
then:
params instanceof List<Parameter>
(params as List<Parameter>).size() == 2
when:
def outputs = LinUtils.getMetadataObject(lidStore, new URI('lid://testKey#output'))
then:
outputs instanceof List<Parameter>
def param = (outputs as List)[0] as Parameter
param.name == "output"
when:
outputs = LinUtils.getMetadataObject(lidStore, new URI('lid://testKey2#output'))
then:
outputs instanceof List
(outputs as List).isEmpty()
when:
LinUtils.getMetadataObject(lidStore, new URI('lid://testKey#no-exist'))
then:
thrown(IllegalArgumentException)
when:
LinUtils.getMetadataObject(lidStore, new URI('lid://testKey#outputs.no-exist'))
then:
thrown(IllegalArgumentException)
when:
LinUtils.getMetadataObject(lidStore, new URI('lid://no-exist#something'))
then:
thrown(FileNotFoundException)
}
def "should check params in an object"() {
given:
def obj = [ "type": "value", "workflow": ["repository": "subvalue"], "output" : [ ["path":"/to/file"],["path":"file2"] ], "labels": ["a","b"] ]
expect:
LinUtils.checkParams(obj, PARAMS) == EXPECTED
where:
PARAMS | EXPECTED
["type": ["value"]] | true
["type": ["wrong"]] | false
["workflow.repository": ["subvalue"]] | true
["workflow.repository": ["wrong"]] | false
["output.path": ["wrong"]] | false
["output.path": ["/to/file"]] | true
["output.path": ["file2"]] | true
["labels": ["a"]] | true
["labels": ["c"]] | false
["labels": ["a","b"]] | true
["labels": ["a","b","c"]] | false
["type" : ["value", "value2"]] | false
}
def "should navigate in object params"() {
given:
def obj = [
"key1": "value1",
"nested": [
"subkey": "subvalue"
]
]
expect:
LinUtils.navigate(obj, PATH) == EXPECTED
where:
PATH | EXPECTED
"key1" | "value1"
"nested.subkey" | "subvalue"
"wrongKey" | null
}
def "should add objects matching parameters"() {
given:
def results = []
when:
LinUtils.treatObject(OBJECT, PARAMS, results)
then:
results == EXPECTED
where:
OBJECT | PARAMS | EXPECTED
["field": "value"] | ["field": ["value"]] | [["field": "value"]]
["field": "wrong"] | ["field": ["value"]] | []
[["field": "value"], ["field": "x"]] | ["field": ["value"]] | [["field": "value"]]
"string" | [:] | ["string"]
["nested": ["subfield": "match"]] | ["nested.subfield": ["match"]] | [["nested": ["subfield": "match"]]]
["nested": ["subfield": "nomatch"]] | ["nested.subfield": ["match"]] | []
[["nested": ["subfield": "match"]], ["nested": ["subfield": "other"]]] | ["nested.subfield": ["match"]] | [["nested": ["subfield": "match"]]]
}
def 'should navigate' (){
def uniqueId = UUID.randomUUID()
def mainScript = new DataPath("file://path/to/main.nf", new Checksum("78910", "nextflow", "standard"))
def workflow = new Workflow([mainScript], "https://nextflow.io/nf-test/", "123456")
def wfRun = new WorkflowRun(workflow, uniqueId.toString(), "test_run", [new Parameter("String", "param1", [key: "value1"]), new Parameter("String", "param2", "value2")])
expect:
LinUtils.navigate(wfRun, "workflow.commitId") == "123456"
LinUtils.navigate(wfRun, "params.name") == ["param1", "param2"]
LinUtils.navigate(wfRun, "params.value.key") == "value1"
LinUtils.navigate(wfRun, "params.value.no-exist") == null
LinUtils.navigate(wfRun, "params.no-exist") == null
LinUtils.navigate(wfRun, "no-exist") == null
LinUtils.navigate(null, "something") == null
}
}

View File

@@ -0,0 +1,478 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.cli
import java.nio.file.Files
import java.nio.file.Path
import java.time.Instant
import java.time.OffsetDateTime
import java.time.ZoneOffset
import nextflow.SysEnv
import nextflow.config.ConfigMap
import nextflow.dag.MermaidHtmlRenderer
import nextflow.exception.AbortOperationException
import nextflow.file.FileHelper
import nextflow.lineage.DefaultLinHistoryLog
import nextflow.lineage.LinHistoryRecord
import nextflow.lineage.LinStoreFactory
import nextflow.lineage.fs.LinFileSystemProvider
import nextflow.lineage.model.v1beta1.Checksum
import nextflow.lineage.model.v1beta1.DataPath
import nextflow.lineage.model.v1beta1.FileOutput
import nextflow.lineage.model.v1beta1.Parameter
import nextflow.lineage.model.v1beta1.TaskRun
import nextflow.lineage.model.v1beta1.Workflow
import nextflow.lineage.model.v1beta1.WorkflowRun
import nextflow.lineage.serde.LinEncoder
import nextflow.plugin.Plugins
import nextflow.util.CacheHelper
import org.junit.Rule
import spock.lang.Shared
import spock.lang.Specification
import test.OutputCapture
import static test.TestHelper.filterLogNoise
class LinCommandImplTest extends Specification{
@Shared
Path tmpDir
@Shared
Path storeLocation
@Shared
ConfigMap configMap
def reset() {
def provider = FileHelper.getProviderFor('lid') as LinFileSystemProvider
provider?.reset()
LinStoreFactory.reset()
}
def setup() {
reset()
// clear the environment to avoid the local env pollute the test env
SysEnv.push([:])
tmpDir = Files.createTempDirectory('tmp')
storeLocation = tmpDir.resolve("store")
configMap = new ConfigMap([lineage: [enabled: true, store: [location: storeLocation.toString(), logLocation: storeLocation.resolve(".log").toString()]]])
}
def cleanup() {
Plugins.stop()
LinStoreFactory.reset()
SysEnv.pop()
tmpDir?.deleteDir()
}
def setupSpec() {
reset()
}
/*
* Read more http://mrhaki.blogspot.com.es/2015/02/spocklight-capture-and-assert-system.html
*/
@Rule
OutputCapture capture = new OutputCapture()
def 'should print executions lids' (){
given:
def historyFile = storeLocation.resolve(".history")
def lidLog = new DefaultLinHistoryLog(historyFile)
def uniqueId = UUID.randomUUID()
def date = new Date();
def recordEntry = "${LinHistoryRecord.TIMESTAMP_FMT.format(date)}\trun_name\t${uniqueId}\tlid://123456".toString()
lidLog.write("run_name", uniqueId, "lid://123456", date)
when:
new LinCommandImpl().list(configMap)
def stdout = filterLogNoise(capture)
then:
stdout.size() == 2
stdout[1] == recordEntry
}
def 'should print no history' (){
given:
def historyFile = storeLocation.resolve(".meta/.history")
Files.createDirectories(historyFile.parent)
when:
new LinCommandImpl().list(configMap)
def stdout = filterLogNoise(capture, true)
then:
stdout.size() == 1
stdout[0] == "No workflow runs found in lineage history log"
}
def 'should show lid content' (){
given:
def lidFile = storeLocation.resolve("12345/.data.json")
Files.createDirectories(lidFile.parent)
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
def encoder = new LinEncoder().withPrettyPrint(true)
def entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
"lid://123987/file.bam","lid://123987/", null, 1234, time, time, null)
def jsonSer = encoder.encode(entry)
def expectedOutput = jsonSer
lidFile.text = jsonSer
when:
new LinCommandImpl().view(configMap, ["lid://12345"])
def stdout = filterLogNoise(capture)
then:
stdout.size() == expectedOutput.readLines().size()
stdout.join('\n') == expectedOutput
}
def 'should show empty lists content' (){
given:
def lidFile = storeLocation.resolve("12345/.data.json")
Files.createDirectories(lidFile.parent)
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
def encoder = new LinEncoder().withPrettyPrint(true)
def entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
"lid://123987/file.bam","lid://123987/", null, 1234, time, time, [])
def jsonSer = encoder.encode(entry)
def expectedOutput = '[]'
lidFile.text = jsonSer
when:
new LinCommandImpl().view(configMap, ["lid://12345#labels"])
def stdout = filterLogNoise(capture)
then:
stdout.size() == expectedOutput.readLines().size()
stdout.join('\n') == expectedOutput
}
def 'should show empty lists when no outputs' () {
def lidFile = storeLocation.resolve("12345/.data.json")
Files.createDirectories(lidFile.parent)
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
def encoder = new LinEncoder().withPrettyPrint(true)
def expectedOutput = '[]'
def wf = new Workflow([new DataPath("/path/to/main.nf)")], "hello-nf", "aasdklk")
def entry = new WorkflowRun(wf, "sessionId", "run_name",
[new Parameter("String", "sample_id", "ggal_gut"),
new Parameter("Integer", "reads", 2)])
lidFile.text = encoder.encode(entry)
when:
new LinCommandImpl().view(configMap, ["lid://12345#output"])
def stdout = filterLogNoise(capture)
then:
stdout.size() == expectedOutput.readLines().size()
stdout.join('\n') == expectedOutput
}
def 'should warn if no lid content' (){
given:
when:
new LinCommandImpl().view(configMap, ["lid://12345"])
def stdout = filterLogNoise(capture)
then:
stdout.size() == 1
stdout[0] == "Error loading lid://12345 - Lineage record 12345 not found"
}
def 'should get lineage lid content' (){
given:
def outputHtml = tmpDir.resolve('lineage.html')
def lidFile = storeLocation.resolve("12345/file.bam/.data.json")
def lidFile2 = storeLocation.resolve("123987/file.bam/.data.json")
def lidFile3 = storeLocation.resolve("123987/.data.json")
def lidFile4 = storeLocation.resolve("45678/output.txt/.data.json")
def lidFile5 = storeLocation.resolve("45678/.data.json")
Files.createDirectories(lidFile.parent)
Files.createDirectories(lidFile2.parent)
Files.createDirectories(lidFile3.parent)
Files.createDirectories(lidFile4.parent)
Files.createDirectories(lidFile5.parent)
def encoder = new LinEncoder()
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
def entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
"lid://123987/file.bam", "lid://45678", null, 1234, time, time, null)
lidFile.text = encoder.encode(entry)
entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
"lid://123987", "lid://45678", "lid://123987", 1234, time, time, null)
lidFile2.text = encoder.encode(entry)
entry = new TaskRun("u345-2346-1stw2", "foo",
new Checksum("abcde2345","nextflow","standard"),
'this is a script',
[new Parameter( "val", "sample_id","ggal_gut"),
new Parameter("path","reads", ["lid://45678/output.txt"] ),
new Parameter("path","input", [new DataPath("path/to/file",new Checksum("45372qe","nextflow","standard"))])
],
null, null, null, null, [:],[])
lidFile3.text = encoder.encode(entry)
entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
"lid://45678", "lid://45678", null, 1234, time, time, null)
lidFile4.text = encoder.encode(entry)
entry = new TaskRun("u345-2346-1stw2", "bar",
new Checksum("abfs2556","nextflow","standard"),
'this is a script',
null,null, null, null, null, [:],[])
lidFile5.text = encoder.encode(entry)
final network = """\
flowchart TB
lid://12345/file.bam["lid://12345/file.bam"]
lid://123987/file.bam["lid://123987/file.bam"]
lid://123987(["foo [lid://123987]"])
ggal_gut["ggal_gut"]
path/to/file["path/to/file"]
lid://45678/output.txt["lid://45678/output.txt"]
lid://45678(["bar [lid://45678]"])
lid://123987/file.bam --> lid://12345/file.bam
lid://123987 --> lid://123987/file.bam
ggal_gut --> lid://123987
lid://45678/output.txt --> lid://123987
path/to/file --> lid://123987
lid://45678 --> lid://45678/output.txt""".stripIndent()
final template = MermaidHtmlRenderer.readTemplate()
def expectedOutput = template.replace('REPLACE_WITH_NETWORK_DATA', network)
when:
new LinCommandImpl().render(configMap, ["lid://12345/file.bam", outputHtml.toString()])
def stdout = filterLogNoise(capture)
then:
stdout.size() == 1
stdout[0] == "Rendered lineage graph for lid://12345/file.bam to ${outputHtml}"
outputHtml.exists()
outputHtml.text == expectedOutput
}
def 'should get lineage from workflow lid content' (){
given:
def outputHtml = tmpDir.resolve('lineage.html')
def lidFile = storeLocation.resolve("12345/file.bam/.data.json")
def lidFile3 = storeLocation.resolve("12345/.data.json")
Files.createDirectories(lidFile.parent)
Files.createDirectories(lidFile3.parent)
def encoder = new LinEncoder()
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
def entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
"lid://12345", "lid://12345", null, 1234, time, time, null)
lidFile.text = encoder.encode(entry)
def wf = new Workflow([new DataPath("/path/to/main.nf)")], "hello-nf", "aasdklk")
entry = new WorkflowRun(wf,"sessionId","run_name",
[new Parameter( "String", "sample_id","ggal_gut"),
new Parameter("Integer","reads",2)])
lidFile3.text = encoder.encode(entry)
final network = """\
flowchart TB
lid://12345/file.bam["lid://12345/file.bam"]
lid://12345(["run_name [lid://12345]"])
ggal_gut["ggal_gut"]
2.0["2.0"]
lid://12345 --> lid://12345/file.bam
ggal_gut --> lid://12345
2.0 --> lid://12345""".stripIndent()
final template = MermaidHtmlRenderer.readTemplate()
def expectedOutput = template.replace('REPLACE_WITH_NETWORK_DATA', network)
when:
new LinCommandImpl().render(configMap, ["lid://12345/file.bam", outputHtml.toString()])
def stdout = filterLogNoise(capture)
then:
stdout.size() == 1
stdout[0] == "Rendered lineage graph for lid://12345/file.bam to ${outputHtml}"
outputHtml.exists()
outputHtml.text == expectedOutput
}
def 'should show an error if trying to do a query'(){
given:
def lidFile = storeLocation.resolve("12345/.data.json")
Files.createDirectories(lidFile.parent)
def encoder = new LinEncoder().withPrettyPrint(true)
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
def entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
"lid://123987/file.bam", "lid://123987/", null, 1234, time, time, null)
def jsonSer = encoder.encode(entry)
def expectedOutput = "Error loading lid:///?type=FileOutput - Cannot get record from the root LID URI"
lidFile.text = jsonSer
when:
new LinCommandImpl().view(configMap, ["lid:///?type=FileOutput"])
def stdout = filterLogNoise(capture)
then:
stdout.size() == expectedOutput.readLines().size()
stdout.join('\n') == expectedOutput
}
def 'should diff'(){
given:
def lidFile = storeLocation.resolve("12345/.data.json")
Files.createDirectories(lidFile.parent)
def lidFile2 = storeLocation.resolve("67890/.data.json")
Files.createDirectories(lidFile2.parent)
def encoder = new LinEncoder().withPrettyPrint(true)
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
def entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
"lid://123987/file.bam", "lid://123987/", null, 1234, time, time, null)
def entry2 = new FileOutput("path/to/file2",new Checksum("42472qet","nextflow","standard"),
"lid://123987/file2.bam", "lid://123987/", null, 1235, time, time, null)
lidFile.text = encoder.encode(entry)
lidFile2.text = encoder.encode(entry2)
def expectedOutput = '''diff --git 12345 67890
--- 12345
+++ 67890
@@ -2,16 +2,16 @@
"version": "lineage/v1beta1",
"kind": "FileOutput",
"spec": {
- "path": "path/to/file",
+ "path": "path/to/file2",
"checksum": {
- "value": "45372qe",
+ "value": "42472qet",
"algorithm": "nextflow",
"mode": "standard"
},
- "source": "lid://123987/file.bam",
+ "source": "lid://123987/file2.bam",
"workflowRun": "lid://123987/",
"taskRun": null,
- "size": 1234,
+ "size": 1235,
"createdAt": "1970-01-02T10:17:36.789Z",
"modifiedAt": "1970-01-02T10:17:36.789Z",
"labels": null
'''
when:
new LinCommandImpl().diff(configMap, ["lid://12345", "lid://67890"])
def stdout = filterLogNoise(capture)
then:
stdout.join('\n') == expectedOutput
}
def 'should print error if no entry found diff'(){
given:
def lidFile = storeLocation.resolve("12345/.data.json")
Files.createDirectories(lidFile.parent)
def encoder = new LinEncoder().withPrettyPrint(true)
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
def entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
"lid://123987/file.bam", "lid://123987/", null, 1234, time, time, null)
lidFile.text = encoder.encode(entry)
when:
new LinCommandImpl().diff(configMap, ["lid://89012", "lid://12345"])
new LinCommandImpl().diff(configMap, ["lid://12345", "lid://67890"])
def stdout = filterLogNoise(capture)
then:
stdout.size() == 2
stdout[0] == "No entry found for lid://89012."
stdout[1] == "No entry found for lid://67890."
}
def 'should print error store is not found in diff'(){
when:
def config = new ConfigMap()
new LinCommandImpl().list(config)
new LinCommandImpl().view(config, ["lid:///12345"])
new LinCommandImpl().render(config, ["lid://12345", "output.html"])
new LinCommandImpl().diff(config, ["lid://89012", "lid://12345"])
def stdout = filterLogNoise(capture)
def expectedOutput = "Error lineage store not loaded - Check Nextflow configuration"
then:
stdout.size() == 4
stdout[0] == expectedOutput
stdout[1] == expectedOutput
stdout[2] == expectedOutput
stdout[3] == expectedOutput
}
def 'should find metadata descriptions'(){
given:
def lidFile = storeLocation.resolve("123987/file.bam/.data.json")
Files.createDirectories(lidFile.parent)
def lidFile2 = storeLocation.resolve("123987/file2.bam/.data.json")
Files.createDirectories(lidFile2.parent)
def lidFile3 = storeLocation.resolve(".meta/123987/file3.bam/.data.json")
Files.createDirectories(lidFile3.parent)
def encoder = new LinEncoder().withPrettyPrint(true)
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
def entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
"lid://123987/file.bam", "lid://123987/", null, 1234, time, time, ["experiment=test"])
def entry2 = new FileOutput("path/to/file2",new Checksum("42472qet","nextflow","standard"),
"lid://123987/file2.bam", "lid://123987/", null, 1235, time, time, ["experiment=test"])
def entry3 = new FileOutput("path/to/file3",new Checksum("42472qet","nextflow","standard"),
"lid://123987/file2.bam", "lid://123987/", null, 1235, time, time, null)
def expectedOutput1 = 'lid://123987/file.bam\nlid://123987/file2.bam'
def expectedOutput2 = 'lid://123987/file2.bam\nlid://123987/file.bam'
lidFile.text = encoder.encode(entry)
lidFile2.text = encoder.encode(entry2)
lidFile3.text = encoder.encode(entry3)
when:
new LinCommandImpl().find(configMap, ["type=FileOutput", "label=experiment=test"])
def stdout = filterLogNoise(capture)
then:
stdout.join('\n') == expectedOutput1 || stdout.join('\n') == expectedOutput2
}
def 'should print correct validate path' () {
given:
def outputFolder = tmpDir.resolve('output')
Files.createDirectories(outputFolder)
def outputFile = outputFolder.resolve('file1.txt')
outputFile.text = "this is file1 == "
and:
def encoder = new LinEncoder().withPrettyPrint(true)
def hash = CacheHelper.hasher(outputFile).hash().toString()
def correctData = new FileOutput(outputFile.toString(), new Checksum(hash,"nextflow", "standard"))
def incorrectData = new FileOutput(outputFile.toString(), new Checksum("incorrectHash","nextflow", "standard"))
def lid1 = storeLocation.resolve('12345/output/file1.txt/.data.json')
Files.createDirectories(lid1.parent)
lid1.text = encoder.encode(correctData)
def lid2 = storeLocation.resolve('12345/output/file2.txt/.data.json')
Files.createDirectories(lid2.parent)
lid2.text = encoder.encode(incorrectData)
when:
new LinCommandImpl().check(configMap, ["lid://12345/output/file1.txt"])
def stdout = filterLogNoise(capture)
def expectedOutput1 = "Checksum validation succeed"
then:
stdout.size() == 1
stdout[0] == expectedOutput1
when:
new LinCommandImpl().check(configMap, ["lid://12345/output/file2.txt"])
then:
def err = thrown(AbortOperationException)
err.message == "Checksum of '${outputFile}' does not match with lineage metadata"
}
}

View File

@@ -0,0 +1,50 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.config
import spock.lang.Specification
/**
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
class LineageConfigTest extends Specification {
def 'should create default config' () {
when:
def config = new LineageConfig(Map.of())
then:
!config.enabled
!config.store.location
}
def 'should create default with enable' () {
when:
def config = new LineageConfig([enabled: true])
then:
config.enabled
!config.store.location
}
def 'should create data config with location' () {
when:
def config = new LineageConfig(enabled: true, store: [location: "/some/data/store"])
then:
config.enabled
config.store.location == '/some/data/store'
}
}

View File

@@ -0,0 +1,496 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.fs
import java.nio.ByteBuffer
import java.nio.channels.NonWritableChannelException
import java.nio.file.AccessDeniedException
import java.nio.file.AccessMode
import java.nio.file.FileSystemNotFoundException
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.ProviderMismatchException
import java.nio.file.StandardOpenOption
import java.nio.file.attribute.BasicFileAttributes
import nextflow.Global
import nextflow.Session
import nextflow.lineage.DefaultLinStore
import spock.lang.Shared
import spock.lang.Specification
/**
* LID File system provider tests
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
class LinFileSystemProviderTest extends Specification {
@Shared def wdir = Files.createTempDirectory('wdir')
@Shared def data = Files.createTempDirectory('work')
def cleanupSpec(){
wdir.deleteDir()
data.deleteDir()
}
def 'should return lid scheme' () {
given:
def provider = new LinFileSystemProvider()
expect:
provider.getScheme() == 'lid'
}
def 'should get lid path' () {
given:
def lid = Mock(LinPath)
and:
def provider = new LinFileSystemProvider()
expect:
provider.toLinPath(lid) == lid
when:
provider.toLinPath(Path.of('foo'))
then:
thrown(ProviderMismatchException)
}
def 'should create new file system' () {
given:
def provider = new LinFileSystemProvider()
def config = [store:[location:data.toString()]]
def lid = LinPath.asUri('lid://12345')
when:
def fs = provider.newFileSystem(lid, config) as LinFileSystem
then:
(fs.store as DefaultLinStore).location == data
}
def 'should get a file system' () {
given:
def provider = new LinFileSystemProvider()
def config = [store:[location: data.toString()]]
def uri = LinPath.asUri('lid://12345')
when:
provider.getFileSystem(uri)
then:
thrown(FileSystemNotFoundException)
when:
provider.newFileSystem(uri, config) as LinFileSystem
and:
def fs = provider.getFileSystem(uri) as LinFileSystem
then:
(fs.store as DefaultLinStore).location == data
}
def 'should get or create a file system' () {
given:
def config = [lineage:[store:[location: data.toString()]]]
Global.session = Mock(Session) { getConfig()>>config }
and:
def uri = LinPath.asUri('lid://12345')
def provider = new LinFileSystemProvider()
when:
def fs = provider.getFileSystemOrCreate(uri) as LinFileSystem
then:
(fs.store as DefaultLinStore).location == data
when:
def fs2 = provider.getFileSystemOrCreate(uri) as LinFileSystem
then:
fs2.is(fs)
}
def 'should create new byte channel' () {
given:
def config = [lineage:[store:[location:wdir.toString()]]]
def outputMeta = wdir.resolve("12345/output.txt")
def output = data.resolve("output.txt")
output.text = "Hello, World!"
outputMeta.mkdirs()
outputMeta.resolve(".data.json").text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path":"'+output.toString()+'"}}'
Global.session = Mock(Session) { getConfig()>>config }
and:
def provider = new LinFileSystemProvider()
def lid = provider.getPath(LinPath.asUri('lid://12345/output.txt'))
def opts = Set.of(StandardOpenOption.READ)
when:
def channel = provider.newByteChannel(lid, opts)
then:
channel.isOpen()
channel.position() == 0
channel.size() == "Hello, World!".getBytes().size()
when:
channel.truncate(25)
then:
thrown(NonWritableChannelException)
when:
def buffer = ByteBuffer.allocate(1000);
def read = channel.read(buffer)
def bytes = new byte[read]
buffer.get(0,bytes)
then:
bytes == "Hello, World!".getBytes()
when:
channel.position(2)
then:
channel.position() == 2
when:
channel.write(buffer)
then:
thrown(NonWritableChannelException)
when:
provider.newByteChannel(lid, Set.of(StandardOpenOption.WRITE))
then:
thrown(UnsupportedOperationException)
when:
provider.newByteChannel(lid, Set.of(StandardOpenOption.APPEND))
then:
thrown(UnsupportedOperationException)
cleanup:
channel.close()
outputMeta.deleteDir()
output.delete()
}
def 'should create new byte channel for LinMetadata' () {
given:
def config = [lineage:[store:[location:wdir.toString()]]]
def outputMeta = wdir.resolve("12345")
outputMeta.mkdirs()
outputMeta.resolve(".data.json").text = '{"version":"lineage/v1beta1","kind":"WorkflowRun","spec":{"sessionId":"session","name":"run_name","params":[{"type":"String","name":"param1","value":"value1"}]}}'
Global.session = Mock(Session) { getConfig()>>config }
and:
def provider = new LinFileSystemProvider()
def lid = provider.getPath(LinPath.asUri('lid://12345#name'))
when:
def channel = provider.newByteChannel(lid, Set.of(StandardOpenOption.READ))
then:
channel.isOpen()
channel.position() == 0
channel.size() == '"run_name"'.getBytes().size()
when:
channel.truncate(25)
then:
thrown(NonWritableChannelException)
when:
def buffer = ByteBuffer.allocate(1000);
def read = channel.read(buffer)
def bytes = new byte[read]
buffer.get(0,bytes)
then:
bytes =='"run_name"'.getBytes()
when:
channel.position(2)
then:
channel.position() == 2
when:
channel.write(buffer)
then:
thrown(NonWritableChannelException)
when:
provider.newByteChannel(lid, Set.of(StandardOpenOption.WRITE))
then:
thrown(UnsupportedOperationException)
when:
provider.newByteChannel(lid, Set.of(StandardOpenOption.APPEND))
then:
thrown(UnsupportedOperationException)
cleanup:
channel.close()
outputMeta.deleteDir()
}
def 'should read lid' () {
given:
def config = [lineage:[store:[location:wdir.toString()]]]
def outputMeta = wdir.resolve("12345/output.txt")
def output = data.resolve("output.txt")
output.text = "Hello, World!"
outputMeta.mkdirs()
outputMeta.resolve(".data.json").text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path":"'+output.toString()+'"}}'
Global.session = Mock(Session) { getConfig()>>config }
and:
def provider = new LinFileSystemProvider()
def lid = provider.getPath(LinPath.asUri('lid://12345/output.txt'))
def opts = Set.of(StandardOpenOption.READ)
expect:
lid.text == "Hello, World!"
cleanup:
outputMeta.deleteDir()
output.delete()
}
def 'should not create a directory' () {
given:
def config = [lineage:[store:[location:wdir.toString()]]]
Global.session = Mock(Session) { getConfig()>>config }
and:
def provider = new LinFileSystemProvider()
def lid = provider.getPath(LinPath.asUri('lid://12345'))
when:
provider.createDirectory(lid)
then:
thrown(UnsupportedOperationException)
}
def 'should create directory stream' () {
given:
def output1 = data.resolve('path')
output1.mkdir()
output1.resolve('file1.txt').text = 'file1'
output1.resolve('file2.txt').text = 'file2'
output1.resolve('file3.txt').text = 'file3'
wdir.resolve('12345/output1').mkdirs()
wdir.resolve('12345/output2').mkdirs()
wdir.resolve('12345/.data.json').text = '{"version":"lineage/v1beta1","kind":"TaskRun","spec":{"name":"dummy"}}'
wdir.resolve('12345/output1/.data.json').text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path": "' + output1.toString() + '"}}'
and:
def config = [lineage:[store:[location:wdir.toString()]]]
Global.session = Mock(Session) { getConfig()>>config }
and:
def provider = new LinFileSystemProvider()
def lid = provider.getPath(LinPath.asUri('lid://12345/output1'))
def lid2 = provider.getPath(LinPath.asUri('lid://12345'))
def lid3 = provider.getPath(LinPath.asUri('lid://678'))
expect:
Files.exists(lid)
Files.exists(lid.resolve('file1.txt'))
Files.exists(lid.resolve('file2.txt'))
Files.exists(lid.resolve('file3.txt'))
when:
provider.newDirectoryStream(lid3, (p) -> true)
then:
thrown(FileNotFoundException)
when:
def stream = provider.newDirectoryStream(lid, (p) -> true)
and:
def result = stream.toList()
then:
result.toSet() == [
lid.resolve('file1.txt'),
lid.resolve('file2.txt'),
lid.resolve('file3.txt')
] as Set
when:
stream = provider.newDirectoryStream(lid2, (p) -> true)
and:
result = stream.toList()
then:
result.size() == 1
result[0] == lid2.resolve('output1')
cleanup:
wdir.resolve('12345').deleteDir()
output1.deleteDir()
}
def 'should not delete a file' () {
given:
def config = [lineage:[store:[location:wdir.toString()]]]
Global.session = Mock(Session) { getConfig()>>config }
and:
def provider = new LinFileSystemProvider()
def lid = provider.getPath(LinPath.asUri('lid://12345'))
when:
provider.delete(lid)
then:
thrown(UnsupportedOperationException)
}
def 'should not copy a file' () {
given:
def config = [lineage:[store:[location:wdir.toString()]]]
Global.session = Mock(Session) { getConfig()>>config }
and:
def provider = new LinFileSystemProvider()
def lid1 = provider.getPath(LinPath.asUri('lid://12345/abc'))
def lid2 = provider.getPath(LinPath.asUri('lid://54321/foo'))
when:
provider.copy(lid1, lid2)
then:
thrown(UnsupportedOperationException)
}
def 'should not move a file' () {
given:
def config = [lineage:[store:[location:wdir.toString()]]]
Global.session = Mock(Session) { getConfig()>>config }
and:
def provider = new LinFileSystemProvider()
def lid1 = provider.getPath(LinPath.asUri('lid://12345/abc'))
def lid2 = provider.getPath(LinPath.asUri('lid://54321/foo'))
when:
provider.move(lid1, lid2)
then:
thrown(UnsupportedOperationException)
}
def 'should check is same file' () {
given:
def folder = Files.createTempDirectory('test')
def config = [lineage:[store:[location:folder.toString()]]]
Global.session = Mock(Session) { getConfig()>>config }
and:
def provider = new LinFileSystemProvider()
def lid1 = provider.getPath(LinPath.asUri('lid://12345/abc'))
def lid2 = provider.getPath(LinPath.asUri('lid://54321/foo'))
def lid3 = provider.getPath(LinPath.asUri('lid://54321/foo'))
expect:
!provider.isSameFile(lid1, lid2)
!provider.isSameFile(lid1, lid3)
and:
provider.isSameFile(lid2, lid3)
cleanup:
folder?.deleteDir()
}
def 'should check is hidden file' () {
given:
def folder = Files.createTempDirectory('test')
def config = [lineage:[store:[location:wdir.toString()]]]
Global.session = Mock(Session) { getConfig()>>config }
and:
def output = folder.resolve('path')
output.mkdir()
output.resolve('abc').text = 'file1'
output.resolve('.foo').text = 'file2'
wdir.resolve('12345/output').mkdirs()
wdir.resolve('12345/output/.data.json').text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path": "' + output.toString() + '"}}'
and:
def provider = new LinFileSystemProvider()
def lid1 = provider.getPath(LinPath.asUri('lid://12345/output/abc'))
def lid2 = provider.getPath(LinPath.asUri('lid://12345/output/.foo'))
expect:
!provider.isHidden(lid1)
provider.isHidden(lid2)
cleanup:
folder?.deleteDir()
}
def 'should read file attributes' () {
given:
def config = [lineage:[store:[location:wdir.toString()]]]
def file = data.resolve('abc')
file.text = 'Hello'
wdir.resolve('12345/abc').mkdirs()
wdir.resolve('12345/abc/.data.json').text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path":"' + file.toString() + '"}}'
and:
Global.session = Mock(Session) { getConfig()>>config }
and:
def provider = new LinFileSystemProvider()
def lid1 = provider.getPath(LinPath.asUri('lid://12345/abc'))
when:
def attr1 = provider.readAttributes(lid1, BasicFileAttributes)
def real1= Files.readAttributes(file,BasicFileAttributes)
then:
!attr1.directory
attr1.isRegularFile()
attr1.size() == real1.size()
attr1.creationTime() == real1.creationTime()
attr1.lastModifiedTime() == real1.lastModifiedTime()
attr1.lastAccessTime() == real1.lastAccessTime()
cleanup:
file?.delete()
wdir.resolve('12345').deleteDir()
}
def 'should throw exception in unsupported methods'() {
given:
def config = [lineage:[store:[location:wdir.toString()]]]
Global.session = Mock(Session) { getConfig()>>config }
def provider = new LinFileSystemProvider()
when:
provider.newOutputStream(null)
then:
thrown(UnsupportedOperationException)
when:
provider.getFileStore(null)
then:
thrown(UnsupportedOperationException)
when:
provider.readAttributes(null, "attrib")
then:
thrown(UnsupportedOperationException)
when:
provider.setAttribute(null, "attrib", null)
then:
thrown(UnsupportedOperationException)
}
def 'should throw exception when checking access mode'(){
given:
def config = [lineage:[store:[location:wdir.toString()]]]
Global.session = Mock(Session) { getConfig()>>config }
def provider = new LinFileSystemProvider()
def lid1 = provider.getPath(LinPath.asUri('lid://12345/abc'))
when:
provider.checkAccess(lid1, AccessMode.WRITE)
then:
def ex1 = thrown(AccessDeniedException)
ex1.message == "Write mode not supported"
when:
provider.checkAccess(lid1, AccessMode.EXECUTE)
then:
def ex2 = thrown(AccessDeniedException)
ex2.message == "Execute mode not supported"
}
}

View File

@@ -0,0 +1,89 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.fs
import java.nio.file.Files
import java.nio.file.Path
import nextflow.Global
import nextflow.Session
import spock.lang.Specification
import spock.lang.Unroll
/**
* LID Path Factory tests.
*
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
class LinPathFactoryTest extends Specification {
Path tmp
def setup() {
tmp = Files.createTempDirectory("data")
Global.session = Mock(Session) { getConfig()>> [workflow:[lineage:[store:[location: tmp.toString()]]]] }
}
def cleanup() {
Global.session = null
tmp.deleteDir()
}
def 'should create lin path' () {
given:
def factory = new LinPathFactory()
expect:
factory.parseUri('foo') == null
when:
def p1 = factory.parseUri('lid://12345')
then:
p1.toUriString() == 'lid://12345'
when:
def p2 = factory.parseUri('lid://12345/x/y/z')
then:
p2.toUriString() == 'lid://12345/x/y/z'
when:
def p3 = factory.parseUri('lid://12345//x///y/z//')
then:
p3.toUriString() == 'lid://12345/x/y/z'
when:
factory.parseUri('lid:///12345')
then:
thrown(IllegalArgumentException)
}
@Unroll
def 'should convert get lid uri string' () {
given:
def factory = new LinPathFactory()
when:
def lid = LinPathFactory.create(EXPECTED)
then:
factory.toUriString(lid) == EXPECTED
where:
_ | EXPECTED
_ | 'lid://123'
_ | 'lid://123/a/b/c'
}
}

View File

@@ -0,0 +1,718 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.fs
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.ProviderMismatchException
import java.time.OffsetDateTime
import nextflow.lineage.LinUtils
import nextflow.lineage.model.v1beta1.Checksum
import nextflow.lineage.model.v1beta1.FileOutput
import nextflow.lineage.model.v1beta1.Parameter
import nextflow.lineage.model.v1beta1.Workflow
import nextflow.lineage.model.v1beta1.WorkflowOutput
import nextflow.lineage.model.v1beta1.WorkflowRun
import nextflow.lineage.serde.LinEncoder
import nextflow.util.CacheHelper
import org.junit.Rule
import spock.lang.Shared
import spock.lang.Specification
import spock.lang.Unroll
import test.OutputCapture
import static test.TestHelper.filterLogNoise
/**
* LID Path Tests
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
*/
class LinPathTest extends Specification {
@Shared
Path wdir
@Shared
Path data
@Shared
def fs = Mock(LinFileSystem)
def setupSpec(){
wdir = Files.createTempDirectory("wdir")
data = wdir.resolve('work')
}
def cleanupSpec(){
wdir.deleteDir()
}
@Rule
OutputCapture capture = new OutputCapture()
def 'should create from URI' () {
when:
def path = new LinPath(fs, new URI( URI_STRING ))
then:
path.filePath == PATH
path.fragment == FRAGMENT
path.query == QUERY
where:
URI_STRING | PATH | QUERY | FRAGMENT
"lid://1234/hola" | "1234/hola" | null | null
"lid://1234/hola#workflow.repository" | "1234/hola" | null | "workflow.repository"
"lid://1234/#workflow.repository" | "1234" | null | "workflow.repository"
"lid://1234/?q=a&b=c" | "1234" | "q=a&b=c" | null
"lid://1234/?q=a&b=c#workflow.repository" | "1234" | "q=a&b=c" | "workflow.repository"
"lid:///" | "/" | null | null
}
def 'should throw exception if fragment contains an unknown property'() {
when:
new LinPath(fs, new URI ("lid://1234/hola#no-exist"))
then:
thrown(IllegalArgumentException)
}
def 'should warn if query is specified'() {
when:
new LinPath(fs, new URI("lid://1234/hola?query"))
def stdout = filterLogNoise(capture)
then:
stdout.size() == 1
stdout[0].endsWith("Query string is not supported for Lineage URI: `lid://1234/hola?query` -- it will be ignored")
}
def 'should create correct lid Path' () {
when:
def lid = new LinPath(FS, PATH, MORE)
then:
lid.filePath == EXPECTED_FILE
where:
FS | PATH | MORE | EXPECTED_FILE
fs | '/' | [] as String[] | '/'
null | '/' | [] as String[] | '/'
fs | '/' | ['a','b'] as String[] | 'a/b'
null | '/' | ['a','b'] as String[] | 'a/b'
fs | '' | [] as String[] | '/'
null | '' | [] as String[] | '/'
fs | '' | ['a','b'] as String[] | 'a/b'
null | '' | ['a','b'] as String[] | 'a/b'
fs | '1234' | [] as String[] | '1234'
null | '1234' | [] as String[] | '1234'
fs | '1234' | ['a','b'] as String[] | '1234/a/b'
null | '1234' | ['a','b'] as String[] | '1234/a/b'
fs | '1234/c' | [] as String[] | '1234/c'
null | '1234/c' | [] as String[] | '1234/c'
fs | '1234/c' | ['a','b'] as String[] | '1234/c/a/b'
null | '1234/c' | ['a','b'] as String[] | '1234/c/a/b'
fs | '/1234/c' | [] as String[] | '1234/c'
null | '/1234/c' | [] as String[] | '1234/c'
fs | '/1234/c' | ['a','b'] as String[] | '1234/c/a/b'
null | '/1234/c' | ['a','b'] as String[] | '1234/c/a/b'
fs | '../c' | ['a','b'] as String[] | 'c/a/b'
null | '../c' | ['a','b'] as String[] | '../c/a/b'
fs | '../c' | [] as String[] | 'c'
null | '../c' | [] as String[] | '../c'
fs | '..' | [] as String[] | '/'
null | '..' | [] as String[] | '..'
fs | '/..' | [] as String[] | '/'
null | '/..' | [] as String[] | '/'
fs | './1234/c' | ['a','b'] as String[] | '1234/c/a/b'
null | './1234/c' | ['a','b'] as String[] | '1234/c/a/b'
fs | './1234/c' | [] as String[] | '1234/c'
null | './1234/c' | [] as String[] | '1234/c'
fs | '1234' | ['/'] as String[] | '1234'
null | '1234' | ['/'] as String[] | '1234'
null | '../../a/b' | [] as String[] | '../../a/b'
fs | '1234/' | [] as String[] | '1234'
null | '1234/' | [] as String[] | '1234'
}
def 'should get target path' () {
given:
def outputFolder = data.resolve('output')
def outputSubFolder = outputFolder.resolve('some/path')
outputSubFolder.mkdirs()
def outputSubFolderFile = outputSubFolder.resolve('file1.txt')
outputSubFolderFile.text = "this is file1"
def outputFile = data.resolve('file2.txt')
outputFile.text = "this is file2"
def lidFs = new LinFileSystemProvider().newFileSystem(new URI("lid:///"), [enabled: true, store: [location: wdir.toString()]])
wdir.resolve('12345/output1').mkdirs()
wdir.resolve('12345/path/to/file2.txt').mkdirs()
wdir.resolve('12345/.data.json').text = '{"version":"lineage/v1beta1","kind":"TaskRun","spec":{"name":"test"}}'
wdir.resolve('12345/output1/.data.json').text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path": "' + outputFolder.toString() + '"}}'
wdir.resolve('12345/path/to/file2.txt/.data.json').text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path": "' + outputFile.toString() + '"}}'
def time = OffsetDateTime.now()
def wfResultsMetadata = new LinEncoder().withPrettyPrint(true).encode(new WorkflowOutput(time, "lid://1234", [new Parameter( "Path", "a", "lid://1234/a.txt")]))
wdir.resolve('5678/').mkdirs()
wdir.resolve('5678/.data.json').text = wfResultsMetadata
expect: 'Get real path when LinPath is the output data or a subfolder'
new LinPath(lidFs, '12345/output1').getTargetPath() == outputFolder
new LinPath(lidFs,'12345/output1/some/path').getTargetPath() == outputSubFolder
new LinPath(lidFs,'12345/output1/some/path/file1.txt').getTargetPath().text == outputSubFolderFile.text
new LinPath(lidFs, '12345/path/to/file2.txt').getTargetPath().text == outputFile.text
when: 'LinPath fs is null'
new LinPath(null, '12345').getTargetPath()
then:
thrown(IllegalArgumentException)
when: 'LinPath is empty'
new LinPath(lidFs, '/').getTargetPath()
then:
thrown(IllegalArgumentException)
when: 'LinPath is not an output data description'
new LinPath(lidFs, '12345').getTargetPath()
then:
thrown(FileNotFoundException)
when: 'LinPath is not subfolder of an output data description'
new LinPath(lidFs, '12345/path').getTargetPath()
then:
thrown(FileNotFoundException)
when: 'LinPath subfolder of an output data description does not exist'
new LinPath(lidFs, '12345/output1/other/path').getTargetPath()
then:
thrown(FileNotFoundException)
when: 'Lid does not exist'
new LinPath(lidFs, '23456').getTargetPath()
then:
thrown(FileNotFoundException)
when: 'LinPath is not an output data description but is a workflow/task run'
def result = new LinPath(lidFs, '12345').getTargetOrIntermediatePath()
then:
result instanceof LinIntermediatePath
when: 'LinPath is a subpath of a workflow/task run data description'
result = new LinPath(lidFs, '12345/path').getTargetOrIntermediatePath()
then:
result instanceof LinIntermediatePath
when: 'Lid description'
result = new LinPath(lidFs, '5678').getTargetOrMetadataPath()
then:
result instanceof LinMetadataPath
result.text == wfResultsMetadata
when: 'Lid description subobject'
def result2 = new LinPath(lidFs, '5678#output').getTargetOrMetadataPath()
then:
result2 instanceof LinMetadataPath
result2.text == LinUtils.encodeSearchOutputs([new Parameter("Path","a", "lid://1234/a.txt")], true)
when: 'Lid subobject does not exist'
new LinPath(lidFs, '23456#notexists').getTargetOrMetadataPath()
then:
thrown(IllegalArgumentException)
}
def 'should get subobjects as path' (){
given:
def lidFs = new LinFileSystemProvider().newFileSystem(new URI("lid:///"), [enabled: true, store: [location: wdir.toString()]])
def wf = new WorkflowRun(new Workflow([],"repo", "commit"), "sessionId", "runId", [new Parameter("String", "param1", "value1")])
when: 'workflow repo in workflow run'
Path p = LinPath.getMetadataAsTargetPath(wf, lidFs, "123456", "workflow.repository")
then:
p instanceof LinMetadataPath
p.text == '"repo"'
when: 'outputs'
def outputs = new WorkflowOutput(OffsetDateTime.now(), "lid://123456", [new Parameter("Collection", "samples", ["sample1", "sample2"])])
lidFs.store.save("123456#output", outputs)
Path p2 = LinPath.getMetadataAsTargetPath(wf, lidFs, "123456", "output")
then:
p2 instanceof LinMetadataPath
p2.text == LinUtils.encodeSearchOutputs([new Parameter("Collection", "samples", ["sample1", "sample2"])], true)
when: 'child does not exists'
LinPath.getMetadataAsTargetPath(wf, lidFs, "123456", "no-exist")
then:
def exception = thrown(FileNotFoundException)
exception.message == "Target path '123456#no-exist' does not exist"
when: 'outputs does not exists'
LinPath.getMetadataAsTargetPath(wf, lidFs, "6789", "output")
then:
def exception1 = thrown(FileNotFoundException)
exception1.message == "Target path '6789#output' does not exist"
when: 'null object'
LinPath.getMetadataAsTargetPath(null, lidFs, "123456", "no-exist")
then:
def exception2 = thrown(FileNotFoundException)
exception2.message == "Target path '123456' does not exist"
cleanup:
wdir.resolve("123456").deleteDir()
}
def 'should get file name' () {
expect:
new LinPath(fs, PATH).getFileName() == EXPECTED
where:
PATH | EXPECTED
'1234567890/this/file.bam' | new LinPath(null, 'file.bam')
'12345/hola?query#output' | new LinPath("query", "output", "hola", null)
}
def 'should get file parent' () {
when:
def lid1 = new LinPath(fs, '1234567890/this/file.bam')
then:
lid1.getParent() == new LinPath(fs, '1234567890/this')
lid1.getParent().getParent() == new LinPath(fs, '1234567890')
lid1.getParent().getParent().getParent() == new LinPath(fs, "/")
lid1.getParent().getParent().getParent().getParent() == null
}
@Unroll
def 'should get name count' () {
expect:
new LinPath(fs, PATH).getNameCount() == EXPECTED
where:
PATH | EXPECTED
'/' | 0
'123' | 1
'123/a' | 2
'123/a/' | 2
'123/a/b' | 3
'' | 0
}
@Unroll
def 'should get name by index' () {
expect:
new LinPath(fs, PATH).getName(INDEX) == EXPECTED
where:
PATH | INDEX | EXPECTED
'123' | 0 | new LinPath(fs, '123')
'123/a' | 1 | new LinPath(null, 'a')
'123/a/' | 1 | new LinPath(null, 'a')
'123/a/b' | 2 | new LinPath(null, 'b')
'123/a?q#output' | 1 | new LinPath(null, 'a?q#output')
}
@Unroll
def 'should get subpath' () {
expect:
new LinPath(fs, PATH).subpath(BEGIN,END) == EXPECTED
where:
PATH | BEGIN | END | EXPECTED
'123' | 0 | 1 | new LinPath(fs, '123')
'123/a' | 0 | 2 | new LinPath(fs, '123/a')
'123/a/' | 0 | 2 | new LinPath(fs, '123/a')
'123/a' | 1 | 2 | new LinPath(null, 'a')
'123/a/' | 1 | 2 | new LinPath(null, 'a')
'123/a/b' | 2 | 3 | new LinPath(null, 'b')
'123/a/b' | 1 | 3 | new LinPath(null, 'a/b')
}
def 'should normalize a path' () {
expect:
new LinPath(fs, '123').normalize() == new LinPath(fs, '123')
new LinPath(fs, '123/a/b').normalize() == new LinPath(fs, '123/a/b')
new LinPath(fs, '123/./a/b').normalize() == new LinPath(fs, '123/a/b')
new LinPath(fs, '123/a/../a/b').normalize() == new LinPath(fs, '123/a/b')
}
@Unroll
def 'should validate startWith' () {
expect:
new LinPath(fs,PATH).startsWith(OTHER) == EXPECTED
where:
PATH | OTHER | EXPECTED
'12345/a/b' | '12345' | true
'12345/a/b' | '12345/a' | true
'12345/a/b' | '12345/a/b' | true
and:
'12345/a/b' | '12345/b' | false
'12345/a/b' | 'xyz' | false
}
@Unroll
def 'should validate endsWith' () {
expect:
new LinPath(fs,PATH).endsWith(OTHER) == EXPECTED
where:
PATH | OTHER | EXPECTED
'12345/a/b' | 'b' | true
'12345/a/b' | 'a/b' | true
'12345/a/b' | '12345/a/b' | true
and:
'12345/a/b' | '12345/b' | false
'12345/a/b' | 'xyz' | false
}
def 'should validate isAbsolute' () {
expect:
new LinPath(fs,'1234/a/b/c').isAbsolute()
new LinPath(fs,'1234/a/b/c').getRoot().isAbsolute()
new LinPath(fs,'1234/a/b/c').getParent().isAbsolute()
new LinPath(fs,'1234/a/b/c').normalize().isAbsolute()
new LinPath(fs,'1234/a/b/c').getName(0).isAbsolute()
new LinPath(fs,'1234/a/b/c').subpath(0,2).isAbsolute()
and:
!new LinPath(fs,'1234/a/b/c').getFileName().isAbsolute()
!new LinPath(fs,'1234/a/b/c').getName(1).isAbsolute()
!new LinPath(fs,'1234/a/b/c').subpath(1,3).isAbsolute()
}
@Unroll
def 'should get root path' () {
expect:
new LinPath(fs,PATH).getRoot() == new LinPath(fs,EXPECTED)
where:
PATH | EXPECTED
'12345' | '/'
'12345/a' | '/'
}
def 'should relativize path' () {
expect:
BASE_PATH.relativize(PATH) == EXPECTED
where :
BASE_PATH | PATH | EXPECTED
new LinPath(fs, '/') | new LinPath(fs, '123/a/b/c') | new LinPath(null, '123/a/b/c')
new LinPath(fs,'123/a/') | new LinPath(fs, '123/a/b/c') | new LinPath(null, 'b/c')
new LinPath(fs,'123/a/') | new LinPath(fs, '321/a/') | new LinPath(null, '../../321/a')
new LinPath(null,'123/a') | new LinPath(null, '123/a/b/c') | new LinPath(null, 'b/c')
new LinPath(null,'123/a') | new LinPath(null, '321/a') | new LinPath(null, '../../321/a')
new LinPath(fs,'../a/') | new LinPath(fs, '321/a') | new LinPath(null, '../321/a')
new LinPath(fs,'321/a/') | new LinPath(fs, '../a') | new LinPath(null, '../../a')
new LinPath(null,'321/a/') | new LinPath(null, '../a') | new LinPath(null, '../../../a')
}
def 'relativize should throw exception' () {
given:
def lid1 = new LinPath(fs,'123/a/')
def lid2 = new LinPath(null,'123/a/')
def lid3 = new LinPath(null, '../a/b')
when: 'comparing relative with absolute'
lid1.relativize(lid2)
then:
thrown(IllegalArgumentException)
when: 'undefined base path'
lid3.relativize(lid2)
then:
thrown(IllegalArgumentException)
}
def 'should resolve path' () {
when:
def lid1 = new LinPath(fs, '123/a/b/c')
def lid2 = new LinPath(fs, '321/x/y/z')
def rel1 = new LinPath(null, 'foo')
def rel2 = new LinPath(null, 'bar/')
then:
lid1.resolve(lid2) == lid2
lid2.resolve(lid1) == lid1
and:
lid1.resolve(rel1) == new LinPath(fs,'123/a/b/c/foo')
lid1.resolve(rel2) == new LinPath(fs,'123/a/b/c/bar')
and:
rel1.resolve(rel2) == new LinPath(null, 'foo/bar')
rel2.resolve(rel1) == new LinPath(null, 'bar/foo')
}
def 'should resolve path as string' () {
given:
def pr = Mock(LinFileSystemProvider)
def lidfs = Mock(LinFileSystem){
provider() >> pr}
def lid1 = new LinPath(lidfs, '123/a/b/c')
expect:
lid1.resolve('x/y') == new LinPath(lidfs, '123/a/b/c/x/y')
lid1.resolve('/x/y/') == new LinPath(lidfs, '123/a/b/c/x/y')
when:
def result = lid1.resolve('lid://321')
then:
pr.getPath(LinPath.asUri('lid://321')) >> new LinPath(lidfs, '321')
and:
result == new LinPath(lidfs, '321')
}
def 'should throw illegal exception when not correct scheme' (){
when: 'creation'
new LinPath(fs, new URI("http://1234"))
then:
thrown(IllegalArgumentException)
when: 'asUri'
LinPath.asUri("http://1234")
then:
thrown(IllegalArgumentException)
when: 'asUri'
LinPath.asUri("")
then:
thrown(IllegalArgumentException)
}
def 'should throw provider mismatch exception when different path types' () {
given:
def pr = Mock(LinFileSystemProvider)
def fs = Mock(LinFileSystem){
provider() >> pr}
and:
def lid = new LinPath(fs, '123/a/b/c')
when: 'resolve with path'
lid.resolve(Path.of('d'))
then:
thrown(ProviderMismatchException)
when: 'resolve with uri string'
lid.resolve(Path.of('http://1234'))
then:
thrown(ProviderMismatchException)
when: 'relativize'
lid.relativize(Path.of('d'))
then:
thrown(ProviderMismatchException)
}
def 'should throw exception for unsupported methods' () {
given:
def pr = Mock(LinFileSystemProvider)
def fs = Mock(LinFileSystem){
provider() >> pr}
and:
def lid = new LinPath(fs, '123/a/b/c')
when: 'to file'
lid.toFile()
then:
thrown(UnsupportedOperationException)
when: 'register'
lid.register(null, null,null)
then:
thrown(UnsupportedOperationException)
}
def 'should throw exception for incorrect index'() {
when: 'getting name with negative index'
new LinPath(fs, "1234").getName(-1)
then:
thrown(IllegalArgumentException)
when: 'getting name with larger index tha namecount'
new LinPath(fs, "1234").getName(2)
then:
thrown(IllegalArgumentException)
when: 'getting subpath with negative index'
new LinPath(fs, "1234").subpath(-1,1)
then:
thrown(IllegalArgumentException)
when: 'getting subpath with larger index tha namecount'
new LinPath(fs, "1234").subpath(0,2)
then:
thrown(IllegalArgumentException)
}
@Unroll
def 'should get to uri string' () {
expect:
new LinPath(fs, PATH).toUriString() == EXPECTED
where:
PATH | EXPECTED
'/' | 'lid:///'
'1234' | 'lid://1234'
'1234/a/b/c' | 'lid://1234/a/b/c'
'' | 'lid:///'
}
@Unroll
def 'should get string' () {
expect:
new LinPath(fs, PATH).toString() == EXPECTED
where:
PATH | EXPECTED
'/' | '/'
'1234' | '1234'
'1234/a/b/c' | '1234/a/b/c'
'' | '/'
}
@Unroll
def 'should validate asString method'() {
expect:
LinPath.asUriString(FIRST, MORE as String[]) == EXPECTED
where:
FIRST | MORE | EXPECTED
'foo' | [] | 'lid://foo'
'foo/' | [] | 'lid://foo'
'/foo' | [] | 'lid://foo'
and:
'a' | ['/b/'] | 'lid://a/b'
'a' | ['/b','c'] | 'lid://a/b/c'
'a' | ['/b','//c'] | 'lid://a/b/c'
'a' | ['/b/c', 'd'] | 'lid://a/b/c/d'
'/a/' | ['/b/c', 'd'] | 'lid://a/b/c/d'
}
@Unroll
def 'should check is lid uri string' () {
expect:
LinPath.isLidUri(STR) == EXPECTED
where:
STR | EXPECTED
null | false
'' | false
'foo' | false
'/foo' | false
'lid:/foo' | false
'lid:foo' | false
'lid/foo' | false
and:
'lid://' | true
'lid:///' | true
'lid://foo/bar' | true
}
def 'should detect equals'(){
expect:
new LinPath(FS1, PATH1).equals(new LinPath(FS2, PATH2)) == EXPECTED
where:
FS1 | FS2 | PATH1 | PATH2 | EXPECTED
null | fs | "12345/path" | "12345/path" | false
fs | null | "12345/path" | "12345/path" | false
null | null | "12345/" | "12345/path" | false
fs | fs | "12345/" | "12345/path" | false
and:
null | null | "12345/path" | "12345/path" | true
fs | fs | "12345/path" | "12345/path" | true
null | null | "12345/" | "12345" | true
fs | fs | "12345/" | "12345 " | true
}
def 'should validate correct hash'(){
given:
def file = wdir.resolve("file.txt")
file.text = "this is a data file"
def hash = CacheHelper.hasher(file).hash().toString()
def correctData = new FileOutput(file.toString(), new Checksum(hash,"nextflow", "standard"))
when:
def error = LinPath.validateDataOutput(correctData)
then:
!error
cleanup:
file.delete()
}
def 'should warn with incorrect hash'(){
given:
def file = wdir.resolve("file.txt")
file.text = "this is a data file"
def hash = CacheHelper.hasher(file).hash().toString()
def correctData = new FileOutput(file.toString(), new Checksum("abscd","nextflow", "standard"))
when:
def error = LinPath.validateDataOutput(correctData)
then:
error == "Checksum of '$file' does not match with lineage metadata"
cleanup:
file.delete()
}
def 'should warn when hash algorithm is not supported'(){
given:
def file = wdir.resolve("file.txt")
file.text = "this is a data file"
def hash = CacheHelper.hasher(file).hash().toString()
def correctData = new FileOutput(file.toString(), new Checksum(hash,"not-supported", "standard"))
when:
def error = LinPath.validateDataOutput(correctData)
then:
error == "Checksum of '$file' can't be validated - algorithm 'not-supported' is not supported"
cleanup:
file.delete()
}
def 'should throw exception when file not found validating hash'(){
when:
def correctData = new FileOutput("not/existing/file", new Checksum("120741","nextflow", "standard"))
LinPath.validateDataOutput(correctData)
then:
thrown(FileNotFoundException)
}
def 'should validate path' () {
given:
def outputFolder = data.resolve('output')
outputFolder.mkdirs()
def outputFile = outputFolder.resolve('file1.txt')
outputFile.text = "this is file1"
def encoder = new LinEncoder()
def hash = CacheHelper.hasher(outputFile).hash().toString()
def correctData = new FileOutput(outputFile.toString(), new Checksum(hash,"nextflow", "standard"))
def incorrectData = new FileOutput(outputFile.toString(), new Checksum("incorrectHash","nextflow", "standard"))
wdir.resolve('12345/output/file1.txt').mkdirs()
wdir.resolve('12345/output/file2.txt').mkdirs()
wdir.resolve('12345/output/file1.txt/.data.json').text = encoder.encode(correctData)
wdir.resolve('12345/output/file2.txt/.data.json').text = encoder.encode(incorrectData)
def lidFs = new LinFileSystemProvider().newFileSystem(new URI("lid:///"), [enabled: true, store: [location: wdir.toString()]])
when:
def succeed = new LinPath(lidFs, '12345/output/file1.txt').validate()
then:
succeed
when:
succeed = new LinPath(lidFs, '12345/output/file2.txt').validate()
then:
!succeed
when:
succeed = new LinPath(lidFs, '12345/output/file3.txt').validate()
then:
!succeed
cleanup:
outputFile.delete()
}
}

View File

@@ -0,0 +1,58 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.model
import nextflow.lineage.model.v1beta1.Checksum
import nextflow.util.CacheHelper
import spock.lang.Specification
/**
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
class ChecksumTest extends Specification {
def 'should create a checksum'() {
given:
def checksum = new Checksum(algorithm: 'sha1', value: '1234567890abcdef', mode: 'hex')
expect:
checksum.algorithm == 'sha1'
checksum.value == '1234567890abcdef'
checksum.mode == 'hex'
}
def 'should create a checksum with of factory method'() {
given:
def checksum1 = Checksum.of('1234567890abcdef','sha1', CacheHelper.HashMode.DEFAULT())
expect:
checksum1.algorithm == 'sha1'
checksum1.value == '1234567890abcdef'
checksum1.mode == 'standard'
}
def 'should create checksum with ofNextflow factory method'() {
given:
def checksum1 = Checksum.ofNextflow('1234567890abcdef')
expect:
checksum1.algorithm == 'nextflow'
checksum1.value == CacheHelper.hasher('1234567890abcdef').hash().toString()
checksum1.mode == 'standard'
}
}

View File

@@ -0,0 +1,266 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.serde
import java.nio.file.Path
import java.time.OffsetDateTime
import nextflow.lineage.model.v1beta1.Checksum
import nextflow.lineage.model.v1beta1.DataPath
import nextflow.lineage.model.v1beta1.FileOutput
import nextflow.lineage.model.v1beta1.Parameter
import nextflow.lineage.model.v1beta1.TaskOutput
import nextflow.lineage.model.v1beta1.TaskRun
import nextflow.lineage.model.v1beta1.Workflow
import nextflow.lineage.model.v1beta1.WorkflowOutput
import nextflow.lineage.model.v1beta1.WorkflowRun
import spock.lang.Specification
class LinEncoderTest extends Specification{
def 'should encode and decode Outputs'(){
given:
def encoder = new LinEncoder()
and:
def output = new FileOutput("/path/to/file", new Checksum("hash_value", "hash_algorithm", "standard"),
"lid://source", "lid://workflow", "lid://task", 1234)
when:
def encoded = encoder.encode(output)
def object = encoder.decode(encoded)
then:
object instanceof FileOutput
def result = object as FileOutput
result.path == "/path/to/file"
result.checksum instanceof Checksum
result.checksum.value == "hash_value"
result.checksum.algorithm == "hash_algorithm"
result.checksum.mode == "standard"
result.source == "lid://source"
result.size == 1234
}
def 'should encode and decode WorkflowRuns'(){
given:
def encoder = new LinEncoder()
and:
def config = [
process: [
container : "quay.io/nextflow/bash",
executor : "local",
resourceLabels: ["owner": "xxx"],
scratch : false
]
]
def metadata = [
runName : "big_kare",
start : "2025-11-06T13:35:42.049135334Z",
container : "quay.io/nextflow/bash",
commandLine : "nextflow run 'https://github.com/nextflow-io/hello' -name big_kare -with-tower -r master",
nextflow : [version: "25.10.0", enable: [dsl: 2.0]],
containerEngine: "docker",
wave : [enabled: true],
fusion : [enabled: true, version: "2.4"],
platform : [
workflowId: "wf1234",
user : [id: "xxx", userName: "john-smith", email: "john.smith@acme.com", firstName: "John", lastName: "Smith", organization: "acme"],
workspace : [id: "1234", name: "test-workspace", fullName: "Test workspace", organization: "acme"],
computeEnv: [id: "ce3456", name: "test-ce", platform: "aws-cloud"],
pipeline : [id: "pipe294", name: "https://github.com/nextflow-io/hello", revision: "master", commitId: null],
labels : []
],
failOnIgnore : false
]
def uniqueId = UUID.randomUUID()
def mainScript = new DataPath("file://path/to/main.nf", new Checksum("78910", "nextflow", "standard"))
def workflow = new Workflow([mainScript], "https://nextflow.io/nf-test/", "123456")
def wfRun = new WorkflowRun(workflow, uniqueId.toString(), "test_run",
[new Parameter("String", "param1", "value1"), new Parameter("String", "param2", "value2")],
config, metadata
)
when:
def encoded = encoder.encode(wfRun)
def object = encoder.decode(encoded)
then:
object instanceof WorkflowRun
def result = object as WorkflowRun
result.workflow instanceof Workflow
result.workflow.scriptFiles.first instanceof DataPath
result.workflow.scriptFiles.first.path == "file://path/to/main.nf"
result.workflow.scriptFiles.first.checksum instanceof Checksum
result.workflow.scriptFiles.first.checksum.value == "78910"
result.workflow.commitId == "123456"
result.sessionId == uniqueId.toString()
result.name == "test_run"
result.params.size() == 2
result.params.get(0).name == "param1"
result.config.process.container == "quay.io/nextflow/bash"
result.config.process.executor == "local"
result.metadata.platform.workflowId == "wf1234"
}
def 'should decode WorkflowRuns without metadata'(){
given:
def encoder = new LinEncoder()
def wfRunStr = '''
{
"version": "lineage/v1beta1",
"kind": "WorkflowRun",
"spec": {
"workflow": {
"scriptFiles": [
{
"path": "https://github.com/nextflow-io/hello/main.nf",
"checksum": {
"value": "78910",
"algorithm": "nextflow",
"mode": "standard"
}
}
],
"repository": "https://github.com/nextflow-io/hello",
"commitId": "2ce0b0e2943449188092a0e25102f0dadc70cb0a"
},
"sessionId": "4f02559e-9ebd-41d8-8ee2-a8d1e4f09c67",
"name": "test_run",
"params": [],
"config": {
"process": {
"container": "quay.io/nextflow/bash",
"executor": "local",
"resourceLabels": {
"owner": "xxx"
},
"scratch": false
}
}
}
}
'''
when:
def object = encoder.decode(wfRunStr)
then:
object instanceof WorkflowRun
def result = object as WorkflowRun
result.workflow instanceof Workflow
result.workflow.scriptFiles.first instanceof DataPath
result.workflow.scriptFiles.first.path == "https://github.com/nextflow-io/hello/main.nf"
result.workflow.scriptFiles.first.checksum instanceof Checksum
result.workflow.scriptFiles.first.checksum.value == "78910"
result.workflow.commitId == "2ce0b0e2943449188092a0e25102f0dadc70cb0a"
result.sessionId == "4f02559e-9ebd-41d8-8ee2-a8d1e4f09c67"
result.name == "test_run"
result.params.size() == 0
result.config.process.container == "quay.io/nextflow/bash"
result.config.process.executor == "local"
result.metadata == null
}
def 'should encode and decode WorkflowOutputs'(){
given:
def encoder = new LinEncoder()
and:
def time = OffsetDateTime.now()
def wfResults = new WorkflowOutput(time, "lid://1234", [
new Parameter("Collection", "a", [id: 'id', file: 'sample.txt' as Path]),
new Parameter("String", "b", "B")
])
when:
def encoded = encoder.encode(wfResults)
def object = encoder.decode(encoded)
then:
object instanceof WorkflowOutput
def result = object as WorkflowOutput
result.createdAt == time
result.workflowRun == "lid://1234"
result.output == [new Parameter("Collection", "a", [id: 'id', file: 'sample.txt']), new Parameter("String", "b", "B")]
}
def 'should encode and decode TaskRun'() {
given:
def encoder = new LinEncoder()
and:
def uniqueId = UUID.randomUUID()
def taskRun = new TaskRun(
uniqueId.toString(),"name", new Checksum("78910", "nextflow", "standard"), 'this is a script',
[new Parameter("String", "param1", "value1")], "container:version", "conda", "spack", "amd64",
[a: "A", b: "B"], [new DataPath("path/to/file", new Checksum("78910", "nextflow", "standard"))]
)
when:
def encoded = encoder.encode(taskRun)
def object = encoder.decode(encoded)
then:
object instanceof TaskRun
def result = object as TaskRun
result.sessionId == uniqueId.toString()
result.name == "name"
result.codeChecksum.value == "78910"
result.script == "this is a script"
result.input.size() == 1
result.input.get(0).name == "param1"
result.container == "container:version"
result.conda == "conda"
result.spack == "spack"
result.architecture == "amd64"
result.globalVars == [a: "A", b: "B"]
result.binEntries.size() == 1
result.binEntries.get(0).path == "path/to/file"
result.binEntries.get(0).checksum.value == "78910"
}
def 'should encode and decode TaskOutputs'(){
given:
def encoder = new LinEncoder()
and:
def time = OffsetDateTime.now()
def parameter = new Parameter("a","b", "c")
def wfResults = new TaskOutput("lid://1234", "lid://5678", time, [parameter], null)
when:
def encoded = encoder.encode(wfResults)
def object = encoder.decode(encoded)
then:
object instanceof TaskOutput
def result = object as TaskOutput
result.createdAt == time
result.taskRun == "lid://1234"
result.workflowRun == "lid://5678"
result.output.size() == 1
result.output[0] == parameter
}
def 'object with null date attributes' () {
given:
def encoder = new LinEncoder()
and:
def wfResults = new WorkflowOutput(null, "lid://1234")
when:
def encoded = encoder.encode(wfResults)
def object = encoder.decode(encoded)
then:
encoded == '{"version":"lineage/v1beta1","kind":"WorkflowOutput","spec":{"createdAt":null,"workflowRun":"lid://1234","output":null}}'
def result = object as WorkflowOutput
result.createdAt == null
}
}

View File

@@ -0,0 +1,181 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.lineage.serde
import nextflow.lineage.model.v1beta1.FileOutput
import nextflow.lineage.model.v1beta1.LinModel
import nextflow.lineage.model.v1beta1.TaskRun
import spock.lang.Specification
class LinTypeAdapterFactoryTest extends Specification {
def 'should read both new and old JSON formats'() {
given:
def encoder = new LinEncoder()
when: 'create old format JSON (without spec wrapper)'
def oldFormatJson = """
{
"version": "${LinModel.VERSION}",
"kind": "FileOutput",
"path": "/path/to/file",
"checksum": {
"value": "hash_value",
"algorithm": "hash_algorithm",
"mode": "standard"
},
"source": "lid://source",
"workflow": "lid://workflow",
"task": "lid://task",
"size": 1234
}
"""
and: 'deserialize old format'
def oldResult = encoder.decode(oldFormatJson)
then: 'should work correctly with backward compatibility'
oldResult instanceof FileOutput
oldResult.path == "/path/to/file"
oldResult.checksum.value == "hash_value"
oldResult.source == "lid://source"
oldResult.size == 1234
when: 'create new format JSON (with spec wrapper)'
def newFormatJson = """
{
"version": "${LinModel.VERSION}",
"kind": "FileOutput",
"spec": {
"path": "/path/to/file",
"checksum": {
"value": "hash_value",
"algorithm": "hash_algorithm",
"mode": "standard"
},
"source": "lid://source",
"workflow": "lid://workflow",
"task": "lid://task",
"size": 1234
}
}
"""
and: 'deserialize new format'
def newResult = encoder.decode(newFormatJson)
then: 'should work correctly'
newResult instanceof FileOutput
newResult.path == "/path/to/file"
newResult.checksum.value == "hash_value"
newResult.source == "lid://source"
newResult.size == 1234
}
def 'should handle TaskRun old format'() {
given:
def encoder = new LinEncoder()
when: 'create old format TaskRun JSON'
def oldFormatJson = """
{
"version": "${LinModel.VERSION}",
"kind": "TaskRun",
"sessionId": "session123",
"name": "testTask",
"codeChecksum": {
"value": "hash123",
"algorithm": "nextflow",
"mode": "standard"
},
"script": "echo hello",
"input": [],
"container": "ubuntu:latest",
"conda": null,
"spack": null,
"architecture": "amd64",
"globalVars": {},
"binEntries": []
}
"""
and: 'deserialize old format'
def result = encoder.decode(oldFormatJson)
then: 'should work correctly'
result instanceof TaskRun
result.sessionId == "session123"
result.name == "testTask"
result.codeChecksum.value == "hash123"
result.script == "echo hello"
result.container == "ubuntu:latest"
result.architecture == "amd64"
}
def 'should reject JSON without version field'() {
given:
def encoder = new LinEncoder()
when: 'try to deserialize JSON without version'
def invalidJson = """
{
"kind": "FileOutput",
"path": "/path/to/file"
}
"""
encoder.decode(invalidJson)
then: 'should throw exception'
thrown(Exception)
}
def 'should reject JSON with wrong version'() {
given:
def encoder = new LinEncoder()
when: 'try to deserialize JSON with wrong version'
def invalidJson = """
{
"version": "wrong/version",
"kind": "FileOutput",
"path": "/path/to/file"
}
"""
encoder.decode(invalidJson)
then: 'should throw exception'
thrown(Exception)
}
def 'should reject JSON without kind field'() {
given:
def encoder = new LinEncoder()
when: 'try to deserialize JSON without kind'
def invalidJson = """
{
"version": "${LinModel.VERSION}",
"path": "/path/to/file"
}
"""
encoder.decode(invalidJson)
then: 'should throw exception'
thrown(Exception)
}
}