add nextflow d30e48d
This commit is contained in:
@@ -0,0 +1,79 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package nextflow.lineage
|
||||
|
||||
import spock.lang.Specification
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
|
||||
/**
|
||||
* Lineage History file tests
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
class DefaultLinHistoryLogTest extends Specification {
|
||||
|
||||
Path tempDir
|
||||
Path historyFile
|
||||
DefaultLinHistoryLog linHistoryLog
|
||||
|
||||
def setup() {
|
||||
tempDir = Files.createTempDirectory("wdir")
|
||||
historyFile = tempDir.resolve("lin-history")
|
||||
linHistoryLog = new DefaultLinHistoryLog(historyFile)
|
||||
}
|
||||
|
||||
def cleanup(){
|
||||
tempDir?.deleteDir()
|
||||
}
|
||||
|
||||
def "write should add a new file to the history folder"() {
|
||||
given:
|
||||
UUID sessionId = UUID.randomUUID()
|
||||
String runName = "TestRun"
|
||||
String runLid = "lid://123"
|
||||
|
||||
when:
|
||||
linHistoryLog.write(runName, sessionId, runLid)
|
||||
|
||||
then:
|
||||
def files = historyFile.listFiles()
|
||||
files.size() == 1
|
||||
def parsedRecord = LinHistoryRecord.parse(files[0].text)
|
||||
parsedRecord.sessionId == sessionId
|
||||
parsedRecord.runName == runName
|
||||
parsedRecord.runLid == runLid
|
||||
}
|
||||
|
||||
def 'should get records' () {
|
||||
given:
|
||||
UUID sessionId = UUID.randomUUID()
|
||||
String runName = "Run1"
|
||||
String runLid = "lid://123"
|
||||
and:
|
||||
linHistoryLog.write(runName, sessionId, runLid)
|
||||
|
||||
when:
|
||||
def records = linHistoryLog.getRecords()
|
||||
then:
|
||||
records.size() == 1
|
||||
records[0].sessionId == sessionId
|
||||
records[0].runName == runName
|
||||
records[0].runLid == runLid
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage
|
||||
|
||||
import nextflow.lineage.config.LineageConfig
|
||||
import spock.lang.Specification
|
||||
import spock.lang.Unroll
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class DefaultLinStoreFactoryTest extends Specification {
|
||||
|
||||
@Unroll
|
||||
def 'should validate can open' () {
|
||||
given:
|
||||
def factory = new DefaultLinStoreFactory()
|
||||
def config = new LineageConfig(CONFIG)
|
||||
|
||||
expect:
|
||||
factory.canOpen(config) == EXPECTED
|
||||
|
||||
where:
|
||||
EXPECTED | CONFIG
|
||||
true | [:]
|
||||
true | [store:[location:'/some/path']]
|
||||
true | [store:[location:'some/rel/path']]
|
||||
true | [store:[location:'file:/this/that']]
|
||||
true | [store:[location:'s3://some/path']]
|
||||
false | [store:[location:'http://some/path']]
|
||||
false | [store:[location:'jdbc:foo']]
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.time.Instant
|
||||
import java.time.OffsetDateTime
|
||||
import java.time.ZoneOffset
|
||||
|
||||
import nextflow.lineage.model.v1beta1.Checksum
|
||||
import nextflow.lineage.model.v1beta1.DataPath
|
||||
import nextflow.lineage.model.v1beta1.FileOutput
|
||||
import nextflow.lineage.model.v1beta1.Parameter
|
||||
import nextflow.lineage.model.v1beta1.Workflow
|
||||
import nextflow.lineage.model.v1beta1.WorkflowRun
|
||||
import nextflow.lineage.serde.LinEncoder
|
||||
import nextflow.lineage.config.LineageConfig
|
||||
import spock.lang.Specification
|
||||
import spock.lang.TempDir
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
class DefaultLinStoreTest extends Specification {
|
||||
|
||||
@TempDir
|
||||
Path tempDir
|
||||
|
||||
Path storeLocation
|
||||
LineageConfig config
|
||||
|
||||
def setup() {
|
||||
storeLocation = tempDir.resolve("store")
|
||||
def configMap = [enabled: true, store: [location: storeLocation.toString(), logLocation: storeLocation.resolve(".log").toString()]]
|
||||
config = new LineageConfig(configMap)
|
||||
}
|
||||
|
||||
def 'should open store'() {
|
||||
given:
|
||||
def store = new DefaultLinStore()
|
||||
when:
|
||||
store.open(config)
|
||||
def historyLog = store.getHistoryLog()
|
||||
then:
|
||||
store.getLocation() == storeLocation
|
||||
historyLog != null
|
||||
historyLog instanceof DefaultLinHistoryLog
|
||||
}
|
||||
|
||||
def "save should store value in the correct file location"() {
|
||||
given:
|
||||
def key = "testKey"
|
||||
def value = new FileOutput("/path/to/file", new Checksum("hash_value", "hash_algorithm", "standard"), "lid://source", "lid://workflow", "lid://task", 1234)
|
||||
def lidStore = new DefaultLinStore()
|
||||
lidStore.open(config)
|
||||
|
||||
when:
|
||||
lidStore.save(key, value)
|
||||
|
||||
then:
|
||||
def filePath = storeLocation.resolve("$key/.data.json")
|
||||
Files.exists(filePath)
|
||||
filePath.text == new LinEncoder().encode(value)
|
||||
}
|
||||
|
||||
def "load should retrieve stored value correctly"() {
|
||||
given:
|
||||
def key = "testKey"
|
||||
def value = new FileOutput("/path/to/file", new Checksum("hash_value", "hash_algorithm", "standard"), "lid://source", "lid://workflow", "lid://task", 1234)
|
||||
def lidStore = new DefaultLinStore()
|
||||
lidStore.open(config)
|
||||
lidStore.save(key, value)
|
||||
|
||||
expect:
|
||||
lidStore.load(key).toString() == value.toString()
|
||||
}
|
||||
|
||||
def "load should return null if key does not exist"() {
|
||||
given:
|
||||
def lidStore = new DefaultLinStore()
|
||||
lidStore.open(config)
|
||||
|
||||
expect:
|
||||
lidStore.load("nonexistentKey") == null
|
||||
}
|
||||
|
||||
def 'should query' () {
|
||||
given:
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(1234567), ZoneOffset.UTC)
|
||||
def mainScript = new DataPath("file://path/to/main.nf", new Checksum("78910", "nextflow", "standard"))
|
||||
def workflow = new Workflow([mainScript],"https://nextflow.io/nf-test/", "123456" )
|
||||
def key = "testKey"
|
||||
def value1 = new WorkflowRun(workflow, uniqueId.toString(), "test_run", [ new Parameter("String", "param1", "value1"), new Parameter("String", "param2", "value2")] )
|
||||
def key2 = "testKey2"
|
||||
def value2 = new FileOutput("/path/tp/file1", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", null, 1234, time, time, ["value1", "value2"])
|
||||
def key3 = "testKey3"
|
||||
def value3 = new FileOutput("/path/tp/file2", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", null, 1234, time, time, ["value2", "value3"])
|
||||
def key4 = "testKey4"
|
||||
def value4 = new FileOutput("/path/tp/file", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", null, 1234, time, time, ["value4", "value3"])
|
||||
|
||||
def lidStore = new DefaultLinStore()
|
||||
lidStore.open(config)
|
||||
lidStore.save(key, value1)
|
||||
lidStore.save(key2, value2)
|
||||
lidStore.save(key3, value3)
|
||||
lidStore.save(key4, value4)
|
||||
|
||||
when:
|
||||
def results = lidStore.search( [type:['FileOutput'], labels:['value2']]).toList()
|
||||
then:
|
||||
results.size() == 2
|
||||
results.containsAll([key2,key3])
|
||||
}
|
||||
|
||||
def 'should search subkeys' () {
|
||||
given:
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(1234567), ZoneOffset.UTC)
|
||||
def mainScript = new DataPath("file://path/to/main.nf", new Checksum("78910", "nextflow", "standard"))
|
||||
def workflow = new Workflow([mainScript], "https://nextflow.io/nf-test/", "123456")
|
||||
def key = "testKey"
|
||||
def value1 = new WorkflowRun(workflow, uniqueId.toString(), "test_run", [new Parameter("String", "param1", "value1"), new Parameter("String", "param2", "value2")])
|
||||
def key2 = "testKey/file1"
|
||||
def value2 = new FileOutput("/path/file1", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", null, 1234, time, time, ["value1", "value2"])
|
||||
def key3 = "testKey/subfolder/file3"
|
||||
def value3 = new FileOutput("/path//file2", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", null, 1234, time, time, ["value2", "value3"])
|
||||
def key4 = "testKey2/file2"
|
||||
def value4 = new FileOutput("/path/tp/file", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", null, 1234, time, time, ["value4", "value3"])
|
||||
|
||||
def lidStore = new DefaultLinStore()
|
||||
lidStore.open(config)
|
||||
lidStore.save(key, value1)
|
||||
lidStore.save(key2, value2)
|
||||
lidStore.save(key3, value3)
|
||||
lidStore.save(key4, value4)
|
||||
|
||||
when:
|
||||
def results = lidStore.getSubKeys("testKey").toList()
|
||||
then:
|
||||
results.size() == 2
|
||||
results.containsAll([key2, key3])
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage
|
||||
|
||||
import java.nio.file.Path
|
||||
import java.time.Instant
|
||||
import java.time.OffsetDateTime
|
||||
import java.time.ZoneOffset
|
||||
|
||||
import groovyx.gpars.dataflow.DataflowQueue
|
||||
import nextflow.Channel
|
||||
import nextflow.Session
|
||||
import nextflow.lineage.config.LineageConfig
|
||||
import nextflow.lineage.fs.LinPathFactory
|
||||
import nextflow.lineage.model.v1beta1.Checksum
|
||||
import nextflow.lineage.model.v1beta1.DataPath
|
||||
import nextflow.lineage.model.v1beta1.FileOutput
|
||||
import nextflow.lineage.model.v1beta1.Parameter
|
||||
import nextflow.lineage.model.v1beta1.Workflow
|
||||
import nextflow.lineage.model.v1beta1.WorkflowRun
|
||||
import spock.lang.Specification
|
||||
import spock.lang.TempDir
|
||||
|
||||
import static nextflow.lineage.fs.LinPath.*
|
||||
import static test.ScriptHelper.runDataflow
|
||||
|
||||
/**
|
||||
* Lineage channel extensions tests
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
class LinExtensionImplTest extends Specification {
|
||||
|
||||
@TempDir
|
||||
Path tempDir
|
||||
|
||||
Path storeLocation
|
||||
Map configMap
|
||||
|
||||
def setup() {
|
||||
storeLocation = tempDir.resolve("store")
|
||||
configMap = [linage: [enabled: true, store: [location: storeLocation.toString()]]]
|
||||
}
|
||||
|
||||
def 'should return global query results' () {
|
||||
given:
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(1234567), ZoneOffset.UTC)
|
||||
def mainScript = new DataPath("file://path/to/main.nf", new Checksum("78910", "nextflow", "standard"))
|
||||
def workflow = new Workflow([mainScript],"https://nextflow.io/nf-test/", "123456" )
|
||||
def key = "testKey"
|
||||
def value1 = new WorkflowRun(workflow, uniqueId.toString(), "test_run", [ new Parameter("String", "param1", "value1"), new Parameter("String", "param2", "value2")] )
|
||||
def key2 = "testKey2"
|
||||
def value2 = new FileOutput("/path/tp/file1", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", "taskid", 1234, time, time, ["value1","value2"])
|
||||
def key3 = "testKey3"
|
||||
def value3 = new FileOutput("/path/tp/file2", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", null, 1234, time, time, ["value2", "value3"])
|
||||
def key4 = "testKey4"
|
||||
def value4 = new FileOutput("/path/tp/file", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", "taskid", 1234, time, time, ["value4","value3"])
|
||||
def lidStore = new DefaultLinStore()
|
||||
def session = Mock(Session) {
|
||||
getConfig() >> configMap
|
||||
}
|
||||
lidStore.open(LineageConfig.create(session))
|
||||
lidStore.save(key, value1)
|
||||
lidStore.save(key2, value2)
|
||||
lidStore.save(key3, value3)
|
||||
lidStore.save(key4, value4)
|
||||
def linExt = Spy(new LinExtensionImpl())
|
||||
when:
|
||||
def results = new DataflowQueue()
|
||||
linExt.fromLineage(session, results, [label: ["value2", "value3"]])
|
||||
then:
|
||||
linExt.getStore(session) >> lidStore
|
||||
and:
|
||||
results.val == LinPathFactory.create( asUriString(key3) )
|
||||
results.val == Channel.STOP
|
||||
|
||||
when:
|
||||
results = new DataflowQueue()
|
||||
linExt.fromLineage(session, results, [taskRun: "taskid", label: ["value4"]])
|
||||
then:
|
||||
linExt.getStore(session) >> lidStore
|
||||
and:
|
||||
results.val == LinPathFactory.create( asUriString(key4) )
|
||||
results.val == Channel.STOP
|
||||
|
||||
when:
|
||||
results = new DataflowQueue()
|
||||
linExt.fromLineage(session, results, [workflowRun: "testkey", taskRun: "taskid", label: "value2"])
|
||||
then:
|
||||
linExt.getStore(session) >> lidStore
|
||||
and:
|
||||
results.val == LinPathFactory.create( asUriString(key2) )
|
||||
results.val == Channel.STOP
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package nextflow.lineage
|
||||
|
||||
import spock.lang.Specification
|
||||
|
||||
/**
|
||||
* Lineage History Record tests
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
class LinHistoryRecordTest extends Specification {
|
||||
def "LinHistoryRecord parse should throw for invalid record"() {
|
||||
when:
|
||||
LinHistoryRecord.parse("invalid-record")
|
||||
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
}
|
||||
|
||||
def "LinHistoryRecord parse should handle 4-column record"() {
|
||||
given:
|
||||
def timestamp = new Date()
|
||||
def formattedTimestamp = LinHistoryRecord.TIMESTAMP_FMT.format(timestamp)
|
||||
def line = "${formattedTimestamp}\trun-1\t${UUID.randomUUID()}\tlid://123"
|
||||
|
||||
when:
|
||||
def record = LinHistoryRecord.parse(line)
|
||||
|
||||
then:
|
||||
record.timestamp != null
|
||||
record.runName == "run-1"
|
||||
record.runLid == "lid://123"
|
||||
}
|
||||
|
||||
def "LinHistoryRecord toString should produce tab-separated format"() {
|
||||
given:
|
||||
UUID sessionId = UUID.randomUUID()
|
||||
def record = new LinHistoryRecord(new Date(), "TestRun", sessionId, "lid://123")
|
||||
|
||||
when:
|
||||
def line = record.toString()
|
||||
|
||||
then:
|
||||
line.contains("\t")
|
||||
line.split("\t").size() == 4
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,983 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage
|
||||
|
||||
import nextflow.extension.FilesEx
|
||||
import nextflow.lineage.exception.OutputRelativePathException
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.attribute.BasicFileAttributes
|
||||
|
||||
import com.google.common.hash.HashCode
|
||||
import nextflow.NextflowMeta
|
||||
import nextflow.Session
|
||||
import nextflow.file.FileHolder
|
||||
import nextflow.lineage.model.v1beta1.Checksum
|
||||
import nextflow.lineage.model.v1beta1.FileOutput
|
||||
import nextflow.lineage.model.v1beta1.DataPath
|
||||
import nextflow.lineage.model.v1beta1.Parameter
|
||||
import nextflow.lineage.model.v1beta1.TaskOutput
|
||||
import nextflow.lineage.model.v1beta1.Workflow
|
||||
import nextflow.lineage.model.v1beta1.WorkflowOutput
|
||||
import nextflow.lineage.model.v1beta1.WorkflowRun
|
||||
import nextflow.lineage.serde.LinEncoder
|
||||
import nextflow.lineage.config.LineageConfig
|
||||
import nextflow.processor.TaskConfig
|
||||
import nextflow.processor.TaskHandler
|
||||
import nextflow.processor.TaskId
|
||||
import nextflow.processor.TaskRun
|
||||
import nextflow.script.ScriptBinding
|
||||
import nextflow.script.PlatformMetadata
|
||||
import nextflow.script.ScriptMeta
|
||||
import nextflow.script.TokenVar
|
||||
import nextflow.script.WorkflowMetadata
|
||||
import nextflow.script.params.EnvOutParam
|
||||
import nextflow.script.params.FileInParam
|
||||
import nextflow.script.params.FileOutParam
|
||||
import nextflow.script.params.InParam
|
||||
import nextflow.script.params.OutParam
|
||||
import nextflow.script.params.StdInParam
|
||||
import nextflow.script.params.StdOutParam
|
||||
import nextflow.script.params.ValueInParam
|
||||
import nextflow.script.params.ValueOutParam
|
||||
import nextflow.trace.event.FilePublishEvent
|
||||
import nextflow.trace.event.TaskEvent
|
||||
import nextflow.trace.event.WorkflowOutputEvent
|
||||
import nextflow.util.CacheHelper
|
||||
import nextflow.util.PathNormalizer
|
||||
import spock.lang.Shared
|
||||
import spock.lang.Specification
|
||||
import spock.lang.Unroll
|
||||
|
||||
import static nextflow.lineage.fs.LinPath.*
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class LinObserverTest extends Specification {
|
||||
@Shared
|
||||
Path lidFolder = Files.createTempDirectory("lid")
|
||||
def cleanupSpec(){
|
||||
lidFolder.deleteDir()
|
||||
}
|
||||
|
||||
def 'should normalize paths' (){
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def workDir = folder.resolve("workDir")
|
||||
def projectDir = folder.resolve("projectDir")
|
||||
def metadata = Mock(WorkflowMetadata){
|
||||
getRepository() >> "https://nextflow.io/nf-test/"
|
||||
getCommitId() >> "123456"
|
||||
getScriptId() >> "78910"
|
||||
getProjectDir() >> projectDir
|
||||
getWorkDir() >> workDir
|
||||
}
|
||||
def params = [path: workDir.resolve("path/file.txt"), sequence: projectDir.resolve("file2.txt").toString(), value: 12]
|
||||
when:
|
||||
def results = LinObserver.getNormalizedParams(params, new PathNormalizer(metadata))
|
||||
then:
|
||||
results.size() == 3
|
||||
results.get(0).name == "path"
|
||||
results.get(0).type == Path.simpleName
|
||||
results.get(0).value == "work/path/file.txt"
|
||||
results.get(1).name == "sequence"
|
||||
results.get(1).type == "String"
|
||||
results.get(1).value == projectDir.resolve("file2.txt").toString()
|
||||
results.get(2).name == "value"
|
||||
results.get(2).type == "Integer"
|
||||
results.get(2).value == 12
|
||||
|
||||
cleanup:
|
||||
ScriptMeta.reset()
|
||||
folder?.deleteDir()
|
||||
}
|
||||
def 'should collect script files' () {
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
and:
|
||||
def config = [workflow:[lineage:[enabled: true, store:[location:folder.toString()]]]]
|
||||
def store = new DefaultLinStore();
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def scriptFile = folder.resolve("main.nf")
|
||||
def module1 = folder.resolve("a_script1.nf"); module1.text = 'hola'
|
||||
def module2 = folder.resolve("b_script2.nf"); module2.text = 'world'
|
||||
and:
|
||||
|
||||
def metadata = Mock(WorkflowMetadata){
|
||||
getRepository() >> "https://nextflow.io/nf-test/"
|
||||
getCommitId() >> "123456"
|
||||
getScriptId() >> "78910"
|
||||
getScriptFile() >> scriptFile
|
||||
getProjectDir() >> folder.resolve("projectDir")
|
||||
getWorkDir() >> folder.resolve("workDir")
|
||||
}
|
||||
def session = Mock(Session) {
|
||||
getConfig() >> config
|
||||
getUniqueId() >> uniqueId
|
||||
getRunName() >> "test_run"
|
||||
getWorkflowMetadata() >> metadata
|
||||
getParams() >> new ScriptBinding.ParamsMap()
|
||||
}
|
||||
store.open(LineageConfig.create(session))
|
||||
def observer = Spy(new LinObserver(session, store))
|
||||
|
||||
when:
|
||||
def files = observer.collectScriptDataPaths(new PathNormalizer(metadata))
|
||||
then:
|
||||
observer.allScriptFiles() >> [ module2, scriptFile, module1 ]
|
||||
and:
|
||||
files.size() == 3
|
||||
and:
|
||||
files[0].path == "file://${scriptFile.toString()}"
|
||||
files[0].checksum == new Checksum("78910", "nextflow", "standard")
|
||||
and:
|
||||
files[1].path == "file://$module1"
|
||||
files[1].checksum == Checksum.ofNextflow(module1.text)
|
||||
and:
|
||||
files[2].path == "file://$module2"
|
||||
files[2].checksum == Checksum.ofNextflow(module2.text)
|
||||
|
||||
cleanup:
|
||||
ScriptMeta.reset()
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
def 'should save workflow' (){
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
|
||||
def store = new DefaultLinStore();
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def scriptFile = folder.resolve("main.nf")
|
||||
def map = [
|
||||
repository: "https://nextflow.io/nf-test/",
|
||||
commitId: "123456",
|
||||
scriptId: "78910",
|
||||
scriptFile: scriptFile,
|
||||
projectDir: folder.resolve("projectDir"),
|
||||
revision: "main",
|
||||
projectName: "nextflow.io/nf-test",
|
||||
workDir: folder.resolve("workDir")
|
||||
]
|
||||
def metadata = Mock(WorkflowMetadata){
|
||||
getRepository() >> map.repository
|
||||
getCommitId() >> map.commitId
|
||||
getScriptId() >> map.scriptId
|
||||
getScriptFile() >> map.scriptFile
|
||||
getProjectDir() >> map.projectDir
|
||||
getRevision() >> map.revision
|
||||
getProjectName() >> map.projectName
|
||||
getWorkDir() >> map.workDir
|
||||
toMap() >> map
|
||||
}
|
||||
def session = Mock(Session) {
|
||||
getConfig() >> config
|
||||
getUniqueId() >> uniqueId
|
||||
getRunName() >> "test_run"
|
||||
getWorkflowMetadata() >> metadata
|
||||
getParams() >> new ScriptBinding.ParamsMap()
|
||||
}
|
||||
store.open(LineageConfig.create(session))
|
||||
def observer = new LinObserver(session, store)
|
||||
def mainScript = new DataPath("file://${scriptFile.toString()}", new Checksum("78910", "nextflow", "standard"))
|
||||
def workflow = new Workflow([mainScript], map.repository, map.commitId)
|
||||
def workflowRun = new WorkflowRun(workflow, uniqueId.toString(), "test_run", [], config, map)
|
||||
when:
|
||||
observer.onFlowCreate(session)
|
||||
observer.onFlowBegin()
|
||||
then:
|
||||
folder.resolve("${observer.executionHash}/.data.json").text == new LinEncoder().encode(workflowRun)
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
def 'should strip sensitive user data from platform metadata in lineage' () {
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
|
||||
def store = new DefaultLinStore()
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def scriptFile = folder.resolve("main.nf")
|
||||
and:
|
||||
def platformMeta = new PlatformMetadata()
|
||||
platformMeta.user = new PlatformMetadata.User([
|
||||
id: 'u1234', userName: 'john-smith', email: 'john.smith@acme.com',
|
||||
firstName: 'John', lastName: 'Smith', organization: 'ACME'
|
||||
])
|
||||
platformMeta.workspace = new PlatformMetadata.Workspace([
|
||||
workspaceId: '1234', workspaceName: 'my-ws', workspaceFullName: 'My WS', orgName: 'ACME'
|
||||
])
|
||||
def map = [
|
||||
repository: "https://nextflow.io/nf-test/",
|
||||
commitId: "123456",
|
||||
scriptId: "78910",
|
||||
scriptFile: scriptFile,
|
||||
projectDir: folder.resolve("projectDir"),
|
||||
revision: "main",
|
||||
projectName: "nextflow.io/nf-test",
|
||||
workDir: folder.resolve("workDir"),
|
||||
platform: platformMeta
|
||||
]
|
||||
def metadata = Mock(WorkflowMetadata) {
|
||||
getRepository() >> map.repository
|
||||
getCommitId() >> map.commitId
|
||||
getScriptId() >> map.scriptId
|
||||
getScriptFile() >> map.scriptFile
|
||||
getProjectDir() >> map.projectDir
|
||||
getWorkDir() >> map.workDir
|
||||
toMap() >> map
|
||||
}
|
||||
def session = Mock(Session) {
|
||||
getConfig() >> config
|
||||
getUniqueId() >> uniqueId
|
||||
getRunName() >> "test_run"
|
||||
getWorkflowMetadata() >> metadata
|
||||
getParams() >> new ScriptBinding.ParamsMap()
|
||||
}
|
||||
store.open(LineageConfig.create(session))
|
||||
def observer = new LinObserver(session, store)
|
||||
|
||||
when:
|
||||
observer.onFlowCreate(session)
|
||||
observer.onFlowBegin()
|
||||
def stored = store.load(observer.executionHash) as WorkflowRun
|
||||
|
||||
then:
|
||||
def storedPlatform = stored.metadata.platform as Map
|
||||
storedPlatform.user.id == 'u1234'
|
||||
storedPlatform.user.userName == 'john-smith'
|
||||
storedPlatform.user.organization == 'ACME'
|
||||
!storedPlatform.user.containsKey('email')
|
||||
!storedPlatform.user.containsKey('firstName')
|
||||
!storedPlatform.user.containsKey('lastName')
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
def 'should convert Path values to URI strings in workflow metadata' () {
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
|
||||
def store = new DefaultLinStore()
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def scriptFile = folder.resolve("main.nf")
|
||||
def workDir = folder.resolve("workDir")
|
||||
def somePath = folder.resolve("data/input.txt")
|
||||
def map = [
|
||||
repository : "https://nextflow.io/nf-test/",
|
||||
commitId : "abc123",
|
||||
scriptId : "78910",
|
||||
scriptFile : scriptFile,
|
||||
projectDir : folder.resolve("projectDir"),
|
||||
workDir : workDir,
|
||||
somePathKey : somePath,
|
||||
someStringKey: "hello"
|
||||
]
|
||||
def metadata = Mock(WorkflowMetadata) {
|
||||
getRepository() >> map.repository
|
||||
getCommitId() >> map.commitId
|
||||
getScriptId() >> map.scriptId
|
||||
getScriptFile() >> map.scriptFile
|
||||
getProjectDir() >> map.projectDir
|
||||
getWorkDir() >> map.workDir
|
||||
toMap() >> map
|
||||
}
|
||||
def session = Mock(Session) {
|
||||
getConfig() >> config
|
||||
getUniqueId() >> uniqueId
|
||||
getRunName() >> "test_run"
|
||||
getWorkflowMetadata() >> metadata
|
||||
getParams() >> new ScriptBinding.ParamsMap()
|
||||
}
|
||||
store.open(LineageConfig.create(session))
|
||||
def observer = new LinObserver(session, store)
|
||||
|
||||
when:
|
||||
observer.onFlowCreate(session)
|
||||
observer.onFlowBegin()
|
||||
def stored = store.load(observer.executionHash) as WorkflowRun
|
||||
|
||||
then:
|
||||
stored.metadata.somePathKey == FilesEx.toUriString(somePath)
|
||||
stored.metadata.someStringKey == "hello"
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
def 'should remove transient properties from workflow metadata' () {
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
|
||||
def store = new DefaultLinStore()
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def scriptFile = folder.resolve("main.nf")
|
||||
def map = [
|
||||
repository : "https://nextflow.io/nf-test/",
|
||||
commitId : "abc123",
|
||||
scriptId : "78910",
|
||||
scriptFile : scriptFile,
|
||||
projectDir : folder.resolve("projectDir"),
|
||||
workDir : folder.resolve("workDir"),
|
||||
// transient properties that must be removed
|
||||
completed : new Date(),
|
||||
duration : 1000L,
|
||||
exitStatus : 0,
|
||||
errorMessage: "none",
|
||||
errorReport : "none",
|
||||
stats : [:],
|
||||
success : true
|
||||
]
|
||||
def metadata = Mock(WorkflowMetadata) {
|
||||
getRepository() >> map.repository
|
||||
getCommitId() >> map.commitId
|
||||
getScriptId() >> map.scriptId
|
||||
getScriptFile() >> map.scriptFile
|
||||
getProjectDir() >> map.projectDir
|
||||
getWorkDir() >> map.workDir
|
||||
toMap() >> map
|
||||
}
|
||||
def session = Mock(Session) {
|
||||
getConfig() >> config
|
||||
getUniqueId() >> uniqueId
|
||||
getRunName() >> "test_run"
|
||||
getWorkflowMetadata() >> metadata
|
||||
getParams() >> new ScriptBinding.ParamsMap()
|
||||
}
|
||||
store.open(LineageConfig.create(session))
|
||||
def observer = new LinObserver(session, store)
|
||||
|
||||
when:
|
||||
observer.onFlowCreate(session)
|
||||
observer.onFlowBegin()
|
||||
def stored = store.load(observer.executionHash) as WorkflowRun
|
||||
|
||||
then:
|
||||
!stored.metadata.containsKey('completed')
|
||||
!stored.metadata.containsKey('duration')
|
||||
!stored.metadata.containsKey('exitStatus')
|
||||
!stored.metadata.containsKey('errorMessage')
|
||||
!stored.metadata.containsKey('errorReport')
|
||||
!stored.metadata.containsKey('stats')
|
||||
!stored.metadata.containsKey('success')
|
||||
stored.metadata.containsKey('commitId')
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
def 'should convert NextflowMeta to JSON map in workflow metadata' () {
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
|
||||
def store = new DefaultLinStore()
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def scriptFile = folder.resolve("main.nf")
|
||||
def nfMeta = new NextflowMeta("24.10.0", 9999, "01-01-2024 00:00 UTC")
|
||||
def map = [
|
||||
repository : "https://nextflow.io/nf-test/",
|
||||
commitId : "abc123",
|
||||
scriptId : "78910",
|
||||
scriptFile : scriptFile,
|
||||
projectDir : folder.resolve("projectDir"),
|
||||
workDir : folder.resolve("workDir"),
|
||||
nextflow : nfMeta
|
||||
]
|
||||
def metadata = Mock(WorkflowMetadata) {
|
||||
getRepository() >> map.repository
|
||||
getCommitId() >> map.commitId
|
||||
getScriptId() >> map.scriptId
|
||||
getScriptFile() >> map.scriptFile
|
||||
getProjectDir() >> map.projectDir
|
||||
getWorkDir() >> map.workDir
|
||||
toMap() >> map
|
||||
}
|
||||
def session = Mock(Session) {
|
||||
getConfig() >> config
|
||||
getUniqueId() >> uniqueId
|
||||
getRunName() >> "test_run"
|
||||
getWorkflowMetadata() >> metadata
|
||||
getParams() >> new ScriptBinding.ParamsMap()
|
||||
}
|
||||
store.open(LineageConfig.create(session))
|
||||
def observer = new LinObserver(session, store)
|
||||
|
||||
when:
|
||||
observer.onFlowCreate(session)
|
||||
observer.onFlowBegin()
|
||||
def stored = store.load(observer.executionHash) as WorkflowRun
|
||||
|
||||
then:
|
||||
stored.metadata.nextflow instanceof Map
|
||||
(stored.metadata.nextflow as Map).version == "24.10.0"
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
def 'should normalize configFiles in workflow metadata' () {
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
|
||||
def store = new DefaultLinStore()
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def scriptFile = folder.resolve("main.nf")
|
||||
def workDir = folder.resolve("workDir")
|
||||
def projectDir = folder.resolve("projectDir")
|
||||
def configFile = projectDir.resolve("nextflow.config")
|
||||
def map = [
|
||||
repository : "https://nextflow.io/nf-test/",
|
||||
commitId : "abc123",
|
||||
scriptId : "78910",
|
||||
scriptFile : scriptFile,
|
||||
projectDir : projectDir,
|
||||
workDir : workDir,
|
||||
configFiles : [configFile]
|
||||
]
|
||||
def metadata = Mock(WorkflowMetadata) {
|
||||
getRepository() >> map.repository
|
||||
getCommitId() >> map.commitId
|
||||
getScriptId() >> map.scriptId
|
||||
getScriptFile() >> map.scriptFile
|
||||
getProjectDir() >> map.projectDir
|
||||
getWorkDir() >> map.workDir
|
||||
toMap() >> map
|
||||
}
|
||||
def session = Mock(Session) {
|
||||
getConfig() >> config
|
||||
getUniqueId() >> uniqueId
|
||||
getRunName() >> "test_run"
|
||||
getWorkflowMetadata() >> metadata
|
||||
getParams() >> new ScriptBinding.ParamsMap()
|
||||
}
|
||||
store.open(LineageConfig.create(session))
|
||||
def observer = new LinObserver(session, store)
|
||||
|
||||
when:
|
||||
observer.onFlowCreate(session)
|
||||
observer.onFlowBegin()
|
||||
def stored = store.load(observer.executionHash) as WorkflowRun
|
||||
|
||||
then:
|
||||
def normalizer = new PathNormalizer(metadata)
|
||||
(stored.metadata.configFiles as List) == [normalizer.normalizePath(configFile)]
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
def 'should handle null user in platform metadata' () {
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
|
||||
def store = new DefaultLinStore()
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def scriptFile = folder.resolve("main.nf")
|
||||
def platformMeta = new PlatformMetadata()
|
||||
platformMeta.workflowId = 'wf-999'
|
||||
platformMeta.workflowUrl = 'https://tower.nf/orgs/org/workspaces/ws/watch/wf-999'
|
||||
// user is intentionally left null
|
||||
def map = [
|
||||
repository : "https://nextflow.io/nf-test/",
|
||||
commitId : "abc123",
|
||||
scriptId : "78910",
|
||||
scriptFile : scriptFile,
|
||||
projectDir : folder.resolve("projectDir"),
|
||||
workDir : folder.resolve("workDir"),
|
||||
platform : platformMeta
|
||||
]
|
||||
def metadata = Mock(WorkflowMetadata) {
|
||||
getRepository() >> map.repository
|
||||
getCommitId() >> map.commitId
|
||||
getScriptId() >> map.scriptId
|
||||
getScriptFile() >> map.scriptFile
|
||||
getProjectDir() >> map.projectDir
|
||||
getWorkDir() >> map.workDir
|
||||
toMap() >> map
|
||||
}
|
||||
def session = Mock(Session) {
|
||||
getConfig() >> config
|
||||
getUniqueId() >> uniqueId
|
||||
getRunName() >> "test_run"
|
||||
getWorkflowMetadata() >> metadata
|
||||
getParams() >> new ScriptBinding.ParamsMap()
|
||||
}
|
||||
store.open(LineageConfig.create(session))
|
||||
def observer = new LinObserver(session, store)
|
||||
|
||||
when:
|
||||
observer.onFlowCreate(session)
|
||||
observer.onFlowBegin()
|
||||
def stored = store.load(observer.executionHash) as WorkflowRun
|
||||
|
||||
then:
|
||||
def storedPlatform = stored.metadata.platform as Map
|
||||
storedPlatform.workflowId == 'wf-999'
|
||||
storedPlatform.workflowUrl == 'https://tower.nf/orgs/org/workspaces/ws/watch/wf-999'
|
||||
storedPlatform.user == null
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
def 'should return empty metadata when toMap throws' () {
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
|
||||
def store = new DefaultLinStore()
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def scriptFile = folder.resolve("main.nf")
|
||||
def metadata = Mock(WorkflowMetadata) {
|
||||
getRepository() >> "https://nextflow.io/nf-test/"
|
||||
getCommitId() >> "abc123"
|
||||
getScriptId() >> "78910"
|
||||
getScriptFile() >> scriptFile
|
||||
getProjectDir() >> folder.resolve("projectDir")
|
||||
getWorkDir() >> folder.resolve("workDir")
|
||||
toMap() >> { throw new RuntimeException("metadata error") }
|
||||
}
|
||||
def session = Mock(Session) {
|
||||
getConfig() >> config
|
||||
getUniqueId() >> uniqueId
|
||||
getRunName() >> "test_run"
|
||||
getWorkflowMetadata() >> metadata
|
||||
getParams() >> new ScriptBinding.ParamsMap()
|
||||
}
|
||||
store.open(LineageConfig.create(session))
|
||||
def observer = new LinObserver(session, store)
|
||||
|
||||
when:
|
||||
observer.onFlowCreate(session)
|
||||
observer.onFlowBegin()
|
||||
def stored = store.load(observer.executionHash) as WorkflowRun
|
||||
|
||||
then:
|
||||
stored.metadata == null
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should get parameter type' () {
|
||||
expect:
|
||||
LinObserver.getParameterType(PARAM) == STRING
|
||||
where:
|
||||
PARAM | STRING
|
||||
null | null
|
||||
new FileInParam(null, []) | "path"
|
||||
new ValueOutParam(null, []) | "val"
|
||||
new EnvOutParam(null, []) | "env"
|
||||
new StdInParam(null, []) | "stdin"
|
||||
new StdOutParam(null, []) | "stdout"
|
||||
Path.of("test") | "Path"
|
||||
["test"] | "Collection"
|
||||
[key:"value"] | "Map"
|
||||
}
|
||||
|
||||
def 'should save task run' () {
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test').toRealPath()
|
||||
def config = [workflow:[lineage:[enabled: true, store:[location:folder.toString()]]]]
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def workDir = folder.resolve("work")
|
||||
def session = Mock(Session) {
|
||||
getConfig()>>config
|
||||
getUniqueId()>>uniqueId
|
||||
getRunName()>>"test_run"
|
||||
getWorkDir() >> workDir
|
||||
}
|
||||
def metadata = Mock(WorkflowMetadata){
|
||||
getRepository() >> "https://nextflow.io/nf-test/"
|
||||
getCommitId() >> "123456"
|
||||
getScriptId() >> "78910"
|
||||
getProjectDir() >> folder.resolve("projectDir")
|
||||
getWorkDir() >> workDir
|
||||
}
|
||||
and:
|
||||
def store = new DefaultLinStore();
|
||||
store.open(LineageConfig.create(session))
|
||||
and:
|
||||
def observer = Spy(new LinObserver(session, store))
|
||||
def normalizer = new PathNormalizer(metadata)
|
||||
observer.executionHash = "hash"
|
||||
observer.normalizer = normalizer
|
||||
and:
|
||||
def hash = HashCode.fromString("1234567890")
|
||||
def taskWd = workDir.resolve('12/34567890')
|
||||
Files.createDirectories(taskWd)
|
||||
and:
|
||||
observer.getTaskGlobalVars(_) >> [:]
|
||||
observer.getTaskBinEntries(_) >> []
|
||||
|
||||
and: 'Task Inputs'
|
||||
def inputs = new LinkedHashMap<InParam, Object>()
|
||||
// File from task
|
||||
inputs.put(new FileInParam(null, []).bind("file1"), [new FileHolder(workDir.resolve('78/567890/file1.txt'))])
|
||||
// Normal file
|
||||
def file = folder.resolve("file2.txt")
|
||||
file.text = "this is a test file"
|
||||
def fileHash = CacheHelper.hasher(file).hash().toString()
|
||||
inputs.put(new FileInParam(null, []).bind("file2"), [new FileHolder(file)])
|
||||
//Value input
|
||||
inputs.put(new ValueInParam(null, []).bind("id"), "value")
|
||||
|
||||
and: 'Task Outputs'
|
||||
def outputs = new LinkedHashMap<OutParam, Object>()
|
||||
// Single Path output
|
||||
def outFile1 = taskWd.resolve('fileOut1.txt')
|
||||
outFile1.text = 'some data'
|
||||
def fileHash1 = CacheHelper.hasher(outFile1).hash().toString()
|
||||
def attrs1 = Files.readAttributes(outFile1, BasicFileAttributes)
|
||||
outputs.put(new FileOutParam(null, []).bind(new TokenVar("file1")), outFile1)
|
||||
// Collection Path output
|
||||
def outFile2 = taskWd.resolve('fileOut2.txt')
|
||||
outFile2.text = 'some other data'
|
||||
def fileHash2 = CacheHelper.hasher(outFile2).hash().toString()
|
||||
def attrs2 = Files.readAttributes(outFile2, BasicFileAttributes)
|
||||
outputs.put(new FileOutParam(null, []).bind(new TokenVar("file2")), [outFile2])
|
||||
outputs.put(new ValueOutParam(null, []).bind(new TokenVar("id")), "value")
|
||||
|
||||
and: 'Task description'
|
||||
def task = Mock(TaskRun) {
|
||||
getId() >> TaskId.of(100)
|
||||
getName() >> 'foo'
|
||||
getHash() >> hash
|
||||
getSource() >> 'echo task source'
|
||||
getScript() >> 'this is the script'
|
||||
getInputs() >> inputs
|
||||
getOutputs() >> outputs
|
||||
getWorkDir() >> taskWd
|
||||
}
|
||||
def handler = Mock(TaskHandler){
|
||||
getTask() >> task
|
||||
}
|
||||
|
||||
and: 'Expected LID objects'
|
||||
def sourceHash = CacheHelper.hasher('echo task source').hash().toString()
|
||||
def script = 'this is the script'
|
||||
def taskDescription = new nextflow.lineage.model.v1beta1.TaskRun(uniqueId.toString(), "foo",
|
||||
new Checksum(sourceHash, "nextflow", "standard"),
|
||||
script,
|
||||
[
|
||||
new Parameter("path", "file1", ['lid://78567890/file1.txt']),
|
||||
new Parameter("path", "file2", [[path: normalizer.normalizePath(file), checksum: [value:fileHash, algorithm: "nextflow", mode: "standard"]]]),
|
||||
new Parameter("val", "id", "value")
|
||||
], null, null, null, null, [:], [], "lid://hash")
|
||||
def dataOutput1 = new FileOutput(outFile1.toString(), new Checksum(fileHash1, "nextflow", "standard"),
|
||||
"lid://1234567890", "lid://hash", "lid://1234567890", attrs1.size(), LinUtils.toDate(attrs1.creationTime()), LinUtils.toDate(attrs1.lastModifiedTime()) )
|
||||
def dataOutput2 = new FileOutput(outFile2.toString(), new Checksum(fileHash2, "nextflow", "standard"),
|
||||
"lid://1234567890", "lid://hash", "lid://1234567890", attrs2.size(), LinUtils.toDate(attrs2.creationTime()), LinUtils.toDate(attrs2.lastModifiedTime()) )
|
||||
|
||||
when:
|
||||
observer.onTaskComplete(new TaskEvent(handler, null))
|
||||
def taskRunResult = store.load("${hash.toString()}")
|
||||
def dataOutputResult1 = store.load("${hash}/fileOut1.txt") as FileOutput
|
||||
def dataOutputResult2 = store.load("${hash}/fileOut2.txt") as FileOutput
|
||||
def taskOutputsResult = store.load("${hash}#output") as TaskOutput
|
||||
then:
|
||||
taskRunResult == taskDescription
|
||||
dataOutputResult1 == dataOutput1
|
||||
dataOutputResult2 == dataOutput2
|
||||
taskOutputsResult.taskRun == "lid://1234567890"
|
||||
taskOutputsResult.workflowRun == "lid://hash"
|
||||
taskOutputsResult.output.size() == 3
|
||||
taskOutputsResult.output.get(0).type == "path"
|
||||
taskOutputsResult.output.get(0).name == "file1"
|
||||
taskOutputsResult.output.get(0).value == "lid://1234567890/fileOut1.txt"
|
||||
taskOutputsResult.output.get(1).type == "path"
|
||||
taskOutputsResult.output.get(1).name == "file2"
|
||||
taskOutputsResult.output.get(1).value == ["lid://1234567890/fileOut2.txt"]
|
||||
taskOutputsResult.output.get(2).type == "val"
|
||||
taskOutputsResult.output.get(2).name == "id"
|
||||
taskOutputsResult.output.get(2).value == "value"
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
def 'should save task data output' () {
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
|
||||
def store = new DefaultLinStore();
|
||||
def session = Mock(Session) {
|
||||
getConfig()>>config
|
||||
}
|
||||
store.open(LineageConfig.create(session))
|
||||
def observer = Spy(new LinObserver(session, store))
|
||||
observer.executionHash = "hash"
|
||||
and:
|
||||
def workDir = folder.resolve('12/34567890')
|
||||
Files.createDirectories(workDir)
|
||||
and:
|
||||
def outFile = workDir.resolve('foo/bar/file.bam')
|
||||
Files.createDirectories(outFile.parent)
|
||||
outFile.text = 'some data'
|
||||
def fileHash = CacheHelper.hasher(outFile).hash().toString()
|
||||
and:
|
||||
def hash = HashCode.fromInt(123456789)
|
||||
and:
|
||||
def task = Mock(TaskRun) {
|
||||
getId() >> TaskId.of(100)
|
||||
getName() >> 'foo'
|
||||
getHash() >> hash
|
||||
getWorkDir() >> workDir
|
||||
}
|
||||
and:
|
||||
def attrs = Files.readAttributes(outFile, BasicFileAttributes)
|
||||
def output = new FileOutput(outFile.toString(), new Checksum(fileHash, "nextflow", "standard"),
|
||||
"lid://15cd5b07", "lid://hash", "lid://15cd5b07", attrs.size(), LinUtils.toDate(attrs.creationTime()), LinUtils.toDate(attrs.lastModifiedTime()) )
|
||||
and:
|
||||
observer.readAttributes(outFile) >> attrs
|
||||
|
||||
when:
|
||||
observer.storeTaskOutput(task, outFile)
|
||||
then:
|
||||
folder.resolve("${hash}/foo/bar/file.bam/.data.json").text == new LinEncoder().encode(output)
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
def 'should relativise task output dirs' (){
|
||||
when:
|
||||
def config = [workflow:[lineage:[enabled: true, store:[location:lidFolder.toString()]]]]
|
||||
def store = new DefaultLinStore();
|
||||
def session = Mock(Session) {
|
||||
getConfig()>>config
|
||||
}
|
||||
def hash = HashCode.fromInt(123456789)
|
||||
def taskConfig = Mock(TaskConfig){
|
||||
getStoreDir() >> STORE_DIR
|
||||
}
|
||||
def task = Mock(TaskRun) {
|
||||
getId() >> TaskId.of(100)
|
||||
getName() >> 'foo'
|
||||
getHash() >> hash
|
||||
getWorkDir() >> WORK_DIR
|
||||
getConfig() >> taskConfig
|
||||
}
|
||||
store.open(LineageConfig.create(session))
|
||||
def observer = new LinObserver(session, store)
|
||||
then:
|
||||
observer.getTaskRelative(task, PATH) == EXPECTED
|
||||
where:
|
||||
WORK_DIR | STORE_DIR | PATH | EXPECTED
|
||||
Path.of('/path/to/work/12/3456789') | Path.of('/path/to/storeDir') | Path.of('/path/to/work/12/3456789/relative') | "relative"
|
||||
Path.of('/path/to/work/12/3456789') | Path.of('/path/to/storeDir') | Path.of('/path/to/storeDir/relative') | "relative"
|
||||
Path.of('work/12/3456789') | Path.of('storeDir') | Path.of('work/12/3456789/relative') | "relative"
|
||||
Path.of('work/12/3456789') | Path.of('storeDir') | Path.of('storeDir/relative') | "relative"
|
||||
Path.of('work/12/3456789') | Path.of('storeDir') | Path.of('results/relative') | "results/relative"
|
||||
Path.of('/path/to/work/12/3456789') | Path.of('storeDir') | Path.of('./relative') | "relative"
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should return exception when relativize task output dirs'() {
|
||||
when:
|
||||
def config = [workflow:[lineage:[enabled: true, store:[location:lidFolder.toString()]]]]
|
||||
def store = new DefaultLinStore();
|
||||
def session = Mock(Session) {
|
||||
getConfig()>>config
|
||||
}
|
||||
def hash = HashCode.fromInt(123456789)
|
||||
def taskConfig = Mock(TaskConfig){
|
||||
getStoreDir() >> STORE_DIR
|
||||
}
|
||||
def task = Mock(TaskRun) {
|
||||
getId() >> TaskId.of(100)
|
||||
getName() >> 'foo'
|
||||
getHash() >> hash
|
||||
getWorkDir() >> WORK_DIR
|
||||
getConfig() >> taskConfig
|
||||
}
|
||||
store.open(LineageConfig.create(session))
|
||||
def observer = new LinObserver(session, store)
|
||||
observer.getTaskRelative(task, PATH)
|
||||
then:
|
||||
def e = thrown(IllegalArgumentException)
|
||||
e.message == "Cannot access the relative path for output '$PATH' and task '${task.name}'".toString()
|
||||
|
||||
where:
|
||||
WORK_DIR | STORE_DIR | PATH
|
||||
Path.of('/path/to/work/12/3456789') | Path.of('/path/to/storeDir') | Path.of('/another/path/relative')
|
||||
Path.of('/path/to/work/12/3456789') | Path.of('/path/to/storeDir') | Path.of('../path/to/storeDir/relative')
|
||||
}
|
||||
|
||||
def 'should relativize workflow output dirs' (){
|
||||
when:
|
||||
def config = [workflow:[lineage:[enabled: true, store:[location:lidFolder.toString()]]]]
|
||||
def store = new DefaultLinStore();
|
||||
def session = Mock(Session) {
|
||||
getOutputDir()>>OUTPUT_DIR
|
||||
getConfig()>>config
|
||||
}
|
||||
store.open(LineageConfig.create(session))
|
||||
def observer = new LinObserver(session, store)
|
||||
then:
|
||||
observer.getWorkflowRelative(PATH) == EXPECTED
|
||||
where:
|
||||
OUTPUT_DIR | PATH | EXPECTED
|
||||
Path.of('/path/to/outDir') | Path.of('/path/to/outDir/relative') | "relative"
|
||||
Path.of('outDir') | Path.of('outDir/relative') | "relative"
|
||||
Path.of('/path/to/outDir') | Path.of('results/relative') | "results/relative"
|
||||
Path.of('/path/to/outDir') | Path.of('./relative') | "relative"
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should return exception when relativize workflow output dirs' (){
|
||||
when:
|
||||
def config = [workflow:[lineage:[enabled: true, store:[location:lidFolder.toString()]]]]
|
||||
def store = new DefaultLinStore();
|
||||
def session = Mock(Session) {
|
||||
getOutputDir()>>OUTPUT_DIR
|
||||
getConfig()>>config
|
||||
}
|
||||
def observer = new LinObserver(session, store)
|
||||
observer.getWorkflowRelative(PATH)
|
||||
then:
|
||||
thrown(OutputRelativePathException)
|
||||
where:
|
||||
OUTPUT_DIR | PATH | EXPECTED
|
||||
Path.of('/path/to/outDir') | Path.of('/another/path/') | "relative"
|
||||
Path.of('/path/to/outDir') | Path.of('../relative') | "relative"
|
||||
}
|
||||
|
||||
def 'should save workflow output'() {
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def config = [lineage:[enabled: true, store:[location:folder.toString()]]]
|
||||
def store = new DefaultLinStore();
|
||||
def outputDir = folder.resolve('results')
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def scriptFile = folder.resolve("main.nf")
|
||||
def workDir= folder.resolve("work")
|
||||
def metadata = Mock(WorkflowMetadata){
|
||||
getRepository() >> "https://nextflow.io/nf-test/"
|
||||
getCommitId() >> "123456"
|
||||
getScriptId() >> "78910"
|
||||
getScriptFile() >> scriptFile
|
||||
getProjectDir() >> folder.resolve("projectDir")
|
||||
getWorkDir() >> workDir
|
||||
}
|
||||
def session = Mock(Session) {
|
||||
getConfig()>>config
|
||||
getOutputDir()>>outputDir
|
||||
getWorkDir() >> workDir
|
||||
getWorkflowMetadata()>>metadata
|
||||
getUniqueId()>>uniqueId
|
||||
getRunName()>>"test_run"
|
||||
getParams() >> new ScriptBinding.ParamsMap()
|
||||
}
|
||||
store.open(LineageConfig.create(session))
|
||||
def observer = new LinObserver(session, store)
|
||||
def encoder = new LinEncoder()
|
||||
|
||||
when: 'Starting workflow'
|
||||
observer.onFlowCreate(session)
|
||||
observer.onFlowBegin()
|
||||
then: 'History file should contain execution hash'
|
||||
def lid = LinHistoryRecord.parse(folder.resolve(".history/${observer.executionHash}").text)
|
||||
lid.runLid == asUriString(observer.executionHash)
|
||||
lid.sessionId == uniqueId
|
||||
lid.runName == "test_run"
|
||||
|
||||
when: ' publish output with source file'
|
||||
def outFile1 = outputDir.resolve('foo/file.bam')
|
||||
Files.createDirectories(outFile1.parent)
|
||||
outFile1.text = 'some data1'
|
||||
def sourceFile1 = workDir.resolve('12/3987/file.bam')
|
||||
Files.createDirectories(sourceFile1.parent)
|
||||
sourceFile1.text = 'some data1'
|
||||
observer.onFilePublish(new FilePublishEvent(sourceFile1, outFile1))
|
||||
observer.onWorkflowOutput(new WorkflowOutputEvent("a", outFile1))
|
||||
|
||||
then: 'check file 1 output metadata in lid store'
|
||||
def attrs1 = Files.readAttributes(outFile1, BasicFileAttributes)
|
||||
def fileHash1 = CacheHelper.hasher(outFile1).hash().toString()
|
||||
def output1 = new FileOutput(outFile1.toString(), new Checksum(fileHash1, "nextflow", "standard"),
|
||||
"lid://123987/file.bam", "$LID_PROT${observer.executionHash}", null,
|
||||
attrs1.size(), LinUtils.toDate(attrs1.creationTime()), LinUtils.toDate(attrs1.lastModifiedTime()) )
|
||||
folder.resolve("${observer.executionHash}/foo/file.bam/.data.json").text == encoder.encode(output1)
|
||||
|
||||
when: 'publish without source path'
|
||||
def outFile2 = outputDir.resolve('foo/file2.bam')
|
||||
Files.createDirectories(outFile2.parent)
|
||||
outFile2.text = 'some data2'
|
||||
def attrs2 = Files.readAttributes(outFile2, BasicFileAttributes)
|
||||
def fileHash2 = CacheHelper.hasher(outFile2).hash().toString()
|
||||
observer.onFilePublish(new FilePublishEvent(null, outFile2))
|
||||
observer.onWorkflowOutput(new WorkflowOutputEvent("b", outFile2))
|
||||
then: 'Check outFile2 metadata in lid store'
|
||||
def output2 = new FileOutput(outFile2.toString(), new Checksum(fileHash2, "nextflow", "standard"),
|
||||
"lid://${observer.executionHash}" , "lid://${observer.executionHash}", null,
|
||||
attrs2.size(), LinUtils.toDate(attrs2.creationTime()), LinUtils.toDate(attrs2.lastModifiedTime()) )
|
||||
folder.resolve("${observer.executionHash}/foo/file2.bam/.data.json").text == encoder.encode(output2)
|
||||
|
||||
when: 'Workflow complete'
|
||||
observer.onFlowComplete()
|
||||
then: 'Check WorkflowOutput is written in the lid store'
|
||||
def resultsRetrieved = store.load("${observer.executionHash}#output") as WorkflowOutput
|
||||
resultsRetrieved.output == [new Parameter(Path.simpleName, "a", "lid://${observer.executionHash}/foo/file.bam"), new Parameter(Path.simpleName, "b", "lid://${observer.executionHash}/foo/file2.bam")]
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
def 'should not save workflow output entry when no outputs'() {
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def config = [lineage: [enabled: true, store: [location: folder.toString()]]]
|
||||
def store = new DefaultLinStore();
|
||||
def outputDir = folder.resolve('results')
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def scriptFile = folder.resolve("main.nf")
|
||||
def workDir = folder.resolve("work")
|
||||
def metadata = Mock(WorkflowMetadata) {
|
||||
getRepository() >> "https://nextflow.io/nf-test/"
|
||||
getCommitId() >> "123456"
|
||||
getScriptId() >> "78910"
|
||||
getScriptFile() >> scriptFile
|
||||
getProjectDir() >> folder.resolve("projectDir")
|
||||
getWorkDir() >> workDir
|
||||
}
|
||||
def session = Mock(Session) {
|
||||
getConfig() >> config
|
||||
getOutputDir() >> outputDir
|
||||
getWorkDir() >> workDir
|
||||
getWorkflowMetadata() >> metadata
|
||||
getUniqueId() >> uniqueId
|
||||
getRunName() >> "test_run"
|
||||
getParams() >> new ScriptBinding.ParamsMap()
|
||||
}
|
||||
store.open(LineageConfig.create(session))
|
||||
def observer = new LinObserver(session, store)
|
||||
|
||||
when:
|
||||
observer.onFlowCreate(session)
|
||||
observer.onFlowBegin()
|
||||
observer.onFlowComplete()
|
||||
def resultFile = folder.resolve("${observer.executionHash}#output")
|
||||
then:
|
||||
!resultFile.exists()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package nextflow.lineage
|
||||
|
||||
import spock.lang.Specification
|
||||
|
||||
/**
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
class LinPropertyValidationTest extends Specification{
|
||||
|
||||
def 'should throw exception when property does not exist'(){
|
||||
when:
|
||||
new LinPropertyValidator().validate(['value', 'not_existing'])
|
||||
then:
|
||||
def e = thrown(IllegalArgumentException)
|
||||
e.message.startsWith( "Property 'not_existing' doesn't exist in the lineage model")
|
||||
}
|
||||
|
||||
def 'should not throw exception when property exist'(){
|
||||
when:
|
||||
new LinPropertyValidator().validate(['value', 'output'])
|
||||
then:
|
||||
noExceptionThrown()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage
|
||||
|
||||
import java.time.ZoneId
|
||||
|
||||
import nextflow.lineage.model.v1beta1.Checksum
|
||||
import nextflow.lineage.model.v1beta1.DataPath
|
||||
import nextflow.lineage.model.v1beta1.Parameter
|
||||
import nextflow.lineage.model.v1beta1.Workflow
|
||||
import nextflow.lineage.model.v1beta1.WorkflowOutput
|
||||
import nextflow.lineage.model.v1beta1.WorkflowRun
|
||||
import nextflow.lineage.config.LineageConfig
|
||||
import spock.lang.Specification
|
||||
import spock.lang.TempDir
|
||||
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.attribute.FileTime
|
||||
import java.time.Instant
|
||||
import java.time.OffsetDateTime
|
||||
import java.time.ZoneOffset
|
||||
|
||||
class LinUtilsTest extends Specification{
|
||||
|
||||
@TempDir
|
||||
Path tempDir
|
||||
|
||||
Path storeLocation
|
||||
LineageConfig config
|
||||
|
||||
def setup() {
|
||||
storeLocation = tempDir.resolve("store")
|
||||
def configMap = [enabled: true, store: [location: storeLocation.toString()]]
|
||||
config = new LineageConfig(configMap)
|
||||
}
|
||||
|
||||
def 'should convert to Date'(){
|
||||
expect:
|
||||
LinUtils.toDate(FILE_TIME) == DATE
|
||||
where:
|
||||
FILE_TIME | DATE
|
||||
null | null
|
||||
FileTime.fromMillis(1234) | Instant.ofEpochMilli(1234).atZone(ZoneId.systemDefault())?.toOffsetDateTime()
|
||||
}
|
||||
|
||||
def 'should convert to FileTime'(){
|
||||
expect:
|
||||
LinUtils.toFileTime(DATE) == FILE_TIME
|
||||
where:
|
||||
FILE_TIME | DATE
|
||||
null | null
|
||||
FileTime.fromMillis(1234) | OffsetDateTime.ofInstant(Instant.ofEpochMilli(1234), ZoneOffset.UTC)
|
||||
}
|
||||
|
||||
|
||||
def 'should get lineage record'() {
|
||||
given:
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def mainScript = new DataPath("file://path/to/main.nf", new Checksum("78910", "nextflow", "standard"))
|
||||
def workflow = new Workflow([mainScript], "https://nextflow.io/nf-test/", "123456")
|
||||
def key1 = "testKey"
|
||||
def value1 = new WorkflowRun(workflow, uniqueId.toString(), "test_run", [new Parameter("String", "param1", "value1"), new Parameter("String", "param2", "value2")])
|
||||
def outputs1 = new WorkflowOutput(OffsetDateTime.now(), "lid://testKey", [new Parameter( "String", "output", "name")] )
|
||||
def key2 = "testKey2"
|
||||
def value2 = new WorkflowRun(workflow, uniqueId.toString(), "test_run2", [new Parameter("String", "param1", "value1"), new Parameter("String", "param2", "value2")])
|
||||
def lidStore = new DefaultLinStore()
|
||||
lidStore.open(config)
|
||||
lidStore.save(key1, value1)
|
||||
lidStore.save("$key1#output", outputs1)
|
||||
lidStore.save(key2, value2)
|
||||
|
||||
when:
|
||||
def params = LinUtils.getMetadataObject(lidStore, new URI('lid://testKey#params'))
|
||||
then:
|
||||
params instanceof List<Parameter>
|
||||
(params as List<Parameter>).size() == 2
|
||||
|
||||
when:
|
||||
def outputs = LinUtils.getMetadataObject(lidStore, new URI('lid://testKey#output'))
|
||||
then:
|
||||
outputs instanceof List<Parameter>
|
||||
def param = (outputs as List)[0] as Parameter
|
||||
param.name == "output"
|
||||
|
||||
when:
|
||||
outputs = LinUtils.getMetadataObject(lidStore, new URI('lid://testKey2#output'))
|
||||
then:
|
||||
outputs instanceof List
|
||||
(outputs as List).isEmpty()
|
||||
|
||||
when:
|
||||
LinUtils.getMetadataObject(lidStore, new URI('lid://testKey#no-exist'))
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
|
||||
when:
|
||||
LinUtils.getMetadataObject(lidStore, new URI('lid://testKey#outputs.no-exist'))
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
|
||||
when:
|
||||
LinUtils.getMetadataObject(lidStore, new URI('lid://no-exist#something'))
|
||||
then:
|
||||
thrown(FileNotFoundException)
|
||||
}
|
||||
|
||||
|
||||
def "should check params in an object"() {
|
||||
given:
|
||||
def obj = [ "type": "value", "workflow": ["repository": "subvalue"], "output" : [ ["path":"/to/file"],["path":"file2"] ], "labels": ["a","b"] ]
|
||||
|
||||
expect:
|
||||
LinUtils.checkParams(obj, PARAMS) == EXPECTED
|
||||
|
||||
where:
|
||||
PARAMS | EXPECTED
|
||||
["type": ["value"]] | true
|
||||
["type": ["wrong"]] | false
|
||||
["workflow.repository": ["subvalue"]] | true
|
||||
["workflow.repository": ["wrong"]] | false
|
||||
["output.path": ["wrong"]] | false
|
||||
["output.path": ["/to/file"]] | true
|
||||
["output.path": ["file2"]] | true
|
||||
["labels": ["a"]] | true
|
||||
["labels": ["c"]] | false
|
||||
["labels": ["a","b"]] | true
|
||||
["labels": ["a","b","c"]] | false
|
||||
["type" : ["value", "value2"]] | false
|
||||
|
||||
}
|
||||
|
||||
def "should navigate in object params"() {
|
||||
given:
|
||||
def obj = [
|
||||
"key1": "value1",
|
||||
"nested": [
|
||||
"subkey": "subvalue"
|
||||
]
|
||||
]
|
||||
|
||||
expect:
|
||||
LinUtils.navigate(obj, PATH) == EXPECTED
|
||||
|
||||
where:
|
||||
PATH | EXPECTED
|
||||
"key1" | "value1"
|
||||
"nested.subkey" | "subvalue"
|
||||
"wrongKey" | null
|
||||
}
|
||||
|
||||
def "should add objects matching parameters"() {
|
||||
given:
|
||||
def results = []
|
||||
|
||||
when:
|
||||
LinUtils.treatObject(OBJECT, PARAMS, results)
|
||||
|
||||
then:
|
||||
results == EXPECTED
|
||||
|
||||
where:
|
||||
OBJECT | PARAMS | EXPECTED
|
||||
["field": "value"] | ["field": ["value"]] | [["field": "value"]]
|
||||
["field": "wrong"] | ["field": ["value"]] | []
|
||||
[["field": "value"], ["field": "x"]] | ["field": ["value"]] | [["field": "value"]]
|
||||
"string" | [:] | ["string"]
|
||||
["nested": ["subfield": "match"]] | ["nested.subfield": ["match"]] | [["nested": ["subfield": "match"]]]
|
||||
["nested": ["subfield": "nomatch"]] | ["nested.subfield": ["match"]] | []
|
||||
[["nested": ["subfield": "match"]], ["nested": ["subfield": "other"]]] | ["nested.subfield": ["match"]] | [["nested": ["subfield": "match"]]]
|
||||
}
|
||||
|
||||
def 'should navigate' (){
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def mainScript = new DataPath("file://path/to/main.nf", new Checksum("78910", "nextflow", "standard"))
|
||||
def workflow = new Workflow([mainScript], "https://nextflow.io/nf-test/", "123456")
|
||||
def wfRun = new WorkflowRun(workflow, uniqueId.toString(), "test_run", [new Parameter("String", "param1", [key: "value1"]), new Parameter("String", "param2", "value2")])
|
||||
|
||||
expect:
|
||||
LinUtils.navigate(wfRun, "workflow.commitId") == "123456"
|
||||
LinUtils.navigate(wfRun, "params.name") == ["param1", "param2"]
|
||||
LinUtils.navigate(wfRun, "params.value.key") == "value1"
|
||||
LinUtils.navigate(wfRun, "params.value.no-exist") == null
|
||||
LinUtils.navigate(wfRun, "params.no-exist") == null
|
||||
LinUtils.navigate(wfRun, "no-exist") == null
|
||||
LinUtils.navigate(null, "something") == null
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,478 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.cli
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.time.Instant
|
||||
import java.time.OffsetDateTime
|
||||
import java.time.ZoneOffset
|
||||
|
||||
import nextflow.SysEnv
|
||||
import nextflow.config.ConfigMap
|
||||
import nextflow.dag.MermaidHtmlRenderer
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.file.FileHelper
|
||||
import nextflow.lineage.DefaultLinHistoryLog
|
||||
import nextflow.lineage.LinHistoryRecord
|
||||
import nextflow.lineage.LinStoreFactory
|
||||
import nextflow.lineage.fs.LinFileSystemProvider
|
||||
import nextflow.lineage.model.v1beta1.Checksum
|
||||
import nextflow.lineage.model.v1beta1.DataPath
|
||||
import nextflow.lineage.model.v1beta1.FileOutput
|
||||
import nextflow.lineage.model.v1beta1.Parameter
|
||||
import nextflow.lineage.model.v1beta1.TaskRun
|
||||
import nextflow.lineage.model.v1beta1.Workflow
|
||||
import nextflow.lineage.model.v1beta1.WorkflowRun
|
||||
import nextflow.lineage.serde.LinEncoder
|
||||
import nextflow.plugin.Plugins
|
||||
import nextflow.util.CacheHelper
|
||||
import org.junit.Rule
|
||||
import spock.lang.Shared
|
||||
import spock.lang.Specification
|
||||
import test.OutputCapture
|
||||
|
||||
import static test.TestHelper.filterLogNoise
|
||||
|
||||
class LinCommandImplTest extends Specification{
|
||||
|
||||
@Shared
|
||||
Path tmpDir
|
||||
|
||||
@Shared
|
||||
Path storeLocation
|
||||
|
||||
@Shared
|
||||
ConfigMap configMap
|
||||
|
||||
def reset() {
|
||||
def provider = FileHelper.getProviderFor('lid') as LinFileSystemProvider
|
||||
provider?.reset()
|
||||
LinStoreFactory.reset()
|
||||
}
|
||||
|
||||
def setup() {
|
||||
reset()
|
||||
// clear the environment to avoid the local env pollute the test env
|
||||
SysEnv.push([:])
|
||||
tmpDir = Files.createTempDirectory('tmp')
|
||||
storeLocation = tmpDir.resolve("store")
|
||||
configMap = new ConfigMap([lineage: [enabled: true, store: [location: storeLocation.toString(), logLocation: storeLocation.resolve(".log").toString()]]])
|
||||
}
|
||||
|
||||
def cleanup() {
|
||||
Plugins.stop()
|
||||
LinStoreFactory.reset()
|
||||
SysEnv.pop()
|
||||
tmpDir?.deleteDir()
|
||||
}
|
||||
|
||||
def setupSpec() {
|
||||
reset()
|
||||
}
|
||||
|
||||
/*
|
||||
* Read more http://mrhaki.blogspot.com.es/2015/02/spocklight-capture-and-assert-system.html
|
||||
*/
|
||||
@Rule
|
||||
OutputCapture capture = new OutputCapture()
|
||||
|
||||
def 'should print executions lids' (){
|
||||
given:
|
||||
def historyFile = storeLocation.resolve(".history")
|
||||
def lidLog = new DefaultLinHistoryLog(historyFile)
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def date = new Date();
|
||||
def recordEntry = "${LinHistoryRecord.TIMESTAMP_FMT.format(date)}\trun_name\t${uniqueId}\tlid://123456".toString()
|
||||
lidLog.write("run_name", uniqueId, "lid://123456", date)
|
||||
when:
|
||||
new LinCommandImpl().list(configMap)
|
||||
def stdout = filterLogNoise(capture)
|
||||
|
||||
then:
|
||||
stdout.size() == 2
|
||||
stdout[1] == recordEntry
|
||||
}
|
||||
|
||||
def 'should print no history' (){
|
||||
given:
|
||||
def historyFile = storeLocation.resolve(".meta/.history")
|
||||
Files.createDirectories(historyFile.parent)
|
||||
|
||||
when:
|
||||
new LinCommandImpl().list(configMap)
|
||||
def stdout = filterLogNoise(capture, true)
|
||||
|
||||
then:
|
||||
stdout.size() == 1
|
||||
stdout[0] == "No workflow runs found in lineage history log"
|
||||
}
|
||||
|
||||
def 'should show lid content' (){
|
||||
given:
|
||||
def lidFile = storeLocation.resolve("12345/.data.json")
|
||||
Files.createDirectories(lidFile.parent)
|
||||
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
|
||||
def encoder = new LinEncoder().withPrettyPrint(true)
|
||||
def entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
|
||||
"lid://123987/file.bam","lid://123987/", null, 1234, time, time, null)
|
||||
def jsonSer = encoder.encode(entry)
|
||||
def expectedOutput = jsonSer
|
||||
lidFile.text = jsonSer
|
||||
when:
|
||||
new LinCommandImpl().view(configMap, ["lid://12345"])
|
||||
def stdout = filterLogNoise(capture)
|
||||
|
||||
then:
|
||||
stdout.size() == expectedOutput.readLines().size()
|
||||
stdout.join('\n') == expectedOutput
|
||||
}
|
||||
|
||||
def 'should show empty lists content' (){
|
||||
given:
|
||||
def lidFile = storeLocation.resolve("12345/.data.json")
|
||||
Files.createDirectories(lidFile.parent)
|
||||
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
|
||||
def encoder = new LinEncoder().withPrettyPrint(true)
|
||||
def entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
|
||||
"lid://123987/file.bam","lid://123987/", null, 1234, time, time, [])
|
||||
def jsonSer = encoder.encode(entry)
|
||||
def expectedOutput = '[]'
|
||||
lidFile.text = jsonSer
|
||||
when:
|
||||
new LinCommandImpl().view(configMap, ["lid://12345#labels"])
|
||||
def stdout = filterLogNoise(capture)
|
||||
|
||||
then:
|
||||
stdout.size() == expectedOutput.readLines().size()
|
||||
stdout.join('\n') == expectedOutput
|
||||
}
|
||||
|
||||
def 'should show empty lists when no outputs' () {
|
||||
def lidFile = storeLocation.resolve("12345/.data.json")
|
||||
Files.createDirectories(lidFile.parent)
|
||||
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
|
||||
def encoder = new LinEncoder().withPrettyPrint(true)
|
||||
def expectedOutput = '[]'
|
||||
def wf = new Workflow([new DataPath("/path/to/main.nf)")], "hello-nf", "aasdklk")
|
||||
def entry = new WorkflowRun(wf, "sessionId", "run_name",
|
||||
[new Parameter("String", "sample_id", "ggal_gut"),
|
||||
new Parameter("Integer", "reads", 2)])
|
||||
lidFile.text = encoder.encode(entry)
|
||||
|
||||
when:
|
||||
new LinCommandImpl().view(configMap, ["lid://12345#output"])
|
||||
def stdout = filterLogNoise(capture)
|
||||
|
||||
then:
|
||||
stdout.size() == expectedOutput.readLines().size()
|
||||
stdout.join('\n') == expectedOutput
|
||||
}
|
||||
|
||||
def 'should warn if no lid content' (){
|
||||
given:
|
||||
|
||||
when:
|
||||
new LinCommandImpl().view(configMap, ["lid://12345"])
|
||||
def stdout = filterLogNoise(capture)
|
||||
|
||||
then:
|
||||
stdout.size() == 1
|
||||
stdout[0] == "Error loading lid://12345 - Lineage record 12345 not found"
|
||||
}
|
||||
|
||||
def 'should get lineage lid content' (){
|
||||
given:
|
||||
|
||||
def outputHtml = tmpDir.resolve('lineage.html')
|
||||
|
||||
def lidFile = storeLocation.resolve("12345/file.bam/.data.json")
|
||||
def lidFile2 = storeLocation.resolve("123987/file.bam/.data.json")
|
||||
def lidFile3 = storeLocation.resolve("123987/.data.json")
|
||||
def lidFile4 = storeLocation.resolve("45678/output.txt/.data.json")
|
||||
def lidFile5 = storeLocation.resolve("45678/.data.json")
|
||||
Files.createDirectories(lidFile.parent)
|
||||
Files.createDirectories(lidFile2.parent)
|
||||
Files.createDirectories(lidFile3.parent)
|
||||
Files.createDirectories(lidFile4.parent)
|
||||
Files.createDirectories(lidFile5.parent)
|
||||
def encoder = new LinEncoder()
|
||||
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
|
||||
def entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
|
||||
"lid://123987/file.bam", "lid://45678", null, 1234, time, time, null)
|
||||
lidFile.text = encoder.encode(entry)
|
||||
entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
|
||||
"lid://123987", "lid://45678", "lid://123987", 1234, time, time, null)
|
||||
lidFile2.text = encoder.encode(entry)
|
||||
entry = new TaskRun("u345-2346-1stw2", "foo",
|
||||
new Checksum("abcde2345","nextflow","standard"),
|
||||
'this is a script',
|
||||
[new Parameter( "val", "sample_id","ggal_gut"),
|
||||
new Parameter("path","reads", ["lid://45678/output.txt"] ),
|
||||
new Parameter("path","input", [new DataPath("path/to/file",new Checksum("45372qe","nextflow","standard"))])
|
||||
],
|
||||
null, null, null, null, [:],[])
|
||||
lidFile3.text = encoder.encode(entry)
|
||||
entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
|
||||
"lid://45678", "lid://45678", null, 1234, time, time, null)
|
||||
lidFile4.text = encoder.encode(entry)
|
||||
entry = new TaskRun("u345-2346-1stw2", "bar",
|
||||
new Checksum("abfs2556","nextflow","standard"),
|
||||
'this is a script',
|
||||
null,null, null, null, null, [:],[])
|
||||
lidFile5.text = encoder.encode(entry)
|
||||
final network = """\
|
||||
flowchart TB
|
||||
lid://12345/file.bam["lid://12345/file.bam"]
|
||||
lid://123987/file.bam["lid://123987/file.bam"]
|
||||
lid://123987(["foo [lid://123987]"])
|
||||
ggal_gut["ggal_gut"]
|
||||
path/to/file["path/to/file"]
|
||||
lid://45678/output.txt["lid://45678/output.txt"]
|
||||
lid://45678(["bar [lid://45678]"])
|
||||
lid://123987/file.bam --> lid://12345/file.bam
|
||||
lid://123987 --> lid://123987/file.bam
|
||||
ggal_gut --> lid://123987
|
||||
lid://45678/output.txt --> lid://123987
|
||||
path/to/file --> lid://123987
|
||||
lid://45678 --> lid://45678/output.txt""".stripIndent()
|
||||
final template = MermaidHtmlRenderer.readTemplate()
|
||||
def expectedOutput = template.replace('REPLACE_WITH_NETWORK_DATA', network)
|
||||
|
||||
when:
|
||||
new LinCommandImpl().render(configMap, ["lid://12345/file.bam", outputHtml.toString()])
|
||||
def stdout = filterLogNoise(capture)
|
||||
|
||||
then:
|
||||
stdout.size() == 1
|
||||
stdout[0] == "Rendered lineage graph for lid://12345/file.bam to ${outputHtml}"
|
||||
outputHtml.exists()
|
||||
outputHtml.text == expectedOutput
|
||||
}
|
||||
|
||||
def 'should get lineage from workflow lid content' (){
|
||||
given:
|
||||
|
||||
def outputHtml = tmpDir.resolve('lineage.html')
|
||||
|
||||
def lidFile = storeLocation.resolve("12345/file.bam/.data.json")
|
||||
def lidFile3 = storeLocation.resolve("12345/.data.json")
|
||||
Files.createDirectories(lidFile.parent)
|
||||
Files.createDirectories(lidFile3.parent)
|
||||
def encoder = new LinEncoder()
|
||||
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
|
||||
def entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
|
||||
"lid://12345", "lid://12345", null, 1234, time, time, null)
|
||||
lidFile.text = encoder.encode(entry)
|
||||
def wf = new Workflow([new DataPath("/path/to/main.nf)")], "hello-nf", "aasdklk")
|
||||
entry = new WorkflowRun(wf,"sessionId","run_name",
|
||||
[new Parameter( "String", "sample_id","ggal_gut"),
|
||||
new Parameter("Integer","reads",2)])
|
||||
lidFile3.text = encoder.encode(entry)
|
||||
final network = """\
|
||||
flowchart TB
|
||||
lid://12345/file.bam["lid://12345/file.bam"]
|
||||
lid://12345(["run_name [lid://12345]"])
|
||||
ggal_gut["ggal_gut"]
|
||||
2.0["2.0"]
|
||||
lid://12345 --> lid://12345/file.bam
|
||||
ggal_gut --> lid://12345
|
||||
2.0 --> lid://12345""".stripIndent()
|
||||
final template = MermaidHtmlRenderer.readTemplate()
|
||||
def expectedOutput = template.replace('REPLACE_WITH_NETWORK_DATA', network)
|
||||
|
||||
when:
|
||||
new LinCommandImpl().render(configMap, ["lid://12345/file.bam", outputHtml.toString()])
|
||||
def stdout = filterLogNoise(capture)
|
||||
|
||||
then:
|
||||
stdout.size() == 1
|
||||
stdout[0] == "Rendered lineage graph for lid://12345/file.bam to ${outputHtml}"
|
||||
outputHtml.exists()
|
||||
outputHtml.text == expectedOutput
|
||||
}
|
||||
|
||||
def 'should show an error if trying to do a query'(){
|
||||
given:
|
||||
def lidFile = storeLocation.resolve("12345/.data.json")
|
||||
Files.createDirectories(lidFile.parent)
|
||||
def encoder = new LinEncoder().withPrettyPrint(true)
|
||||
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
|
||||
def entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
|
||||
"lid://123987/file.bam", "lid://123987/", null, 1234, time, time, null)
|
||||
def jsonSer = encoder.encode(entry)
|
||||
def expectedOutput = "Error loading lid:///?type=FileOutput - Cannot get record from the root LID URI"
|
||||
lidFile.text = jsonSer
|
||||
when:
|
||||
new LinCommandImpl().view(configMap, ["lid:///?type=FileOutput"])
|
||||
def stdout = filterLogNoise(capture)
|
||||
|
||||
then:
|
||||
stdout.size() == expectedOutput.readLines().size()
|
||||
stdout.join('\n') == expectedOutput
|
||||
}
|
||||
|
||||
def 'should diff'(){
|
||||
given:
|
||||
def lidFile = storeLocation.resolve("12345/.data.json")
|
||||
Files.createDirectories(lidFile.parent)
|
||||
def lidFile2 = storeLocation.resolve("67890/.data.json")
|
||||
Files.createDirectories(lidFile2.parent)
|
||||
def encoder = new LinEncoder().withPrettyPrint(true)
|
||||
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
|
||||
def entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
|
||||
"lid://123987/file.bam", "lid://123987/", null, 1234, time, time, null)
|
||||
def entry2 = new FileOutput("path/to/file2",new Checksum("42472qet","nextflow","standard"),
|
||||
"lid://123987/file2.bam", "lid://123987/", null, 1235, time, time, null)
|
||||
lidFile.text = encoder.encode(entry)
|
||||
lidFile2.text = encoder.encode(entry2)
|
||||
def expectedOutput = '''diff --git 12345 67890
|
||||
--- 12345
|
||||
+++ 67890
|
||||
@@ -2,16 +2,16 @@
|
||||
"version": "lineage/v1beta1",
|
||||
"kind": "FileOutput",
|
||||
"spec": {
|
||||
- "path": "path/to/file",
|
||||
+ "path": "path/to/file2",
|
||||
"checksum": {
|
||||
- "value": "45372qe",
|
||||
+ "value": "42472qet",
|
||||
"algorithm": "nextflow",
|
||||
"mode": "standard"
|
||||
},
|
||||
- "source": "lid://123987/file.bam",
|
||||
+ "source": "lid://123987/file2.bam",
|
||||
"workflowRun": "lid://123987/",
|
||||
"taskRun": null,
|
||||
- "size": 1234,
|
||||
+ "size": 1235,
|
||||
"createdAt": "1970-01-02T10:17:36.789Z",
|
||||
"modifiedAt": "1970-01-02T10:17:36.789Z",
|
||||
"labels": null
|
||||
'''
|
||||
|
||||
when:
|
||||
new LinCommandImpl().diff(configMap, ["lid://12345", "lid://67890"])
|
||||
def stdout = filterLogNoise(capture)
|
||||
|
||||
then:
|
||||
stdout.join('\n') == expectedOutput
|
||||
}
|
||||
|
||||
def 'should print error if no entry found diff'(){
|
||||
given:
|
||||
def lidFile = storeLocation.resolve("12345/.data.json")
|
||||
Files.createDirectories(lidFile.parent)
|
||||
def encoder = new LinEncoder().withPrettyPrint(true)
|
||||
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
|
||||
def entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
|
||||
"lid://123987/file.bam", "lid://123987/", null, 1234, time, time, null)
|
||||
lidFile.text = encoder.encode(entry)
|
||||
|
||||
when:
|
||||
new LinCommandImpl().diff(configMap, ["lid://89012", "lid://12345"])
|
||||
new LinCommandImpl().diff(configMap, ["lid://12345", "lid://67890"])
|
||||
def stdout = filterLogNoise(capture)
|
||||
|
||||
then:
|
||||
stdout.size() == 2
|
||||
stdout[0] == "No entry found for lid://89012."
|
||||
stdout[1] == "No entry found for lid://67890."
|
||||
}
|
||||
|
||||
def 'should print error store is not found in diff'(){
|
||||
when:
|
||||
def config = new ConfigMap()
|
||||
new LinCommandImpl().list(config)
|
||||
new LinCommandImpl().view(config, ["lid:///12345"])
|
||||
new LinCommandImpl().render(config, ["lid://12345", "output.html"])
|
||||
new LinCommandImpl().diff(config, ["lid://89012", "lid://12345"])
|
||||
|
||||
def stdout = filterLogNoise(capture)
|
||||
def expectedOutput = "Error lineage store not loaded - Check Nextflow configuration"
|
||||
then:
|
||||
stdout.size() == 4
|
||||
stdout[0] == expectedOutput
|
||||
stdout[1] == expectedOutput
|
||||
stdout[2] == expectedOutput
|
||||
stdout[3] == expectedOutput
|
||||
}
|
||||
|
||||
def 'should find metadata descriptions'(){
|
||||
given:
|
||||
def lidFile = storeLocation.resolve("123987/file.bam/.data.json")
|
||||
Files.createDirectories(lidFile.parent)
|
||||
def lidFile2 = storeLocation.resolve("123987/file2.bam/.data.json")
|
||||
Files.createDirectories(lidFile2.parent)
|
||||
def lidFile3 = storeLocation.resolve(".meta/123987/file3.bam/.data.json")
|
||||
Files.createDirectories(lidFile3.parent)
|
||||
def encoder = new LinEncoder().withPrettyPrint(true)
|
||||
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
|
||||
def entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
|
||||
"lid://123987/file.bam", "lid://123987/", null, 1234, time, time, ["experiment=test"])
|
||||
def entry2 = new FileOutput("path/to/file2",new Checksum("42472qet","nextflow","standard"),
|
||||
"lid://123987/file2.bam", "lid://123987/", null, 1235, time, time, ["experiment=test"])
|
||||
def entry3 = new FileOutput("path/to/file3",new Checksum("42472qet","nextflow","standard"),
|
||||
"lid://123987/file2.bam", "lid://123987/", null, 1235, time, time, null)
|
||||
def expectedOutput1 = 'lid://123987/file.bam\nlid://123987/file2.bam'
|
||||
def expectedOutput2 = 'lid://123987/file2.bam\nlid://123987/file.bam'
|
||||
lidFile.text = encoder.encode(entry)
|
||||
lidFile2.text = encoder.encode(entry2)
|
||||
lidFile3.text = encoder.encode(entry3)
|
||||
when:
|
||||
new LinCommandImpl().find(configMap, ["type=FileOutput", "label=experiment=test"])
|
||||
def stdout = filterLogNoise(capture)
|
||||
|
||||
then:
|
||||
stdout.join('\n') == expectedOutput1 || stdout.join('\n') == expectedOutput2
|
||||
}
|
||||
|
||||
def 'should print correct validate path' () {
|
||||
given:
|
||||
def outputFolder = tmpDir.resolve('output')
|
||||
Files.createDirectories(outputFolder)
|
||||
def outputFile = outputFolder.resolve('file1.txt')
|
||||
outputFile.text = "this is file1 == "
|
||||
|
||||
and:
|
||||
def encoder = new LinEncoder().withPrettyPrint(true)
|
||||
def hash = CacheHelper.hasher(outputFile).hash().toString()
|
||||
def correctData = new FileOutput(outputFile.toString(), new Checksum(hash,"nextflow", "standard"))
|
||||
def incorrectData = new FileOutput(outputFile.toString(), new Checksum("incorrectHash","nextflow", "standard"))
|
||||
def lid1 = storeLocation.resolve('12345/output/file1.txt/.data.json')
|
||||
Files.createDirectories(lid1.parent)
|
||||
lid1.text = encoder.encode(correctData)
|
||||
def lid2 = storeLocation.resolve('12345/output/file2.txt/.data.json')
|
||||
Files.createDirectories(lid2.parent)
|
||||
lid2.text = encoder.encode(incorrectData)
|
||||
when:
|
||||
new LinCommandImpl().check(configMap, ["lid://12345/output/file1.txt"])
|
||||
def stdout = filterLogNoise(capture)
|
||||
def expectedOutput1 = "Checksum validation succeed"
|
||||
then:
|
||||
stdout.size() == 1
|
||||
stdout[0] == expectedOutput1
|
||||
|
||||
when:
|
||||
new LinCommandImpl().check(configMap, ["lid://12345/output/file2.txt"])
|
||||
then:
|
||||
def err = thrown(AbortOperationException)
|
||||
err.message == "Checksum of '${outputFile}' does not match with lineage metadata"
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.config
|
||||
|
||||
|
||||
import spock.lang.Specification
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class LineageConfigTest extends Specification {
|
||||
|
||||
def 'should create default config' () {
|
||||
when:
|
||||
def config = new LineageConfig(Map.of())
|
||||
then:
|
||||
!config.enabled
|
||||
!config.store.location
|
||||
}
|
||||
|
||||
def 'should create default with enable' () {
|
||||
when:
|
||||
def config = new LineageConfig([enabled: true])
|
||||
then:
|
||||
config.enabled
|
||||
!config.store.location
|
||||
}
|
||||
|
||||
def 'should create data config with location' () {
|
||||
when:
|
||||
def config = new LineageConfig(enabled: true, store: [location: "/some/data/store"])
|
||||
then:
|
||||
config.enabled
|
||||
config.store.location == '/some/data/store'
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,496 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.fs
|
||||
|
||||
import java.nio.ByteBuffer
|
||||
import java.nio.channels.NonWritableChannelException
|
||||
import java.nio.file.AccessDeniedException
|
||||
import java.nio.file.AccessMode
|
||||
import java.nio.file.FileSystemNotFoundException
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.ProviderMismatchException
|
||||
import java.nio.file.StandardOpenOption
|
||||
import java.nio.file.attribute.BasicFileAttributes
|
||||
|
||||
import nextflow.Global
|
||||
import nextflow.Session
|
||||
import nextflow.lineage.DefaultLinStore
|
||||
import spock.lang.Shared
|
||||
import spock.lang.Specification
|
||||
/**
|
||||
* LID File system provider tests
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
class LinFileSystemProviderTest extends Specification {
|
||||
|
||||
@Shared def wdir = Files.createTempDirectory('wdir')
|
||||
@Shared def data = Files.createTempDirectory('work')
|
||||
|
||||
|
||||
def cleanupSpec(){
|
||||
wdir.deleteDir()
|
||||
data.deleteDir()
|
||||
}
|
||||
|
||||
def 'should return lid scheme' () {
|
||||
given:
|
||||
def provider = new LinFileSystemProvider()
|
||||
expect:
|
||||
provider.getScheme() == 'lid'
|
||||
}
|
||||
|
||||
def 'should get lid path' () {
|
||||
given:
|
||||
def lid = Mock(LinPath)
|
||||
and:
|
||||
def provider = new LinFileSystemProvider()
|
||||
expect:
|
||||
provider.toLinPath(lid) == lid
|
||||
|
||||
when:
|
||||
provider.toLinPath(Path.of('foo'))
|
||||
then:
|
||||
thrown(ProviderMismatchException)
|
||||
}
|
||||
|
||||
def 'should create new file system' () {
|
||||
given:
|
||||
def provider = new LinFileSystemProvider()
|
||||
def config = [store:[location:data.toString()]]
|
||||
def lid = LinPath.asUri('lid://12345')
|
||||
when:
|
||||
def fs = provider.newFileSystem(lid, config) as LinFileSystem
|
||||
then:
|
||||
(fs.store as DefaultLinStore).location == data
|
||||
}
|
||||
|
||||
def 'should get a file system' () {
|
||||
given:
|
||||
def provider = new LinFileSystemProvider()
|
||||
def config = [store:[location: data.toString()]]
|
||||
def uri = LinPath.asUri('lid://12345')
|
||||
when:
|
||||
provider.getFileSystem(uri)
|
||||
then:
|
||||
thrown(FileSystemNotFoundException)
|
||||
|
||||
when:
|
||||
provider.newFileSystem(uri, config) as LinFileSystem
|
||||
and:
|
||||
def fs = provider.getFileSystem(uri) as LinFileSystem
|
||||
then:
|
||||
(fs.store as DefaultLinStore).location == data
|
||||
}
|
||||
|
||||
def 'should get or create a file system' () {
|
||||
given:
|
||||
def config = [lineage:[store:[location: data.toString()]]]
|
||||
Global.session = Mock(Session) { getConfig()>>config }
|
||||
and:
|
||||
def uri = LinPath.asUri('lid://12345')
|
||||
def provider = new LinFileSystemProvider()
|
||||
|
||||
when:
|
||||
def fs = provider.getFileSystemOrCreate(uri) as LinFileSystem
|
||||
then:
|
||||
(fs.store as DefaultLinStore).location == data
|
||||
|
||||
when:
|
||||
def fs2 = provider.getFileSystemOrCreate(uri) as LinFileSystem
|
||||
then:
|
||||
fs2.is(fs)
|
||||
}
|
||||
|
||||
def 'should create new byte channel' () {
|
||||
given:
|
||||
def config = [lineage:[store:[location:wdir.toString()]]]
|
||||
def outputMeta = wdir.resolve("12345/output.txt")
|
||||
def output = data.resolve("output.txt")
|
||||
output.text = "Hello, World!"
|
||||
outputMeta.mkdirs()
|
||||
outputMeta.resolve(".data.json").text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path":"'+output.toString()+'"}}'
|
||||
|
||||
Global.session = Mock(Session) { getConfig()>>config }
|
||||
and:
|
||||
def provider = new LinFileSystemProvider()
|
||||
def lid = provider.getPath(LinPath.asUri('lid://12345/output.txt'))
|
||||
def opts = Set.of(StandardOpenOption.READ)
|
||||
when:
|
||||
def channel = provider.newByteChannel(lid, opts)
|
||||
then:
|
||||
channel.isOpen()
|
||||
channel.position() == 0
|
||||
channel.size() == "Hello, World!".getBytes().size()
|
||||
when:
|
||||
channel.truncate(25)
|
||||
then:
|
||||
thrown(NonWritableChannelException)
|
||||
|
||||
when:
|
||||
def buffer = ByteBuffer.allocate(1000);
|
||||
def read = channel.read(buffer)
|
||||
def bytes = new byte[read]
|
||||
buffer.get(0,bytes)
|
||||
then:
|
||||
bytes == "Hello, World!".getBytes()
|
||||
when:
|
||||
channel.position(2)
|
||||
then:
|
||||
channel.position() == 2
|
||||
|
||||
when:
|
||||
channel.write(buffer)
|
||||
then:
|
||||
thrown(NonWritableChannelException)
|
||||
|
||||
when:
|
||||
provider.newByteChannel(lid, Set.of(StandardOpenOption.WRITE))
|
||||
then:
|
||||
thrown(UnsupportedOperationException)
|
||||
|
||||
when:
|
||||
provider.newByteChannel(lid, Set.of(StandardOpenOption.APPEND))
|
||||
then:
|
||||
thrown(UnsupportedOperationException)
|
||||
|
||||
cleanup:
|
||||
channel.close()
|
||||
outputMeta.deleteDir()
|
||||
output.delete()
|
||||
}
|
||||
|
||||
def 'should create new byte channel for LinMetadata' () {
|
||||
given:
|
||||
def config = [lineage:[store:[location:wdir.toString()]]]
|
||||
def outputMeta = wdir.resolve("12345")
|
||||
outputMeta.mkdirs()
|
||||
outputMeta.resolve(".data.json").text = '{"version":"lineage/v1beta1","kind":"WorkflowRun","spec":{"sessionId":"session","name":"run_name","params":[{"type":"String","name":"param1","value":"value1"}]}}'
|
||||
|
||||
Global.session = Mock(Session) { getConfig()>>config }
|
||||
and:
|
||||
def provider = new LinFileSystemProvider()
|
||||
def lid = provider.getPath(LinPath.asUri('lid://12345#name'))
|
||||
|
||||
when:
|
||||
def channel = provider.newByteChannel(lid, Set.of(StandardOpenOption.READ))
|
||||
then:
|
||||
channel.isOpen()
|
||||
channel.position() == 0
|
||||
channel.size() == '"run_name"'.getBytes().size()
|
||||
|
||||
when:
|
||||
channel.truncate(25)
|
||||
then:
|
||||
thrown(NonWritableChannelException)
|
||||
|
||||
when:
|
||||
def buffer = ByteBuffer.allocate(1000);
|
||||
def read = channel.read(buffer)
|
||||
def bytes = new byte[read]
|
||||
buffer.get(0,bytes)
|
||||
then:
|
||||
bytes =='"run_name"'.getBytes()
|
||||
|
||||
when:
|
||||
channel.position(2)
|
||||
then:
|
||||
channel.position() == 2
|
||||
|
||||
when:
|
||||
channel.write(buffer)
|
||||
then:
|
||||
thrown(NonWritableChannelException)
|
||||
|
||||
when:
|
||||
provider.newByteChannel(lid, Set.of(StandardOpenOption.WRITE))
|
||||
then:
|
||||
thrown(UnsupportedOperationException)
|
||||
|
||||
when:
|
||||
provider.newByteChannel(lid, Set.of(StandardOpenOption.APPEND))
|
||||
then:
|
||||
thrown(UnsupportedOperationException)
|
||||
|
||||
cleanup:
|
||||
channel.close()
|
||||
outputMeta.deleteDir()
|
||||
}
|
||||
|
||||
def 'should read lid' () {
|
||||
given:
|
||||
def config = [lineage:[store:[location:wdir.toString()]]]
|
||||
def outputMeta = wdir.resolve("12345/output.txt")
|
||||
def output = data.resolve("output.txt")
|
||||
output.text = "Hello, World!"
|
||||
outputMeta.mkdirs()
|
||||
outputMeta.resolve(".data.json").text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path":"'+output.toString()+'"}}'
|
||||
|
||||
Global.session = Mock(Session) { getConfig()>>config }
|
||||
and:
|
||||
def provider = new LinFileSystemProvider()
|
||||
def lid = provider.getPath(LinPath.asUri('lid://12345/output.txt'))
|
||||
def opts = Set.of(StandardOpenOption.READ)
|
||||
|
||||
expect:
|
||||
lid.text == "Hello, World!"
|
||||
|
||||
cleanup:
|
||||
outputMeta.deleteDir()
|
||||
output.delete()
|
||||
}
|
||||
|
||||
def 'should not create a directory' () {
|
||||
given:
|
||||
def config = [lineage:[store:[location:wdir.toString()]]]
|
||||
Global.session = Mock(Session) { getConfig()>>config }
|
||||
and:
|
||||
def provider = new LinFileSystemProvider()
|
||||
def lid = provider.getPath(LinPath.asUri('lid://12345'))
|
||||
|
||||
when:
|
||||
provider.createDirectory(lid)
|
||||
then:
|
||||
thrown(UnsupportedOperationException)
|
||||
|
||||
}
|
||||
|
||||
def 'should create directory stream' () {
|
||||
given:
|
||||
def output1 = data.resolve('path')
|
||||
output1.mkdir()
|
||||
output1.resolve('file1.txt').text = 'file1'
|
||||
output1.resolve('file2.txt').text = 'file2'
|
||||
output1.resolve('file3.txt').text = 'file3'
|
||||
wdir.resolve('12345/output1').mkdirs()
|
||||
wdir.resolve('12345/output2').mkdirs()
|
||||
wdir.resolve('12345/.data.json').text = '{"version":"lineage/v1beta1","kind":"TaskRun","spec":{"name":"dummy"}}'
|
||||
wdir.resolve('12345/output1/.data.json').text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path": "' + output1.toString() + '"}}'
|
||||
|
||||
and:
|
||||
def config = [lineage:[store:[location:wdir.toString()]]]
|
||||
Global.session = Mock(Session) { getConfig()>>config }
|
||||
and:
|
||||
def provider = new LinFileSystemProvider()
|
||||
def lid = provider.getPath(LinPath.asUri('lid://12345/output1'))
|
||||
def lid2 = provider.getPath(LinPath.asUri('lid://12345'))
|
||||
def lid3 = provider.getPath(LinPath.asUri('lid://678'))
|
||||
|
||||
expect:
|
||||
Files.exists(lid)
|
||||
Files.exists(lid.resolve('file1.txt'))
|
||||
Files.exists(lid.resolve('file2.txt'))
|
||||
Files.exists(lid.resolve('file3.txt'))
|
||||
|
||||
when:
|
||||
provider.newDirectoryStream(lid3, (p) -> true)
|
||||
then:
|
||||
thrown(FileNotFoundException)
|
||||
|
||||
when:
|
||||
def stream = provider.newDirectoryStream(lid, (p) -> true)
|
||||
and:
|
||||
def result = stream.toList()
|
||||
then:
|
||||
result.toSet() == [
|
||||
lid.resolve('file1.txt'),
|
||||
lid.resolve('file2.txt'),
|
||||
lid.resolve('file3.txt')
|
||||
] as Set
|
||||
|
||||
when:
|
||||
stream = provider.newDirectoryStream(lid2, (p) -> true)
|
||||
and:
|
||||
result = stream.toList()
|
||||
then:
|
||||
result.size() == 1
|
||||
result[0] == lid2.resolve('output1')
|
||||
|
||||
cleanup:
|
||||
wdir.resolve('12345').deleteDir()
|
||||
output1.deleteDir()
|
||||
|
||||
}
|
||||
|
||||
def 'should not delete a file' () {
|
||||
given:
|
||||
def config = [lineage:[store:[location:wdir.toString()]]]
|
||||
Global.session = Mock(Session) { getConfig()>>config }
|
||||
and:
|
||||
def provider = new LinFileSystemProvider()
|
||||
def lid = provider.getPath(LinPath.asUri('lid://12345'))
|
||||
|
||||
when:
|
||||
provider.delete(lid)
|
||||
then:
|
||||
thrown(UnsupportedOperationException)
|
||||
|
||||
}
|
||||
|
||||
def 'should not copy a file' () {
|
||||
given:
|
||||
def config = [lineage:[store:[location:wdir.toString()]]]
|
||||
Global.session = Mock(Session) { getConfig()>>config }
|
||||
and:
|
||||
def provider = new LinFileSystemProvider()
|
||||
def lid1 = provider.getPath(LinPath.asUri('lid://12345/abc'))
|
||||
def lid2 = provider.getPath(LinPath.asUri('lid://54321/foo'))
|
||||
|
||||
when:
|
||||
provider.copy(lid1, lid2)
|
||||
then:
|
||||
thrown(UnsupportedOperationException)
|
||||
}
|
||||
|
||||
def 'should not move a file' () {
|
||||
given:
|
||||
def config = [lineage:[store:[location:wdir.toString()]]]
|
||||
Global.session = Mock(Session) { getConfig()>>config }
|
||||
and:
|
||||
def provider = new LinFileSystemProvider()
|
||||
def lid1 = provider.getPath(LinPath.asUri('lid://12345/abc'))
|
||||
def lid2 = provider.getPath(LinPath.asUri('lid://54321/foo'))
|
||||
|
||||
when:
|
||||
provider.move(lid1, lid2)
|
||||
then:
|
||||
thrown(UnsupportedOperationException)
|
||||
}
|
||||
|
||||
def 'should check is same file' () {
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def config = [lineage:[store:[location:folder.toString()]]]
|
||||
Global.session = Mock(Session) { getConfig()>>config }
|
||||
and:
|
||||
def provider = new LinFileSystemProvider()
|
||||
def lid1 = provider.getPath(LinPath.asUri('lid://12345/abc'))
|
||||
def lid2 = provider.getPath(LinPath.asUri('lid://54321/foo'))
|
||||
def lid3 = provider.getPath(LinPath.asUri('lid://54321/foo'))
|
||||
|
||||
expect:
|
||||
!provider.isSameFile(lid1, lid2)
|
||||
!provider.isSameFile(lid1, lid3)
|
||||
and:
|
||||
provider.isSameFile(lid2, lid3)
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
def 'should check is hidden file' () {
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def config = [lineage:[store:[location:wdir.toString()]]]
|
||||
Global.session = Mock(Session) { getConfig()>>config }
|
||||
and:
|
||||
def output = folder.resolve('path')
|
||||
output.mkdir()
|
||||
output.resolve('abc').text = 'file1'
|
||||
output.resolve('.foo').text = 'file2'
|
||||
wdir.resolve('12345/output').mkdirs()
|
||||
wdir.resolve('12345/output/.data.json').text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path": "' + output.toString() + '"}}'
|
||||
and:
|
||||
def provider = new LinFileSystemProvider()
|
||||
def lid1 = provider.getPath(LinPath.asUri('lid://12345/output/abc'))
|
||||
def lid2 = provider.getPath(LinPath.asUri('lid://12345/output/.foo'))
|
||||
|
||||
expect:
|
||||
!provider.isHidden(lid1)
|
||||
provider.isHidden(lid2)
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
def 'should read file attributes' () {
|
||||
given:
|
||||
def config = [lineage:[store:[location:wdir.toString()]]]
|
||||
def file = data.resolve('abc')
|
||||
file.text = 'Hello'
|
||||
wdir.resolve('12345/abc').mkdirs()
|
||||
wdir.resolve('12345/abc/.data.json').text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path":"' + file.toString() + '"}}'
|
||||
and:
|
||||
Global.session = Mock(Session) { getConfig()>>config }
|
||||
and:
|
||||
def provider = new LinFileSystemProvider()
|
||||
def lid1 = provider.getPath(LinPath.asUri('lid://12345/abc'))
|
||||
|
||||
when:
|
||||
def attr1 = provider.readAttributes(lid1, BasicFileAttributes)
|
||||
def real1= Files.readAttributes(file,BasicFileAttributes)
|
||||
then:
|
||||
!attr1.directory
|
||||
attr1.isRegularFile()
|
||||
attr1.size() == real1.size()
|
||||
attr1.creationTime() == real1.creationTime()
|
||||
attr1.lastModifiedTime() == real1.lastModifiedTime()
|
||||
attr1.lastAccessTime() == real1.lastAccessTime()
|
||||
|
||||
cleanup:
|
||||
file?.delete()
|
||||
wdir.resolve('12345').deleteDir()
|
||||
}
|
||||
|
||||
def 'should throw exception in unsupported methods'() {
|
||||
given:
|
||||
def config = [lineage:[store:[location:wdir.toString()]]]
|
||||
Global.session = Mock(Session) { getConfig()>>config }
|
||||
def provider = new LinFileSystemProvider()
|
||||
|
||||
when:
|
||||
provider.newOutputStream(null)
|
||||
then:
|
||||
thrown(UnsupportedOperationException)
|
||||
|
||||
when:
|
||||
provider.getFileStore(null)
|
||||
then:
|
||||
thrown(UnsupportedOperationException)
|
||||
|
||||
when:
|
||||
provider.readAttributes(null, "attrib")
|
||||
then:
|
||||
thrown(UnsupportedOperationException)
|
||||
|
||||
when:
|
||||
provider.setAttribute(null, "attrib", null)
|
||||
then:
|
||||
thrown(UnsupportedOperationException)
|
||||
}
|
||||
|
||||
def 'should throw exception when checking access mode'(){
|
||||
given:
|
||||
def config = [lineage:[store:[location:wdir.toString()]]]
|
||||
Global.session = Mock(Session) { getConfig()>>config }
|
||||
def provider = new LinFileSystemProvider()
|
||||
def lid1 = provider.getPath(LinPath.asUri('lid://12345/abc'))
|
||||
|
||||
when:
|
||||
provider.checkAccess(lid1, AccessMode.WRITE)
|
||||
then:
|
||||
def ex1 = thrown(AccessDeniedException)
|
||||
ex1.message == "Write mode not supported"
|
||||
|
||||
when:
|
||||
provider.checkAccess(lid1, AccessMode.EXECUTE)
|
||||
then:
|
||||
def ex2 = thrown(AccessDeniedException)
|
||||
ex2.message == "Execute mode not supported"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.fs
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
|
||||
import nextflow.Global
|
||||
import nextflow.Session
|
||||
import spock.lang.Specification
|
||||
import spock.lang.Unroll
|
||||
|
||||
/**
|
||||
* LID Path Factory tests.
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
class LinPathFactoryTest extends Specification {
|
||||
|
||||
Path tmp
|
||||
|
||||
def setup() {
|
||||
tmp = Files.createTempDirectory("data")
|
||||
Global.session = Mock(Session) { getConfig()>> [workflow:[lineage:[store:[location: tmp.toString()]]]] }
|
||||
}
|
||||
|
||||
def cleanup() {
|
||||
Global.session = null
|
||||
tmp.deleteDir()
|
||||
}
|
||||
|
||||
def 'should create lin path' () {
|
||||
given:
|
||||
def factory = new LinPathFactory()
|
||||
|
||||
expect:
|
||||
factory.parseUri('foo') == null
|
||||
|
||||
when:
|
||||
def p1 = factory.parseUri('lid://12345')
|
||||
then:
|
||||
p1.toUriString() == 'lid://12345'
|
||||
|
||||
when:
|
||||
def p2 = factory.parseUri('lid://12345/x/y/z')
|
||||
then:
|
||||
p2.toUriString() == 'lid://12345/x/y/z'
|
||||
|
||||
when:
|
||||
def p3 = factory.parseUri('lid://12345//x///y/z//')
|
||||
then:
|
||||
p3.toUriString() == 'lid://12345/x/y/z'
|
||||
|
||||
when:
|
||||
factory.parseUri('lid:///12345')
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should convert get lid uri string' () {
|
||||
given:
|
||||
def factory = new LinPathFactory()
|
||||
|
||||
when:
|
||||
def lid = LinPathFactory.create(EXPECTED)
|
||||
then:
|
||||
factory.toUriString(lid) == EXPECTED
|
||||
|
||||
where:
|
||||
_ | EXPECTED
|
||||
_ | 'lid://123'
|
||||
_ | 'lid://123/a/b/c'
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,718 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.fs
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.ProviderMismatchException
|
||||
import java.time.OffsetDateTime
|
||||
|
||||
import nextflow.lineage.LinUtils
|
||||
import nextflow.lineage.model.v1beta1.Checksum
|
||||
import nextflow.lineage.model.v1beta1.FileOutput
|
||||
import nextflow.lineage.model.v1beta1.Parameter
|
||||
import nextflow.lineage.model.v1beta1.Workflow
|
||||
import nextflow.lineage.model.v1beta1.WorkflowOutput
|
||||
import nextflow.lineage.model.v1beta1.WorkflowRun
|
||||
import nextflow.lineage.serde.LinEncoder
|
||||
import nextflow.util.CacheHelper
|
||||
import org.junit.Rule
|
||||
import spock.lang.Shared
|
||||
import spock.lang.Specification
|
||||
import spock.lang.Unroll
|
||||
import test.OutputCapture
|
||||
|
||||
import static test.TestHelper.filterLogNoise
|
||||
/**
|
||||
* LID Path Tests
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
class LinPathTest extends Specification {
|
||||
|
||||
@Shared
|
||||
Path wdir
|
||||
@Shared
|
||||
Path data
|
||||
@Shared
|
||||
def fs = Mock(LinFileSystem)
|
||||
|
||||
def setupSpec(){
|
||||
wdir = Files.createTempDirectory("wdir")
|
||||
data = wdir.resolve('work')
|
||||
}
|
||||
|
||||
def cleanupSpec(){
|
||||
wdir.deleteDir()
|
||||
}
|
||||
|
||||
@Rule
|
||||
OutputCapture capture = new OutputCapture()
|
||||
|
||||
def 'should create from URI' () {
|
||||
when:
|
||||
def path = new LinPath(fs, new URI( URI_STRING ))
|
||||
then:
|
||||
path.filePath == PATH
|
||||
path.fragment == FRAGMENT
|
||||
path.query == QUERY
|
||||
|
||||
where:
|
||||
URI_STRING | PATH | QUERY | FRAGMENT
|
||||
"lid://1234/hola" | "1234/hola" | null | null
|
||||
"lid://1234/hola#workflow.repository" | "1234/hola" | null | "workflow.repository"
|
||||
"lid://1234/#workflow.repository" | "1234" | null | "workflow.repository"
|
||||
"lid://1234/?q=a&b=c" | "1234" | "q=a&b=c" | null
|
||||
"lid://1234/?q=a&b=c#workflow.repository" | "1234" | "q=a&b=c" | "workflow.repository"
|
||||
"lid:///" | "/" | null | null
|
||||
}
|
||||
|
||||
def 'should throw exception if fragment contains an unknown property'() {
|
||||
when:
|
||||
new LinPath(fs, new URI ("lid://1234/hola#no-exist"))
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
|
||||
}
|
||||
|
||||
def 'should warn if query is specified'() {
|
||||
when:
|
||||
new LinPath(fs, new URI("lid://1234/hola?query"))
|
||||
def stdout = filterLogNoise(capture)
|
||||
|
||||
then:
|
||||
stdout.size() == 1
|
||||
stdout[0].endsWith("Query string is not supported for Lineage URI: `lid://1234/hola?query` -- it will be ignored")
|
||||
}
|
||||
|
||||
def 'should create correct lid Path' () {
|
||||
when:
|
||||
def lid = new LinPath(FS, PATH, MORE)
|
||||
then:
|
||||
lid.filePath == EXPECTED_FILE
|
||||
where:
|
||||
FS | PATH | MORE | EXPECTED_FILE
|
||||
fs | '/' | [] as String[] | '/'
|
||||
null | '/' | [] as String[] | '/'
|
||||
fs | '/' | ['a','b'] as String[] | 'a/b'
|
||||
null | '/' | ['a','b'] as String[] | 'a/b'
|
||||
fs | '' | [] as String[] | '/'
|
||||
null | '' | [] as String[] | '/'
|
||||
fs | '' | ['a','b'] as String[] | 'a/b'
|
||||
null | '' | ['a','b'] as String[] | 'a/b'
|
||||
fs | '1234' | [] as String[] | '1234'
|
||||
null | '1234' | [] as String[] | '1234'
|
||||
fs | '1234' | ['a','b'] as String[] | '1234/a/b'
|
||||
null | '1234' | ['a','b'] as String[] | '1234/a/b'
|
||||
fs | '1234/c' | [] as String[] | '1234/c'
|
||||
null | '1234/c' | [] as String[] | '1234/c'
|
||||
fs | '1234/c' | ['a','b'] as String[] | '1234/c/a/b'
|
||||
null | '1234/c' | ['a','b'] as String[] | '1234/c/a/b'
|
||||
fs | '/1234/c' | [] as String[] | '1234/c'
|
||||
null | '/1234/c' | [] as String[] | '1234/c'
|
||||
fs | '/1234/c' | ['a','b'] as String[] | '1234/c/a/b'
|
||||
null | '/1234/c' | ['a','b'] as String[] | '1234/c/a/b'
|
||||
fs | '../c' | ['a','b'] as String[] | 'c/a/b'
|
||||
null | '../c' | ['a','b'] as String[] | '../c/a/b'
|
||||
fs | '../c' | [] as String[] | 'c'
|
||||
null | '../c' | [] as String[] | '../c'
|
||||
fs | '..' | [] as String[] | '/'
|
||||
null | '..' | [] as String[] | '..'
|
||||
fs | '/..' | [] as String[] | '/'
|
||||
null | '/..' | [] as String[] | '/'
|
||||
fs | './1234/c' | ['a','b'] as String[] | '1234/c/a/b'
|
||||
null | './1234/c' | ['a','b'] as String[] | '1234/c/a/b'
|
||||
fs | './1234/c' | [] as String[] | '1234/c'
|
||||
null | './1234/c' | [] as String[] | '1234/c'
|
||||
fs | '1234' | ['/'] as String[] | '1234'
|
||||
null | '1234' | ['/'] as String[] | '1234'
|
||||
null | '../../a/b' | [] as String[] | '../../a/b'
|
||||
fs | '1234/' | [] as String[] | '1234'
|
||||
null | '1234/' | [] as String[] | '1234'
|
||||
}
|
||||
|
||||
def 'should get target path' () {
|
||||
given:
|
||||
def outputFolder = data.resolve('output')
|
||||
def outputSubFolder = outputFolder.resolve('some/path')
|
||||
outputSubFolder.mkdirs()
|
||||
def outputSubFolderFile = outputSubFolder.resolve('file1.txt')
|
||||
outputSubFolderFile.text = "this is file1"
|
||||
def outputFile = data.resolve('file2.txt')
|
||||
outputFile.text = "this is file2"
|
||||
|
||||
def lidFs = new LinFileSystemProvider().newFileSystem(new URI("lid:///"), [enabled: true, store: [location: wdir.toString()]])
|
||||
|
||||
wdir.resolve('12345/output1').mkdirs()
|
||||
wdir.resolve('12345/path/to/file2.txt').mkdirs()
|
||||
wdir.resolve('12345/.data.json').text = '{"version":"lineage/v1beta1","kind":"TaskRun","spec":{"name":"test"}}'
|
||||
wdir.resolve('12345/output1/.data.json').text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path": "' + outputFolder.toString() + '"}}'
|
||||
wdir.resolve('12345/path/to/file2.txt/.data.json').text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path": "' + outputFile.toString() + '"}}'
|
||||
def time = OffsetDateTime.now()
|
||||
def wfResultsMetadata = new LinEncoder().withPrettyPrint(true).encode(new WorkflowOutput(time, "lid://1234", [new Parameter( "Path", "a", "lid://1234/a.txt")]))
|
||||
wdir.resolve('5678/').mkdirs()
|
||||
wdir.resolve('5678/.data.json').text = wfResultsMetadata
|
||||
|
||||
expect: 'Get real path when LinPath is the output data or a subfolder'
|
||||
new LinPath(lidFs, '12345/output1').getTargetPath() == outputFolder
|
||||
new LinPath(lidFs,'12345/output1/some/path').getTargetPath() == outputSubFolder
|
||||
new LinPath(lidFs,'12345/output1/some/path/file1.txt').getTargetPath().text == outputSubFolderFile.text
|
||||
new LinPath(lidFs, '12345/path/to/file2.txt').getTargetPath().text == outputFile.text
|
||||
|
||||
when: 'LinPath fs is null'
|
||||
new LinPath(null, '12345').getTargetPath()
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
|
||||
when: 'LinPath is empty'
|
||||
new LinPath(lidFs, '/').getTargetPath()
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
|
||||
when: 'LinPath is not an output data description'
|
||||
new LinPath(lidFs, '12345').getTargetPath()
|
||||
then:
|
||||
thrown(FileNotFoundException)
|
||||
|
||||
when: 'LinPath is not subfolder of an output data description'
|
||||
new LinPath(lidFs, '12345/path').getTargetPath()
|
||||
then:
|
||||
thrown(FileNotFoundException)
|
||||
|
||||
when: 'LinPath subfolder of an output data description does not exist'
|
||||
new LinPath(lidFs, '12345/output1/other/path').getTargetPath()
|
||||
then:
|
||||
thrown(FileNotFoundException)
|
||||
|
||||
when: 'Lid does not exist'
|
||||
new LinPath(lidFs, '23456').getTargetPath()
|
||||
then:
|
||||
thrown(FileNotFoundException)
|
||||
|
||||
when: 'LinPath is not an output data description but is a workflow/task run'
|
||||
def result = new LinPath(lidFs, '12345').getTargetOrIntermediatePath()
|
||||
then:
|
||||
result instanceof LinIntermediatePath
|
||||
|
||||
when: 'LinPath is a subpath of a workflow/task run data description'
|
||||
result = new LinPath(lidFs, '12345/path').getTargetOrIntermediatePath()
|
||||
then:
|
||||
result instanceof LinIntermediatePath
|
||||
|
||||
when: 'Lid description'
|
||||
result = new LinPath(lidFs, '5678').getTargetOrMetadataPath()
|
||||
then:
|
||||
result instanceof LinMetadataPath
|
||||
result.text == wfResultsMetadata
|
||||
|
||||
when: 'Lid description subobject'
|
||||
def result2 = new LinPath(lidFs, '5678#output').getTargetOrMetadataPath()
|
||||
then:
|
||||
result2 instanceof LinMetadataPath
|
||||
result2.text == LinUtils.encodeSearchOutputs([new Parameter("Path","a", "lid://1234/a.txt")], true)
|
||||
|
||||
when: 'Lid subobject does not exist'
|
||||
new LinPath(lidFs, '23456#notexists').getTargetOrMetadataPath()
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
}
|
||||
|
||||
def 'should get subobjects as path' (){
|
||||
given:
|
||||
def lidFs = new LinFileSystemProvider().newFileSystem(new URI("lid:///"), [enabled: true, store: [location: wdir.toString()]])
|
||||
def wf = new WorkflowRun(new Workflow([],"repo", "commit"), "sessionId", "runId", [new Parameter("String", "param1", "value1")])
|
||||
|
||||
when: 'workflow repo in workflow run'
|
||||
Path p = LinPath.getMetadataAsTargetPath(wf, lidFs, "123456", "workflow.repository")
|
||||
then:
|
||||
p instanceof LinMetadataPath
|
||||
p.text == '"repo"'
|
||||
|
||||
when: 'outputs'
|
||||
def outputs = new WorkflowOutput(OffsetDateTime.now(), "lid://123456", [new Parameter("Collection", "samples", ["sample1", "sample2"])])
|
||||
lidFs.store.save("123456#output", outputs)
|
||||
Path p2 = LinPath.getMetadataAsTargetPath(wf, lidFs, "123456", "output")
|
||||
then:
|
||||
p2 instanceof LinMetadataPath
|
||||
p2.text == LinUtils.encodeSearchOutputs([new Parameter("Collection", "samples", ["sample1", "sample2"])], true)
|
||||
|
||||
when: 'child does not exists'
|
||||
LinPath.getMetadataAsTargetPath(wf, lidFs, "123456", "no-exist")
|
||||
then:
|
||||
def exception = thrown(FileNotFoundException)
|
||||
exception.message == "Target path '123456#no-exist' does not exist"
|
||||
|
||||
when: 'outputs does not exists'
|
||||
LinPath.getMetadataAsTargetPath(wf, lidFs, "6789", "output")
|
||||
then:
|
||||
def exception1 = thrown(FileNotFoundException)
|
||||
exception1.message == "Target path '6789#output' does not exist"
|
||||
|
||||
when: 'null object'
|
||||
LinPath.getMetadataAsTargetPath(null, lidFs, "123456", "no-exist")
|
||||
then:
|
||||
def exception2 = thrown(FileNotFoundException)
|
||||
exception2.message == "Target path '123456' does not exist"
|
||||
|
||||
cleanup:
|
||||
wdir.resolve("123456").deleteDir()
|
||||
}
|
||||
|
||||
def 'should get file name' () {
|
||||
expect:
|
||||
new LinPath(fs, PATH).getFileName() == EXPECTED
|
||||
where:
|
||||
PATH | EXPECTED
|
||||
'1234567890/this/file.bam' | new LinPath(null, 'file.bam')
|
||||
'12345/hola?query#output' | new LinPath("query", "output", "hola", null)
|
||||
|
||||
}
|
||||
|
||||
def 'should get file parent' () {
|
||||
when:
|
||||
def lid1 = new LinPath(fs, '1234567890/this/file.bam')
|
||||
then:
|
||||
lid1.getParent() == new LinPath(fs, '1234567890/this')
|
||||
lid1.getParent().getParent() == new LinPath(fs, '1234567890')
|
||||
lid1.getParent().getParent().getParent() == new LinPath(fs, "/")
|
||||
lid1.getParent().getParent().getParent().getParent() == null
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should get name count' () {
|
||||
expect:
|
||||
new LinPath(fs, PATH).getNameCount() == EXPECTED
|
||||
where:
|
||||
PATH | EXPECTED
|
||||
'/' | 0
|
||||
'123' | 1
|
||||
'123/a' | 2
|
||||
'123/a/' | 2
|
||||
'123/a/b' | 3
|
||||
'' | 0
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should get name by index' () {
|
||||
expect:
|
||||
new LinPath(fs, PATH).getName(INDEX) == EXPECTED
|
||||
where:
|
||||
PATH | INDEX | EXPECTED
|
||||
'123' | 0 | new LinPath(fs, '123')
|
||||
'123/a' | 1 | new LinPath(null, 'a')
|
||||
'123/a/' | 1 | new LinPath(null, 'a')
|
||||
'123/a/b' | 2 | new LinPath(null, 'b')
|
||||
'123/a?q#output' | 1 | new LinPath(null, 'a?q#output')
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should get subpath' () {
|
||||
expect:
|
||||
new LinPath(fs, PATH).subpath(BEGIN,END) == EXPECTED
|
||||
where:
|
||||
PATH | BEGIN | END | EXPECTED
|
||||
'123' | 0 | 1 | new LinPath(fs, '123')
|
||||
'123/a' | 0 | 2 | new LinPath(fs, '123/a')
|
||||
'123/a/' | 0 | 2 | new LinPath(fs, '123/a')
|
||||
'123/a' | 1 | 2 | new LinPath(null, 'a')
|
||||
'123/a/' | 1 | 2 | new LinPath(null, 'a')
|
||||
'123/a/b' | 2 | 3 | new LinPath(null, 'b')
|
||||
'123/a/b' | 1 | 3 | new LinPath(null, 'a/b')
|
||||
}
|
||||
|
||||
def 'should normalize a path' () {
|
||||
expect:
|
||||
new LinPath(fs, '123').normalize() == new LinPath(fs, '123')
|
||||
new LinPath(fs, '123/a/b').normalize() == new LinPath(fs, '123/a/b')
|
||||
new LinPath(fs, '123/./a/b').normalize() == new LinPath(fs, '123/a/b')
|
||||
new LinPath(fs, '123/a/../a/b').normalize() == new LinPath(fs, '123/a/b')
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should validate startWith' () {
|
||||
expect:
|
||||
new LinPath(fs,PATH).startsWith(OTHER) == EXPECTED
|
||||
where:
|
||||
PATH | OTHER | EXPECTED
|
||||
'12345/a/b' | '12345' | true
|
||||
'12345/a/b' | '12345/a' | true
|
||||
'12345/a/b' | '12345/a/b' | true
|
||||
and:
|
||||
'12345/a/b' | '12345/b' | false
|
||||
'12345/a/b' | 'xyz' | false
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should validate endsWith' () {
|
||||
expect:
|
||||
new LinPath(fs,PATH).endsWith(OTHER) == EXPECTED
|
||||
where:
|
||||
PATH | OTHER | EXPECTED
|
||||
'12345/a/b' | 'b' | true
|
||||
'12345/a/b' | 'a/b' | true
|
||||
'12345/a/b' | '12345/a/b' | true
|
||||
and:
|
||||
'12345/a/b' | '12345/b' | false
|
||||
'12345/a/b' | 'xyz' | false
|
||||
}
|
||||
|
||||
def 'should validate isAbsolute' () {
|
||||
expect:
|
||||
new LinPath(fs,'1234/a/b/c').isAbsolute()
|
||||
new LinPath(fs,'1234/a/b/c').getRoot().isAbsolute()
|
||||
new LinPath(fs,'1234/a/b/c').getParent().isAbsolute()
|
||||
new LinPath(fs,'1234/a/b/c').normalize().isAbsolute()
|
||||
new LinPath(fs,'1234/a/b/c').getName(0).isAbsolute()
|
||||
new LinPath(fs,'1234/a/b/c').subpath(0,2).isAbsolute()
|
||||
and:
|
||||
!new LinPath(fs,'1234/a/b/c').getFileName().isAbsolute()
|
||||
!new LinPath(fs,'1234/a/b/c').getName(1).isAbsolute()
|
||||
!new LinPath(fs,'1234/a/b/c').subpath(1,3).isAbsolute()
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should get root path' () {
|
||||
expect:
|
||||
new LinPath(fs,PATH).getRoot() == new LinPath(fs,EXPECTED)
|
||||
where:
|
||||
PATH | EXPECTED
|
||||
'12345' | '/'
|
||||
'12345/a' | '/'
|
||||
}
|
||||
|
||||
def 'should relativize path' () {
|
||||
expect:
|
||||
BASE_PATH.relativize(PATH) == EXPECTED
|
||||
where :
|
||||
BASE_PATH | PATH | EXPECTED
|
||||
new LinPath(fs, '/') | new LinPath(fs, '123/a/b/c') | new LinPath(null, '123/a/b/c')
|
||||
new LinPath(fs,'123/a/') | new LinPath(fs, '123/a/b/c') | new LinPath(null, 'b/c')
|
||||
new LinPath(fs,'123/a/') | new LinPath(fs, '321/a/') | new LinPath(null, '../../321/a')
|
||||
new LinPath(null,'123/a') | new LinPath(null, '123/a/b/c') | new LinPath(null, 'b/c')
|
||||
new LinPath(null,'123/a') | new LinPath(null, '321/a') | new LinPath(null, '../../321/a')
|
||||
new LinPath(fs,'../a/') | new LinPath(fs, '321/a') | new LinPath(null, '../321/a')
|
||||
new LinPath(fs,'321/a/') | new LinPath(fs, '../a') | new LinPath(null, '../../a')
|
||||
new LinPath(null,'321/a/') | new LinPath(null, '../a') | new LinPath(null, '../../../a')
|
||||
}
|
||||
|
||||
def 'relativize should throw exception' () {
|
||||
given:
|
||||
def lid1 = new LinPath(fs,'123/a/')
|
||||
def lid2 = new LinPath(null,'123/a/')
|
||||
def lid3 = new LinPath(null, '../a/b')
|
||||
when: 'comparing relative with absolute'
|
||||
lid1.relativize(lid2)
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
|
||||
when: 'undefined base path'
|
||||
lid3.relativize(lid2)
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
}
|
||||
|
||||
def 'should resolve path' () {
|
||||
when:
|
||||
def lid1 = new LinPath(fs, '123/a/b/c')
|
||||
def lid2 = new LinPath(fs, '321/x/y/z')
|
||||
def rel1 = new LinPath(null, 'foo')
|
||||
def rel2 = new LinPath(null, 'bar/')
|
||||
|
||||
then:
|
||||
lid1.resolve(lid2) == lid2
|
||||
lid2.resolve(lid1) == lid1
|
||||
and:
|
||||
lid1.resolve(rel1) == new LinPath(fs,'123/a/b/c/foo')
|
||||
lid1.resolve(rel2) == new LinPath(fs,'123/a/b/c/bar')
|
||||
and:
|
||||
rel1.resolve(rel2) == new LinPath(null, 'foo/bar')
|
||||
rel2.resolve(rel1) == new LinPath(null, 'bar/foo')
|
||||
}
|
||||
|
||||
def 'should resolve path as string' () {
|
||||
given:
|
||||
def pr = Mock(LinFileSystemProvider)
|
||||
def lidfs = Mock(LinFileSystem){
|
||||
provider() >> pr}
|
||||
|
||||
|
||||
def lid1 = new LinPath(lidfs, '123/a/b/c')
|
||||
|
||||
expect:
|
||||
lid1.resolve('x/y') == new LinPath(lidfs, '123/a/b/c/x/y')
|
||||
lid1.resolve('/x/y/') == new LinPath(lidfs, '123/a/b/c/x/y')
|
||||
|
||||
when:
|
||||
def result = lid1.resolve('lid://321')
|
||||
then:
|
||||
pr.getPath(LinPath.asUri('lid://321')) >> new LinPath(lidfs, '321')
|
||||
and:
|
||||
result == new LinPath(lidfs, '321')
|
||||
}
|
||||
|
||||
def 'should throw illegal exception when not correct scheme' (){
|
||||
when: 'creation'
|
||||
new LinPath(fs, new URI("http://1234"))
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
|
||||
when: 'asUri'
|
||||
LinPath.asUri("http://1234")
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
|
||||
when: 'asUri'
|
||||
LinPath.asUri("")
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
|
||||
}
|
||||
|
||||
def 'should throw provider mismatch exception when different path types' () {
|
||||
given:
|
||||
def pr = Mock(LinFileSystemProvider)
|
||||
def fs = Mock(LinFileSystem){
|
||||
provider() >> pr}
|
||||
and:
|
||||
def lid = new LinPath(fs, '123/a/b/c')
|
||||
|
||||
when: 'resolve with path'
|
||||
lid.resolve(Path.of('d'))
|
||||
then:
|
||||
thrown(ProviderMismatchException)
|
||||
|
||||
when: 'resolve with uri string'
|
||||
lid.resolve(Path.of('http://1234'))
|
||||
then:
|
||||
thrown(ProviderMismatchException)
|
||||
|
||||
when: 'relativize'
|
||||
lid.relativize(Path.of('d'))
|
||||
then:
|
||||
thrown(ProviderMismatchException)
|
||||
}
|
||||
|
||||
def 'should throw exception for unsupported methods' () {
|
||||
given:
|
||||
def pr = Mock(LinFileSystemProvider)
|
||||
def fs = Mock(LinFileSystem){
|
||||
provider() >> pr}
|
||||
and:
|
||||
def lid = new LinPath(fs, '123/a/b/c')
|
||||
|
||||
when: 'to file'
|
||||
lid.toFile()
|
||||
then:
|
||||
thrown(UnsupportedOperationException)
|
||||
|
||||
when: 'register'
|
||||
lid.register(null, null,null)
|
||||
then:
|
||||
thrown(UnsupportedOperationException)
|
||||
}
|
||||
|
||||
def 'should throw exception for incorrect index'() {
|
||||
when: 'getting name with negative index'
|
||||
new LinPath(fs, "1234").getName(-1)
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
|
||||
when: 'getting name with larger index tha namecount'
|
||||
new LinPath(fs, "1234").getName(2)
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
|
||||
when: 'getting subpath with negative index'
|
||||
new LinPath(fs, "1234").subpath(-1,1)
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
|
||||
when: 'getting subpath with larger index tha namecount'
|
||||
new LinPath(fs, "1234").subpath(0,2)
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should get to uri string' () {
|
||||
expect:
|
||||
new LinPath(fs, PATH).toUriString() == EXPECTED
|
||||
where:
|
||||
PATH | EXPECTED
|
||||
'/' | 'lid:///'
|
||||
'1234' | 'lid://1234'
|
||||
'1234/a/b/c' | 'lid://1234/a/b/c'
|
||||
'' | 'lid:///'
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should get string' () {
|
||||
expect:
|
||||
new LinPath(fs, PATH).toString() == EXPECTED
|
||||
where:
|
||||
PATH | EXPECTED
|
||||
'/' | '/'
|
||||
'1234' | '1234'
|
||||
'1234/a/b/c' | '1234/a/b/c'
|
||||
'' | '/'
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should validate asString method'() {
|
||||
expect:
|
||||
LinPath.asUriString(FIRST, MORE as String[]) == EXPECTED
|
||||
|
||||
where:
|
||||
FIRST | MORE | EXPECTED
|
||||
'foo' | [] | 'lid://foo'
|
||||
'foo/' | [] | 'lid://foo'
|
||||
'/foo' | [] | 'lid://foo'
|
||||
and:
|
||||
'a' | ['/b/'] | 'lid://a/b'
|
||||
'a' | ['/b','c'] | 'lid://a/b/c'
|
||||
'a' | ['/b','//c'] | 'lid://a/b/c'
|
||||
'a' | ['/b/c', 'd'] | 'lid://a/b/c/d'
|
||||
'/a/' | ['/b/c', 'd'] | 'lid://a/b/c/d'
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should check is lid uri string' () {
|
||||
expect:
|
||||
LinPath.isLidUri(STR) == EXPECTED
|
||||
|
||||
where:
|
||||
STR | EXPECTED
|
||||
null | false
|
||||
'' | false
|
||||
'foo' | false
|
||||
'/foo' | false
|
||||
'lid:/foo' | false
|
||||
'lid:foo' | false
|
||||
'lid/foo' | false
|
||||
and:
|
||||
'lid://' | true
|
||||
'lid:///' | true
|
||||
'lid://foo/bar' | true
|
||||
}
|
||||
|
||||
def 'should detect equals'(){
|
||||
expect:
|
||||
new LinPath(FS1, PATH1).equals(new LinPath(FS2, PATH2)) == EXPECTED
|
||||
where:
|
||||
FS1 | FS2 | PATH1 | PATH2 | EXPECTED
|
||||
null | fs | "12345/path" | "12345/path" | false
|
||||
fs | null | "12345/path" | "12345/path" | false
|
||||
null | null | "12345/" | "12345/path" | false
|
||||
fs | fs | "12345/" | "12345/path" | false
|
||||
and:
|
||||
null | null | "12345/path" | "12345/path" | true
|
||||
fs | fs | "12345/path" | "12345/path" | true
|
||||
null | null | "12345/" | "12345" | true
|
||||
fs | fs | "12345/" | "12345 " | true
|
||||
}
|
||||
|
||||
def 'should validate correct hash'(){
|
||||
given:
|
||||
def file = wdir.resolve("file.txt")
|
||||
file.text = "this is a data file"
|
||||
def hash = CacheHelper.hasher(file).hash().toString()
|
||||
def correctData = new FileOutput(file.toString(), new Checksum(hash,"nextflow", "standard"))
|
||||
when:
|
||||
def error = LinPath.validateDataOutput(correctData)
|
||||
then:
|
||||
!error
|
||||
|
||||
cleanup:
|
||||
file.delete()
|
||||
}
|
||||
|
||||
def 'should warn with incorrect hash'(){
|
||||
given:
|
||||
def file = wdir.resolve("file.txt")
|
||||
file.text = "this is a data file"
|
||||
def hash = CacheHelper.hasher(file).hash().toString()
|
||||
def correctData = new FileOutput(file.toString(), new Checksum("abscd","nextflow", "standard"))
|
||||
when:
|
||||
def error = LinPath.validateDataOutput(correctData)
|
||||
then:
|
||||
error == "Checksum of '$file' does not match with lineage metadata"
|
||||
|
||||
cleanup:
|
||||
file.delete()
|
||||
}
|
||||
|
||||
def 'should warn when hash algorithm is not supported'(){
|
||||
given:
|
||||
def file = wdir.resolve("file.txt")
|
||||
file.text = "this is a data file"
|
||||
def hash = CacheHelper.hasher(file).hash().toString()
|
||||
def correctData = new FileOutput(file.toString(), new Checksum(hash,"not-supported", "standard"))
|
||||
when:
|
||||
def error = LinPath.validateDataOutput(correctData)
|
||||
then:
|
||||
error == "Checksum of '$file' can't be validated - algorithm 'not-supported' is not supported"
|
||||
|
||||
cleanup:
|
||||
file.delete()
|
||||
}
|
||||
|
||||
def 'should throw exception when file not found validating hash'(){
|
||||
when:
|
||||
def correctData = new FileOutput("not/existing/file", new Checksum("120741","nextflow", "standard"))
|
||||
LinPath.validateDataOutput(correctData)
|
||||
|
||||
then:
|
||||
thrown(FileNotFoundException)
|
||||
}
|
||||
|
||||
def 'should validate path' () {
|
||||
given:
|
||||
def outputFolder = data.resolve('output')
|
||||
outputFolder.mkdirs()
|
||||
def outputFile = outputFolder.resolve('file1.txt')
|
||||
outputFile.text = "this is file1"
|
||||
def encoder = new LinEncoder()
|
||||
def hash = CacheHelper.hasher(outputFile).hash().toString()
|
||||
def correctData = new FileOutput(outputFile.toString(), new Checksum(hash,"nextflow", "standard"))
|
||||
def incorrectData = new FileOutput(outputFile.toString(), new Checksum("incorrectHash","nextflow", "standard"))
|
||||
wdir.resolve('12345/output/file1.txt').mkdirs()
|
||||
wdir.resolve('12345/output/file2.txt').mkdirs()
|
||||
wdir.resolve('12345/output/file1.txt/.data.json').text = encoder.encode(correctData)
|
||||
wdir.resolve('12345/output/file2.txt/.data.json').text = encoder.encode(incorrectData)
|
||||
def lidFs = new LinFileSystemProvider().newFileSystem(new URI("lid:///"), [enabled: true, store: [location: wdir.toString()]])
|
||||
|
||||
when:
|
||||
def succeed = new LinPath(lidFs, '12345/output/file1.txt').validate()
|
||||
then:
|
||||
succeed
|
||||
|
||||
when:
|
||||
succeed = new LinPath(lidFs, '12345/output/file2.txt').validate()
|
||||
then:
|
||||
!succeed
|
||||
|
||||
when:
|
||||
succeed = new LinPath(lidFs, '12345/output/file3.txt').validate()
|
||||
then:
|
||||
!succeed
|
||||
|
||||
cleanup:
|
||||
outputFile.delete()
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.model
|
||||
|
||||
import nextflow.lineage.model.v1beta1.Checksum
|
||||
import nextflow.util.CacheHelper
|
||||
import spock.lang.Specification
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class ChecksumTest extends Specification {
|
||||
|
||||
def 'should create a checksum'() {
|
||||
given:
|
||||
def checksum = new Checksum(algorithm: 'sha1', value: '1234567890abcdef', mode: 'hex')
|
||||
|
||||
expect:
|
||||
checksum.algorithm == 'sha1'
|
||||
checksum.value == '1234567890abcdef'
|
||||
checksum.mode == 'hex'
|
||||
}
|
||||
|
||||
def 'should create a checksum with of factory method'() {
|
||||
given:
|
||||
def checksum1 = Checksum.of('1234567890abcdef','sha1', CacheHelper.HashMode.DEFAULT())
|
||||
|
||||
expect:
|
||||
checksum1.algorithm == 'sha1'
|
||||
checksum1.value == '1234567890abcdef'
|
||||
checksum1.mode == 'standard'
|
||||
}
|
||||
|
||||
def 'should create checksum with ofNextflow factory method'() {
|
||||
given:
|
||||
def checksum1 = Checksum.ofNextflow('1234567890abcdef')
|
||||
|
||||
expect:
|
||||
checksum1.algorithm == 'nextflow'
|
||||
checksum1.value == CacheHelper.hasher('1234567890abcdef').hash().toString()
|
||||
checksum1.mode == 'standard'
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,266 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.serde
|
||||
|
||||
import java.nio.file.Path
|
||||
import java.time.OffsetDateTime
|
||||
|
||||
import nextflow.lineage.model.v1beta1.Checksum
|
||||
import nextflow.lineage.model.v1beta1.DataPath
|
||||
import nextflow.lineage.model.v1beta1.FileOutput
|
||||
import nextflow.lineage.model.v1beta1.Parameter
|
||||
import nextflow.lineage.model.v1beta1.TaskOutput
|
||||
import nextflow.lineage.model.v1beta1.TaskRun
|
||||
import nextflow.lineage.model.v1beta1.Workflow
|
||||
import nextflow.lineage.model.v1beta1.WorkflowOutput
|
||||
import nextflow.lineage.model.v1beta1.WorkflowRun
|
||||
import spock.lang.Specification
|
||||
|
||||
class LinEncoderTest extends Specification{
|
||||
|
||||
def 'should encode and decode Outputs'(){
|
||||
given:
|
||||
def encoder = new LinEncoder()
|
||||
and:
|
||||
def output = new FileOutput("/path/to/file", new Checksum("hash_value", "hash_algorithm", "standard"),
|
||||
"lid://source", "lid://workflow", "lid://task", 1234)
|
||||
|
||||
when:
|
||||
def encoded = encoder.encode(output)
|
||||
def object = encoder.decode(encoded)
|
||||
|
||||
then:
|
||||
object instanceof FileOutput
|
||||
def result = object as FileOutput
|
||||
result.path == "/path/to/file"
|
||||
result.checksum instanceof Checksum
|
||||
result.checksum.value == "hash_value"
|
||||
result.checksum.algorithm == "hash_algorithm"
|
||||
result.checksum.mode == "standard"
|
||||
result.source == "lid://source"
|
||||
result.size == 1234
|
||||
|
||||
}
|
||||
|
||||
def 'should encode and decode WorkflowRuns'(){
|
||||
given:
|
||||
def encoder = new LinEncoder()
|
||||
and:
|
||||
def config = [
|
||||
process: [
|
||||
container : "quay.io/nextflow/bash",
|
||||
executor : "local",
|
||||
resourceLabels: ["owner": "xxx"],
|
||||
scratch : false
|
||||
]
|
||||
]
|
||||
def metadata = [
|
||||
runName : "big_kare",
|
||||
start : "2025-11-06T13:35:42.049135334Z",
|
||||
container : "quay.io/nextflow/bash",
|
||||
commandLine : "nextflow run 'https://github.com/nextflow-io/hello' -name big_kare -with-tower -r master",
|
||||
nextflow : [version: "25.10.0", enable: [dsl: 2.0]],
|
||||
containerEngine: "docker",
|
||||
wave : [enabled: true],
|
||||
fusion : [enabled: true, version: "2.4"],
|
||||
platform : [
|
||||
workflowId: "wf1234",
|
||||
user : [id: "xxx", userName: "john-smith", email: "john.smith@acme.com", firstName: "John", lastName: "Smith", organization: "acme"],
|
||||
workspace : [id: "1234", name: "test-workspace", fullName: "Test workspace", organization: "acme"],
|
||||
computeEnv: [id: "ce3456", name: "test-ce", platform: "aws-cloud"],
|
||||
pipeline : [id: "pipe294", name: "https://github.com/nextflow-io/hello", revision: "master", commitId: null],
|
||||
labels : []
|
||||
],
|
||||
failOnIgnore : false
|
||||
]
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def mainScript = new DataPath("file://path/to/main.nf", new Checksum("78910", "nextflow", "standard"))
|
||||
def workflow = new Workflow([mainScript], "https://nextflow.io/nf-test/", "123456")
|
||||
def wfRun = new WorkflowRun(workflow, uniqueId.toString(), "test_run",
|
||||
[new Parameter("String", "param1", "value1"), new Parameter("String", "param2", "value2")],
|
||||
config, metadata
|
||||
)
|
||||
|
||||
when:
|
||||
def encoded = encoder.encode(wfRun)
|
||||
def object = encoder.decode(encoded)
|
||||
|
||||
then:
|
||||
object instanceof WorkflowRun
|
||||
def result = object as WorkflowRun
|
||||
result.workflow instanceof Workflow
|
||||
result.workflow.scriptFiles.first instanceof DataPath
|
||||
result.workflow.scriptFiles.first.path == "file://path/to/main.nf"
|
||||
result.workflow.scriptFiles.first.checksum instanceof Checksum
|
||||
result.workflow.scriptFiles.first.checksum.value == "78910"
|
||||
result.workflow.commitId == "123456"
|
||||
result.sessionId == uniqueId.toString()
|
||||
result.name == "test_run"
|
||||
result.params.size() == 2
|
||||
result.params.get(0).name == "param1"
|
||||
result.config.process.container == "quay.io/nextflow/bash"
|
||||
result.config.process.executor == "local"
|
||||
result.metadata.platform.workflowId == "wf1234"
|
||||
}
|
||||
|
||||
def 'should decode WorkflowRuns without metadata'(){
|
||||
given:
|
||||
def encoder = new LinEncoder()
|
||||
def wfRunStr = '''
|
||||
{
|
||||
"version": "lineage/v1beta1",
|
||||
"kind": "WorkflowRun",
|
||||
"spec": {
|
||||
"workflow": {
|
||||
"scriptFiles": [
|
||||
{
|
||||
"path": "https://github.com/nextflow-io/hello/main.nf",
|
||||
"checksum": {
|
||||
"value": "78910",
|
||||
"algorithm": "nextflow",
|
||||
"mode": "standard"
|
||||
}
|
||||
}
|
||||
],
|
||||
"repository": "https://github.com/nextflow-io/hello",
|
||||
"commitId": "2ce0b0e2943449188092a0e25102f0dadc70cb0a"
|
||||
},
|
||||
"sessionId": "4f02559e-9ebd-41d8-8ee2-a8d1e4f09c67",
|
||||
"name": "test_run",
|
||||
"params": [],
|
||||
"config": {
|
||||
"process": {
|
||||
"container": "quay.io/nextflow/bash",
|
||||
"executor": "local",
|
||||
"resourceLabels": {
|
||||
"owner": "xxx"
|
||||
},
|
||||
"scratch": false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
'''
|
||||
when:
|
||||
def object = encoder.decode(wfRunStr)
|
||||
|
||||
then:
|
||||
object instanceof WorkflowRun
|
||||
def result = object as WorkflowRun
|
||||
result.workflow instanceof Workflow
|
||||
result.workflow.scriptFiles.first instanceof DataPath
|
||||
result.workflow.scriptFiles.first.path == "https://github.com/nextflow-io/hello/main.nf"
|
||||
result.workflow.scriptFiles.first.checksum instanceof Checksum
|
||||
result.workflow.scriptFiles.first.checksum.value == "78910"
|
||||
result.workflow.commitId == "2ce0b0e2943449188092a0e25102f0dadc70cb0a"
|
||||
result.sessionId == "4f02559e-9ebd-41d8-8ee2-a8d1e4f09c67"
|
||||
result.name == "test_run"
|
||||
result.params.size() == 0
|
||||
result.config.process.container == "quay.io/nextflow/bash"
|
||||
result.config.process.executor == "local"
|
||||
result.metadata == null
|
||||
}
|
||||
|
||||
def 'should encode and decode WorkflowOutputs'(){
|
||||
given:
|
||||
def encoder = new LinEncoder()
|
||||
and:
|
||||
def time = OffsetDateTime.now()
|
||||
def wfResults = new WorkflowOutput(time, "lid://1234", [
|
||||
new Parameter("Collection", "a", [id: 'id', file: 'sample.txt' as Path]),
|
||||
new Parameter("String", "b", "B")
|
||||
])
|
||||
|
||||
when:
|
||||
def encoded = encoder.encode(wfResults)
|
||||
def object = encoder.decode(encoded)
|
||||
then:
|
||||
object instanceof WorkflowOutput
|
||||
def result = object as WorkflowOutput
|
||||
result.createdAt == time
|
||||
result.workflowRun == "lid://1234"
|
||||
result.output == [new Parameter("Collection", "a", [id: 'id', file: 'sample.txt']), new Parameter("String", "b", "B")]
|
||||
}
|
||||
|
||||
def 'should encode and decode TaskRun'() {
|
||||
given:
|
||||
def encoder = new LinEncoder()
|
||||
and:
|
||||
def uniqueId = UUID.randomUUID()
|
||||
def taskRun = new TaskRun(
|
||||
uniqueId.toString(),"name", new Checksum("78910", "nextflow", "standard"), 'this is a script',
|
||||
[new Parameter("String", "param1", "value1")], "container:version", "conda", "spack", "amd64",
|
||||
[a: "A", b: "B"], [new DataPath("path/to/file", new Checksum("78910", "nextflow", "standard"))]
|
||||
)
|
||||
when:
|
||||
def encoded = encoder.encode(taskRun)
|
||||
def object = encoder.decode(encoded)
|
||||
then:
|
||||
object instanceof TaskRun
|
||||
def result = object as TaskRun
|
||||
result.sessionId == uniqueId.toString()
|
||||
result.name == "name"
|
||||
result.codeChecksum.value == "78910"
|
||||
result.script == "this is a script"
|
||||
result.input.size() == 1
|
||||
result.input.get(0).name == "param1"
|
||||
result.container == "container:version"
|
||||
result.conda == "conda"
|
||||
result.spack == "spack"
|
||||
result.architecture == "amd64"
|
||||
result.globalVars == [a: "A", b: "B"]
|
||||
result.binEntries.size() == 1
|
||||
result.binEntries.get(0).path == "path/to/file"
|
||||
result.binEntries.get(0).checksum.value == "78910"
|
||||
}
|
||||
|
||||
def 'should encode and decode TaskOutputs'(){
|
||||
given:
|
||||
def encoder = new LinEncoder()
|
||||
and:
|
||||
def time = OffsetDateTime.now()
|
||||
def parameter = new Parameter("a","b", "c")
|
||||
def wfResults = new TaskOutput("lid://1234", "lid://5678", time, [parameter], null)
|
||||
when:
|
||||
def encoded = encoder.encode(wfResults)
|
||||
def object = encoder.decode(encoded)
|
||||
|
||||
then:
|
||||
object instanceof TaskOutput
|
||||
def result = object as TaskOutput
|
||||
result.createdAt == time
|
||||
result.taskRun == "lid://1234"
|
||||
result.workflowRun == "lid://5678"
|
||||
result.output.size() == 1
|
||||
result.output[0] == parameter
|
||||
}
|
||||
|
||||
def 'object with null date attributes' () {
|
||||
given:
|
||||
def encoder = new LinEncoder()
|
||||
and:
|
||||
def wfResults = new WorkflowOutput(null, "lid://1234")
|
||||
when:
|
||||
def encoded = encoder.encode(wfResults)
|
||||
def object = encoder.decode(encoded)
|
||||
then:
|
||||
encoded == '{"version":"lineage/v1beta1","kind":"WorkflowOutput","spec":{"createdAt":null,"workflowRun":"lid://1234","output":null}}'
|
||||
def result = object as WorkflowOutput
|
||||
result.createdAt == null
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.lineage.serde
|
||||
|
||||
|
||||
import nextflow.lineage.model.v1beta1.FileOutput
|
||||
import nextflow.lineage.model.v1beta1.LinModel
|
||||
import nextflow.lineage.model.v1beta1.TaskRun
|
||||
import spock.lang.Specification
|
||||
|
||||
class LinTypeAdapterFactoryTest extends Specification {
|
||||
|
||||
def 'should read both new and old JSON formats'() {
|
||||
given:
|
||||
def encoder = new LinEncoder()
|
||||
|
||||
when: 'create old format JSON (without spec wrapper)'
|
||||
def oldFormatJson = """
|
||||
{
|
||||
"version": "${LinModel.VERSION}",
|
||||
"kind": "FileOutput",
|
||||
"path": "/path/to/file",
|
||||
"checksum": {
|
||||
"value": "hash_value",
|
||||
"algorithm": "hash_algorithm",
|
||||
"mode": "standard"
|
||||
},
|
||||
"source": "lid://source",
|
||||
"workflow": "lid://workflow",
|
||||
"task": "lid://task",
|
||||
"size": 1234
|
||||
}
|
||||
"""
|
||||
|
||||
and: 'deserialize old format'
|
||||
def oldResult = encoder.decode(oldFormatJson)
|
||||
|
||||
then: 'should work correctly with backward compatibility'
|
||||
oldResult instanceof FileOutput
|
||||
oldResult.path == "/path/to/file"
|
||||
oldResult.checksum.value == "hash_value"
|
||||
oldResult.source == "lid://source"
|
||||
oldResult.size == 1234
|
||||
|
||||
when: 'create new format JSON (with spec wrapper)'
|
||||
def newFormatJson = """
|
||||
{
|
||||
"version": "${LinModel.VERSION}",
|
||||
"kind": "FileOutput",
|
||||
"spec": {
|
||||
"path": "/path/to/file",
|
||||
"checksum": {
|
||||
"value": "hash_value",
|
||||
"algorithm": "hash_algorithm",
|
||||
"mode": "standard"
|
||||
},
|
||||
"source": "lid://source",
|
||||
"workflow": "lid://workflow",
|
||||
"task": "lid://task",
|
||||
"size": 1234
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
and: 'deserialize new format'
|
||||
def newResult = encoder.decode(newFormatJson)
|
||||
|
||||
then: 'should work correctly'
|
||||
newResult instanceof FileOutput
|
||||
newResult.path == "/path/to/file"
|
||||
newResult.checksum.value == "hash_value"
|
||||
newResult.source == "lid://source"
|
||||
newResult.size == 1234
|
||||
}
|
||||
|
||||
def 'should handle TaskRun old format'() {
|
||||
given:
|
||||
def encoder = new LinEncoder()
|
||||
|
||||
when: 'create old format TaskRun JSON'
|
||||
def oldFormatJson = """
|
||||
{
|
||||
"version": "${LinModel.VERSION}",
|
||||
"kind": "TaskRun",
|
||||
"sessionId": "session123",
|
||||
"name": "testTask",
|
||||
"codeChecksum": {
|
||||
"value": "hash123",
|
||||
"algorithm": "nextflow",
|
||||
"mode": "standard"
|
||||
},
|
||||
"script": "echo hello",
|
||||
"input": [],
|
||||
"container": "ubuntu:latest",
|
||||
"conda": null,
|
||||
"spack": null,
|
||||
"architecture": "amd64",
|
||||
"globalVars": {},
|
||||
"binEntries": []
|
||||
}
|
||||
"""
|
||||
|
||||
and: 'deserialize old format'
|
||||
def result = encoder.decode(oldFormatJson)
|
||||
|
||||
then: 'should work correctly'
|
||||
result instanceof TaskRun
|
||||
result.sessionId == "session123"
|
||||
result.name == "testTask"
|
||||
result.codeChecksum.value == "hash123"
|
||||
result.script == "echo hello"
|
||||
result.container == "ubuntu:latest"
|
||||
result.architecture == "amd64"
|
||||
}
|
||||
|
||||
def 'should reject JSON without version field'() {
|
||||
given:
|
||||
def encoder = new LinEncoder()
|
||||
|
||||
when: 'try to deserialize JSON without version'
|
||||
def invalidJson = """
|
||||
{
|
||||
"kind": "FileOutput",
|
||||
"path": "/path/to/file"
|
||||
}
|
||||
"""
|
||||
encoder.decode(invalidJson)
|
||||
|
||||
then: 'should throw exception'
|
||||
thrown(Exception)
|
||||
}
|
||||
|
||||
def 'should reject JSON with wrong version'() {
|
||||
given:
|
||||
def encoder = new LinEncoder()
|
||||
|
||||
when: 'try to deserialize JSON with wrong version'
|
||||
def invalidJson = """
|
||||
{
|
||||
"version": "wrong/version",
|
||||
"kind": "FileOutput",
|
||||
"path": "/path/to/file"
|
||||
}
|
||||
"""
|
||||
encoder.decode(invalidJson)
|
||||
|
||||
then: 'should throw exception'
|
||||
thrown(Exception)
|
||||
}
|
||||
|
||||
def 'should reject JSON without kind field'() {
|
||||
given:
|
||||
def encoder = new LinEncoder()
|
||||
|
||||
when: 'try to deserialize JSON without kind'
|
||||
def invalidJson = """
|
||||
{
|
||||
"version": "${LinModel.VERSION}",
|
||||
"path": "/path/to/file"
|
||||
}
|
||||
"""
|
||||
encoder.decode(invalidJson)
|
||||
|
||||
then: 'should throw exception'
|
||||
thrown(Exception)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user