add nextflow d30e48d
This commit is contained in:
@@ -0,0 +1,88 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin
|
||||
|
||||
import groovy.json.JsonSlurper
|
||||
import groovy.transform.Memoized
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.Const
|
||||
import nextflow.SysEnv
|
||||
import nextflow.config.ConfigBuilder
|
||||
import nextflow.util.Duration
|
||||
|
||||
@Slf4j
|
||||
class BaseCommandImpl {
|
||||
|
||||
protected static final int API_TIMEOUT_MS = 10_000
|
||||
|
||||
/**
|
||||
* Creates a TowerClient instance with optional authentication token.
|
||||
*
|
||||
* @param apiUrl Seqera Platform API url
|
||||
* @param accessToken Optional personal access token for authentication (PAT)
|
||||
* @return Configured TowerClient instance with timeout settings
|
||||
*/
|
||||
@Memoized
|
||||
protected TowerClient createTowerClient(String apiUrl, String accessToken) {
|
||||
return new TowerClient( new TowerConfig( [accessToken: accessToken, endpoint: apiUrl, httpConnectTimeout: Duration.of(API_TIMEOUT_MS)], SysEnv.get()))
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert API endpoint to web URL
|
||||
* e.g., https://api.cloud.seqera.io -> https://cloud.seqera.io
|
||||
* https://cloud.seqera.io/api -> https://cloud.seqera.io
|
||||
*/
|
||||
protected String getWebUrlFromApiEndpoint(String apiEndpoint) {
|
||||
return apiEndpoint.replace('://api.', '://').replace('/api', '')
|
||||
}
|
||||
|
||||
protected Map readConfig() {
|
||||
final builder = new ConfigBuilder().setHomeDir(Const.APP_HOME_DIR).setCurrentDir(Const.APP_HOME_DIR)
|
||||
return builder.buildConfigObject().flatten()
|
||||
}
|
||||
|
||||
protected List<Map> listUserWorkspaces(TowerClient client, String userId) {
|
||||
return client.listUserWorkspacesAndOrgs(userId).findAll { ((Map) it).workspaceId != null }
|
||||
}
|
||||
|
||||
protected List listComputeEnvironments(TowerClient client, String workspaceId) {
|
||||
try {
|
||||
final json = client.apiGet("/compute-envs", workspaceId ? [workspaceId: workspaceId] : [:])
|
||||
return json.computeEnvs as List ?: []
|
||||
} catch ( Exception e ) {
|
||||
throw new RuntimeException("Failed to get compute environments: ${e.message}", e)
|
||||
}
|
||||
}
|
||||
|
||||
protected Map getComputeEnvironment(TowerClient client, String computeEnvId, String workspaceId) {
|
||||
try {
|
||||
final json = client.apiGet(workspaceId ? "/compute-envs/${computeEnvId}" : "/compute-envs", workspaceId ? [workspaceId: workspaceId] : [:])
|
||||
return unifyComputeEnvDescription(json.computeEnv as Map ?: [:])
|
||||
} catch ( Exception e ) {
|
||||
throw new RuntimeException("Failed to get compute environments: ${e.message}", e)
|
||||
}
|
||||
}
|
||||
|
||||
private Map unifyComputeEnvDescription(Map computeEnv) {
|
||||
if (computeEnv && !computeEnv.workDir) {
|
||||
final config = computeEnv?.config as Map
|
||||
log.debug("Config $config")
|
||||
computeEnv.workDir = config?.workDir as String
|
||||
}
|
||||
return computeEnv
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.SysEnv
|
||||
import nextflow.cli.PluginAbstractExec
|
||||
/**
|
||||
* Implements nextflow cache and restore commands
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class CacheCommand implements PluginAbstractExec {
|
||||
|
||||
List<String> getCommands() { ['cache-backup', 'cache-restore'] }
|
||||
|
||||
@Override
|
||||
int exec(String cmd, List<String> args) {
|
||||
|
||||
if( cmd == 'cache-backup') {
|
||||
cacheBackup()
|
||||
}
|
||||
if( cmd == 'cache-restore' )
|
||||
cacheRestore()
|
||||
return 0
|
||||
}
|
||||
|
||||
protected void cacheBackup() {
|
||||
// note: use directly NXF_CLOUDCACHE_PATH along with `session.cloudCachePath`
|
||||
// because the latter required to be initialized via the execution
|
||||
// CmdRun. However this command is only executed to be used via Seqera Platform
|
||||
// that's providing the cache path via the env variable
|
||||
if( !getSession().cloudCachePath && !SysEnv.get('NXF_CLOUDCACHE_PATH') ) {
|
||||
log.debug "Running Nextflow cache backup (CacheManager)"
|
||||
// legacy cache manager
|
||||
new CacheManager(System.getenv()).saveCacheFiles()
|
||||
}
|
||||
else {
|
||||
log.debug "Running Nextflow cache backup (LogsHandler)"
|
||||
new LogsHandler(getSession(), System.getenv()).saveFiles()
|
||||
}
|
||||
}
|
||||
|
||||
protected void cacheRestore() {
|
||||
if( !getSession().cloudCachePath ) {
|
||||
log.debug "Running Nextflow cache restore"
|
||||
// legacy cache manager
|
||||
new CacheManager(System.getenv()).restoreCacheFiles()
|
||||
}
|
||||
else {
|
||||
// this command is only kept for backward compatibility
|
||||
log.debug "Running Nextflow cache restore - DO NOTHING"
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin
|
||||
|
||||
import static java.nio.file.StandardCopyOption.*
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.NoSuchFileException
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.PackageScope
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.Const
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.extension.FilesEx
|
||||
import nextflow.file.FileHelper
|
||||
/**
|
||||
* Back and restore Nextflow cache
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class CacheManager {
|
||||
|
||||
@PackageScope String sessionUuid
|
||||
@PackageScope Path localCachePath
|
||||
@PackageScope Path localOutFile
|
||||
@PackageScope Path localLogFile
|
||||
@PackageScope Path localTimelineFile
|
||||
@PackageScope Path localTowerConfig
|
||||
@PackageScope Path localTowerReports
|
||||
@PackageScope Path remoteWorkDir
|
||||
|
||||
@PackageScope Path getRemoteCachePath() { remoteWorkDir.resolve(".nextflow/cache/${sessionUuid}") }
|
||||
@PackageScope Path getRemoteOutFile() { remoteWorkDir.resolve(localOutFile.getName()) }
|
||||
@PackageScope Path getRemoteLogFile() { remoteWorkDir.resolve(localLogFile.getName()) }
|
||||
@PackageScope Path getRemoteTimelineFile() { remoteWorkDir.resolve(localTimelineFile.getName()) }
|
||||
@PackageScope Path getRemoteTowerConfig() { remoteWorkDir.resolve(localTowerConfig.getName()) }
|
||||
@PackageScope Path getRemoteTowerReports() { remoteWorkDir.resolve(localTowerReports.getName()) }
|
||||
|
||||
CacheManager(Map<String,String> env) {
|
||||
final work = env.get('NXF_WORK') ?: env.get('NXF_TEST_WORK')
|
||||
if( !work )
|
||||
throw new AbortOperationException("Missing target work dir - cache sync cannot be performed")
|
||||
this.remoteWorkDir = FileHelper.asPath(work)
|
||||
|
||||
this.sessionUuid = env.get('NXF_UUID')
|
||||
if( !sessionUuid )
|
||||
throw new AbortOperationException("Missing target uuid - cache sync cannot be performed")
|
||||
|
||||
this.localCachePath = Const.appCacheDir.resolve("cache/${sessionUuid}")
|
||||
|
||||
if( env.NXF_OUT_FILE )
|
||||
localOutFile = Paths.get(env.NXF_OUT_FILE)
|
||||
if( env.NXF_LOG_FILE )
|
||||
localLogFile = Paths.get(env.NXF_LOG_FILE)
|
||||
if( env.NXF_TML_FILE )
|
||||
localTimelineFile = Paths.get(env.NXF_TML_FILE)
|
||||
if( env.TOWER_CONFIG_FILE )
|
||||
localTowerConfig = Paths.get(env.TOWER_CONFIG_FILE)
|
||||
if( env.TOWER_REPORTS_FILE )
|
||||
localTowerReports = Paths.get(env.TOWER_REPORTS_FILE)
|
||||
}
|
||||
|
||||
protected void restoreCacheFiles() {
|
||||
if( !remoteWorkDir || !sessionUuid )
|
||||
return
|
||||
|
||||
if( !Files.exists(remoteCachePath) ) {
|
||||
log.debug "Remote cache path does not exist: $remoteCachePath -- skipping cache restore"
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
log.info "Restoring cache: ${remoteCachePath.toUriString()} => ${localCachePath.toUriString()}"
|
||||
localCachePath.deleteDir()
|
||||
localCachePath.parent.mkdirs()
|
||||
FileHelper.copyPath(remoteCachePath, localCachePath, REPLACE_EXISTING)
|
||||
}
|
||||
catch (NoSuchFileException e) {
|
||||
log.info "Remote cache restore ignored -- reason: ${e.message ?: e}"
|
||||
}
|
||||
}
|
||||
|
||||
protected void saveCacheFiles() {
|
||||
if( !remoteWorkDir || !sessionUuid )
|
||||
return
|
||||
|
||||
// -- upload nextflow metadata cache
|
||||
try {
|
||||
if( localCachePath?.exists() ) {
|
||||
log.info "Saving cache: ${localCachePath.toUriString()} => ${remoteCachePath.toUriString()}"
|
||||
remoteCachePath.deleteDir()
|
||||
remoteCachePath.parent.mkdirs()
|
||||
FilesEx.copyTo(localCachePath, remoteCachePath)
|
||||
}
|
||||
else {
|
||||
log.debug "Local cache path does not exist: $localCachePath -- skipping cache backup"
|
||||
}
|
||||
}
|
||||
catch (Throwable e) {
|
||||
log.warn "Failed to backup resume metadata to remote store path: ${remoteCachePath.toUriString()} -- cause: ${e}", e
|
||||
}
|
||||
|
||||
// -- upload out file
|
||||
try {
|
||||
if( localOutFile?.exists() )
|
||||
FileHelper.copyPath(localOutFile, remoteOutFile, REPLACE_EXISTING)
|
||||
}
|
||||
catch (Throwable e) {
|
||||
log.warn "Unable to upload nextflow out file: $localOutFile -- reason: ${e.message ?: e}", e
|
||||
}
|
||||
// -- upload log file
|
||||
try {
|
||||
if( localLogFile?.exists() )
|
||||
FileHelper.copyPath(localLogFile, remoteLogFile, REPLACE_EXISTING)
|
||||
}
|
||||
catch (Throwable e) {
|
||||
log.warn "Unable to upload nextflow log file: $localLogFile -- reason: ${e.message ?: e}", e
|
||||
}
|
||||
// -- upload timeline file
|
||||
try {
|
||||
if( localTimelineFile?.exists() )
|
||||
FileHelper.copyPath(localTimelineFile, remoteTimelineFile, REPLACE_EXISTING)
|
||||
}
|
||||
catch (Throwable e) {
|
||||
log.warn "Unable to upload nextflow timeline file: $localTimelineFile -- reason: ${e.message ?: e}", e
|
||||
}
|
||||
// -- upload tower config file
|
||||
try {
|
||||
if( localTowerConfig?.exists() )
|
||||
FileHelper.copyPath(localTowerConfig, remoteTowerConfig, REPLACE_EXISTING)
|
||||
}
|
||||
catch (Throwable e) {
|
||||
log.warn "Unable to upload tower config file: $localTowerConfig -- reason: ${e.message ?: e}", e
|
||||
}
|
||||
// -- upload tower reports file
|
||||
try {
|
||||
if( localTowerReports?.exists() )
|
||||
FileHelper.copyPath(localTowerReports, remoteTowerReports, REPLACE_EXISTING)
|
||||
}
|
||||
catch (Throwable e) {
|
||||
log.warn "Unable to upload tower reports file: $localTowerReports -- reason: ${e.message ?: e}", e
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.Session
|
||||
import nextflow.SysEnv
|
||||
import nextflow.trace.TraceObserverV2
|
||||
import nextflow.trace.event.TaskEvent
|
||||
import nextflow.util.Duration
|
||||
import nextflow.util.Threads
|
||||
/**
|
||||
* Implements a nextflow observer that periodically checkpoint
|
||||
* log, report and timeline files
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class LogsCheckpoint implements TraceObserverV2 {
|
||||
|
||||
private Session session
|
||||
private Map config
|
||||
private Thread thread
|
||||
private Duration interval
|
||||
private LogsHandler handler
|
||||
private final Object lock = new Object()
|
||||
|
||||
@Override
|
||||
void onFlowCreate(Session session) {
|
||||
this.session = session
|
||||
this.config = session.config
|
||||
this.handler = new LogsHandler(session, SysEnv.get())
|
||||
this.interval = config.navigate('tower.logs.checkpoint.interval', defaultInterval()) as Duration
|
||||
thread = Threads.start('tower-logs-checkpoint', this.&run)
|
||||
}
|
||||
|
||||
private String defaultInterval() {
|
||||
SysEnv.get('TOWER_LOGS_CHECKPOINT_INTERVAL','90s')
|
||||
}
|
||||
|
||||
@Override
|
||||
void onFlowComplete() {
|
||||
synchronized(lock) {
|
||||
thread.interrupt()
|
||||
}
|
||||
thread.join()
|
||||
}
|
||||
|
||||
@Override
|
||||
void onFlowError(TaskEvent event) {
|
||||
synchronized(lock) {
|
||||
thread.interrupt()
|
||||
}
|
||||
thread.join()
|
||||
}
|
||||
|
||||
protected void run() {
|
||||
log.debug "Starting logs checkpoint thread - interval: ${interval}"
|
||||
try {
|
||||
while( true ) {
|
||||
await(interval)
|
||||
if( Thread.currentThread().isInterrupted() )
|
||||
break
|
||||
synchronized(lock) {
|
||||
if( Thread.currentThread().isInterrupted() )
|
||||
break
|
||||
handler.saveFiles()
|
||||
}
|
||||
}
|
||||
}
|
||||
finally {
|
||||
log.debug "Terminating logs checkpoint thread"
|
||||
}
|
||||
}
|
||||
|
||||
protected void await(Duration interval) {
|
||||
try {
|
||||
Thread.sleep(interval.toMillis())
|
||||
}
|
||||
catch (InterruptedException e) {
|
||||
log.debug "Interrupted logs checkpoint thread"
|
||||
Thread.currentThread().interrupt()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin
|
||||
|
||||
import static java.nio.file.StandardCopyOption.*
|
||||
|
||||
import java.nio.file.FileSystems
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.PackageScope
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.Session
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.file.FileHelper
|
||||
/**
|
||||
* Backup Nextflow logs, timeline and reports files
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class LogsHandler {
|
||||
|
||||
@PackageScope Path localOutFile
|
||||
@PackageScope Path localLogFile
|
||||
@PackageScope Path localTimelineFile
|
||||
@PackageScope Path localTowerConfig
|
||||
@PackageScope Path localTowerReports
|
||||
@PackageScope Path remoteWorkDir
|
||||
|
||||
@PackageScope Path getRemoteOutFile() { remoteWorkDir.resolve(localOutFile.getName()) }
|
||||
@PackageScope Path getRemoteLogFile() { remoteWorkDir.resolve(localLogFile.getName()) }
|
||||
@PackageScope Path getRemoteTimelineFile() { remoteWorkDir.resolve(localTimelineFile.getName()) }
|
||||
@PackageScope Path getRemoteTowerConfig() { remoteWorkDir.resolve(localTowerConfig.getName()) }
|
||||
@PackageScope Path getRemoteTowerReports() { remoteWorkDir.resolve(localTowerReports.getName()) }
|
||||
|
||||
LogsHandler(Session session, Map<String,String> env) {
|
||||
if( !session.workDir )
|
||||
throw new AbortOperationException("Missing workflow work directory")
|
||||
if( session.workDir.fileSystem == FileSystems.default )
|
||||
throw new AbortOperationException("Logs handler is only meant to be used with a remote workflow work directory")
|
||||
this.remoteWorkDir = session.workDir
|
||||
|
||||
if( env.NXF_OUT_FILE )
|
||||
localOutFile = Paths.get(env.NXF_OUT_FILE)
|
||||
if( env.NXF_LOG_FILE )
|
||||
localLogFile = Paths.get(env.NXF_LOG_FILE)
|
||||
if( env.NXF_TML_FILE )
|
||||
localTimelineFile = Paths.get(env.NXF_TML_FILE)
|
||||
if( env.TOWER_CONFIG_FILE )
|
||||
localTowerConfig = Paths.get(env.TOWER_CONFIG_FILE)
|
||||
if( env.TOWER_REPORTS_FILE )
|
||||
localTowerReports = Paths.get(env.TOWER_REPORTS_FILE)
|
||||
}
|
||||
|
||||
void saveFiles() {
|
||||
log.trace "Checkpointing logs, timeline and report files"
|
||||
// — upload out file
|
||||
try {
|
||||
if( localOutFile?.exists() )
|
||||
FileHelper.copyPath(localOutFile, remoteOutFile, REPLACE_EXISTING)
|
||||
}
|
||||
catch (Throwable e) {
|
||||
log.warn "Unable to upload nextflow out file: $localOutFile — reason: ${e.message ?: e}", e
|
||||
}
|
||||
// — upload log file
|
||||
try {
|
||||
if( localLogFile?.exists() )
|
||||
FileHelper.copyPath(localLogFile, remoteLogFile, REPLACE_EXISTING)
|
||||
}
|
||||
catch (Throwable e) {
|
||||
log.warn "Unable to upload nextflow log file: $localLogFile — reason: ${e.message ?: e}", e
|
||||
}
|
||||
// — upload timeline file
|
||||
try {
|
||||
if( localTimelineFile?.exists() )
|
||||
FileHelper.copyPath(localTimelineFile, remoteTimelineFile, REPLACE_EXISTING)
|
||||
}
|
||||
catch (Throwable e) {
|
||||
log.warn "Unable to upload nextflow timeline file: $localTimelineFile — reason: ${e.message ?: e}", e
|
||||
}
|
||||
// — upload tower config file
|
||||
try {
|
||||
if( localTowerConfig?.exists() )
|
||||
FileHelper.copyPath(localTowerConfig, remoteTowerConfig, REPLACE_EXISTING)
|
||||
}
|
||||
catch (Throwable e) {
|
||||
log.warn "Unable to upload tower config file: $localTowerConfig — reason: ${e.message ?: e}", e
|
||||
}
|
||||
// — upload tower reports file
|
||||
try {
|
||||
if( localTowerReports?.exists() )
|
||||
FileHelper.copyPath(localTowerReports, remoteTowerReports, REPLACE_EXISTING)
|
||||
}
|
||||
catch (Throwable e) {
|
||||
log.warn "Unable to upload tower reprts file: $localTowerReports — reason: ${e.message ?: e}", e
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,520 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin
|
||||
|
||||
import java.net.http.HttpClient
|
||||
import java.net.http.HttpRequest
|
||||
|
||||
import groovy.json.JsonGenerator
|
||||
import groovy.json.JsonOutput
|
||||
import groovy.json.JsonSlurper
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.TupleConstructor
|
||||
import groovy.util.logging.Slf4j
|
||||
import io.seqera.http.HxClient
|
||||
import io.seqera.tower.plugin.exception.ForbiddenException
|
||||
import io.seqera.tower.plugin.exception.NotFoundException
|
||||
import io.seqera.tower.plugin.exception.UnauthorizedException
|
||||
import io.seqera.util.trace.TraceUtils
|
||||
import nextflow.BuildInfo
|
||||
import nextflow.SysEnv
|
||||
import nextflow.exception.AbortRunException
|
||||
import nextflow.util.Duration
|
||||
import nextflow.util.TestOnly
|
||||
/**
|
||||
* Perform HTTP call to Seqera platform.
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class TowerClient {
|
||||
|
||||
static final public String DEF_ENDPOINT_URL = 'https://api.cloud.seqera.io'
|
||||
|
||||
static private final String TOKEN_PREFIX = '@token:'
|
||||
|
||||
@TupleConstructor
|
||||
static class Response {
|
||||
final int code
|
||||
final String message
|
||||
final String cause
|
||||
boolean isError() { code < 200 || code >= 300 }
|
||||
}
|
||||
|
||||
private HxClient httpClient
|
||||
|
||||
private JsonGenerator generator
|
||||
|
||||
private String endpoint
|
||||
|
||||
private Map<String,Integer> schema = Collections.emptyMap()
|
||||
|
||||
private String accessToken
|
||||
|
||||
private TowerRetryPolicy retryPolicy
|
||||
|
||||
private Duration readTimeout = TowerConfig.DEFAULT_READ_TIMEOUT
|
||||
|
||||
private Duration connectTimeout = TowerConfig.DEFAULT_CONNECT_TIMEOUT
|
||||
|
||||
TowerClient(TowerConfig config) {
|
||||
this.endpoint = checkUrl(config.endpoint)
|
||||
this.accessToken = config.accessToken
|
||||
this.retryPolicy = config.retryPolicy
|
||||
this.readTimeout = config.httpReadTimeout
|
||||
this.connectTimeout = config.httpConnectTimeout
|
||||
this.schema = loadSchema()
|
||||
this.generator = TowerJsonGenerator.create(schema)
|
||||
initHttpClient()
|
||||
}
|
||||
|
||||
@TestOnly
|
||||
protected TowerClient() {
|
||||
this.generator = TowerJsonGenerator.create(Collections.EMPTY_MAP)
|
||||
}
|
||||
|
||||
String getEndpoint() { endpoint }
|
||||
|
||||
/**
|
||||
* Check the URL and create an HttpPost() object. If a invalid i.e. protocol is used,
|
||||
* the constructor will raise an exception.
|
||||
*
|
||||
* The RegEx was taken and adapted from http://urlregex.com
|
||||
*
|
||||
* @param url String with target URL
|
||||
* @return The requested url or the default url, if invalid
|
||||
*/
|
||||
protected String checkUrl(String url) {
|
||||
// report a warning for legacy endpoint
|
||||
if( url.contains('https://api.tower.nf') ) {
|
||||
log.warn "The endpoint `https://api.tower.nf` is deprecated - Please use `https://api.cloud.seqera.io` instead"
|
||||
}
|
||||
if( url =~ "^(https|http)://[-a-zA-Z0-9+&@#/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]" ) {
|
||||
while( url.endsWith('/') )
|
||||
url = url[0..-2]
|
||||
return url
|
||||
}
|
||||
throw new IllegalArgumentException("Only http and https are supported -- The given URL was: ${url}")
|
||||
}
|
||||
|
||||
protected String getHostUrl(String endpoint) {
|
||||
def url = new URL(endpoint)
|
||||
return "${url.protocol}://${url.authority}"
|
||||
}
|
||||
|
||||
Map traceCreate(Map req, String workspaceId){
|
||||
return sendAndProcessRequest( getUrlTraceCreate(workspaceId), req, 'POST')
|
||||
}
|
||||
|
||||
Map traceBegin(Map req, String workspaceId, String workflowId){
|
||||
return sendAndProcessRequest( getUrlTraceBegin(workspaceId, workflowId), req, 'PUT')
|
||||
}
|
||||
|
||||
void traceComplete(Map req, String workspaceId, String workflowId) {
|
||||
final url = getUrlTraceComplete(workspaceId, workflowId)
|
||||
final resp = sendHttpMessage(url, req, 'PUT')
|
||||
logHttpResponse(url, resp)
|
||||
}
|
||||
|
||||
void traceHeartbeat(Map req, String workspaceId, String workflowId) {
|
||||
final url = getUrlTraceHeartbeat( workspaceId, workflowId)
|
||||
final resp = sendHttpMessage(url, req, 'PUT')
|
||||
logHttpResponse(url, resp)
|
||||
}
|
||||
|
||||
void traceProgress(Map req, String workspaceId, String workflowId) {
|
||||
final url = getUrlTraceProgress( workspaceId, workflowId )
|
||||
final resp = sendHttpMessage(url, req, 'PUT')
|
||||
if( resp.error ) {
|
||||
final message = """\
|
||||
Unexpected HTTP response
|
||||
- endpoint : $url
|
||||
- status code : $resp.code
|
||||
- response msg: $resp.message
|
||||
""".stripIndent(true)
|
||||
throw new AbortRunException(message)
|
||||
}
|
||||
}
|
||||
|
||||
protected Map sendAndProcessRequest(String url, Map req, String method){
|
||||
final resp = sendHttpMessage(url, req, method)
|
||||
if( resp.error ) {
|
||||
final message = """\
|
||||
Unexpected HTTP response
|
||||
- endpoint : $url
|
||||
- status code : $resp.code
|
||||
- response msg: $resp.message
|
||||
""".stripIndent(true)
|
||||
throw new AbortRunException(message)
|
||||
}
|
||||
return parseTowerResponse(resp)
|
||||
}
|
||||
|
||||
protected String getUrlTraceCreate(String workspaceId) {
|
||||
def result = this.endpoint + '/trace/create'
|
||||
if( workspaceId )
|
||||
result += "?workspaceId=$workspaceId"
|
||||
return result
|
||||
}
|
||||
|
||||
protected String getUrlTraceBegin(String workspaceId, String workflowId) {
|
||||
def result = "$endpoint/trace/$workflowId/begin"
|
||||
if( workspaceId )
|
||||
result += "?workspaceId=$workspaceId"
|
||||
return result
|
||||
}
|
||||
|
||||
protected String getUrlTraceComplete(String workspaceId, String workflowId) {
|
||||
def result = "$endpoint/trace/$workflowId/complete"
|
||||
if( workspaceId )
|
||||
result += "?workspaceId=$workspaceId"
|
||||
return result
|
||||
}
|
||||
|
||||
protected String getUrlTraceHeartbeat(String workspaceId, String workflowId) {
|
||||
def result = "$endpoint/trace/$workflowId/heartbeat"
|
||||
if( workspaceId )
|
||||
result += "?workspaceId=$workspaceId"
|
||||
return result
|
||||
}
|
||||
|
||||
protected String getUrlTraceProgress(String workspaceId, String workflowId) {
|
||||
def result = "$endpoint/trace/$workflowId/progress"
|
||||
if( workspaceId )
|
||||
result += "?workspaceId=$workspaceId"
|
||||
return result
|
||||
}
|
||||
|
||||
protected void initHttpClient() {
|
||||
final builder = HxClient.newBuilder()
|
||||
// auth settings
|
||||
setupClientAuth(builder, getAccessToken())
|
||||
// retry settings
|
||||
this.httpClient = builder
|
||||
.retryConfig(this.retryPolicy)
|
||||
.followRedirects(HttpClient.Redirect.NORMAL)
|
||||
.version(HttpClient.Version.HTTP_1_1)
|
||||
.connectTimeout(java.time.Duration.ofMillis(connectTimeout.millis))
|
||||
.build()
|
||||
}
|
||||
|
||||
protected void setupClientAuth(HxClient.Builder config, String token) {
|
||||
// check for plain jwt token
|
||||
final refreshToken = SysEnv.get('TOWER_REFRESH_TOKEN')
|
||||
final refreshUrl = refreshToken ? "$endpoint/oauth/access_token" : null
|
||||
if( token.count('.')==2 ) {
|
||||
config.bearerToken(token)
|
||||
config.refreshToken(refreshToken)
|
||||
config.refreshTokenUrl(refreshUrl)
|
||||
config.refreshCookiePolicy(CookiePolicy.ACCEPT_ALL)
|
||||
return
|
||||
}
|
||||
|
||||
// try checking personal access token
|
||||
try {
|
||||
final plain = new String(token.decodeBase64())
|
||||
final p = plain.indexOf('.')
|
||||
if( p!=-1 && new JsonSlurper().parseText( plain.substring(0, p) ) ) {
|
||||
// ok this is bearer token
|
||||
config.bearerToken(token)
|
||||
// setup the refresh
|
||||
config.refreshToken(refreshToken)
|
||||
config.refreshTokenUrl(refreshUrl)
|
||||
config.refreshCookiePolicy(CookiePolicy.ACCEPT_ALL)
|
||||
return
|
||||
}
|
||||
}
|
||||
catch ( Exception e ) {
|
||||
log.trace "Enable to set bearer token ~ Reason: $e.message"
|
||||
}
|
||||
|
||||
// fallback on simple token
|
||||
config.basicAuth(TOKEN_PREFIX + token)
|
||||
}
|
||||
|
||||
String getAccessToken() {
|
||||
if( !accessToken )
|
||||
throw new AbortRunException("Missing Seqera Platform access token -- Make sure there's a variable TOWER_ACCESS_TOKEN in your environment")
|
||||
return accessToken
|
||||
}
|
||||
|
||||
/**
|
||||
* Little helper method that sends a HTTP POST message as JSON with
|
||||
* the current run status, ISO 8601 UTC timestamp, run name and the TraceRecord
|
||||
* object, if present.
|
||||
* @param event The current run status. One of {'started', 'process_submit', 'process_start',
|
||||
* 'process_complete', 'error', 'completed'}
|
||||
* @param payload An additional object to send. Must be of type TraceRecord or Manifest
|
||||
*/
|
||||
protected Response sendHttpMessage(String url, Map payload, String method='POST') {
|
||||
|
||||
// The actual HTTP request
|
||||
final String json = payload != null ? generator.toJson(payload) : null
|
||||
final String debug = json != null ? JsonOutput.prettyPrint(json).indent() : '-'
|
||||
log.trace "HTTP url=$url; payload:\n${debug}\n"
|
||||
try {
|
||||
final resp = httpClient.sendAsString(makeRequest(url, json, method))
|
||||
final status = resp.statusCode()
|
||||
if( status == 401 ) {
|
||||
final msg = 'Unauthorized Seqera Platform API access -- Make sure you have specified the correct access token'
|
||||
return new Response(status, msg)
|
||||
}
|
||||
if( status>=400 ) {
|
||||
final msg = parseCause(resp?.body()) ?: "Unexpected response for request $url"
|
||||
return new Response(status, msg as String)
|
||||
}
|
||||
else {
|
||||
return new Response(status, resp.body())
|
||||
}
|
||||
}
|
||||
catch( IOException e ) {
|
||||
String msg = "Unable to connect to Seqera Platform API: ${getHostUrl(url)}"
|
||||
return new Response(0, msg)
|
||||
}
|
||||
}
|
||||
|
||||
Response sendApiRequest(String url, Map payload=null, String method='GET') {
|
||||
sendHttpMessage(url, payload, method)
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a GET request and return the response body as a streaming {@link InputStream}
|
||||
* instead of buffering the entire response into a {@link String}.
|
||||
* Uses {@code HxClient.sendAsStream()} which goes through the same retry and
|
||||
* auth chain as {@code sendAsString()}.
|
||||
*
|
||||
* Status codes are checked before returning — on error the stream is closed and
|
||||
* the same exceptions as {@link #checkResponse} are thrown.
|
||||
*
|
||||
* @param url the full API URL to GET
|
||||
* @return an InputStream over the response body
|
||||
* @throws UnauthorizedException on 401
|
||||
* @throws ForbiddenException on 403
|
||||
* @throws NotFoundException on 404
|
||||
*/
|
||||
InputStream sendStreamingRequest(String url) {
|
||||
log.trace "HTTP streaming GET url=$url"
|
||||
final req = makeRequest(url, null, 'GET')
|
||||
final resp = httpClient.sendAsStream(req)
|
||||
final status = resp.statusCode()
|
||||
if( status >= 200 && status < 300 )
|
||||
return resp.body()
|
||||
// Error — close the stream and throw
|
||||
resp.body()?.close()
|
||||
if( status == 401 )
|
||||
throw new UnauthorizedException("Seqera authentication failed — check tower.accessToken or TOWER_ACCESS_TOKEN")
|
||||
if( status == 403 )
|
||||
throw new ForbiddenException("Forbidden — check permissions")
|
||||
if( status == 404 )
|
||||
throw new NotFoundException("Resource $url not found")
|
||||
throw new IOException("Seqera API error: HTTP ${status} for ${url}")
|
||||
}
|
||||
|
||||
protected HttpRequest makeRequest(String url, String payload, String verb) {
|
||||
final builder = HttpRequest.newBuilder(URI.create(url))
|
||||
.header('User-Agent', "Nextflow/$BuildInfo.version")
|
||||
.header('Traceparent', TraceUtils.rndTrace())
|
||||
.timeout(java.time.Duration.ofMillis(readTimeout.millis))
|
||||
|
||||
if( verb == 'GET' )
|
||||
return builder.GET().build()
|
||||
|
||||
if( verb == 'DELETE' )
|
||||
return builder.DELETE().build()
|
||||
|
||||
assert payload, "Tower request cannot be empty"
|
||||
builder.header('Content-Type', 'application/json; charset=utf-8')
|
||||
|
||||
if( verb == 'PUT' )
|
||||
return builder.PUT(HttpRequest.BodyPublishers.ofString(payload)).build()
|
||||
|
||||
if( verb == 'POST' )
|
||||
return builder.POST(HttpRequest.BodyPublishers.ofString(payload)).build()
|
||||
|
||||
throw new IllegalArgumentException("Unsupported HTTP verb: $verb")
|
||||
}
|
||||
|
||||
/**
|
||||
* Little helper function that can be called for logging upon an incoming HTTP response
|
||||
*/
|
||||
protected void logHttpResponse(String url, Response resp) {
|
||||
if (resp.code >= 200 && resp.code < 300) {
|
||||
log.trace "Successfully send message to ${url} -- received status code ${resp.code}"
|
||||
}
|
||||
else {
|
||||
def cause = parseCause(resp.cause)
|
||||
def msg = """\
|
||||
Unexpected HTTP response.
|
||||
Failed to send message to ${endpoint} -- received
|
||||
- status code : $resp.code
|
||||
- response msg: $resp.message
|
||||
""".stripIndent(true)
|
||||
// append separately otherwise formatting get broken
|
||||
msg += "- error cause : ${cause ?: '-'}"
|
||||
log.warn(msg)
|
||||
}
|
||||
}
|
||||
|
||||
protected Map parseTowerResponse(Response resp) {
|
||||
if (resp.code >= 200 && resp.code < 300) {
|
||||
return (Map)new JsonSlurper().parseText(resp.message)
|
||||
}
|
||||
|
||||
def cause = parseCause(resp.cause)
|
||||
|
||||
def msg = """\
|
||||
Unexpected Seqera Platform API response
|
||||
- endpoint url: $endpoint
|
||||
- status code : $resp.code
|
||||
- response msg: ${resp.message}
|
||||
""".stripIndent(true)
|
||||
// append separately otherwise formatting get broken
|
||||
msg += "- error cause : ${cause ?: '-'}"
|
||||
throw new AbortRunException(msg)
|
||||
}
|
||||
|
||||
protected String parseCause(String cause) {
|
||||
if( !cause )
|
||||
return null
|
||||
try {
|
||||
def map = (Map)new JsonSlurper().parseText(cause)
|
||||
return map.message
|
||||
}
|
||||
catch ( Exception ) {
|
||||
log.debug "Unable to parse error cause as JSON object: $cause"
|
||||
return cause
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
protected Map<String,Integer> loadSchema() {
|
||||
final props = new Properties()
|
||||
props.load(this.getClass().getResourceAsStream('/tower-schema.properties'))
|
||||
final result = new HashMap<String,Integer>(props.size())
|
||||
for( String key : props.keySet() ) {
|
||||
final value = props.getProperty(key)
|
||||
result.put( key, value ? value as Integer : null )
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
String buildUrl( String path, Map queryParams) {
|
||||
def url = new StringBuilder(endpoint)
|
||||
if( !path.startsWith('/') ) {
|
||||
url.append('/')
|
||||
}
|
||||
url.append(path)
|
||||
|
||||
if( queryParams && !queryParams.isEmpty() ) {
|
||||
url.append('?')
|
||||
url.append(queryParams.collect { k, v -> "${URLEncoder.encode(k.toString(), 'UTF-8')}=${URLEncoder.encode(v.toString(), 'UTF-8')}" }.join('&'))
|
||||
}
|
||||
|
||||
return url.toString()
|
||||
}
|
||||
|
||||
Map apiGet(String path, Map queryParams = [:]) {
|
||||
final url = buildUrl( path, queryParams)
|
||||
|
||||
final response = sendApiRequest(url)
|
||||
checkResponse(response, url)
|
||||
return new JsonSlurper().parseText(response.message) as Map
|
||||
}
|
||||
|
||||
Map apiPost(String path, Map queryParams, Map payload) {
|
||||
final url = buildUrl( path, queryParams)
|
||||
final response= sendApiRequest(url, payload, 'POST')
|
||||
checkResponse(response, url)
|
||||
return response.message ? new JsonSlurper().parseText(response.message) as Map : [:]
|
||||
}
|
||||
|
||||
/**
|
||||
* @return current user info (id, userName, etc.) from GET /user-info
|
||||
*/
|
||||
Map<String, Object> getUserInfo() {
|
||||
final json = apiGet("/user-info")
|
||||
return json.user as Map<String, Object>
|
||||
}
|
||||
|
||||
/**
|
||||
* Calls the Seqera Platform to retrieve the workflow information.
|
||||
*
|
||||
* @param workflowId Id of the workflow
|
||||
* @return Map containing workflow information
|
||||
* @throws RuntimeException if the API call fails
|
||||
*/
|
||||
Map getWorkflowDetails( String workflowId, Map queryParams = [:]) {
|
||||
final json = apiGet("/workflow/${workflowId}", queryParams)
|
||||
return json.workflow as Map
|
||||
}
|
||||
|
||||
|
||||
List<Map> listUserWorkspacesAndOrgs(String userId) {
|
||||
final json = apiGet("/user/${userId}/workspaces")
|
||||
return json.orgsAndWorkspaces as List<Map>
|
||||
}
|
||||
|
||||
private static void checkResponse(Response resp, String url) {
|
||||
if (!resp.error) return
|
||||
final code = resp.code
|
||||
if (code == 401)
|
||||
throw new UnauthorizedException("Seqera authentication failed — check tower.accessToken or TOWER_ACCESS_TOKEN")
|
||||
if (code == 403)
|
||||
throw new ForbiddenException("Forbidden — check permissions")
|
||||
if (code == 404)
|
||||
throw new NotFoundException("Resource $url not found")
|
||||
throw new Exception("Seqera API error: HTTP ${code} for ${url}${resp.message ? ' - ' + resp.message :''}")
|
||||
}
|
||||
|
||||
/**
|
||||
* Calls the Seqera Platform to retrieve the user's workspaces information
|
||||
* and select the one matching with the workspace Id.
|
||||
*
|
||||
* @param userId Id of the workspace user
|
||||
* @param workspaceId Id of the workspace
|
||||
* @return Map containing workspace information
|
||||
* @throws RuntimeException if the API call fails
|
||||
*/
|
||||
Map getUserWorkspaceDetails( String userId, String workspaceId) {
|
||||
if( !userId || !workspaceId ) {
|
||||
return null
|
||||
}
|
||||
try {
|
||||
final orgsAndWorkspaces = listUserWorkspacesAndOrgs(userId)
|
||||
|
||||
final workspace = orgsAndWorkspaces.find { ((Map) it).workspaceId?.toString() == workspaceId }
|
||||
if( workspace ) {
|
||||
final ws = workspace as Map
|
||||
return [
|
||||
orgName : ws.orgName,
|
||||
workspaceId : ws.workspaceId,
|
||||
workspaceName : ws.workspaceName,
|
||||
workspaceFullName: ws.workspaceFullName,
|
||||
roles : ws.roles
|
||||
]
|
||||
}
|
||||
|
||||
return null
|
||||
} catch( Exception e ) {
|
||||
log.debug("Failed to get workspace details for workspace ${workspaceId}: ${e.message}", e)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.config.spec.ConfigOption
|
||||
import nextflow.config.spec.ConfigScope
|
||||
import nextflow.config.spec.ScopeName
|
||||
import nextflow.script.dsl.Description
|
||||
import nextflow.platform.PlatformHelper
|
||||
import nextflow.util.Duration
|
||||
|
||||
/**
|
||||
* Model Seqera Platform configuration
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@ScopeName("tower")
|
||||
@Description("""
|
||||
The `tower` scope controls the settings for [Seqera Platform](https://seqera.io) (formerly Tower Cloud).
|
||||
""")
|
||||
@CompileStatic
|
||||
class TowerConfig implements ConfigScope {
|
||||
|
||||
static final Duration DEFAULT_CONNECT_TIMEOUT = Duration.of('60s')
|
||||
|
||||
static final Duration DEFAULT_READ_TIMEOUT = Duration.of('60s')
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The unique access token for your Seqera Platform account.
|
||||
""")
|
||||
final String accessToken
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The Compute Environment ID in Seqera Platform in which to launch the run (default: the primary environment in the workspace).
|
||||
""")
|
||||
final String computeEnvId
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Enable workflow monitoring with Seqera Platform (default: `false`).
|
||||
""")
|
||||
final boolean enabled
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The endpoint of your Seqera Platform instance (default: `https://api.cloud.seqera.io`).
|
||||
""")
|
||||
final String endpoint
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The HTTP connection timeout for Seqera Platform API requests (default: `'60s'`).
|
||||
""")
|
||||
final Duration httpConnectTimeout
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The HTTP read timeout for Seqera Platform API requests (default: `'60s'`).
|
||||
""")
|
||||
final Duration httpReadTimeout
|
||||
|
||||
final TowerRetryPolicy retryPolicy
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The workspace ID in Seqera Platform in which to save the run (default: the launching user's personal workspace).
|
||||
""")
|
||||
final String workspaceId
|
||||
|
||||
/* required by extension point -- do not remove */
|
||||
TowerConfig() {}
|
||||
|
||||
TowerConfig(Map opts, Map<String,String> env) {
|
||||
this.accessToken = PlatformHelper.getAccessToken(opts, env)
|
||||
if( opts.computeEnvId )
|
||||
this.computeEnvId = opts.computeEnvId as String
|
||||
this.enabled = opts.enabled as boolean
|
||||
this.endpoint = PlatformHelper.getEndpoint(opts, env)
|
||||
this.httpConnectTimeout = opts.httpConnectTimeout ? opts.httpConnectTimeout as Duration : DEFAULT_CONNECT_TIMEOUT
|
||||
this.httpReadTimeout = opts.httpReadTimeout ? opts.httpReadTimeout as Duration : DEFAULT_READ_TIMEOUT
|
||||
this.retryPolicy = new TowerRetryPolicy(opts.retryPolicy as Map ?: Map.of(), opts)
|
||||
this.workspaceId = PlatformHelper.getWorkspaceId(opts, env)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.Memoized
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.Global
|
||||
import nextflow.Session
|
||||
import nextflow.SysEnv
|
||||
import nextflow.file.http.XAuthProvider
|
||||
import nextflow.file.http.XAuthRegistry
|
||||
import nextflow.trace.TraceObserverFactoryV2
|
||||
import nextflow.trace.TraceObserverV2
|
||||
import nextflow.util.Duration
|
||||
/**
|
||||
* Create and register the Tower observer instance
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class TowerFactory implements TraceObserverFactoryV2 {
|
||||
|
||||
private Map<String,String> env
|
||||
|
||||
TowerFactory(){
|
||||
env = SysEnv.get()
|
||||
}
|
||||
|
||||
@Override
|
||||
Collection<TraceObserverV2> create(Session session) {
|
||||
final config = new TowerConfig(session.config.tower as Map ?: Collections.emptyMap(), env)
|
||||
if( !isEnabled(session, config, env) )
|
||||
return Collections.emptyList()
|
||||
final result = new ArrayList<TraceObserverV2>(1)
|
||||
// create the tower observer
|
||||
result.add( new TowerObserver(session, client(session, env), config.workspaceId, env))
|
||||
// create the logs checkpoint
|
||||
if( session.cloudCachePath )
|
||||
result.add( new LogsCheckpoint() )
|
||||
return result
|
||||
}
|
||||
|
||||
@Memoized
|
||||
static TowerClient client(Session session, Map env) {
|
||||
final opts = session.config.tower as Map ?: Collections.emptyMap()
|
||||
final config = new TowerConfig(opts, env)
|
||||
final tower = new TowerClient(config)
|
||||
// register auth provider
|
||||
// note: this is needed to authorize access to resources via XFileSystemProvider used by NF
|
||||
// it's not needed by the tower client logic
|
||||
XAuthRegistry.instance.register(provider(config.endpoint, config.accessToken))
|
||||
return tower
|
||||
}
|
||||
|
||||
static protected XAuthProvider provider(String endpoint, String accessToken) {
|
||||
if (endpoint.endsWith('/'))
|
||||
throw new IllegalArgumentException("Seqera Platform endpoint URL should not end with a `/` character -- offending value: $endpoint")
|
||||
final refreshToken = SysEnv.get('TOWER_REFRESH_TOKEN')
|
||||
return new TowerXAuth(endpoint, accessToken, refreshToken)
|
||||
}
|
||||
|
||||
private static boolean isEnabled(Session session, TowerConfig config, Map<String,String> env) {
|
||||
return config.enabled || env.get('TOWER_WORKFLOW_ID') || session.config.navigate('fusion.enabled') as Boolean
|
||||
}
|
||||
|
||||
static TowerClient client() {
|
||||
return client(Global.session as Session, SysEnv.get())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin
|
||||
|
||||
import java.time.Duration
|
||||
import java.time.Instant
|
||||
import java.time.temporal.ChronoUnit
|
||||
|
||||
import com.google.common.cache.Cache
|
||||
import com.google.common.cache.CacheBuilder
|
||||
import com.google.common.util.concurrent.UncheckedExecutionException
|
||||
import com.google.gson.JsonSyntaxException
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import io.seqera.tower.plugin.exception.BadResponseException
|
||||
import io.seqera.tower.plugin.exception.UnauthorizedException
|
||||
import io.seqera.tower.plugin.exchange.GetLicenseTokenRequest
|
||||
import io.seqera.tower.plugin.exchange.GetLicenseTokenResponse
|
||||
import nextflow.SysEnv
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.exception.ReportWarningException
|
||||
import nextflow.fusion.FusionConfig
|
||||
import nextflow.fusion.FusionToken
|
||||
import nextflow.platform.PlatformHelper
|
||||
import nextflow.plugin.Priority
|
||||
import nextflow.serde.gson.GsonEncoder
|
||||
import org.pf4j.Extension
|
||||
/**
|
||||
* Environment provider for Platform-specific environment variables.
|
||||
*
|
||||
* @author Alberto Miranda <alberto.miranda@seqera.io>
|
||||
*/
|
||||
@Slf4j
|
||||
@Extension
|
||||
@CompileStatic
|
||||
@Priority(-10)
|
||||
class TowerFusionToken implements FusionToken {
|
||||
|
||||
// The path relative to the Platform endpoint where license-scoped JWT tokens are obtained
|
||||
private static final String LICENSE_TOKEN_PATH = 'license/token/'
|
||||
|
||||
// The TowerClient instance used to send requests
|
||||
private TowerClient client
|
||||
|
||||
// Time-to-live for cached tokens
|
||||
private Duration tokenTTL = Duration.of(1, ChronoUnit.HOURS)
|
||||
|
||||
// Cache used for storing license tokens
|
||||
private Cache<String, GetLicenseTokenResponse> tokenCache = CacheBuilder.newBuilder()
|
||||
.expireAfterWrite(tokenTTL)
|
||||
.build()
|
||||
|
||||
// Platform endpoint to use for requests
|
||||
private String endpoint
|
||||
|
||||
// Platform access token to use for requests
|
||||
private volatile String accessToken
|
||||
|
||||
private volatile String refreshToken
|
||||
|
||||
// Platform workflowId
|
||||
private String workspaceId
|
||||
|
||||
// Platform workflowId
|
||||
private String workflowId
|
||||
|
||||
TowerFusionToken() {
|
||||
final config = PlatformHelper.config()
|
||||
final env = SysEnv.get()
|
||||
this.endpoint = PlatformHelper.getEndpoint(config, env)
|
||||
this.accessToken = PlatformHelper.getAccessToken(config, env)
|
||||
this.refreshToken = PlatformHelper.getRefreshToken(config, env)
|
||||
this.workflowId = env.get('TOWER_WORKFLOW_ID')
|
||||
this.workspaceId = PlatformHelper.getWorkspaceId(config, env)
|
||||
this.client = TowerFactory.client()
|
||||
}
|
||||
|
||||
protected void validateConfig() {
|
||||
if( !endpoint )
|
||||
throw new IllegalArgumentException("Missing Seqera Platform endpoint")
|
||||
if( !accessToken )
|
||||
throw new IllegalArgumentException("Seqera Platform access token is required to use Fusion -- see https://docs.seqera.io/fusion/licensing for more information")
|
||||
}
|
||||
|
||||
/**
|
||||
* Return any environment variables relevant to Fusion execution. This method is called
|
||||
* by {@link nextflow.fusion.FusionEnvProvider#getEnvironment} to determine which
|
||||
* environment variables are needed for the current run.
|
||||
*
|
||||
* @param scheme The scheme for which the environment variables are needed (currently unused)
|
||||
* @param config The Fusion configuration object
|
||||
* @return A map of environment variables
|
||||
*/
|
||||
@Override
|
||||
Map<String, String> getEnvironment(String scheme, FusionConfig config) {
|
||||
try {
|
||||
return getEnvironment0(scheme, config)
|
||||
}
|
||||
catch (Exception e) {
|
||||
final msg = "Unable to validate Fusion license - reason: ${e.message}"
|
||||
throw new ReportWarningException(msg, 'getFusionLicenseException', e)
|
||||
}
|
||||
}
|
||||
|
||||
protected Map<String,String> getEnvironment0(String scheme, FusionConfig config) {
|
||||
validateConfig()
|
||||
final product = config.sku()
|
||||
final version = config.version()
|
||||
final token = getLicenseToken(product, version)
|
||||
return Map.of('FUSION_LICENSE_TOKEN', token)
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a request to Platform to obtain a license-scoped JWT for Fusion. The request is authenticated using the
|
||||
* Platform access token provided in the configuration of the current session.
|
||||
*
|
||||
* @throws AbortOperationException if a Platform access token cannot be found
|
||||
*
|
||||
* @return The signed JWT token
|
||||
*/
|
||||
protected String getLicenseToken(String product, String version) {
|
||||
final req = new GetLicenseTokenRequest(
|
||||
product: product,
|
||||
version: version ?: 'unknown',
|
||||
workflowId: workflowId,
|
||||
workspaceId: workspaceId
|
||||
)
|
||||
final key = '${product}-${version}'
|
||||
try {
|
||||
final now = Instant.now()
|
||||
int i=0
|
||||
while( i++<2 ) {
|
||||
final resp = tokenCache.get(key, () -> sendRequest(req))
|
||||
if( resp.error )
|
||||
throw resp.error
|
||||
// Check if the cached response has expired
|
||||
// It's needed because the JWT token TTL in the cache (1 hour) and its expiration date (e.g. 1 day?) are not sync'ed,
|
||||
// so it could happen that we get a token from the cache which was valid at the time of insertion but is now expired.
|
||||
if( resp.expiresAt.isBefore(now) ) {
|
||||
log.debug "Cached token already expired; refreshing"
|
||||
tokenCache.invalidate(key)
|
||||
}
|
||||
else
|
||||
return resp.signedToken
|
||||
}
|
||||
}
|
||||
catch (UncheckedExecutionException e) {
|
||||
// most likely the exception is thrown for the lack of license
|
||||
// to avoid to keep requesting it, and error response is added to the cache
|
||||
tokenCache.put(key, new GetLicenseTokenResponse(error: e.cause))
|
||||
throw e.cause
|
||||
}
|
||||
}
|
||||
|
||||
/**************************************************************************
|
||||
* Helper methods
|
||||
*************************************************************************/
|
||||
|
||||
/**
|
||||
* Parse a JSON string into a {@link GetLicenseTokenResponse} object
|
||||
*
|
||||
* @param json The String containing the JSON representation of the LicenseTokenResponse object
|
||||
* @return The resulting LicenseTokenResponse object
|
||||
*
|
||||
* @throws JsonSyntaxException if the JSON string is not well-formed
|
||||
*/
|
||||
protected static GetLicenseTokenResponse parseLicenseTokenResponse(String json) throws JsonSyntaxException {
|
||||
final gson = new GsonEncoder<GetLicenseTokenResponse>() {}
|
||||
return gson.decode(json)
|
||||
}
|
||||
|
||||
/**
|
||||
* Request a license token from Platform.
|
||||
*
|
||||
* @param request The LicenseTokenRequest object
|
||||
* @return The LicenseTokenResponse object
|
||||
*/
|
||||
private GetLicenseTokenResponse sendRequest(GetLicenseTokenRequest request) {
|
||||
final url = "${client.getEndpoint()}/${LICENSE_TOKEN_PATH}"
|
||||
final resp = client.sendHttpMessage(url, request.toMap())
|
||||
|
||||
if( resp.code == 200 ) {
|
||||
final ret = parseLicenseTokenResponse(resp.message)
|
||||
return ret
|
||||
}
|
||||
|
||||
if( resp.code == 401 ) {
|
||||
throw new UnauthorizedException("Unauthorized [401] - Verify you have provided a Seqera Platform valid access token")
|
||||
}
|
||||
|
||||
throw new BadResponseException("Invalid response: ${url} [${resp.code}] ${resp.message} -- ${resp.cause}")
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin
|
||||
|
||||
import java.nio.file.Path
|
||||
import java.time.Instant
|
||||
import java.time.OffsetDateTime
|
||||
|
||||
import groovy.json.DefaultJsonGenerator
|
||||
import groovy.json.JsonGenerator
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.Const
|
||||
import nextflow.NextflowMeta
|
||||
import nextflow.trace.ProgressRecord
|
||||
import nextflow.util.Duration
|
||||
import org.apache.groovy.json.internal.CharBuf
|
||||
/**
|
||||
* Customized json generator that chomp string values
|
||||
* longer than expected size
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class TowerJsonGenerator extends DefaultJsonGenerator {
|
||||
|
||||
private List<String> stack = new ArrayList<>(10)
|
||||
Map<String,Integer> scheme
|
||||
|
||||
static TowerJsonGenerator create(Map<String,Integer> scheme) {
|
||||
final opts = new JsonGenerator.Options()
|
||||
.addConverter(Path) { Path p, String key -> p.toUriString() }
|
||||
.addConverter(Duration) { Duration d, String key -> d.durationInMillis }
|
||||
.addConverter(NextflowMeta) { NextflowMeta m, String key -> m.toJsonMap() }
|
||||
.addConverter(OffsetDateTime) { it.toString() }
|
||||
.addConverter(Instant) { it.toString() }
|
||||
.dateFormat(Const.ISO_8601_DATETIME_FORMAT).timezone("UTC")
|
||||
|
||||
return new TowerJsonGenerator(opts, scheme)
|
||||
}
|
||||
|
||||
protected TowerJsonGenerator(Options options, Map<String,Integer> scheme) {
|
||||
super(options)
|
||||
this.scheme = scheme
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Map<?, ?> getObjectProperties(Object object) {
|
||||
final result = super.getObjectProperties(object)
|
||||
if( object instanceof ProgressRecord ) {
|
||||
result.remove('hash')
|
||||
result.remove('errored')
|
||||
result.remove('completedCount')
|
||||
result.remove('totalCount')
|
||||
result.remove('taskName')
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
@Override
|
||||
protected synchronized void writeObject(String key, Object object, CharBuf buffer) {
|
||||
final pos = stack.size()
|
||||
if(key) stack.add(pos, key)
|
||||
final fqn = stack.join('.')
|
||||
try {
|
||||
if( fqn == 'workflow.manifest.gitmodules' && object instanceof List ) {
|
||||
writeCharSequence(object.join(','), buffer)
|
||||
}
|
||||
else
|
||||
super.writeObject(key,object,buffer)
|
||||
}
|
||||
catch( Exception e ) {
|
||||
log.warn1 ("Unable to serialize key=$fqn; value=${safeString0(object)}; type=${object?.getClass()?.getName()} -- Cause: ${e.message ?: e}", causedBy: e)
|
||||
}
|
||||
finally {
|
||||
if(key) stack.remove(pos)
|
||||
}
|
||||
}
|
||||
|
||||
private String safeString0(Object obj) {
|
||||
try {
|
||||
return obj !=null ? obj.toString() : null
|
||||
}
|
||||
catch( Throwable e ) {
|
||||
log.debug "SafeString error=${e.message ?: e}"
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void writeRaw(CharSequence seq, CharBuf buffer) {
|
||||
super.writeRaw(chompString(seq),buffer)
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void writeCharSequence(CharSequence seq, CharBuf buffer) {
|
||||
super.writeCharSequence(chompString(seq), buffer)
|
||||
}
|
||||
|
||||
/**
|
||||
* Make sure the specified argument is not longer than the expected length
|
||||
* defined in the `scheme` object
|
||||
*
|
||||
* @param seq The string object as {@link CharSequence} instance
|
||||
* @return A string object whose length does not exceed the max for the current key entry
|
||||
*/
|
||||
final protected CharSequence chompString(CharSequence seq) {
|
||||
final key = stack.join('.')
|
||||
final max = scheme.get(key)
|
||||
if( seq!=null && max && seq.length()>max ) {
|
||||
final result = seq.toString().substring(0,max)
|
||||
// show only the first 100 chars in the log as a preview
|
||||
final preview = result.length()>100 ? result.substring(0,100) + '(truncated)' : result
|
||||
log.warn "Seqera Platform request field `$key` exceeds expected size | offending value: `${preview}`, size: ${seq.size()} (max: $max)"
|
||||
return result
|
||||
}
|
||||
return seq
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,582 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.ToString
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.Session
|
||||
import nextflow.container.resolver.ContainerMeta
|
||||
import nextflow.exception.AbortRunException
|
||||
import nextflow.processor.TaskHandler
|
||||
import nextflow.processor.TaskId
|
||||
import nextflow.processor.TaskProcessor
|
||||
import nextflow.script.PlatformMetadata
|
||||
import nextflow.trace.ResourcesAggregator
|
||||
import nextflow.trace.TraceObserverV2
|
||||
import nextflow.trace.TraceRecord
|
||||
import nextflow.trace.event.FilePublishEvent
|
||||
import nextflow.trace.event.TaskEvent
|
||||
import nextflow.util.Duration
|
||||
import nextflow.util.LoggerHelper
|
||||
import nextflow.util.ProcessHelper
|
||||
import nextflow.util.Threads
|
||||
|
||||
import java.time.Instant
|
||||
import java.time.OffsetDateTime
|
||||
import java.time.ZoneId
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.LinkedBlockingQueue
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
/**
|
||||
* Send out messages via HTTP to a configured URL on different workflow
|
||||
* execution events.
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class TowerObserver implements TraceObserverV2 {
|
||||
|
||||
static private final int TASKS_PER_REQUEST = 100
|
||||
|
||||
static private final Duration REQUEST_INTERVAL = Duration.of('1 sec')
|
||||
|
||||
static private final Duration ALIVE_INTERVAL = Duration.of('1 min')
|
||||
|
||||
@ToString(includeNames = true)
|
||||
static class ProcessEvent {
|
||||
TraceRecord trace
|
||||
boolean completed
|
||||
}
|
||||
|
||||
private Session session
|
||||
|
||||
/**
|
||||
* Workflow identifier, will be taken from the Session() object later
|
||||
*/
|
||||
private String runName
|
||||
|
||||
/**
|
||||
* Store the sessions unique ID for downstream reference purposes
|
||||
*/
|
||||
private String runId
|
||||
|
||||
private String workflowId
|
||||
|
||||
private String watchUrl
|
||||
|
||||
private ResourcesAggregator aggregator
|
||||
|
||||
protected Map<String,String> env = System.getenv()
|
||||
|
||||
private LinkedBlockingQueue<ProcessEvent> events = new LinkedBlockingQueue()
|
||||
|
||||
private Thread sender
|
||||
|
||||
private Duration requestInterval = REQUEST_INTERVAL
|
||||
|
||||
private Duration aliveInterval = ALIVE_INTERVAL
|
||||
|
||||
private LinkedHashSet<String> processNames = new LinkedHashSet<>(20)
|
||||
|
||||
private boolean towerLaunch
|
||||
|
||||
private String workspaceId
|
||||
|
||||
private TowerReports reports
|
||||
|
||||
private TowerClient client
|
||||
|
||||
private Map<String,Boolean> allContainers = new ConcurrentHashMap<>()
|
||||
|
||||
TowerObserver(Session session, TowerClient client, String workspaceId, Map env) {
|
||||
this.session = session
|
||||
this.workspaceId = workspaceId
|
||||
this.reports = new TowerReports(session)
|
||||
this.client = client
|
||||
if( env )
|
||||
this.env = env
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
boolean enableMetrics() { true }
|
||||
|
||||
String getWorkflowId() { workflowId }
|
||||
|
||||
boolean getTowerLaunch() { towerLaunch }
|
||||
|
||||
String getRunName() { runName }
|
||||
|
||||
String getRunId() { runId }
|
||||
|
||||
void setAliveInterval(Duration d) {
|
||||
this.aliveInterval = d
|
||||
}
|
||||
|
||||
void setRequestInterval(Duration d) {
|
||||
this.requestInterval = d
|
||||
}
|
||||
|
||||
String getWorkspaceId() { workspaceId }
|
||||
|
||||
|
||||
/**
|
||||
* On workflow start, submit a message with some basic
|
||||
* information, like Id, activity and an ISO 8601 formatted
|
||||
* timestamp.
|
||||
* @param session The current Nextflow session object
|
||||
*/
|
||||
@Override
|
||||
void onFlowCreate(Session session) {
|
||||
log.debug "Creating Seqera Platform observer -- endpoint=$client.endpoint; requestInterval=$requestInterval; aliveInterval=$aliveInterval;"
|
||||
|
||||
this.session = session
|
||||
this.aggregator = new ResourcesAggregator()
|
||||
this.runName = session.getRunName()
|
||||
this.runId = session.getUniqueId()
|
||||
|
||||
// send hello to verify auth
|
||||
final ret = client.traceCreate(makeCreateReq(session), workspaceId)
|
||||
this.workflowId = ret.workflowId
|
||||
if( !workflowId )
|
||||
throw new AbortRunException("Invalid Seqera Platform API response - Missing workflow Id")
|
||||
log.debug "Platform workflow id: $workflowId; workflow url: ${ret.watchUrl}"
|
||||
session.workflowMetadata.platform.workflowId = workflowId
|
||||
// note: `watchUrl` in the create response requires Platform 26.01 or later
|
||||
this.watchUrl = ret.watchUrl as String
|
||||
session.workflowMetadata.platform.workflowUrl = watchUrl
|
||||
if( ret.message )
|
||||
log.warn(ret.message.toString())
|
||||
// populate platform metadata from the create response
|
||||
if( ret.metadata )
|
||||
applyPlatformMetadata(ret.metadata as Map)
|
||||
|
||||
// Prepare to collect report paths if tower configuration has a 'reports' section
|
||||
reports.flowCreate(workflowId)
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Apply platform metadata received inline from the trace create response.
|
||||
* This avoids extra API calls to fetch user, workspace, and launch details.
|
||||
*/
|
||||
protected void applyPlatformMetadata(Map metadata) {
|
||||
try {
|
||||
final platform = session.workflowMetadata.platform
|
||||
// user info
|
||||
if( metadata.userId )
|
||||
platform.user = new PlatformMetadata.User(
|
||||
id: metadata.userId as String,
|
||||
userName: metadata.userName as String,
|
||||
organization: metadata.userOrganization as String
|
||||
)
|
||||
// workspace info
|
||||
if( metadata.workspaceId )
|
||||
platform.workspace = new PlatformMetadata.Workspace(
|
||||
workspaceId: metadata.workspaceId as String,
|
||||
workspaceName: metadata.workspaceName as String,
|
||||
workspaceFullName: metadata.workspaceFullName as String,
|
||||
orgName: metadata.orgName as String
|
||||
)
|
||||
// launch details (only present for Platform-submitted runs)
|
||||
if( metadata.computeEnvId )
|
||||
platform.computeEnv = new PlatformMetadata.ComputeEnv(
|
||||
id: metadata.computeEnvId as String,
|
||||
name: metadata.computeEnvName as String,
|
||||
platform: metadata.computeEnvPlatform as String
|
||||
)
|
||||
if( metadata.pipelineName )
|
||||
platform.pipeline = new PlatformMetadata.Pipeline(
|
||||
id: metadata.pipelineId as String,
|
||||
name: metadata.pipelineName as String,
|
||||
revision: metadata.revision as String,
|
||||
commitId: metadata.commitId as String
|
||||
)
|
||||
if( metadata.labels )
|
||||
platform.labels = metadata.labels as List<String>
|
||||
}
|
||||
catch( Exception e ) {
|
||||
log.debug("Failed to apply platform metadata from create response", e)
|
||||
}
|
||||
}
|
||||
|
||||
protected Map makeCreateReq(Session session) {
|
||||
def result = new HashMap(5)
|
||||
result.sessionId = session.uniqueId.toString()
|
||||
result.runName = session.runName
|
||||
result.projectName = session.workflowMetadata.projectName
|
||||
result.repository = session.workflowMetadata.repository
|
||||
result.workflowId = env.get('TOWER_WORKFLOW_ID')
|
||||
result.instant = Instant.now().toEpochMilli()
|
||||
this.towerLaunch = result.workflowId != null
|
||||
return result
|
||||
}
|
||||
|
||||
@Override
|
||||
void onProcessCreate(TaskProcessor process) {
|
||||
log.trace "Creating process ${process.name}"
|
||||
if( !processNames.add(process.name) )
|
||||
throw new IllegalStateException("Process name `${process.name}` already used")
|
||||
}
|
||||
|
||||
@Override
|
||||
void onFlowBegin() {
|
||||
// configure error retry
|
||||
|
||||
final payload = client.traceBegin(makeBeginReq(session), workspaceId, workflowId)
|
||||
this.watchUrl ?= payload.watchUrl
|
||||
session.workflowMetadata.platform.workflowUrl ?= watchUrl
|
||||
this.sender = Threads.start('Tower-thread', this.&sendTasks0)
|
||||
final msg = "Monitor the execution with Seqera Platform using this URL: ${watchUrl}"
|
||||
log.info(LoggerHelper.STICKY, msg)
|
||||
}
|
||||
|
||||
/**
|
||||
* Send an HTTP message when the workflow is completed.
|
||||
*/
|
||||
@Override
|
||||
void onFlowComplete() {
|
||||
// publish runtime reports
|
||||
reports.publishRuntimeReports()
|
||||
// submit the completion record
|
||||
if( sender ) {
|
||||
events << new ProcessEvent(completed: true)
|
||||
// wait the submission of pending events
|
||||
sender.join()
|
||||
}
|
||||
// wait and flush reports content
|
||||
reports.flowComplete()
|
||||
// notify the workflow completion
|
||||
// note: only send complete if onFlowBegin was invoked (sender is set there)
|
||||
if( workflowId && sender ) {
|
||||
client.traceComplete(makeCompleteReq(session), workspaceId, workflowId)
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
void onTaskPending(TaskEvent event) {
|
||||
events << new ProcessEvent(trace: event.trace)
|
||||
}
|
||||
|
||||
/**
|
||||
* Send an HTTP message when a process has been submitted
|
||||
*
|
||||
* @param handler A {@link TaskHandler} object representing the task submitted
|
||||
* @param trace A {@link TraceRecord} object holding the task metadata and runtime info
|
||||
*/
|
||||
@Override
|
||||
void onTaskSubmit(TaskEvent event) {
|
||||
events << new ProcessEvent(trace: event.trace)
|
||||
}
|
||||
|
||||
/**
|
||||
* Send an HTTP message, when a process has started
|
||||
*
|
||||
* @param handler A {@link TaskHandler} object representing the task started
|
||||
* @param trace A {@link TraceRecord} object holding the task metadata and runtime info
|
||||
*/
|
||||
@Override
|
||||
void onTaskStart(TaskEvent event) {
|
||||
events << new ProcessEvent(trace: event.trace)
|
||||
}
|
||||
|
||||
/**
|
||||
* Send an HTTP message, when a process completed
|
||||
*
|
||||
* @param handler A {@link TaskHandler} object representing the task completed
|
||||
* @param trace A {@link TraceRecord} object holding the task metadata and runtime info
|
||||
*/
|
||||
@Override
|
||||
void onTaskComplete(TaskEvent event) {
|
||||
events << new ProcessEvent(trace: event.trace)
|
||||
|
||||
synchronized (this) {
|
||||
aggregator.aggregate(event.trace)
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
void onTaskCached(TaskEvent event) {
|
||||
// event was triggered by a stored task, ignore it
|
||||
if( !event.trace )
|
||||
return
|
||||
|
||||
// add the cached task event
|
||||
events << new ProcessEvent(trace: event.trace)
|
||||
|
||||
// remove the record from the current records
|
||||
synchronized (this) {
|
||||
aggregator.aggregate(event.trace)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Send an HTTP message, when a workflow has failed
|
||||
*
|
||||
* @param handler A {@link TaskHandler} object representing the task that caused the workflow execution to fail (it may be null)
|
||||
* @param trace A {@link TraceRecord} object holding the task metadata and runtime info (it may be null)
|
||||
*/
|
||||
@Override
|
||||
void onFlowError(TaskEvent event) {
|
||||
events << new ProcessEvent(trace: event.trace)
|
||||
}
|
||||
|
||||
/**
|
||||
* Update reports file when a file is published
|
||||
*
|
||||
* @param destination File path at `publishDir` of the published file.
|
||||
*/
|
||||
@Override
|
||||
void onFilePublish(FilePublishEvent event) {
|
||||
reports.filePublish(event.target)
|
||||
}
|
||||
|
||||
protected boolean isCliLogsEnabled() {
|
||||
return env.get('TOWER_ALLOW_NEXTFLOW_LOGS') == 'true'
|
||||
}
|
||||
|
||||
protected String getOperationId() {
|
||||
if( !isCliLogsEnabled() )
|
||||
return null
|
||||
try {
|
||||
if( env.get('AWS_BATCH_JOB_ID') )
|
||||
return "aws-batch::${env.get('AWS_BATCH_JOB_ID')}"
|
||||
else
|
||||
return "local-platform::${ProcessHelper.selfPid()}"
|
||||
}
|
||||
catch (Exception e) {
|
||||
log.warn "Unable to retrieve native environment operation id", e
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
protected String getLogFile() {
|
||||
return isCliLogsEnabled() ? env.get('NXF_LOG_FILE') : null
|
||||
}
|
||||
|
||||
protected String getOutFile() {
|
||||
return isCliLogsEnabled() ? env.get('NXF_OUT_FILE') : null
|
||||
}
|
||||
|
||||
protected Map makeBeginReq(Session session) {
|
||||
def workflow = session.getWorkflowMetadata().toMap()
|
||||
workflow.params = session.getParams()
|
||||
workflow.id = getWorkflowId()
|
||||
workflow.remove('stats')
|
||||
|
||||
// render as a string
|
||||
workflow.container = mapToString(workflow.container)
|
||||
workflow.configText = session.resolvedConfig
|
||||
// extra metadata
|
||||
workflow.operationId = getOperationId()
|
||||
workflow.logFile = getLogFile()
|
||||
workflow.outFile = getOutFile()
|
||||
|
||||
def result = new LinkedHashMap(5)
|
||||
result.workflow = workflow
|
||||
result.processNames = new ArrayList(processNames)
|
||||
result.towerLaunch = towerLaunch
|
||||
result.instant = Instant.now().toEpochMilli()
|
||||
return result
|
||||
}
|
||||
|
||||
protected Map makeCompleteReq(Session session) {
|
||||
def workflow = session.getWorkflowMetadata().toMap()
|
||||
//Remove retrieved platform info
|
||||
if( workflow.platform )
|
||||
workflow.remove('platform')
|
||||
|
||||
workflow.params = session.getParams()
|
||||
workflow.id = getWorkflowId()
|
||||
// render as a string
|
||||
workflow.container = mapToString(workflow.container)
|
||||
workflow.configText = session.resolvedConfig
|
||||
// extra metadata
|
||||
workflow.operationId = getOperationId()
|
||||
workflow.logFile = getLogFile()
|
||||
workflow.outFile = getOutFile()
|
||||
|
||||
def result = new LinkedHashMap(5)
|
||||
result.workflow = workflow
|
||||
result.metrics = getMetricsList()
|
||||
result.progress = getWorkflowProgress(false)
|
||||
result.instant = Instant.now().toEpochMilli()
|
||||
return result
|
||||
}
|
||||
|
||||
protected Map makeHeartbeatReq() {
|
||||
def result = new HashMap(1)
|
||||
result.progress = getWorkflowProgress(true)
|
||||
result.instant = Instant.now().toEpochMilli()
|
||||
return result
|
||||
}
|
||||
|
||||
protected String mapToString(def obj) {
|
||||
if( obj == null )
|
||||
return null
|
||||
if( obj instanceof CharSequence )
|
||||
return obj.toString()
|
||||
if( obj instanceof Map ) {
|
||||
// turn this off for multiple containers because the string representation is broken
|
||||
return null
|
||||
}
|
||||
throw new IllegalArgumentException("Illegal container attribute type: ${obj.getClass().getName()} = ${obj}" )
|
||||
}
|
||||
|
||||
protected Map makeTaskMap0(TraceRecord trace) {
|
||||
Map<String,?> record = new LinkedHashMap<>(trace.store.size())
|
||||
for( Map.Entry<String,Object> entry : trace.store.entrySet() ) {
|
||||
def name = entry.key
|
||||
// remove '%' char from field prefix
|
||||
if( name.startsWith('%') )
|
||||
name = 'p' + name.substring(1)
|
||||
// normalise to camelCase
|
||||
name = underscoreToCamelCase(name)
|
||||
// put the value
|
||||
record.put(name, fixTaskField(name,entry.value))
|
||||
}
|
||||
|
||||
// prevent invalid tag data
|
||||
if( record.tag!=null && !(record.tag instanceof CharSequence)) {
|
||||
final msg = "Invalid tag value for process: ${record.process} -- A string is expected instead of type: ${record.tag.getClass().getName()}; offending value=${record.tag}"
|
||||
log.warn1(msg, cacheKey: record.process)
|
||||
record.tag = null
|
||||
}
|
||||
|
||||
// add transient fields
|
||||
record.executor = trace.getExecutorName()
|
||||
record.cloudZone = trace.getMachineInfo()?.zone
|
||||
record.machineType = trace.getMachineInfo()?.type
|
||||
record.priceModel = trace.getMachineInfo()?.priceModel?.toString()
|
||||
record.numSpotInterruptions = trace.getNumSpotInterruptions()
|
||||
record.logStreamId = trace.getLogStreamId()
|
||||
record.resourceAllocation = trace.getResourceAllocation()
|
||||
record.gpuMetrics = trace.getGpuMetrics()
|
||||
return record
|
||||
}
|
||||
|
||||
|
||||
static protected Object fixTaskField(String name, value) {
|
||||
if( TraceRecord.FIELDS[name] == 'date' )
|
||||
return value ? OffsetDateTime.ofInstant(Instant.ofEpochMilli(value as long), ZoneId.systemDefault()) : null
|
||||
else
|
||||
return value
|
||||
}
|
||||
|
||||
protected Map makeTasksReq(Collection<TraceRecord> tasks) {
|
||||
|
||||
def payload = new ArrayList(tasks.size())
|
||||
for( TraceRecord rec : tasks ) {
|
||||
payload << makeTaskMap0(rec)
|
||||
}
|
||||
|
||||
final result = new LinkedHashMap(5)
|
||||
result.put('tasks', payload)
|
||||
result.put('progress', getWorkflowProgress(true))
|
||||
result.put('containers', getNewContainers(tasks))
|
||||
result.instant = Instant.now().toEpochMilli()
|
||||
return result
|
||||
}
|
||||
|
||||
protected List<ContainerMeta> getNewContainers(Collection<TraceRecord> tasks) {
|
||||
final result = new ArrayList<ContainerMeta>()
|
||||
for( TraceRecord it : tasks ) {
|
||||
final meta = it.getContainerMeta()
|
||||
if( meta && !allContainers.get(meta.targetImage) ) {
|
||||
allContainers.put(meta.targetImage, Boolean.TRUE)
|
||||
result.add(meta)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
protected List getMetricsList() {
|
||||
return aggregator.computeSummaryList()
|
||||
}
|
||||
|
||||
protected WorkflowProgress getWorkflowProgress(boolean quick) {
|
||||
def stats = quick ? session.getStatsObserver().getQuickStats() : session.getStatsObserver().getStats()
|
||||
new WorkflowProgress(stats)
|
||||
}
|
||||
|
||||
protected String underscoreToCamelCase(String str) {
|
||||
if( !str.contains('_') )
|
||||
return str
|
||||
|
||||
final words = str.tokenize('_')
|
||||
def result = words[0]
|
||||
for( int i=1; i<words.size(); i++ )
|
||||
result+=words[i].capitalize()
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
|
||||
protected void sendTasks0(dummy) {
|
||||
try {
|
||||
final tasks = new HashMap<TaskId, TraceRecord>(TASKS_PER_REQUEST)
|
||||
boolean complete = false
|
||||
long previous = System.currentTimeMillis()
|
||||
final long period = requestInterval.millis
|
||||
final long delay = period / 10 as long
|
||||
|
||||
while( !complete ) {
|
||||
final ProcessEvent ev = events.poll(delay, TimeUnit.MILLISECONDS)
|
||||
// reconcile task events ie. send out only the last event
|
||||
if( ev ) {
|
||||
log.trace "Tower event=$ev"
|
||||
if( ev.trace )
|
||||
tasks[ev.trace.taskId] = ev.trace
|
||||
if( ev.completed )
|
||||
complete = true
|
||||
}
|
||||
|
||||
// check if there's something to send
|
||||
final now = System.currentTimeMillis()
|
||||
final delta = now - previous
|
||||
|
||||
if( !tasks ) {
|
||||
if( delta > aliveInterval.millis ) {
|
||||
final req = makeHeartbeatReq()
|
||||
client.traceHeartbeat(req, workspaceId, workflowId)
|
||||
previous = now
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if( delta > period || tasks.size() >= TASKS_PER_REQUEST || complete ) {
|
||||
// send
|
||||
final req = makeTasksReq(tasks.values())
|
||||
client.traceProgress(req, workspaceId, workflowId)
|
||||
|
||||
// clean up for next iteration
|
||||
previous = now
|
||||
tasks.clear()
|
||||
}
|
||||
}
|
||||
}
|
||||
catch( Exception e ) {
|
||||
this.sender = null
|
||||
log.error("Aborting session due to Seqera Platform telemetry exception - $e.message", e)
|
||||
session.abort(e)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import io.seqera.tower.plugin.fs.SeqeraFileSystemProvider
|
||||
import nextflow.file.FileHelper
|
||||
import nextflow.plugin.BasePlugin
|
||||
import nextflow.cli.PluginExecAware
|
||||
import org.pf4j.PluginWrapper
|
||||
/**
|
||||
* Seqera Platform plugin
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class TowerPlugin extends BasePlugin implements PluginExecAware {
|
||||
|
||||
@Delegate private CacheCommand delegate
|
||||
|
||||
TowerPlugin(PluginWrapper wrapper) {
|
||||
super(wrapper)
|
||||
this.delegate = new CacheCommand()
|
||||
}
|
||||
|
||||
@Override
|
||||
void start() {
|
||||
super.start()
|
||||
FileHelper.getOrInstallProvider(SeqeraFileSystemProvider)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.seqera.tower.plugin
|
||||
|
||||
import java.nio.charset.Charset
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.PathMatcher
|
||||
import java.nio.file.Paths
|
||||
import java.nio.file.StandardCopyOption
|
||||
import java.time.Duration
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import groovy.yaml.YamlRuntimeException
|
||||
import groovy.yaml.YamlSlurper
|
||||
import groovyx.gpars.agent.Agent
|
||||
import nextflow.Session
|
||||
import nextflow.file.FileHelper
|
||||
import nextflow.trace.config.DagConfig
|
||||
import nextflow.trace.config.ReportConfig
|
||||
import nextflow.trace.config.TimelineConfig
|
||||
import nextflow.trace.config.TraceConfig
|
||||
/**
|
||||
* If reports are defined at `nf-<workflow_id>-tower.yml`, collects all published files
|
||||
* that are reports and writes `nf-<workflow_id>-reports.tsv` file with all the paths.
|
||||
*
|
||||
* @author Jordi Deu-Pons
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class TowerReports {
|
||||
|
||||
private Session session
|
||||
private Path launchReportsPath
|
||||
private Path workReportsPath
|
||||
private PrintWriter reportsFile
|
||||
private boolean processReports
|
||||
private Agent<PrintWriter> writer
|
||||
private List<Map.Entry<String, Map<String, String>>> reportsEntries
|
||||
private List<PathMatcher> matchers
|
||||
private YamlSlurper yamlSlurper
|
||||
private Timer timer
|
||||
private AtomicInteger totalReports
|
||||
|
||||
TowerReports(Session session) {
|
||||
this.session = session
|
||||
this.yamlSlurper = new YamlSlurper()
|
||||
this.totalReports = new AtomicInteger(0)
|
||||
}
|
||||
|
||||
/**
|
||||
* On flow create if there is a tower config yaml for current workflow
|
||||
* start writing a reports file at the background.
|
||||
*
|
||||
* @param workflowId Tower workflow ID
|
||||
*/
|
||||
void flowCreate(String workflowId) {
|
||||
Path launchDir = getLaunchDir()
|
||||
reportsEntries = parseReportEntries(launchDir, workflowId)
|
||||
processReports = reportsEntries.size() > 0
|
||||
if (processReports) {
|
||||
final fileName = System.getenv().getOrDefault("TOWER_REPORTS_FILE", "nf-${workflowId}-reports.tsv".toString()) as String
|
||||
this.launchReportsPath = launchDir.resolve(fileName)
|
||||
this.workReportsPath = session?.workDir?.resolve(fileName)
|
||||
this.reportsFile = new PrintWriter(Files.newBufferedWriter(launchReportsPath, Charset.defaultCharset()), true)
|
||||
this.writer = new Agent<PrintWriter>(reportsFile)
|
||||
|
||||
// send header
|
||||
this.writer.send { PrintWriter it -> it.println("key\tpath\tsize\tdisplay\tmime_type") }
|
||||
|
||||
// Schedule a reports copy if launchDir and workDir are different
|
||||
if (this.workReportsPath && this.launchReportsPath != this.workReportsPath) {
|
||||
final lastTotalReports = new AtomicInteger(0)
|
||||
final task = {
|
||||
// Copy the file only if there are new reports
|
||||
if (lastTotalReports.get() < this.totalReports.get()) {
|
||||
try {
|
||||
final total = this.totalReports.get()
|
||||
log.trace("Reports file sync to workdir with ${total} reports")
|
||||
FileHelper.copyPath(launchReportsPath, workReportsPath, StandardCopyOption.REPLACE_EXISTING)
|
||||
lastTotalReports.set(total)
|
||||
} catch (IOException e) {
|
||||
log.error("Error copying reports file ${launchReportsPath.toUriString()} to the workdir ${workReportsPath.toUriString()} -- ${e.message}")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Copy maximum 1 time per minute
|
||||
final oneMinute = Duration.ofMinutes(1).toMillis()
|
||||
this.timer = new Timer()
|
||||
this.timer.schedule(task, oneMinute, oneMinute)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve current launch dir
|
||||
*
|
||||
* @return Launch directory path
|
||||
*/
|
||||
protected Path getLaunchDir() {
|
||||
return Paths.get('.').toRealPath()
|
||||
}
|
||||
|
||||
/**
|
||||
* On flow complete stop writing the reports file at background.
|
||||
*/
|
||||
void flowComplete() {
|
||||
if (processReports) {
|
||||
if (timer) {
|
||||
timer.cancel()
|
||||
}
|
||||
writer.await()
|
||||
// close and upload it
|
||||
reportsFile.flush()
|
||||
reportsFile.close()
|
||||
saveReportsFileUpload()
|
||||
}
|
||||
}
|
||||
|
||||
protected void saveReportsFileUpload() {
|
||||
try {
|
||||
FileHelper.copyPath(launchReportsPath, workReportsPath, StandardCopyOption.REPLACE_EXISTING)
|
||||
log.debug "Saved reports file ${workReportsPath.toUriString()}"
|
||||
}
|
||||
catch (Exception e) {
|
||||
log.error("Error copying reports file ${launchReportsPath.toUriString()} to the workdir ${workReportsPath.toUriString()} -- ${e.message}")
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load all report entries from tower config yaml file.
|
||||
*
|
||||
* @param launchDir Nextflow launch directory
|
||||
* @param workflowId Tower workflow ID
|
||||
*/
|
||||
protected List<Map.Entry<String, Map<String, String>>> parseReportEntries(Path launchDir, String workflowId) {
|
||||
Path towerConfigPath = launchDir.resolve("nf-${workflowId}-tower.yml")
|
||||
|
||||
// Check if Tower config file is define at assets
|
||||
if (!Files.exists(towerConfigPath)) {
|
||||
towerConfigPath = this.session?.baseDir?.resolve("tower.yml")
|
||||
}
|
||||
|
||||
// Load reports definitions if available
|
||||
List<Map.Entry<String, Map<String, String>>> reportsEntries = []
|
||||
if (towerConfigPath && Files.exists(towerConfigPath)) {
|
||||
try {
|
||||
final towerConfig = this.yamlSlurper.parse(towerConfigPath)
|
||||
if (towerConfig instanceof Map && towerConfig.containsKey("reports")) {
|
||||
Map<String, Map<String, String>> reports = (Map<String, Map<String, String>>) towerConfig.get("reports")
|
||||
for (final e : reports) {
|
||||
reportsEntries.add(e)
|
||||
}
|
||||
}
|
||||
} catch (YamlRuntimeException e) {
|
||||
final msg = e?.cause?.message ?: e.message
|
||||
throw new IllegalArgumentException("Invalid tower.yml format -- ${msg}")
|
||||
}
|
||||
}
|
||||
|
||||
return reportsEntries
|
||||
}
|
||||
|
||||
/**
|
||||
* On file publish check if the path matches a report pattern a write it to
|
||||
* the reports file.
|
||||
*
|
||||
* @param destination Path of the published file at destination filesystem.
|
||||
*/
|
||||
boolean filePublish(Path destination) {
|
||||
if (processReports && destination) {
|
||||
|
||||
if (!matchers) {
|
||||
// Initialize report matchers on first event to use the
|
||||
// path matcher of the destination filesystem
|
||||
matchers = reportsEntries.collect {FileHelper.getPathMatcherFor(convertToGlobPattern(it.key), destination.fileSystem) as PathMatcher}
|
||||
}
|
||||
|
||||
for (int p=0; p < matchers.size(); p++) {
|
||||
if (matchers.get(p).matches(destination)) {
|
||||
final reportEntry = this.reportsEntries.get(p)
|
||||
writer.send((PrintWriter it) -> writeRecord(it, reportEntry, destination))
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
private writeRecord(PrintWriter it, Map.Entry<String,Map<String,String>> reportEntry, Path destination) {
|
||||
try {
|
||||
final target = destination.toUriString()
|
||||
final numRep = totalReports.incrementAndGet()
|
||||
log.trace("Adding report [${numRep}] ${reportEntry.key} -- ${target}")
|
||||
// Report properties
|
||||
final display = reportEntry.value.get("display", "")
|
||||
final mimeType = reportEntry.value.get("mimeType", "")
|
||||
it.println("${reportEntry.key}\t${target}\t${destination.size()}\t${display}\t${mimeType}");
|
||||
it.flush()
|
||||
}
|
||||
catch (Throwable e) {
|
||||
log.error ("Unexpected error writing report entry '${destination.toUriString()}'", e)
|
||||
}
|
||||
}
|
||||
|
||||
protected static String convertToGlobPattern(String reportKey) {
|
||||
final prefix = reportKey.startsWith("**/") ? "" : "**/"
|
||||
return "glob:${prefix}${reportKey}"
|
||||
}
|
||||
|
||||
/**
|
||||
* Publish any built-in reports that are enabled to the reports file.
|
||||
*/
|
||||
void publishRuntimeReports() {
|
||||
final config = session.config
|
||||
final files = []
|
||||
|
||||
if( config.navigate('report.enabled') )
|
||||
files << config.navigate('report.file', ReportConfig.defaultFileName())
|
||||
|
||||
if( config.navigate('timeline.enabled') )
|
||||
files << config.navigate('timeline.file', TimelineConfig.defaultFileName())
|
||||
|
||||
if( config.navigate('trace.enabled') )
|
||||
files << config.navigate('trace.file', TraceConfig.defaultFileName())
|
||||
|
||||
if( config.navigate('dag.enabled') )
|
||||
files << config.navigate('dag.file', DagConfig.defaultFileName())
|
||||
|
||||
for( def file : files )
|
||||
filePublish( (file as Path).complete() )
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin
|
||||
|
||||
|
||||
import io.seqera.util.retry.Retryable
|
||||
import nextflow.config.spec.ConfigOption
|
||||
import nextflow.config.spec.ConfigScope
|
||||
import nextflow.script.dsl.Description
|
||||
import nextflow.util.Duration
|
||||
import nextflow.util.RetryConfig
|
||||
|
||||
/**
|
||||
* Configuration class for Tower retry policy settings.
|
||||
*
|
||||
* This class defines the retry behavior for Tower operations including HTTP requests
|
||||
* and other potentially failing operations. It implements exponential backoff with
|
||||
* jitter to handle transient failures gracefully.
|
||||
*
|
||||
* The retry policy supports:
|
||||
* - Configurable initial delay before the first retry attempt
|
||||
* - Maximum delay cap to prevent excessively long wait times
|
||||
* - Limited number of retry attempts to avoid infinite loops
|
||||
* - Jitter randomization to prevent thundering herd problems
|
||||
* - Exponential backoff multiplier for progressive delay increases
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class TowerRetryPolicy implements Retryable.Config, ConfigScope {
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Initial delay before retrying a failed Tower operation (default: `350ms`).
|
||||
""")
|
||||
Duration delay
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Maximum delay between retry attempts for Tower operations (default: `90s`).
|
||||
""")
|
||||
Duration maxDelay
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Maximum number of retry attempts for Tower operations (default: `5`).
|
||||
""")
|
||||
int maxAttempts
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Random jitter factor applied to retry delays to avoid thundering herd issues (default: `0.25`).
|
||||
""")
|
||||
double jitter
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Multiplier factor for exponential backoff between retry attempts (default: `2.0`).
|
||||
""")
|
||||
double multiplier
|
||||
|
||||
TowerRetryPolicy(Map opts, Map legacy=Map.of()) {
|
||||
this.delay = opts.delay as Duration ?: legacy.backOffDelay as Duration ?: RetryConfig.DEFAULT_DELAY
|
||||
this.maxDelay = opts.maxDelay as Duration ?: RetryConfig.DEFAULT_MAX_DELAY
|
||||
this.maxAttempts = opts.maxAttempts as Integer ?: legacy.maxRetries as Integer ?: RetryConfig.DEFAULT_MAX_ATTEMPTS
|
||||
this.jitter = opts.jitter as Double ?: RetryConfig.DEFAULT_JITTER
|
||||
this.multiplier = opts.multiplier as Double ?: legacy.backOffBase as Double ?: RetryConfig.DEFAULT_MULTIPLIER
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin
|
||||
|
||||
import java.net.http.HttpClient
|
||||
import java.net.http.HttpRequest
|
||||
import java.net.http.HttpResponse
|
||||
import java.time.Duration
|
||||
import java.util.regex.Pattern
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.file.http.XAuthProvider
|
||||
|
||||
/**
|
||||
* Implements Tower authentication strategy for resources accessed
|
||||
* via {@link nextflow.file.http.XFileSystemProvider}
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class TowerXAuth implements XAuthProvider {
|
||||
|
||||
final private Pattern pattern
|
||||
final private String endpoint
|
||||
private String accessToken
|
||||
private String refreshToken
|
||||
private CookieManager cookieManager
|
||||
final private HttpClient httpClient
|
||||
|
||||
TowerXAuth(String endpoint, String accessToken, String refreshToken) {
|
||||
this.endpoint = endpoint
|
||||
this.pattern = ~/(?i)^$endpoint\/.*$/
|
||||
this.accessToken = accessToken
|
||||
this.refreshToken = refreshToken
|
||||
//
|
||||
// the cookie manager
|
||||
cookieManager = new CookieManager(null, CookiePolicy.ACCEPT_ALL)
|
||||
// create http client
|
||||
this.httpClient = HttpClient.newBuilder()
|
||||
.version(HttpClient.Version.HTTP_1_1)
|
||||
.followRedirects(HttpClient.Redirect.NORMAL)
|
||||
.cookieHandler(cookieManager)
|
||||
.connectTimeout(Duration.ofSeconds(10))
|
||||
.build()
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean authorize(URLConnection conn) {
|
||||
final req = conn.getURL().toString()
|
||||
if( pattern.matcher(req).matches() && !conn.getRequestProperty('Authorization') ) {
|
||||
log.trace "Authorizing request connection to: $req"
|
||||
conn.setRequestProperty('Authorization', "Bearer $accessToken")
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
boolean refreshToken(URLConnection conn) {
|
||||
if( !refreshToken || !pattern.matcher(conn.getURL().toString()).matches() ) {
|
||||
return false
|
||||
}
|
||||
|
||||
final req = HttpRequest.newBuilder()
|
||||
.uri(new URI("${endpoint}/oauth/access_token"))
|
||||
.headers('Content-Type',"application/x-www-form-urlencoded")
|
||||
.POST(HttpRequest.BodyPublishers.ofString("grant_type=refresh_token&refresh_token=${URLEncoder.encode(refreshToken, 'UTF-8')}"))
|
||||
.build()
|
||||
|
||||
final resp = httpClient.send(req, HttpResponse.BodyHandlers.ofString())
|
||||
log.debug "Refresh cookie response: [${resp.statusCode()}] ${resp.body()}"
|
||||
if( resp.statusCode() != 200 )
|
||||
return false
|
||||
|
||||
final authCookie = getCookie('JWT')
|
||||
final refreshCookie = getCookie('JWT_REFRESH_TOKEN')
|
||||
|
||||
// set the new bearer token in the current client session
|
||||
if( authCookie?.value ) {
|
||||
log.trace "Updating http client bearer token=$authCookie.value"
|
||||
accessToken = authCookie.value
|
||||
}
|
||||
else {
|
||||
log.warn "Missing JWT cookie from refresh token response ~ $authCookie"
|
||||
}
|
||||
|
||||
// set the new refresh token
|
||||
if( refreshCookie?.value ) {
|
||||
log.trace "Updating http client refresh token=$refreshCookie.value"
|
||||
refreshToken = refreshCookie.value
|
||||
}
|
||||
else {
|
||||
log.warn "Missing JWT_REFRESH_TOKEN cookie from refresh token response ~ $refreshCookie"
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
private HttpCookie getCookie(final String cookieName) {
|
||||
for( HttpCookie it : cookieManager.cookieStore.cookies ) {
|
||||
if( it.name == cookieName )
|
||||
return it
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import nextflow.trace.ProgressRecord
|
||||
import nextflow.trace.WorkflowStats
|
||||
/**
|
||||
* Simple facade to format workflow progress payload
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@EqualsAndHashCode
|
||||
@CompileStatic
|
||||
class WorkflowProgress {
|
||||
private WorkflowStats stats
|
||||
|
||||
WorkflowProgress(WorkflowStats stats) {
|
||||
this.stats = stats
|
||||
}
|
||||
|
||||
int getSucceeded() { stats.succeededCount }
|
||||
|
||||
int getFailed() { stats.failedCount }
|
||||
|
||||
int getIgnored() { stats.ignoredCount }
|
||||
|
||||
int getCached() { stats.cachedCount }
|
||||
|
||||
int getPending() { stats.pendingCount }
|
||||
|
||||
int getSubmitted() { stats.submittedCount }
|
||||
|
||||
int getRunning() { stats.runningCount }
|
||||
|
||||
int getRetries() { stats.retriesCount }
|
||||
|
||||
int getAborted() { stats.abortedCount }
|
||||
|
||||
int getLoadCpus() { stats.loadCpus }
|
||||
|
||||
long getLoadMemory() { stats.loadMemory }
|
||||
|
||||
int getPeakRunning() { stats.peakRunning }
|
||||
|
||||
long getPeakCpus() { stats.peakCpus }
|
||||
|
||||
long getPeakMemory() { stats.peakMemory }
|
||||
|
||||
List<ProgressRecord> getProcesses() { stats.getProcesses() }
|
||||
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,202 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin.dataset
|
||||
|
||||
import io.seqera.tower.plugin.exception.ForbiddenException
|
||||
import io.seqera.tower.plugin.exception.NotFoundException
|
||||
import io.seqera.tower.plugin.exception.UnauthorizedException
|
||||
|
||||
import java.nio.file.AccessDeniedException
|
||||
import java.nio.file.NoSuchFileException
|
||||
import java.time.OffsetDateTime
|
||||
|
||||
import groovy.json.JsonSlurper
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import io.seqera.tower.model.DatasetDto
|
||||
import io.seqera.tower.model.DatasetVersionDto
|
||||
import io.seqera.tower.model.OrgAndWorkspaceDto
|
||||
import io.seqera.tower.plugin.TowerClient
|
||||
import nextflow.exception.AbortOperationException
|
||||
|
||||
/**
|
||||
* Typed client for Seqera Platform dataset API endpoints.
|
||||
* Delegates HTTP execution to {@link TowerClient#sendApiRequest}, inheriting its
|
||||
* authentication token management and retry policy without exposing the underlying
|
||||
* HTTP client.
|
||||
*
|
||||
* @author Seqera Labs
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class SeqeraDatasetClient {
|
||||
|
||||
private final TowerClient towerClient
|
||||
|
||||
SeqeraDatasetClient(TowerClient towerClient) {
|
||||
this.towerClient = towerClient
|
||||
}
|
||||
|
||||
private String getEndpoint() { towerClient.endpoint }
|
||||
|
||||
/**
|
||||
* @return current user info (id, userName, etc.) from GET /user-info
|
||||
*/
|
||||
Long getUserId() {
|
||||
try {
|
||||
final info = towerClient.getUserInfo()
|
||||
if( info?.id == null )
|
||||
throw new AbortOperationException("Unable to retrieve user ID from Seqera Platform — check your access token")
|
||||
return info.id as long
|
||||
}catch( UnauthorizedException e ){
|
||||
throw new AbortOperationException(e.getMessage())
|
||||
}catch( ForbiddenException e){
|
||||
throw new AccessDeniedException("${endpoint}/user-info", null, e.message)
|
||||
}catch(NotFoundException e){
|
||||
throw new NoSuchFileException("${endpoint}/user-info")
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return all orgs and workspaces accessible to the given user from GET /user/{userId}/workspaces
|
||||
*/
|
||||
List<OrgAndWorkspaceDto> listUserWorkspacesAndOrgs(long userId) {
|
||||
try {
|
||||
final list = towerClient.listUserWorkspacesAndOrgs(userId as String)
|
||||
return list.collect { m -> mapOrgAndWorkspace(m) }
|
||||
} catch( UnauthorizedException e ){
|
||||
throw new AbortOperationException(e.getMessage())
|
||||
} catch( ForbiddenException e){
|
||||
throw new AccessDeniedException("${endpoint}/user/$userId/workspaces", null, e.message)
|
||||
} catch(NotFoundException e){
|
||||
throw new NoSuchFileException("${endpoint}/user/$userId/workspaces")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @return all datasets in the given workspace from GET /datasets?workspaceId={workspaceId}
|
||||
*/
|
||||
List<DatasetDto> listDatasets(long workspaceId) {
|
||||
final url = "${endpoint}/datasets?workspaceId=${workspaceId}"
|
||||
log.debug "SeqeraDatasetClient GET $url"
|
||||
final resp = towerClient.sendApiRequest(url)
|
||||
checkFsResponse(resp, url)
|
||||
final json = new JsonSlurper().parseText(resp.message) as Map
|
||||
final list = json.datasets as List<Map>
|
||||
return list ? list.collect { m -> mapDataset(m) } : Collections.<DatasetDto>emptyList()
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new dataset in the given workspace via POST /datasets?workspaceId={workspaceId}.
|
||||
* @return the created dataset DTO
|
||||
*/
|
||||
DatasetDto createDataset(long workspaceId, String name) {
|
||||
final url = "${endpoint}/datasets?workspaceId=${workspaceId}"
|
||||
log.debug "SeqeraDatasetClient POST $url name=$name"
|
||||
final resp = towerClient.sendApiRequest(url, [name: name], 'POST')
|
||||
checkFsResponse(resp, url)
|
||||
final json = new JsonSlurper().parseText(resp.message) as Map
|
||||
return mapDataset(json.dataset as Map)
|
||||
}
|
||||
|
||||
/**
|
||||
* @return all versions for the given dataset from GET /datasets/{datasetId}/versions
|
||||
*/
|
||||
List<DatasetVersionDto> listVersions(String datasetId, long workspaceId) {
|
||||
final url = "${endpoint}/datasets/${datasetId}/versions?workspaceId=${workspaceId}"
|
||||
log.debug "SeqeraDatasetClient GET $url"
|
||||
final resp = towerClient.sendApiRequest(url)
|
||||
checkFsResponse(resp, url)
|
||||
final json = new JsonSlurper().parseText(resp.message) as Map
|
||||
final list = json.versions as List<Map>
|
||||
return list ? list.collect { m -> mapVersion(m) } : Collections.<DatasetVersionDto>emptyList()
|
||||
}
|
||||
|
||||
/**
|
||||
* Download a dataset version as an InputStream.
|
||||
* GET /datasets/{datasetId}/v/{version}/n/{fileName}
|
||||
* The fileName must exactly match DatasetVersionDto.fileName from upload time.
|
||||
*/
|
||||
InputStream downloadDataset(String datasetId, String version, String fileName, long workspaceId) {
|
||||
final encodedName = new URI(null, null, fileName, null).rawPath
|
||||
final url = "${endpoint}/datasets/${datasetId}/v/${version}/n/${encodedName}?workspaceId=$workspaceId"
|
||||
log.debug "SeqeraDatasetClient GET $url (streaming)"
|
||||
try {
|
||||
return towerClient.sendStreamingRequest(url)
|
||||
}
|
||||
catch (UnauthorizedException e) {
|
||||
throw new AbortOperationException("Seqera authentication failed — check tower.accessToken or TOWER_ACCESS_TOKEN")
|
||||
}
|
||||
catch (ForbiddenException e) {
|
||||
throw new AccessDeniedException(url, null, "Forbidden — check workspace permissions")
|
||||
}
|
||||
catch (NotFoundException e) {
|
||||
throw new NoSuchFileException(url)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- private helpers ----
|
||||
|
||||
private static void checkFsResponse(TowerClient.Response resp, String url) {
|
||||
if (!resp.error) return
|
||||
final code = resp.code
|
||||
if (code == 401)
|
||||
throw new AbortOperationException("Seqera authentication failed — check tower.accessToken or TOWER_ACCESS_TOKEN")
|
||||
if (code == 403)
|
||||
throw new AccessDeniedException(url, null, "Forbidden — check workspace permissions")
|
||||
if (code == 404)
|
||||
throw new NoSuchFileException(url)
|
||||
throw new IOException("Seqera API error: HTTP ${code} for ${url}")
|
||||
}
|
||||
|
||||
private static OrgAndWorkspaceDto mapOrgAndWorkspace(Map m) {
|
||||
final dto = new OrgAndWorkspaceDto()
|
||||
dto.orgId = (m.orgId as Long) ?: 0L
|
||||
dto.orgName = m.orgName as String
|
||||
dto.workspaceId = (m.workspaceId as Long) ?: 0L
|
||||
dto.workspaceName = m.workspaceName as String
|
||||
dto.workspaceFullName = m.workspaceFullName as String
|
||||
return dto
|
||||
}
|
||||
|
||||
private static DatasetDto mapDataset(Map m) {
|
||||
final dto = new DatasetDto()
|
||||
dto.id = m.id as String
|
||||
dto.name = m.name as String
|
||||
dto.description = m.description as String
|
||||
dto.version = (m.version as Long) ?: 0L
|
||||
dto.mediaType = m.mediaType as String
|
||||
dto.workspaceId = (m.workspaceId as Long) ?: 0L
|
||||
dto.dateCreated = m.dateCreated ? OffsetDateTime.parse(m.dateCreated as String) : null
|
||||
dto.lastUpdated = m.lastUpdated ? OffsetDateTime.parse(m.lastUpdated as String) : null
|
||||
return dto
|
||||
}
|
||||
|
||||
private static DatasetVersionDto mapVersion(Map m) {
|
||||
final dto = new DatasetVersionDto()
|
||||
dto.datasetId = m.datasetId as String
|
||||
dto.version = (m.version as Long) ?: 0L
|
||||
dto.fileName = m.fileName as String
|
||||
dto.mediaType = m.mediaType as String
|
||||
dto.hasHeader = (m.hasHeader as Boolean) ?: false
|
||||
dto.dateCreated = m.dateCreated ? OffsetDateTime.parse(m.dateCreated as String) : null
|
||||
dto.disabled = (m.disabled as Boolean) ?: false
|
||||
return dto
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin.exception
|
||||
|
||||
import groovy.transform.InheritConstructors
|
||||
|
||||
@InheritConstructors
|
||||
class BadResponseException extends RuntimeException{
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.seqera.tower.plugin.exception
|
||||
|
||||
import groovy.transform.InheritConstructors
|
||||
|
||||
@InheritConstructors
|
||||
class ForbiddenException extends RuntimeException {
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.seqera.tower.plugin.exception
|
||||
|
||||
import groovy.transform.InheritConstructors
|
||||
|
||||
@InheritConstructors
|
||||
class NotFoundException extends RuntimeException {
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin.exception
|
||||
|
||||
import groovy.transform.InheritConstructors
|
||||
|
||||
@InheritConstructors
|
||||
class UnauthorizedException extends RuntimeException {
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin.exchange
|
||||
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import groovy.transform.ToString
|
||||
/**
|
||||
* Models a REST request to obtain a license-scoped JWT token from Platform
|
||||
*
|
||||
* @author Alberto Miranda <alberto.miranda@seqera.io>
|
||||
*/
|
||||
@EqualsAndHashCode
|
||||
@ToString(includeNames = true, includePackage = false)
|
||||
@CompileStatic
|
||||
class GetLicenseTokenRequest {
|
||||
|
||||
/** The product code */
|
||||
String product
|
||||
|
||||
/** The product version */
|
||||
String version
|
||||
|
||||
/**
|
||||
* The Platform workflow ID associated with this request
|
||||
*/
|
||||
String workflowId
|
||||
|
||||
/**
|
||||
* The Platform workspace ID associated with this request
|
||||
*/
|
||||
String workspaceId
|
||||
|
||||
/**
|
||||
* @return a Map representation of the request
|
||||
*/
|
||||
Map<String, String> toMap() {
|
||||
final map = new HashMap<String, String>()
|
||||
map.product = this.product
|
||||
map.version = this.version
|
||||
map.workflowId = this.workflowId
|
||||
map.workspaceId = this.workspaceId
|
||||
return map
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin.exchange
|
||||
|
||||
import java.time.Instant
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.ToString
|
||||
|
||||
/**
|
||||
* Models a REST response containing a license-scoped JWT token from Platform
|
||||
*
|
||||
* @author Alberto Miranda <alberto.miranda@seqera.io>
|
||||
*/
|
||||
@CompileStatic
|
||||
@ToString(includeNames = true, includePackage = false)
|
||||
class GetLicenseTokenResponse {
|
||||
/**
|
||||
* The signed JWT token
|
||||
*/
|
||||
String signedToken
|
||||
|
||||
/**
|
||||
* The expiration timestamp of the token
|
||||
*/
|
||||
Instant expiresAt
|
||||
|
||||
/**
|
||||
* The Exception returned while trying to access the token
|
||||
*/
|
||||
Throwable error
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin.fs
|
||||
|
||||
import java.nio.ByteBuffer
|
||||
import java.nio.channels.ClosedChannelException
|
||||
import java.nio.channels.SeekableByteChannel
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
|
||||
/**
|
||||
* Minimal {@link SeekableByteChannel} backed by an {@link InputStream}.
|
||||
* Supports sequential reads only (no seek/position).
|
||||
*
|
||||
* @author Seqera Labs
|
||||
*/
|
||||
@CompileStatic
|
||||
class DatasetInputStream implements SeekableByteChannel {
|
||||
private final InputStream inputStream
|
||||
private long position0 = 0L
|
||||
private boolean open = true
|
||||
private byte[] buf = new byte[0]
|
||||
|
||||
DatasetInputStream(InputStream inputStream) {
|
||||
this.inputStream = inputStream
|
||||
}
|
||||
|
||||
@Override
|
||||
int read(ByteBuffer dst) throws IOException {
|
||||
if( !open )
|
||||
throw new ClosedChannelException()
|
||||
final len = dst.remaining()
|
||||
if (buf.length < len)
|
||||
buf = new byte[len]
|
||||
final n = inputStream.read(buf, 0, len)
|
||||
if (n > 0) {
|
||||
dst.put(buf, 0, n)
|
||||
position0 += n
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
@Override
|
||||
int write(ByteBuffer src) { throw new UnsupportedOperationException() }
|
||||
|
||||
@Override
|
||||
long position() { position0 }
|
||||
|
||||
@Override
|
||||
SeekableByteChannel position(long newPosition) { throw new UnsupportedOperationException("seek not supported") }
|
||||
|
||||
@Override
|
||||
long size() { throw new UnsupportedOperationException("size not available for streaming dataset channel") }
|
||||
|
||||
@Override
|
||||
SeekableByteChannel truncate(long size) { throw new UnsupportedOperationException() }
|
||||
|
||||
@Override
|
||||
boolean isOpen() { open }
|
||||
|
||||
@Override
|
||||
void close() throws IOException {
|
||||
open = false
|
||||
inputStream.close()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin.fs
|
||||
|
||||
import java.nio.file.attribute.BasicFileAttributes
|
||||
import java.nio.file.attribute.FileTime
|
||||
import java.time.Instant
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import io.seqera.tower.model.DatasetDto
|
||||
|
||||
/**
|
||||
* {@link BasicFileAttributes} for {@code seqera://} paths.
|
||||
* For depth < 4 (directory paths): {@code isDirectory=true}, {@code size=0}.
|
||||
* For depth 4 (dataset file paths): {@code isRegularFile=true}, timestamps from {@link DatasetDto}.
|
||||
*
|
||||
* @author Seqera Labs
|
||||
*/
|
||||
@CompileStatic
|
||||
class SeqeraFileAttributes implements BasicFileAttributes {
|
||||
|
||||
private final boolean directory
|
||||
private final DatasetDto dataset
|
||||
|
||||
/** Construct attributes for a virtual directory (depth 0–3). */
|
||||
SeqeraFileAttributes(boolean isDir) {
|
||||
this.directory = isDir
|
||||
this.dataset = null
|
||||
}
|
||||
|
||||
/** Construct attributes for a dataset file (depth 4). */
|
||||
SeqeraFileAttributes(DatasetDto dataset) {
|
||||
this.directory = false
|
||||
this.dataset = dataset
|
||||
}
|
||||
|
||||
@Override
|
||||
FileTime lastModifiedTime() {
|
||||
if (dataset?.lastUpdated) {
|
||||
return FileTime.from(dataset.lastUpdated.toInstant())
|
||||
}
|
||||
return FileTime.from(Instant.EPOCH)
|
||||
}
|
||||
|
||||
@Override
|
||||
FileTime lastAccessTime() { lastModifiedTime() }
|
||||
|
||||
@Override
|
||||
FileTime creationTime() {
|
||||
if (dataset?.dateCreated) {
|
||||
return FileTime.from(dataset.dateCreated.toInstant())
|
||||
}
|
||||
return FileTime.from(Instant.EPOCH)
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean isRegularFile() { !directory }
|
||||
|
||||
@Override
|
||||
boolean isDirectory() { directory }
|
||||
|
||||
@Override
|
||||
boolean isSymbolicLink() { false }
|
||||
|
||||
@Override
|
||||
boolean isOther() { false }
|
||||
|
||||
@Override
|
||||
long size() { 0L }
|
||||
|
||||
@Override
|
||||
Object fileKey() { dataset?.id }
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin.fs
|
||||
|
||||
import java.nio.file.FileStore
|
||||
import java.nio.file.FileSystem
|
||||
import java.nio.file.NoSuchFileException
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.PathMatcher
|
||||
import java.nio.file.WatchService
|
||||
import java.nio.file.attribute.UserPrincipalLookupService
|
||||
import java.nio.file.spi.FileSystemProvider
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import io.seqera.tower.model.DatasetDto
|
||||
import io.seqera.tower.model.DatasetVersionDto
|
||||
import io.seqera.tower.model.OrgAndWorkspaceDto
|
||||
import io.seqera.tower.plugin.dataset.SeqeraDatasetClient
|
||||
|
||||
/**
|
||||
* FileSystem instance for the {@code seqera://} scheme.
|
||||
* One instance per (endpoint + credentials) pair, cached by {@link SeqeraFileSystemProvider}.
|
||||
*
|
||||
* Lazily populates org/workspace/dataset caches on first access.
|
||||
* Cache is invalidated on dataset write operations.
|
||||
*
|
||||
* @author Seqera Labs
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class SeqeraFileSystem extends FileSystem {
|
||||
|
||||
private final SeqeraFileSystemProvider provider0
|
||||
final SeqeraDatasetClient client
|
||||
|
||||
/** orgName → orgId */
|
||||
private final Map<String, Long> orgCache = new LinkedHashMap<>()
|
||||
/** "orgName/workspaceName" → workspaceId */
|
||||
private final Map<String, Long> workspaceCache = new LinkedHashMap<>()
|
||||
/** workspaceId → list of DatasetDto */
|
||||
private final Map<Long, List<DatasetDto>> datasetCache = new LinkedHashMap<>()
|
||||
/** datasetId → list of DatasetVersionDto */
|
||||
private final Map<String, List<DatasetVersionDto>> versionCache = new LinkedHashMap<>()
|
||||
|
||||
private volatile boolean orgWorkspaceCacheLoaded = false
|
||||
|
||||
SeqeraFileSystem(SeqeraFileSystemProvider provider, SeqeraDatasetClient client) {
|
||||
this.provider0 = provider
|
||||
this.client = client
|
||||
}
|
||||
|
||||
@Override
|
||||
FileSystemProvider provider() { provider0 }
|
||||
|
||||
@Override
|
||||
void close() { /* no-op: platform API connection is stateless */ }
|
||||
|
||||
@Override
|
||||
boolean isOpen() { true }
|
||||
|
||||
@Override
|
||||
boolean isReadOnly() { true }
|
||||
|
||||
@Override
|
||||
String getSeparator() { '/' }
|
||||
|
||||
@Override
|
||||
Iterable<Path> getRootDirectories() {
|
||||
return [getPath('seqera://')] as Iterable<Path>
|
||||
}
|
||||
|
||||
@Override
|
||||
Iterable<FileStore> getFileStores() { Collections.emptyList() }
|
||||
|
||||
@Override
|
||||
Set<String> supportedFileAttributeViews() { Collections.singleton('basic') }
|
||||
|
||||
@Override
|
||||
Path getPath(String first, String... more) {
|
||||
final full = more ? ([first] + more.toList()).join(getSeparator()) : first
|
||||
return new SeqeraPath(this, full)
|
||||
}
|
||||
|
||||
@Override
|
||||
PathMatcher getPathMatcher(String syntaxAndPattern) {
|
||||
throw new UnsupportedOperationException("PathMatcher not supported by seqera:// filesystem")
|
||||
}
|
||||
|
||||
@Override
|
||||
UserPrincipalLookupService getUserPrincipalLookupService() {
|
||||
throw new UnsupportedOperationException("UserPrincipalLookupService not supported by seqera:// filesystem")
|
||||
}
|
||||
|
||||
@Override
|
||||
WatchService newWatchService() {
|
||||
throw new UnsupportedOperationException("WatchService not supported by seqera:// filesystem")
|
||||
}
|
||||
|
||||
// ---- cache management ----
|
||||
|
||||
/**
|
||||
* Ensure the org/workspace cache is populated. Thread-safe: loads at most once.
|
||||
* Calls GET /user-info then GET /user/{userId}/workspaces.
|
||||
*/
|
||||
synchronized void loadOrgWorkspaceCache() {
|
||||
if (orgWorkspaceCacheLoaded) return
|
||||
log.debug "Loading Seqera org/workspace cache"
|
||||
final entries = client.listUserWorkspacesAndOrgs(client.getUserId())
|
||||
for (OrgAndWorkspaceDto entry : entries) {
|
||||
if (entry.orgName)
|
||||
orgCache.put(entry.orgName, entry.orgId)
|
||||
if (entry.orgName && entry.workspaceName && entry.workspaceId)
|
||||
workspaceCache.put("${entry.orgName}/${entry.workspaceName}" as String, entry.workspaceId)
|
||||
}
|
||||
orgWorkspaceCacheLoaded = true
|
||||
}
|
||||
|
||||
/**
|
||||
* @return distinct org names visible to the authenticated user
|
||||
*/
|
||||
synchronized Set<String> listOrgNames() {
|
||||
loadOrgWorkspaceCache()
|
||||
return Collections.unmodifiableSet(orgCache.keySet())
|
||||
}
|
||||
|
||||
/**
|
||||
* @return workspace names for the given org
|
||||
*/
|
||||
synchronized List<String> listWorkspaceNames(String org) {
|
||||
loadOrgWorkspaceCache()
|
||||
return workspaceCache.keySet()
|
||||
.findAll { String k -> k.startsWith("${org}/") }
|
||||
.collect { String k -> k.substring(org.length() + 1) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a workspace ID by org and workspace name.
|
||||
* @throws NoSuchFileException if the org or workspace is not in the cache
|
||||
*/
|
||||
synchronized long resolveWorkspaceId(String org, String workspace) throws NoSuchFileException {
|
||||
loadOrgWorkspaceCache()
|
||||
final key = "${org}/${workspace}" as String
|
||||
final id = workspaceCache.get(key)
|
||||
if (id == null)
|
||||
throw new NoSuchFileException("seqera://${key}", null, "Org or workspace not found or not accessible")
|
||||
return id
|
||||
}
|
||||
|
||||
/**
|
||||
* Return datasets for the given workspace, populating the cache on first access.
|
||||
*/
|
||||
synchronized List<DatasetDto> resolveDatasets(long workspaceId) {
|
||||
List<DatasetDto> cached = datasetCache.get(workspaceId)
|
||||
if (cached == null) {
|
||||
cached = client.listDatasets(workspaceId)
|
||||
datasetCache.put(workspaceId, cached)
|
||||
}
|
||||
return cached
|
||||
}
|
||||
|
||||
/**
|
||||
* Invalidate the dataset and version caches for a workspace (call after a write operation).
|
||||
*/
|
||||
synchronized void invalidateDatasetCache(long workspaceId) {
|
||||
// Remove version caches for all datasets in this workspace
|
||||
final datasets = datasetCache.get(workspaceId)
|
||||
if (datasets) {
|
||||
for (DatasetDto ds : datasets) {
|
||||
versionCache.remove(ds.id)
|
||||
}
|
||||
}
|
||||
datasetCache.remove(workspaceId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a DatasetDto by name within a workspace.
|
||||
* @throws NoSuchFileException if no dataset with the given name exists
|
||||
*/
|
||||
synchronized DatasetDto resolveDataset(long workspaceId, String name) throws NoSuchFileException {
|
||||
final datasets = resolveDatasets(workspaceId)
|
||||
return datasets.find { DatasetDto d -> d.name == name }
|
||||
}
|
||||
|
||||
/**
|
||||
* Return versions for the given dataset, populating the cache on first access.
|
||||
* Note: the version cache is only invalidated when the workspace dataset cache is invalidated
|
||||
* (e.g. after a write operation). Versions published externally during a pipeline run will not
|
||||
* be visible until the cache is cleared.
|
||||
*/
|
||||
synchronized List<DatasetVersionDto> resolveVersions(String datasetId, long workspaceId) {
|
||||
List<DatasetVersionDto> cached = versionCache.get(datasetId)
|
||||
if (cached == null) {
|
||||
cached = client.listVersions(datasetId, workspaceId)
|
||||
versionCache.put(datasetId, cached)
|
||||
}
|
||||
return cached
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,325 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin.fs
|
||||
|
||||
import java.nio.channels.SeekableByteChannel
|
||||
import java.nio.file.AccessDeniedException
|
||||
import java.nio.file.AccessMode
|
||||
import java.nio.file.CopyOption
|
||||
import java.nio.file.DirectoryStream
|
||||
import java.nio.file.FileStore
|
||||
import java.nio.file.FileSystem
|
||||
import java.nio.file.FileSystemAlreadyExistsException
|
||||
import java.nio.file.FileSystemNotFoundException
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.LinkOption
|
||||
import java.nio.file.DirectoryIteratorException
|
||||
import java.nio.file.NoSuchFileException
|
||||
import java.nio.file.NotDirectoryException
|
||||
import java.nio.file.OpenOption
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.ProviderMismatchException
|
||||
import java.nio.file.StandardOpenOption
|
||||
import java.nio.file.attribute.BasicFileAttributes
|
||||
import java.nio.file.attribute.FileAttribute
|
||||
import java.nio.file.attribute.FileAttributeView
|
||||
import java.nio.file.spi.FileSystemProvider
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import io.seqera.tower.model.DatasetDto
|
||||
import io.seqera.tower.model.DatasetVersionDto
|
||||
import io.seqera.tower.plugin.TowerClient
|
||||
import io.seqera.tower.plugin.TowerFactory
|
||||
import io.seqera.tower.plugin.dataset.SeqeraDatasetClient
|
||||
|
||||
/**
|
||||
* NIO {@link FileSystemProvider} for the {@code seqera://} scheme.
|
||||
* Registered via {@code META-INF/services/java.nio.file.spi.FileSystemProvider}.
|
||||
*
|
||||
* Enables Nextflow pipelines to read Seqera Platform datasets as ordinary file paths:
|
||||
* {@code seqera://<org>/<workspace>/datasets/<dataset-name>}
|
||||
*
|
||||
* Follows the {@code LinFileSystemProvider} pattern for structure.
|
||||
* Write support follows the {@code AzFileSystemProvider} buffered-upload pattern.
|
||||
*
|
||||
* @author Seqera Labs
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class SeqeraFileSystemProvider extends FileSystemProvider {
|
||||
|
||||
public static final String SCHEME = 'seqera'
|
||||
|
||||
/** Single filesystem instance — TowerClient is a singleton per session */
|
||||
private volatile SeqeraFileSystem fileSystem
|
||||
|
||||
@Override
|
||||
String getScheme() { SCHEME }
|
||||
|
||||
// ---- FileSystem lifecycle ----
|
||||
|
||||
@Override
|
||||
synchronized FileSystem newFileSystem(URI uri, Map<String, ?> env) throws IOException {
|
||||
checkScheme(uri)
|
||||
if (fileSystem)
|
||||
throw new FileSystemAlreadyExistsException("File system `seqera://` already exists")
|
||||
final TowerClient towerClient = TowerFactory.client()
|
||||
if (!towerClient)
|
||||
throw new IllegalStateException("File system `seqera://` requires the Seqera Platform access token to be provided - use `tower.accessToken` config option or TOWER_ACCESS_TOKEN env variable")
|
||||
final client = new SeqeraDatasetClient(towerClient)
|
||||
fileSystem = new SeqeraFileSystem(this, client)
|
||||
return fileSystem
|
||||
}
|
||||
|
||||
@Override
|
||||
synchronized FileSystem getFileSystem(URI uri) {
|
||||
checkScheme(uri)
|
||||
if (!fileSystem) throw new FileSystemNotFoundException("No seqera:// filesystem has been created yet")
|
||||
return fileSystem
|
||||
}
|
||||
|
||||
synchronized SeqeraFileSystem getOrCreateFileSystem(URI uri, Map<String, ?> env) {
|
||||
checkScheme(uri)
|
||||
if (!fileSystem) {
|
||||
final envMap = env ?: Collections.<String, Object>emptyMap()
|
||||
newFileSystem(uri, envMap as Map<String, ?>)
|
||||
}
|
||||
return fileSystem
|
||||
}
|
||||
|
||||
@Override
|
||||
SeqeraPath getPath(URI uri) {
|
||||
final fs = getOrCreateFileSystem(uri, Collections.emptyMap())
|
||||
return new SeqeraPath(fs, uri.toString())
|
||||
}
|
||||
|
||||
// ---- Read operations ----
|
||||
|
||||
@Override
|
||||
InputStream newInputStream(Path path, OpenOption... options) throws IOException {
|
||||
final sp = toSeqeraPath(path)
|
||||
if (sp.depth() != 4)
|
||||
throw new IllegalArgumentException("Operation `newInputStream` requires a dataset path (depth 4): $path")
|
||||
final fs = sp.getFileSystem() as SeqeraFileSystem
|
||||
final workspaceId = fs.resolveWorkspaceId(sp.org, sp.workspace)
|
||||
final dataset = fs.resolveDataset(workspaceId, sp.datasetName)
|
||||
if (!dataset)
|
||||
throw new NoSuchFileException(sp.toString(), null, "Dataset '${sp.datasetName}' not found in workspace $sp.workspace")
|
||||
final version = resolveVersion(fs, dataset, sp)
|
||||
log.debug "Downloading dataset '${sp.datasetName}' version ${version.version} (${version.fileName}) from workspace $workspaceId"
|
||||
return fs.client.downloadDataset(dataset.id, String.valueOf(version.version), version.fileName, dataset.workspaceId)
|
||||
}
|
||||
|
||||
@Override
|
||||
SeekableByteChannel newByteChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException {
|
||||
if (options?.contains(StandardOpenOption.WRITE) || options?.contains(StandardOpenOption.APPEND))
|
||||
throw new UnsupportedOperationException("File system `seqera://` is read-only")
|
||||
final inputStream = newInputStream(path)
|
||||
return new DatasetInputStream(inputStream)
|
||||
}
|
||||
|
||||
// ---- Metadata ----
|
||||
|
||||
@Override
|
||||
<A extends BasicFileAttributes> A readAttributes(Path path, Class<A> type, LinkOption... options) throws IOException {
|
||||
if (!BasicFileAttributes.isAssignableFrom(type))
|
||||
throw new UnsupportedOperationException("Attribute type not supported: $type")
|
||||
final sp = toSeqeraPath(path)
|
||||
final fs = sp.getFileSystem() as SeqeraFileSystem
|
||||
final d = sp.depth()
|
||||
if (d < 4) {
|
||||
// Virtual directory — validate the path exists (throws NoSuchFileException if not)
|
||||
validateDirectoryExists(fs, sp)
|
||||
return (A) new SeqeraFileAttributes(true)
|
||||
}
|
||||
// Dataset file
|
||||
final workspaceId = fs.resolveWorkspaceId(sp.org, sp.workspace)
|
||||
final dataset = fs.resolveDataset(workspaceId, sp.datasetName)
|
||||
if (!dataset)
|
||||
throw new NoSuchFileException(sp.toString(), null, "Dataset '${sp.datasetName}' not found in workspace $sp.workspace")
|
||||
return (A) new SeqeraFileAttributes(dataset)
|
||||
}
|
||||
|
||||
@Override
|
||||
Map<String, Object> readAttributes(Path path, String attributes, LinkOption... options) throws IOException {
|
||||
throw new UnsupportedOperationException("Operation `readAttributes(String)` not supported by `seqera://` file system")
|
||||
}
|
||||
|
||||
// ---- Access check ----
|
||||
|
||||
@Override
|
||||
void checkAccess(Path path, AccessMode... modes) throws IOException {
|
||||
final sp = toSeqeraPath(path)
|
||||
for (AccessMode m : modes) {
|
||||
if (m == AccessMode.WRITE || m == AccessMode.EXECUTE)
|
||||
throw new AccessDeniedException(path.toString(), null, "seqera:// filesystem is read-only")
|
||||
}
|
||||
// For READ, verify the path resolves without throwing NoSuchFileException
|
||||
if (sp.depth() >= 1) {
|
||||
final fs = sp.getFileSystem() as SeqeraFileSystem
|
||||
if (sp.depth() == 1) {
|
||||
fs.loadOrgWorkspaceCache()
|
||||
if (!fs.listOrgNames().contains(sp.org))
|
||||
throw new NoSuchFileException(path.toString(), null, "Organisation not found")
|
||||
} else {
|
||||
fs.resolveWorkspaceId(sp.org, sp.workspace)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Directory stream ----
|
||||
|
||||
@Override
|
||||
DirectoryStream<Path> newDirectoryStream(Path dir, DirectoryStream.Filter<? super Path> filter) throws IOException {
|
||||
final sp = toSeqeraPath(dir)
|
||||
final fs = sp.getFileSystem() as SeqeraFileSystem
|
||||
final d = sp.depth()
|
||||
List<Path> entries
|
||||
if (d == 0) {
|
||||
// Root: list distinct org names
|
||||
fs.loadOrgWorkspaceCache()
|
||||
entries = fs.listOrgNames().collect { String org -> sp.resolve(org) as Path }
|
||||
} else if (d == 1) {
|
||||
// Org: list workspace names
|
||||
fs.loadOrgWorkspaceCache()
|
||||
entries = fs.listWorkspaceNames(sp.org).collect { String ws -> sp.resolve(ws) as Path }
|
||||
} else if (d == 2) {
|
||||
// Workspace: static resource types
|
||||
entries = ['datasets'].collect { String rt -> sp.resolve(rt) as Path }
|
||||
} else if (d == 3) {
|
||||
// Resource type directory: list dataset names
|
||||
final workspaceId = fs.resolveWorkspaceId(sp.org, sp.workspace)
|
||||
entries = fs.resolveDatasets(workspaceId).collect { DatasetDto ds ->
|
||||
sp.resolve(ds.name) as Path
|
||||
}
|
||||
} else {
|
||||
throw new NotDirectoryException(dir.toString())
|
||||
}
|
||||
|
||||
final filtered = filter ? entries.findAll { Path p ->
|
||||
try { filter.accept(p) }
|
||||
catch (IOException e) { throw new DirectoryIteratorException(e) }
|
||||
} : entries
|
||||
|
||||
return new DirectoryStream<Path>() {
|
||||
@Override Iterator<Path> iterator() { filtered.iterator() }
|
||||
@Override void close() {}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Copy ----
|
||||
|
||||
@Override
|
||||
void copy(Path source, Path target, CopyOption... options) throws IOException {
|
||||
toSeqeraPath(source)
|
||||
if (target instanceof SeqeraPath)
|
||||
throw new UnsupportedOperationException("seqera:// filesystem is read-only")
|
||||
// cross-provider (seqera → local): stream to target
|
||||
try (final InputStream is = newInputStream(source)) {
|
||||
Files.copy(is, target, options)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Unsupported mutations ----
|
||||
|
||||
@Override
|
||||
void move(Path source, Path target, CopyOption... options) {
|
||||
throw new UnsupportedOperationException("move() not supported by seqera:// filesystem")
|
||||
}
|
||||
|
||||
@Override
|
||||
void delete(Path path) {
|
||||
throw new UnsupportedOperationException("delete() not supported by seqera:// filesystem")
|
||||
}
|
||||
|
||||
@Override
|
||||
void createDirectory(Path dir, FileAttribute<?>... attrs) {
|
||||
throw new UnsupportedOperationException("createDirectory() not supported by seqera:// filesystem")
|
||||
}
|
||||
|
||||
// ---- Misc ----
|
||||
|
||||
@Override
|
||||
boolean isSameFile(Path path, Path path2) throws IOException {
|
||||
return path == path2
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean isHidden(Path path) { false }
|
||||
|
||||
@Override
|
||||
FileStore getFileStore(Path path) {
|
||||
throw new UnsupportedOperationException("getFileStore() not supported by seqera:// filesystem")
|
||||
}
|
||||
|
||||
@Override
|
||||
<V extends FileAttributeView> V getFileAttributeView(Path path, Class<V> type, LinkOption... options) {
|
||||
return null
|
||||
}
|
||||
|
||||
@Override
|
||||
void setAttribute(Path path, String attribute, Object value, LinkOption... options) {
|
||||
throw new UnsupportedOperationException("setAttribute() not supported by seqera:// filesystem")
|
||||
}
|
||||
|
||||
// ---- private helpers ----
|
||||
|
||||
private static SeqeraPath toSeqeraPath(Path path) {
|
||||
if (path !instanceof SeqeraPath)
|
||||
throw new ProviderMismatchException()
|
||||
return (SeqeraPath) path
|
||||
}
|
||||
|
||||
private static void checkScheme(URI uri) {
|
||||
if (uri.scheme?.toLowerCase() != SCHEME)
|
||||
throw new IllegalArgumentException("Not a seqera:// URI: $uri")
|
||||
}
|
||||
|
||||
private static void validateDirectoryExists(SeqeraFileSystem fs, SeqeraPath sp) throws NoSuchFileException {
|
||||
final d = sp.depth()
|
||||
if (d == 0) return
|
||||
// Depth 1+: ensure org/workspace cache is loaded
|
||||
fs.loadOrgWorkspaceCache()
|
||||
if (d >= 1 && !fs.listOrgNames().contains(sp.org))
|
||||
throw new NoSuchFileException("seqera://${sp.org}", null, "Organisation not found")
|
||||
if (d >= 2)
|
||||
fs.resolveWorkspaceId(sp.org, sp.workspace)
|
||||
if (d >= 3 && sp.resourceType != 'datasets')
|
||||
throw new NoSuchFileException("seqera://${sp.org}/${sp.workspace}/${sp.resourceType}", null, "Unsupported resource type")
|
||||
}
|
||||
|
||||
private static DatasetVersionDto resolveVersion(SeqeraFileSystem fs, DatasetDto dataset, SeqeraPath sp) throws IOException {
|
||||
final pinnedVersion = sp.version
|
||||
final versions = fs.resolveVersions(dataset.id, dataset.workspaceId)
|
||||
if (versions.isEmpty())
|
||||
throw new NoSuchFileException(sp.toString(), null, "No versions available for dataset '${dataset.name}'")
|
||||
if (pinnedVersion) {
|
||||
final found = versions.find { DatasetVersionDto v -> String.valueOf(v.version) == pinnedVersion }
|
||||
if (!found)
|
||||
throw new NoSuchFileException(sp.toString(), null, "Version '${pinnedVersion}' not found for dataset '${dataset.name}'")
|
||||
return found
|
||||
}
|
||||
// Latest non-disabled version
|
||||
final latest = versions.findAll { DatasetVersionDto v -> !v.disabled }
|
||||
.max { DatasetVersionDto v -> v.version }
|
||||
if (!latest)
|
||||
throw new NoSuchFileException(sp.toString(), null, "No enabled versions for dataset '${dataset.name}'")
|
||||
return latest
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,490 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin.fs
|
||||
|
||||
import java.nio.file.FileSystem
|
||||
import java.nio.file.InvalidPathException
|
||||
import java.nio.file.LinkOption
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.ProviderMismatchException
|
||||
import java.nio.file.WatchEvent
|
||||
import java.nio.file.WatchKey
|
||||
import java.nio.file.WatchService
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
|
||||
/**
|
||||
* {@link Path} implementation for the {@code seqera://} scheme.
|
||||
*
|
||||
* Path hierarchy:
|
||||
* <pre>
|
||||
* depth 0 seqera:// (root — directory)
|
||||
* depth 1 seqera://<org> (org — directory)
|
||||
* depth 2 seqera://<org>/<workspace> (workspace — directory)
|
||||
* depth 3 seqera://<org>/<workspace>/datasets (resource type — directory)
|
||||
* depth 4 seqera://<org>/<workspace>/datasets/<name> (dataset file)
|
||||
* seqera://<org>/<workspace>/datasets/<name@ver> (pinned version)
|
||||
* </pre>
|
||||
*
|
||||
* @author Seqera Labs
|
||||
*/
|
||||
@CompileStatic
|
||||
class SeqeraPath implements Path {
|
||||
|
||||
/** URI scheme */
|
||||
public static final String SCHEME = 'seqera'
|
||||
public static final String PROTOCOL = "${SCHEME}://"
|
||||
public static final String SEPARATOR = '/'
|
||||
|
||||
private final SeqeraFileSystem fs
|
||||
/** path segments in order: [org, workspace, resourceType, datasetName] — null for missing levels */
|
||||
private final String org
|
||||
private final String workspace
|
||||
private final String resourceType
|
||||
private final String datasetName
|
||||
/** version string extracted from {@code @version} suffix; null when not pinned */
|
||||
private final String version
|
||||
/**
|
||||
* Raw relative path string — non-null only for relative {@code SeqeraPath} instances
|
||||
* created by {@link #relativize(Path)}. When non-null, {@link #fs} is {@code null}
|
||||
* and all segment fields are {@code null}.
|
||||
*/
|
||||
private final String relPath
|
||||
|
||||
/**
|
||||
* Parse a {@code seqera://} URI string into a SeqeraPath.
|
||||
* The URI authority is the org; path segments are workspace, resourceType, datasetName.
|
||||
* The last segment may contain a {@code @version} suffix.
|
||||
*/
|
||||
SeqeraPath(SeqeraFileSystem fs, String uriString) {
|
||||
this.fs = fs
|
||||
this.relPath = null
|
||||
if (!uriString.startsWith("${SCHEME}://"))
|
||||
throw new InvalidPathException(uriString, "Not a seqera:// URI")
|
||||
// strip scheme: seqera://rest
|
||||
final withoutScheme = uriString.substring("${SCHEME}://".length())
|
||||
// split on '/'
|
||||
final parts = withoutScheme.split('/', -1).toList().findAll { it != null } as List<String>
|
||||
// parts[0]=org, parts[1]=workspace, parts[2]=resourceType, parts[3]=datasetName[@version]
|
||||
this.org = parts.size() > 0 && parts[0] ? parts[0] : null
|
||||
this.workspace = parts.size() > 1 && parts[1] ? parts[1] : null
|
||||
this.resourceType = parts.size() > 2 && parts[2] ? parts[2] : null
|
||||
if (parts.size() > 3 && parts[3]) {
|
||||
final last = parts[3]
|
||||
final atIdx = last.lastIndexOf('@')
|
||||
if (atIdx > 0) {
|
||||
this.datasetName = last.substring(0, atIdx)
|
||||
this.version = last.substring(atIdx + 1)
|
||||
} else {
|
||||
this.datasetName = last
|
||||
this.version = null
|
||||
}
|
||||
} else {
|
||||
this.datasetName = null
|
||||
this.version = null
|
||||
}
|
||||
validatePath(uriString)
|
||||
}
|
||||
|
||||
/** Internal constructor for programmatic absolute path creation */
|
||||
SeqeraPath(SeqeraFileSystem fs, String org, String workspace, String resourceType, String datasetName, String version) {
|
||||
this.fs = fs
|
||||
this.relPath = null
|
||||
this.org = org
|
||||
this.workspace = workspace
|
||||
this.resourceType = resourceType
|
||||
this.datasetName = datasetName
|
||||
this.version = version
|
||||
validatePath(null)
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructor for relative paths produced by {@link #relativize(Path)}.
|
||||
* The {@code relPath} is a slash-separated string of the differing path segments.
|
||||
* All segment fields are {@code null}; {@link #isAbsolute()} returns {@code false}.
|
||||
*/
|
||||
SeqeraPath(String relPath) {
|
||||
this.fs = null
|
||||
this.relPath = relPath ?: ''
|
||||
this.org = null
|
||||
this.workspace = null
|
||||
this.resourceType = null
|
||||
this.datasetName = null
|
||||
this.version = null
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate structural integrity: deeper segments require all shallower ones,
|
||||
* and no segment may contain {@code /}.
|
||||
*
|
||||
* @param original original URI string used in error messages (null → derive from fields)
|
||||
* @throws InvalidPathException if the path is malformed
|
||||
*/
|
||||
private void validatePath(String original) {
|
||||
final label = original ?: rawPath()
|
||||
if (datasetName && !workspace)
|
||||
throw new InvalidPathException(label, "Dataset path requires a workspace segment")
|
||||
if (resourceType && !workspace)
|
||||
throw new InvalidPathException(label, "Resource type requires a workspace segment")
|
||||
if (workspace && !org)
|
||||
throw new InvalidPathException(label, "Workspace requires an org segment")
|
||||
// Segments from URI parsing never contain '/', but guard the internal constructor too
|
||||
if (org?.contains('/'))
|
||||
throw new InvalidPathException(label, "Org name cannot contain '/'")
|
||||
if (workspace?.contains('/'))
|
||||
throw new InvalidPathException(label, "Workspace name cannot contain '/'")
|
||||
if (resourceType?.contains('/'))
|
||||
throw new InvalidPathException(label, "Resource type cannot contain '/'")
|
||||
if (datasetName?.contains('/'))
|
||||
throw new InvalidPathException(label, "Dataset name cannot contain '/'")
|
||||
}
|
||||
|
||||
/** Return a list of name component strings (works for both absolute and relative paths). */
|
||||
private List<String> nameComponents() {
|
||||
if (isAbsolute()) {
|
||||
final d = depth()
|
||||
final result = new ArrayList<String>(d)
|
||||
for (int i = 0; i < d; i++)
|
||||
result.add(getName(i).toString())
|
||||
return result
|
||||
}
|
||||
if (!relPath) return Collections.<String>emptyList()
|
||||
return relPath.split('/').toList().findAll { String s -> s } as List<String>
|
||||
}
|
||||
|
||||
/** Build a raw path string from the current fields, for use in exception messages. */
|
||||
private String rawPath() {
|
||||
final sb = new StringBuilder("${SCHEME}://")
|
||||
if (org) sb.append(org)
|
||||
if (workspace) sb.append('/').append(workspace)
|
||||
if (resourceType) sb.append('/').append(resourceType)
|
||||
if (datasetName) {
|
||||
sb.append('/').append(datasetName)
|
||||
if (version) sb.append('@').append(version)
|
||||
}
|
||||
return sb.toString()
|
||||
}
|
||||
|
||||
// ---- path component accessors ----
|
||||
|
||||
String getOrg() { org }
|
||||
String getWorkspace() { workspace }
|
||||
String getResourceType() { resourceType }
|
||||
String getDatasetName() { datasetName }
|
||||
String getVersion() { version }
|
||||
|
||||
/**
|
||||
* Path depth: 0=root, 1=org, 2=workspace, 3=resourceType, 4=dataset file.
|
||||
*/
|
||||
int depth() {
|
||||
if (datasetName) return 4
|
||||
if (resourceType) return 3
|
||||
if (workspace) return 2
|
||||
if (org) return 1
|
||||
return 0
|
||||
}
|
||||
|
||||
boolean isDirectory() { depth() < 4 }
|
||||
boolean isRegularFile() { depth() == 4 }
|
||||
|
||||
// ---- Path API ----
|
||||
|
||||
@Override
|
||||
FileSystem getFileSystem() { fs }
|
||||
|
||||
@Override
|
||||
boolean isAbsolute() { fs != null }
|
||||
|
||||
@Override
|
||||
Path getRoot() { new SeqeraPath(fs, null, null, null, null, null) }
|
||||
|
||||
@Override
|
||||
Path getFileName() {
|
||||
final d = depth()
|
||||
if (d == 0) return null
|
||||
final name = d == 4 ? (version ? "${datasetName}@${version}" : datasetName)
|
||||
: d == 3 ? resourceType
|
||||
: d == 2 ? workspace
|
||||
: org
|
||||
return new SeqeraPath( name as String)
|
||||
}
|
||||
|
||||
@Override
|
||||
Path getParent() {
|
||||
final d = depth()
|
||||
if (d == 0) return null
|
||||
if (d == 1) return new SeqeraPath(fs, null, null, null, null, null)
|
||||
if (d == 2) return new SeqeraPath(fs, org, null, null, null, null)
|
||||
if (d == 3) return new SeqeraPath(fs, org, workspace, null, null, null)
|
||||
return new SeqeraPath(fs, org, workspace, resourceType, null, null)
|
||||
}
|
||||
|
||||
@Override
|
||||
int getNameCount() { depth() }
|
||||
|
||||
@Override
|
||||
Path getName(int index) {
|
||||
final d = depth()
|
||||
if (index < 0 || index >= d)
|
||||
throw new IllegalArgumentException("Index out of range: $index")
|
||||
if (index == 0) return new SeqeraPath(org)
|
||||
if (index == 1) return new SeqeraPath(workspace)
|
||||
if (index == 2) return new SeqeraPath(resourceType)
|
||||
return new SeqeraPath((version ? "${datasetName}@${version}" : datasetName) as String)
|
||||
}
|
||||
|
||||
@Override
|
||||
Path subpath(int beginIndex, int endIndex) {
|
||||
throw new UnsupportedOperationException("subpath not supported by seqera:// paths")
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean startsWith(Path other) {
|
||||
if (other !instanceof SeqeraPath)
|
||||
return false
|
||||
final that = (SeqeraPath) other
|
||||
if (this.isAbsolute() != that.isAbsolute())
|
||||
return false
|
||||
final thisNames = this.nameComponents()
|
||||
final thatNames = that.nameComponents()
|
||||
if (thatNames.size() > thisNames.size())
|
||||
return false
|
||||
for (int i = 0; i < thatNames.size(); i++) {
|
||||
if (thisNames[i] != thatNames[i])
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean startsWith(String other) {
|
||||
if (!other) return false
|
||||
try {
|
||||
final Path p = SeqeraPath.isSeqeraUri(other) ? new SeqeraPath(fs, other) : new SeqeraPath(other)
|
||||
return startsWith(p)
|
||||
} catch (Exception ignored) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean endsWith(Path other) {
|
||||
if (other !instanceof SeqeraPath)
|
||||
return false
|
||||
final that = (SeqeraPath) other
|
||||
if (that.isAbsolute())
|
||||
return this.equals(that)
|
||||
final thisNames = this.nameComponents()
|
||||
final thatNames = that.nameComponents()
|
||||
if (thatNames.isEmpty() || thatNames.size() > thisNames.size())
|
||||
return false
|
||||
final offset = thisNames.size() - thatNames.size()
|
||||
for (int i = 0; i < thatNames.size(); i++) {
|
||||
if (thisNames[offset + i] != thatNames[i])
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean endsWith(String other) {
|
||||
if (!other) return false
|
||||
try {
|
||||
final Path p = SeqeraPath.isSeqeraUri(other) ? new SeqeraPath(fs, other) : new SeqeraPath(other)
|
||||
return endsWith(p)
|
||||
} catch (Exception ignored) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
Path normalize() { this }
|
||||
|
||||
@Override
|
||||
Path resolve(Path other) {
|
||||
if (other instanceof SeqeraPath) {
|
||||
final that = (SeqeraPath) other
|
||||
if (that.isAbsolute()) return that
|
||||
// Relative SeqeraPath: resolve each segment of relPath against this
|
||||
return resolve(that.relPath)
|
||||
}
|
||||
return resolve(other.toString())
|
||||
}
|
||||
|
||||
@Override
|
||||
Path resolve(String segment) {
|
||||
if (!segment) return this
|
||||
// Absolute seqera:// URI — parse and return directly
|
||||
if (segment.startsWith(PROTOCOL))
|
||||
return new SeqeraPath(fs, segment)
|
||||
// Strip a single leading slash
|
||||
final stripped = segment.startsWith(SEPARATOR) ? segment.substring(1) : segment
|
||||
if (!stripped) return this
|
||||
// Multi-segment: split and resolve one segment at a time
|
||||
final segs = stripped.split(SEPARATOR, -1).findAll { String s -> s } as List<String>
|
||||
SeqeraPath result = this
|
||||
for (String seg : segs) {
|
||||
result = result.resolveOne(seg)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
/** Resolve a single (non-empty, slash-free) segment against this path. */
|
||||
private SeqeraPath resolveOne(String seg) {
|
||||
final d = depth()
|
||||
if (d == 0) return new SeqeraPath(fs, seg, null, null, null, null)
|
||||
if (d == 1) return new SeqeraPath(fs, org, seg, null, null, null)
|
||||
if (d == 2) return new SeqeraPath(fs, org, workspace, seg, null, null)
|
||||
if (d == 3) {
|
||||
final atIdx = seg.lastIndexOf('@')
|
||||
if (atIdx > 0)
|
||||
return new SeqeraPath(fs, org, workspace, resourceType, seg.substring(0, atIdx), seg.substring(atIdx + 1))
|
||||
return new SeqeraPath(fs, org, workspace, resourceType, seg, null)
|
||||
}
|
||||
throw new IllegalStateException("Cannot resolve a path segment on a depth-4 path: $this")
|
||||
}
|
||||
|
||||
@Override
|
||||
Path resolveSibling(Path other) {
|
||||
final parent = getParent()
|
||||
return parent != null ? parent.resolve(other) : other
|
||||
}
|
||||
|
||||
@Override
|
||||
Path resolveSibling(String other) {
|
||||
final parent = getParent()
|
||||
return parent != null ? parent.resolve(other) : new SeqeraPath(fs, other)
|
||||
}
|
||||
|
||||
@Override
|
||||
Path relativize(Path other) {
|
||||
if (other !instanceof SeqeraPath)
|
||||
throw new ProviderMismatchException()
|
||||
final that = (SeqeraPath) other
|
||||
if (!this.isAbsolute() || !that.isAbsolute())
|
||||
throw new IllegalArgumentException("Both paths must be absolute to relativize: ${this} vs ${other}")
|
||||
final thisNames = this.nameComponents()
|
||||
final thatNames = that.nameComponents()
|
||||
// Find common prefix length
|
||||
int common = 0
|
||||
while (common < thisNames.size() && common < thatNames.size()
|
||||
&& thisNames[common] == thatNames[common])
|
||||
common++
|
||||
// Build ".." for each remaining segment in this, then append remaining segments of other
|
||||
final parts = new ArrayList<String>()
|
||||
for (int i = common; i < thisNames.size(); i++)
|
||||
parts.add('..')
|
||||
for (int i = common; i < thatNames.size(); i++)
|
||||
parts.add(thatNames[i])
|
||||
return new SeqeraPath(parts.join(SEPARATOR))
|
||||
}
|
||||
|
||||
@Override
|
||||
URI toUri() {
|
||||
// Build path component for depth >= 2
|
||||
String uriPath = null
|
||||
if (workspace) {
|
||||
final segments = [workspace]
|
||||
if (resourceType) segments.add(resourceType)
|
||||
if (datasetName) segments.add(version ? "${datasetName}@${version}" as String : datasetName)
|
||||
uriPath = '/' + segments.join('/')
|
||||
}
|
||||
// new URI(scheme, authority, path, query, fragment) avoids URI.create() pitfalls for edge cases
|
||||
return new URI(SCHEME, org ?: '', uriPath, null, null)
|
||||
}
|
||||
|
||||
@Override
|
||||
String toString() {
|
||||
if (!isAbsolute()) return relPath
|
||||
// Return the canonical human-readable representation
|
||||
final d = depth()
|
||||
if (d == 0) return "${SCHEME}://"
|
||||
return toUri().toString()
|
||||
}
|
||||
|
||||
@Override
|
||||
Path toAbsolutePath() {
|
||||
if (!isAbsolute())
|
||||
throw new IllegalStateException("Cannot convert relative SeqeraPath to absolute — no default directory context")
|
||||
return this
|
||||
}
|
||||
|
||||
@Override
|
||||
Path toRealPath(LinkOption... options) { this }
|
||||
|
||||
@Override
|
||||
File toFile() {
|
||||
throw new UnsupportedOperationException("toFile() not supported for seqera:// paths")
|
||||
}
|
||||
|
||||
@Override
|
||||
WatchKey register(WatchService watcher, WatchEvent.Kind<?>[] events, WatchEvent.Modifier... modifiers) {
|
||||
throw new UnsupportedOperationException("WatchService not supported by seqera:// paths")
|
||||
}
|
||||
|
||||
@Override
|
||||
WatchKey register(WatchService watcher, WatchEvent.Kind<?>... events) {
|
||||
throw new UnsupportedOperationException("WatchService not supported by seqera:// paths")
|
||||
}
|
||||
|
||||
@Override
|
||||
Iterator<Path> iterator() {
|
||||
final d = depth()
|
||||
final List<Path> parts = new ArrayList<>(d)
|
||||
for (int i = 0; i < d; i++) {
|
||||
parts.add(getName(i))
|
||||
}
|
||||
return parts.iterator()
|
||||
}
|
||||
|
||||
@Override
|
||||
int compareTo(Path other) {
|
||||
return toString().compareTo(other.toString())
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean equals(Object obj) {
|
||||
if (obj == this) return true
|
||||
if (obj !instanceof SeqeraPath) return false
|
||||
return toString() == obj.toString()
|
||||
}
|
||||
|
||||
@Override
|
||||
int hashCode() { toString().hashCode() }
|
||||
|
||||
static URI asUri(String path) {
|
||||
if( !path )
|
||||
throw new IllegalArgumentException("Missing 'path' argument")
|
||||
if( !path.startsWith(PROTOCOL) )
|
||||
throw new IllegalArgumentException("Invalid Seqera file system path URI - it must start with '${PROTOCOL}' prefix - offending value: $path")
|
||||
if( path.startsWith(PROTOCOL + SEPARATOR) && path.length() > PROTOCOL.length() + 1 )
|
||||
throw new IllegalArgumentException("Invalid Seqera file system path URI - make sure the scheme prefix does not contain more than two slash characters or a query in the root '/' - offending value: $path")
|
||||
|
||||
//URI strings like seqera://./something are converted to seqera://something
|
||||
if( path.startsWith(PROTOCOL + './') ) {
|
||||
path = PROTOCOL + path.substring(PROTOCOL.length() + 2)
|
||||
}
|
||||
|
||||
if( path == PROTOCOL || path == PROTOCOL + '.') //Empty path case
|
||||
return new URI(PROTOCOL + '/')
|
||||
return new URI(path)
|
||||
}
|
||||
|
||||
static boolean isSeqeraUri(String path) {
|
||||
return path && path.startsWith(PROTOCOL)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin.fs
|
||||
|
||||
import nextflow.file.FileHelper
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.file.FileSystemPathFactory
|
||||
|
||||
/**
|
||||
* PF4J extension that registers the {@code seqera://} URI scheme with Nextflow's file helper,
|
||||
* allowing pipeline scripts to use {@code file('seqera://org/ws/datasets/name')} transparently.
|
||||
*
|
||||
* Registered as a PF4J extension point via {@code extensionPoints} in {@code build.gradle}.
|
||||
*
|
||||
* @author Seqera Labs
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class SeqeraPathFactory extends FileSystemPathFactory {
|
||||
|
||||
|
||||
@Override
|
||||
Path parseUri(String str) {
|
||||
return str?.startsWith(SeqeraPath.PROTOCOL) ? create(str) : null
|
||||
}
|
||||
|
||||
@Override
|
||||
String toUriString(Path path) {
|
||||
if (path instanceof SeqeraPath)
|
||||
return path.toUri().toString()
|
||||
return null
|
||||
}
|
||||
|
||||
@Override
|
||||
String getBashLib(Path target) {
|
||||
// No bash-level staging for seqera:// — handled via NIO newInputStream/copy
|
||||
return null
|
||||
}
|
||||
|
||||
@Override
|
||||
String getUploadCmd(String source, Path target) {
|
||||
// No bash upload command — seqera:// filesystem is read-only
|
||||
return null
|
||||
}
|
||||
|
||||
static SeqeraPath create(String path) {
|
||||
final uri = SeqeraPath.asUri(path)
|
||||
return (SeqeraPath) FileHelper.getOrCreateFileSystemFor(uri).provider().getPath(uri)
|
||||
}
|
||||
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1 @@
|
||||
io.seqera.tower.plugin.fs.SeqeraFileSystemProvider
|
||||
@@ -0,0 +1,17 @@
|
||||
#
|
||||
# Copyright 2013-2026, Seqera Labs
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
io.seqera.tower.plugin.TowerFactory
|
||||
3
nextflow/plugins/nf-tower/src/resources/build.txt
Normal file
3
nextflow/plugins/nf-tower/src/resources/build.txt
Normal file
@@ -0,0 +1,3 @@
|
||||
version=0.3.0-SNAPSHOT
|
||||
commit=0a96ac3.dirty
|
||||
timestamp=2019-09-02T22:08:52.381+02:00
|
||||
128
nextflow/plugins/nf-tower/src/resources/tower-schema.properties
Normal file
128
nextflow/plugins/nf-tower/src/resources/tower-schema.properties
Normal file
@@ -0,0 +1,128 @@
|
||||
#
|
||||
# Copyright 2013-2026, Seqera Labs
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# workflow
|
||||
workflow.id = 16
|
||||
workflow.projectDir = 255
|
||||
workflow.profile = 100
|
||||
workflow.homeDir = 255
|
||||
workflow.workDir = 255
|
||||
workflow.container = 255
|
||||
workflow.commitId = 40
|
||||
workflow.errorMessage = 255
|
||||
workflow.repository = 255
|
||||
workflow.containerEngine = 255
|
||||
workflow.scriptFile = 255
|
||||
workflow.userName = 40
|
||||
workflow.launchDir = 255
|
||||
workflow.runName = 80
|
||||
workflow.sessionId = 36
|
||||
workflow.scriptId = 40
|
||||
workflow.revision = 100
|
||||
workflow.nextflow.version = 20
|
||||
workflow.nextflow.build = 10
|
||||
workflow.projectName = 100
|
||||
workflow.scriptName = 100
|
||||
workflow.operationId = 110
|
||||
workflow.outFile = 255
|
||||
workflow.logFile = 255
|
||||
workflow.manifest.nextflowVersion = 20
|
||||
workflow.manifest.defaultBranch = 20
|
||||
workflow.manifest.version = 20
|
||||
workflow.manifest.homePage = 200
|
||||
workflow.manifest.description = 1024
|
||||
workflow.manifest.name = 150
|
||||
workflow.manifest.mainScript = 100
|
||||
workflow.manifest.author = 150
|
||||
workflow.manifest.gitmodules = 150
|
||||
workflow.manifest.doi = 100
|
||||
workflow.stats.computeTimeFmt = 50
|
||||
# process names
|
||||
processNames = 255
|
||||
# tasks
|
||||
workflowId = 16
|
||||
tasks.env = 2048
|
||||
tasks.script = 10240
|
||||
tasks.status = 10
|
||||
tasks.hash = 34
|
||||
tasks.name = 255
|
||||
tasks.process = 255
|
||||
tasks.tag = 255
|
||||
tasks.container = 255
|
||||
tasks.scratch = 255
|
||||
tasks.workdir = 512
|
||||
tasks.queue = 100
|
||||
tasks.errorAction = 10
|
||||
tasks.nativeId = 100
|
||||
tasks.module = 255
|
||||
tasks.executor = 25
|
||||
tasks.machineType = 25
|
||||
tasks.cloudZone = 25
|
||||
tasks.priceModel = 15
|
||||
# process progress
|
||||
progress.name = 255
|
||||
# metrics
|
||||
metrics.process = 255
|
||||
metrics.cpuUsage.minLabel = 255
|
||||
metrics.cpuUsage.maxLabel = 255
|
||||
metrics.cpuUsage.q1Label = 255
|
||||
metrics.cpuUsage.q2Label = 255
|
||||
metrics.cpuUsage.q3Label = 255
|
||||
metrics.process.minLabel = 255
|
||||
metrics.process.maxLabel = 255
|
||||
metrics.process.q1Label = 255
|
||||
metrics.process.q2Label = 255
|
||||
metrics.process.q3Label = 255
|
||||
metrics.mem.minLabel = 255
|
||||
metrics.mem.maxLabel = 255
|
||||
metrics.mem.q1Label = 255
|
||||
metrics.mem.q2Label = 255
|
||||
metrics.mem.q3Label = 255
|
||||
metrics.memUsage.minLabel = 255
|
||||
metrics.memUsage.maxLabel = 255
|
||||
metrics.memUsage.q1Label = 255
|
||||
metrics.memUsage.q2Label = 255
|
||||
metrics.memUsage.q3Label = 255
|
||||
metrics.timeUsage.minLabel = 255
|
||||
metrics.timeUsage.maxLabel = 255
|
||||
metrics.timeUsage.q1Label = 255
|
||||
metrics.timeUsage.q2Label = 255
|
||||
metrics.timeUsage.q3Label = 255
|
||||
metrics.vmem.minLabel = 255
|
||||
metrics.vmem.maxLabel = 255
|
||||
metrics.vmem.q1Label = 255
|
||||
metrics.vmem.q2Label = 255
|
||||
metrics.vmem.q3Label = 255
|
||||
metrics.cpu.minLabel = 255
|
||||
metrics.cpu.maxLabel = 255
|
||||
metrics.cpu.q1Label = 255
|
||||
metrics.cpu.q2Label = 255
|
||||
metrics.cpu.q3Label = 255
|
||||
metrics.time.minLabel = 255
|
||||
metrics.time.maxLabel = 255
|
||||
metrics.time.q1Label = 255
|
||||
metrics.time.q2Label = 255
|
||||
metrics.time.q3Label = 255
|
||||
metrics.reads.minLabel = 255
|
||||
metrics.reads.maxLabel = 255
|
||||
metrics.reads.q1Label = 255
|
||||
metrics.reads.q2Label = 255
|
||||
metrics.reads.q3Label = 255
|
||||
metrics.writes.minLabel = 255
|
||||
metrics.writes.maxLabel = 255
|
||||
metrics.writes.q1Label = 255
|
||||
metrics.writes.q2Label = 255
|
||||
metrics.writes.q3Label = 255
|
||||
@@ -0,0 +1,217 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Paths
|
||||
|
||||
import nextflow.SysEnv
|
||||
import nextflow.exception.AbortOperationException
|
||||
import spock.lang.Specification
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class CacheManagerTest extends Specification {
|
||||
|
||||
def setupSpec() {
|
||||
SysEnv.push([:])
|
||||
}
|
||||
|
||||
def cleanupSpec() {
|
||||
SysEnv.pop()
|
||||
}
|
||||
|
||||
def 'should init empty files' () {
|
||||
when:
|
||||
new CacheManager([:])
|
||||
then:
|
||||
thrown(AbortOperationException)
|
||||
}
|
||||
|
||||
def 'should upload cache files' () {
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def remote = folder.resolve('remote'); remote.mkdir()
|
||||
def local = folder.resolve('local'); local.mkdir()
|
||||
def outFile = local.resolve('nf-out.txt'); outFile.text = 'out file'
|
||||
def logFile = local.resolve('nf-log.txt'); logFile.text = 'log file'
|
||||
def tmlFile = local.resolve('nf-tml.txt'); tmlFile.text = 'tml file'
|
||||
def cfgFile = local.resolve('tw-config.txt'); cfgFile.text = 'config file'
|
||||
def repFile = local.resolve('tw-report.txt'); repFile.text = 'report file'
|
||||
and:
|
||||
def uuid = UUID.randomUUID().toString()
|
||||
and:
|
||||
def ENV = [
|
||||
NXF_UUID:uuid,
|
||||
NXF_WORK: remote.toString(),
|
||||
NXF_OUT_FILE: outFile.toString(),
|
||||
NXF_LOG_FILE: logFile.toString(),
|
||||
NXF_TML_FILE: tmlFile.toString(),
|
||||
TOWER_CONFIG_FILE: cfgFile.toString(),
|
||||
TOWER_REPORTS_FILE: repFile.toString(),
|
||||
]
|
||||
|
||||
when:
|
||||
def tower = new CacheManager(ENV)
|
||||
then:
|
||||
tower.sessionUuid == uuid
|
||||
tower.localCachePath == Paths.get(".nextflow/cache/$uuid")
|
||||
tower.localOutFile == outFile
|
||||
tower.localLogFile == logFile
|
||||
tower.localTimelineFile == tmlFile
|
||||
tower.localTowerConfig == cfgFile
|
||||
tower.localTowerReports == repFile
|
||||
and:
|
||||
tower.remoteWorkDir == remote
|
||||
and:
|
||||
tower.remoteCachePath == remote.resolve(".nextflow/cache/$uuid")
|
||||
tower.remoteOutFile == remote.resolve( outFile.name )
|
||||
tower.remoteLogFile == remote.resolve( logFile.name )
|
||||
tower.remoteTimelineFile == remote.resolve( tmlFile.name )
|
||||
tower.remoteTowerConfig == remote.resolve( cfgFile.name )
|
||||
tower.remoteTowerReports == remote.resolve( repFile.name )
|
||||
|
||||
when:
|
||||
// create local cache fake data
|
||||
tower.localCachePath = local.resolve(".nextflow/cache/$uuid");
|
||||
tower.localCachePath.mkdirs()
|
||||
tower.localCachePath.resolve('index-foo').text = 'index foo'
|
||||
tower.localCachePath.resolve('db').mkdir()
|
||||
tower.localCachePath.resolve('db/xxx').text = 'data xxx'
|
||||
tower.localCachePath.resolve('db/yyy').text = 'data yyy'
|
||||
and:
|
||||
tower.saveCacheFiles()
|
||||
then:
|
||||
tower.remoteCachePath.resolve('index-foo').text == 'index foo'
|
||||
tower.remoteCachePath.resolve('db/xxx').text == 'data xxx'
|
||||
tower.remoteCachePath.resolve('db/yyy').text == 'data yyy'
|
||||
and:
|
||||
tower.remoteOutFile.text == outFile.text
|
||||
tower.remoteLogFile.text == logFile.text
|
||||
tower.remoteTimelineFile.text == tmlFile.text
|
||||
tower.remoteTowerConfig.text == cfgFile.text
|
||||
tower.remoteTowerReports.text == repFile.text
|
||||
|
||||
// simulate a 2nd run with different data
|
||||
when:
|
||||
tower.localCachePath.deleteDir()
|
||||
tower.localCachePath.mkdirs()
|
||||
tower.localCachePath.resolve('index-bar').text = 'index bar'
|
||||
tower.localCachePath.resolve('db').mkdir()
|
||||
tower.localCachePath.resolve('db/alpha').text = 'data alpha'
|
||||
tower.localCachePath.resolve('db/delta').text = 'data delta'
|
||||
and:
|
||||
tower.saveCacheFiles()
|
||||
then:
|
||||
tower.remoteCachePath.resolve('index-bar').text == 'index bar'
|
||||
tower.remoteCachePath.resolve('db/alpha').text == 'data alpha'
|
||||
tower.remoteCachePath.resolve('db/delta').text == 'data delta'
|
||||
and:
|
||||
!tower.remoteCachePath.resolve('index-foo').exists()
|
||||
!tower.remoteCachePath.resolve('db/xxx').exists()
|
||||
!tower.remoteCachePath.resolve('db/yyy').exists()
|
||||
and:
|
||||
tower.remoteOutFile.text == outFile.text
|
||||
tower.remoteLogFile.text == logFile.text
|
||||
tower.remoteTimelineFile.text == tmlFile.text
|
||||
tower.remoteTowerConfig.text == cfgFile.text
|
||||
tower.remoteTowerReports.text == repFile.text
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
def 'should upload log files even when local cache path does not exist' () {
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def remote = folder.resolve('remote'); remote.mkdir()
|
||||
def local = folder.resolve('local'); local.mkdir()
|
||||
def outFile = local.resolve('nf-out.txt'); outFile.text = 'out file'
|
||||
def logFile = local.resolve('nf-log.txt'); logFile.text = 'log file'
|
||||
def tmlFile = local.resolve('nf-tml.txt'); tmlFile.text = 'tml file'
|
||||
def cfgFile = local.resolve('tw-config.txt'); cfgFile.text = 'config file'
|
||||
def repFile = local.resolve('tw-report.txt'); repFile.text = 'report file'
|
||||
and:
|
||||
def uuid = UUID.randomUUID().toString()
|
||||
and:
|
||||
def ENV = [
|
||||
NXF_UUID: uuid,
|
||||
NXF_WORK: remote.toString(),
|
||||
NXF_OUT_FILE: outFile.toString(),
|
||||
NXF_LOG_FILE: logFile.toString(),
|
||||
NXF_TML_FILE: tmlFile.toString(),
|
||||
TOWER_CONFIG_FILE: cfgFile.toString(),
|
||||
TOWER_REPORTS_FILE: repFile.toString(),
|
||||
]
|
||||
|
||||
when:
|
||||
def tower = new CacheManager(ENV)
|
||||
// do NOT create localCachePath — simulates K8s where cache may not exist locally
|
||||
tower.saveCacheFiles()
|
||||
then:
|
||||
// metadata cache is not copied
|
||||
!tower.remoteCachePath.exists()
|
||||
and:
|
||||
// log files are still copied
|
||||
tower.remoteOutFile.text == outFile.text
|
||||
tower.remoteLogFile.text == logFile.text
|
||||
tower.remoteTimelineFile.text == tmlFile.text
|
||||
tower.remoteTowerConfig.text == cfgFile.text
|
||||
tower.remoteTowerReports.text == repFile.text
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
def 'should download cache files' () {
|
||||
given:
|
||||
def uuid = UUID.randomUUID().toString()
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def local = folder.resolve('local'); local.mkdir()
|
||||
def outFile = local.resolve('nf-out.txt');
|
||||
def logFile = local.resolve('nf-log.txt')
|
||||
def tmlFile = local.resolve('nf-tml.txt')
|
||||
def cfgFile = local.resolve('tw-config.txt')
|
||||
def repFile = local.resolve('tw-report.txt')
|
||||
and:
|
||||
def remote = folder.resolve('remote'); remote.mkdir()
|
||||
remote.resolve('nf-out.txt').text = 'the out file'
|
||||
remote.resolve('nf-log.txt').text = 'the log file'
|
||||
remote.resolve('nf-tml.txt').text = 'the timeline file'
|
||||
remote.resolve('nf-config.txt').text = 'the config file'
|
||||
remote.resolve('nf-report.txt').text = 'the report file'
|
||||
and:
|
||||
remote.resolve(".nextflow/cache/$uuid").mkdirs()
|
||||
remote.resolve(".nextflow/cache/$uuid").resolve('index-bar').text = 'index bar'
|
||||
remote.resolve(".nextflow/cache/$uuid").resolve('db').mkdirs()
|
||||
remote.resolve(".nextflow/cache/$uuid").resolve('db/alpha').text = 'data alpha'
|
||||
remote.resolve(".nextflow/cache/$uuid").resolve('db/delta').text = 'data delta'
|
||||
and:
|
||||
def tower = new CacheManager([NXF_UUID: uuid, NXF_WORK: remote.toString()])
|
||||
|
||||
when:
|
||||
tower.restoreCacheFiles()
|
||||
then:
|
||||
tower.localCachePath.resolve('index-bar').text == 'index bar'
|
||||
tower.localCachePath.resolve('db/alpha').text == 'data alpha'
|
||||
tower.localCachePath.resolve('db/delta').text == 'data delta'
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin
|
||||
|
||||
import nextflow.Session
|
||||
import nextflow.SysEnv
|
||||
import nextflow.util.Duration
|
||||
import spock.lang.Specification
|
||||
import test.TestHelper
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class LogsCheckpointTest extends Specification {
|
||||
|
||||
def 'should configure default delay' () {
|
||||
given:
|
||||
def session = Mock(Session) {
|
||||
getWorkDir() >> TestHelper.createInMemTempDir()
|
||||
getConfig() >> [:]
|
||||
}
|
||||
and:
|
||||
def checkpoint = new LogsCheckpoint()
|
||||
|
||||
when:
|
||||
checkpoint.onFlowCreate(session)
|
||||
then:
|
||||
checkpoint.@interval == Duration.of('90s')
|
||||
}
|
||||
|
||||
def 'should configure delay via env var' () {
|
||||
given:
|
||||
SysEnv.push(TOWER_LOGS_CHECKPOINT_INTERVAL: '200s')
|
||||
def session = Mock(Session) {
|
||||
getWorkDir() >> TestHelper.createInMemTempDir()
|
||||
getConfig() >> [:]
|
||||
}
|
||||
and:
|
||||
def checkpoint = new LogsCheckpoint()
|
||||
|
||||
when:
|
||||
checkpoint.onFlowCreate(session)
|
||||
then:
|
||||
checkpoint.@interval == Duration.of('200s')
|
||||
|
||||
cleanup:
|
||||
SysEnv.pop()
|
||||
}
|
||||
|
||||
def 'should configure delay via config file' () {
|
||||
given:
|
||||
SysEnv.push(NXF_WORK: '/some/path', TOWER_LOGS_CHECKPOINT_INTERVAL: '200s')
|
||||
def session = Mock(Session) {
|
||||
getConfig()>>[tower:[logs:[checkpoint:[interval: '500s']]]]
|
||||
getWorkDir() >> TestHelper.createInMemTempDir()
|
||||
}
|
||||
and:
|
||||
def checkpoint = new LogsCheckpoint()
|
||||
|
||||
when:
|
||||
checkpoint.onFlowCreate(session)
|
||||
then:
|
||||
checkpoint.@interval == Duration.of('500s')
|
||||
|
||||
cleanup:
|
||||
SysEnv.pop()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin
|
||||
|
||||
import java.nio.file.Files
|
||||
|
||||
import nextflow.Session
|
||||
import nextflow.exception.AbortOperationException
|
||||
import spock.lang.Specification
|
||||
import test.TestHelper
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class LogsHandlerTest extends Specification {
|
||||
|
||||
def 'should init empty files' () {
|
||||
when:
|
||||
new LogsHandler(Mock(Session), [:])
|
||||
then:
|
||||
thrown(AbortOperationException)
|
||||
}
|
||||
|
||||
def 'should upload cache files' () {
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def remote = TestHelper.createInMemTempDir()
|
||||
def local = folder.resolve('local'); local.mkdir()
|
||||
def outFile = local.resolve('nf-out.txt'); outFile.text = 'out file'
|
||||
def logFile = local.resolve('nf-log.txt'); logFile.text = 'log file'
|
||||
def tmlFile = local.resolve('nf-tml.txt'); tmlFile.text = 'tml file'
|
||||
def cfgFile = local.resolve('tw-config.txt'); cfgFile.text = 'config file'
|
||||
def repFile = local.resolve('tw-report.txt'); repFile.text = 'report file'
|
||||
and:
|
||||
def uuid = UUID.randomUUID().toString()
|
||||
and:
|
||||
def session = Mock(Session) {getWorkDir() >> remote }
|
||||
def ENV = [
|
||||
NXF_UUID:uuid,
|
||||
NXF_OUT_FILE: outFile.toString(),
|
||||
NXF_LOG_FILE: logFile.toString(),
|
||||
NXF_TML_FILE: tmlFile.toString(),
|
||||
TOWER_CONFIG_FILE: cfgFile.toString(),
|
||||
TOWER_REPORTS_FILE: repFile.toString(),
|
||||
]
|
||||
|
||||
when:
|
||||
def tower = new LogsHandler(session, ENV)
|
||||
then:
|
||||
tower.localOutFile == outFile
|
||||
tower.localLogFile == logFile
|
||||
tower.localTimelineFile == tmlFile
|
||||
tower.localTowerConfig == cfgFile
|
||||
tower.localTowerReports == repFile
|
||||
and:
|
||||
tower.remoteWorkDir == remote
|
||||
and:
|
||||
tower.remoteOutFile == remote.resolve( outFile.name )
|
||||
tower.remoteLogFile == remote.resolve( logFile.name )
|
||||
tower.remoteTimelineFile == remote.resolve( tmlFile.name )
|
||||
tower.remoteTowerConfig == remote.resolve( cfgFile.name )
|
||||
tower.remoteTowerReports == remote.resolve( repFile.name )
|
||||
|
||||
when:
|
||||
// create local cache fake data
|
||||
tower.saveFiles()
|
||||
then:
|
||||
tower.remoteOutFile.text == outFile.text
|
||||
tower.remoteLogFile.text == logFile.text
|
||||
tower.remoteTimelineFile.text == tmlFile.text
|
||||
tower.remoteTowerConfig.text == cfgFile.text
|
||||
tower.remoteTowerReports.text == repFile.text
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,310 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin
|
||||
|
||||
import java.net.http.HttpResponse
|
||||
import java.time.Instant
|
||||
|
||||
import com.github.tomakehurst.wiremock.WireMockServer
|
||||
import com.github.tomakehurst.wiremock.client.WireMock
|
||||
import io.seqera.http.HxClient
|
||||
import nextflow.exception.AbortRunException
|
||||
import nextflow.util.Duration
|
||||
import spock.lang.Specification
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class TowerClientTest extends Specification {
|
||||
|
||||
protected boolean aroundNow(value) {
|
||||
def now = Instant.now().toEpochMilli()
|
||||
value > now-1_000 && value <= now
|
||||
}
|
||||
|
||||
def 'should parse response' () {
|
||||
given:
|
||||
def tower = new TowerClient()
|
||||
|
||||
when:
|
||||
def resp = new TowerClient.Response(200, '{"status":"OK", "workflowId":"12345", "watchUrl": "http://foo.com/watch/12345"}')
|
||||
def result = tower.parseTowerResponse(resp)
|
||||
then:
|
||||
result.workflowId == '12345'
|
||||
result.watchUrl == 'http://foo.com/watch/12345'
|
||||
|
||||
when:
|
||||
resp = new TowerClient.Response(500, '{"status":"OK", "workflowId":"12345"}')
|
||||
tower.parseTowerResponse(resp)
|
||||
then:
|
||||
thrown(Exception)
|
||||
}
|
||||
|
||||
def 'should validate URL' () {
|
||||
given:
|
||||
def observer = new TowerClient()
|
||||
|
||||
expect:
|
||||
observer.checkUrl('http://localhost') == 'http://localhost'
|
||||
observer.checkUrl('http://google.com') == 'http://google.com'
|
||||
observer.checkUrl('https://google.com') == 'https://google.com'
|
||||
observer.checkUrl('http://google.com:8080') == 'http://google.com:8080'
|
||||
observer.checkUrl('http://google.com:8080/') == 'http://google.com:8080'
|
||||
observer.checkUrl('http://google.com:8080/foo/bar') == 'http://google.com:8080/foo/bar'
|
||||
observer.checkUrl('http://google.com:8080/foo/bar/') == 'http://google.com:8080/foo/bar'
|
||||
observer.checkUrl('http://google.com:8080/foo/bar///') == 'http://google.com:8080/foo/bar'
|
||||
|
||||
when:
|
||||
observer.checkUrl('ftp://localhost')
|
||||
then:
|
||||
def e = thrown(IllegalArgumentException)
|
||||
e.message == 'Only http and https are supported -- The given URL was: ftp://localhost'
|
||||
}
|
||||
|
||||
def 'should get watch url' () {
|
||||
given:
|
||||
def observer = new TowerClient()
|
||||
expect:
|
||||
observer.getHostUrl(STR) == EXPECTED
|
||||
where:
|
||||
STR | EXPECTED
|
||||
'http://foo.com' | 'http://foo.com'
|
||||
'http://foo.com:800/' | 'http://foo.com:800'
|
||||
'https://foo.com:800/' | 'https://foo.com:800'
|
||||
'http://foo.com:8000/this/that' | 'http://foo.com:8000'
|
||||
}
|
||||
|
||||
def 'should get access token' () {
|
||||
when:
|
||||
def config = new TowerConfig([accessToken: 'abc'], [TOWER_ACCESS_TOKEN: 'xyz'])
|
||||
def client = new TowerClient(config)
|
||||
then:
|
||||
// the token in the config overrides the one in the env
|
||||
client.getAccessToken() == 'abc'
|
||||
|
||||
when:
|
||||
config = new TowerConfig([accessToken: 'abc'], [TOWER_ACCESS_TOKEN: 'xyz', TOWER_WORKFLOW_ID: '111222333'])
|
||||
client = new TowerClient(config)
|
||||
then:
|
||||
// the token from the env is taken because is a tower launch aka TOWER_WORKFLOW_ID is set
|
||||
client.getAccessToken() == 'xyz'
|
||||
|
||||
when:
|
||||
config = new TowerConfig([:], [TOWER_ACCESS_TOKEN: 'xyz'])
|
||||
client = new TowerClient(config)
|
||||
then:
|
||||
client.getAccessToken() == 'xyz'
|
||||
|
||||
when:
|
||||
def c = new TowerClient()
|
||||
c.getAccessToken()
|
||||
then:
|
||||
thrown(AbortRunException)
|
||||
}
|
||||
|
||||
def 'should set the auth token' () {
|
||||
given:
|
||||
def http = Mock(HxClient.Builder)
|
||||
def client = new TowerClient()
|
||||
and:
|
||||
def SIMPLE = '4ffbf1009ebabea77db3d72efefa836dfbb71271'
|
||||
def BEARER = 'eyJ0aWQiOiA1fS5jZmM1YjVhOThjZjM2MTk1NjBjZWU1YmMwODUxYzA1ZjkzMDdmN2Iz'
|
||||
|
||||
when:
|
||||
client.setupClientAuth(http, SIMPLE)
|
||||
then:
|
||||
1 * http.basicAuth('@token:' + SIMPLE) >> http
|
||||
|
||||
when:
|
||||
client.setupClientAuth(http, SIMPLE)
|
||||
then:
|
||||
1 * http.basicAuth('@token:' + SIMPLE) >> http
|
||||
|
||||
when:
|
||||
client.setupClientAuth(http, BEARER)
|
||||
then:
|
||||
1 * http.bearerToken(BEARER) >> http
|
||||
1 * http.refreshToken(_) >> http
|
||||
1 * http.refreshTokenUrl(_) >> http
|
||||
}
|
||||
|
||||
def 'should get trace endpoint' () {
|
||||
given:
|
||||
def client = new TowerClient()
|
||||
client.@endpoint = TowerClient.DEF_ENDPOINT_URL
|
||||
|
||||
expect:
|
||||
client.getUrlTraceCreate(null) == 'https://api.cloud.seqera.io/trace/create'
|
||||
client.getUrlTraceBegin(null, '12345') == 'https://api.cloud.seqera.io/trace/12345/begin'
|
||||
client.getUrlTraceProgress(null, '12345') == 'https://api.cloud.seqera.io/trace/12345/progress'
|
||||
client.getUrlTraceHeartbeat(null, '12345') == 'https://api.cloud.seqera.io/trace/12345/heartbeat'
|
||||
client.getUrlTraceComplete(null, '12345') == 'https://api.cloud.seqera.io/trace/12345/complete'
|
||||
}
|
||||
|
||||
def 'should get trace endpoint with workspace' () {
|
||||
given:
|
||||
def client = new TowerClient()
|
||||
client.@endpoint = TowerClient.DEF_ENDPOINT_URL
|
||||
|
||||
expect:
|
||||
client.getUrlTraceCreate('300') == 'https://api.cloud.seqera.io/trace/create?workspaceId=300'
|
||||
client.getUrlTraceBegin('300', '12345') == 'https://api.cloud.seqera.io/trace/12345/begin?workspaceId=300'
|
||||
client.getUrlTraceProgress('300', '12345') == 'https://api.cloud.seqera.io/trace/12345/progress?workspaceId=300'
|
||||
client.getUrlTraceHeartbeat('300', '12345') == 'https://api.cloud.seqera.io/trace/12345/heartbeat?workspaceId=300'
|
||||
client.getUrlTraceComplete('300', '12345') == 'https://api.cloud.seqera.io/trace/12345/complete?workspaceId=300'
|
||||
}
|
||||
|
||||
def 'should load schema col len' () {
|
||||
given:
|
||||
def tower = new TowerClient()
|
||||
|
||||
when:
|
||||
def schema = tower.loadSchema()
|
||||
then:
|
||||
schema.get('workflow.start') == null
|
||||
schema.get('workflow.profile') == 100
|
||||
schema.get('workflow.projectDir') == 255
|
||||
}
|
||||
|
||||
def 'should handle HTTP request with content'() {
|
||||
given: 'a TowerClient'
|
||||
def tower = new TowerClient()
|
||||
def content = '{"test": "data"}'
|
||||
def request = tower.makeRequest('http://example.com/test', content, 'POST')
|
||||
|
||||
expect: 'the request should be created with the content'
|
||||
request != null
|
||||
request.method() == 'POST'
|
||||
request.uri().toString() == 'http://example.com/test'
|
||||
}
|
||||
|
||||
def 'should send http message' () {
|
||||
given:
|
||||
def client = Mock(HxClient)
|
||||
def tower = new TowerClient()
|
||||
tower.@httpClient = client
|
||||
|
||||
when:
|
||||
def resp = tower.sendHttpMessage('http://foo.com', [foo: 'bar'], 'POST')
|
||||
then:
|
||||
1 * client.sendAsString(_) >> Mock(HttpResponse) { statusCode() >> 200; body() >> '{}' }
|
||||
and:
|
||||
!resp.error
|
||||
resp.code == 200
|
||||
}
|
||||
|
||||
def 'should return error response on http request timeout' () {
|
||||
given: 'a WireMock server that hangs for 5 seconds'
|
||||
def wireMock = new WireMockServer(0)
|
||||
wireMock.start()
|
||||
wireMock.stubFor(
|
||||
WireMock.post(WireMock.anyUrl())
|
||||
.willReturn(WireMock.aResponse()
|
||||
.withFixedDelay(5_000)
|
||||
.withStatus(200)
|
||||
.withBody('{}'))
|
||||
)
|
||||
|
||||
and: 'a TowerClient whose requests carry a 200ms timeout'
|
||||
TowerConfig config = Mock(TowerConfig) {
|
||||
getHttpReadTimeout() >> Duration.of('200 ms')
|
||||
getHttpConnectTimeout() >> Duration.of('5 s')
|
||||
getEndpoint() >> wireMock.baseUrl()
|
||||
getAccessToken() >> 'token'
|
||||
}
|
||||
TowerClient client = new TowerClient(config)
|
||||
|
||||
when:
|
||||
def response = client.sendHttpMessage("${wireMock.baseUrl()}/trace/create", [runName: 'test'], 'POST')
|
||||
|
||||
then: 'a timeout produces an error response with code 0'
|
||||
response.code == 0
|
||||
response.message.contains('Unable to connect')
|
||||
|
||||
cleanup:
|
||||
wireMock.stop()
|
||||
}
|
||||
|
||||
def 'should build URL without query params'() {
|
||||
given:
|
||||
def client = new TowerClient()
|
||||
client.@endpoint = 'https://api.cloud.seqera.io'
|
||||
|
||||
when:
|
||||
def url = client.buildUrl( '/workflow/launch', [:])
|
||||
|
||||
then:
|
||||
url == 'https://api.cloud.seqera.io/workflow/launch'
|
||||
}
|
||||
|
||||
def 'should build URL with query params'() {
|
||||
given:
|
||||
def client = new TowerClient()
|
||||
client.@endpoint = 'https://api.cloud.seqera.io'
|
||||
|
||||
when:
|
||||
def url = client.buildUrl( '/workflow/launch', [workspaceId: '12345'])
|
||||
|
||||
then:
|
||||
url.contains('https://api.cloud.seqera.io/workflow/launch?')
|
||||
url.contains('workspaceId=12345')
|
||||
}
|
||||
|
||||
def 'should URL encode query params'() {
|
||||
given:
|
||||
def client = new TowerClient()
|
||||
client.@endpoint = 'https://api.cloud.seqera.io'
|
||||
|
||||
when:
|
||||
def url = client.buildUrl( '/workflow', [name: 'test workflow'])
|
||||
|
||||
then:
|
||||
url.contains('name=test+workflow')
|
||||
}
|
||||
|
||||
def 'should send AbortRunException in selected client calls'() {
|
||||
given:
|
||||
def client = Spy(new TowerClient(new TowerConfig([:], [TOWER_ACCESS_TOKEN: 'token']))){
|
||||
sendHttpMessage(_,_,_) >> new TowerClient.Response(401)
|
||||
}
|
||||
|
||||
when:
|
||||
client.traceCreate([:], '1234')
|
||||
then:
|
||||
thrown(AbortRunException)
|
||||
|
||||
when:
|
||||
client.traceBegin([:], '1234', '5678')
|
||||
then:
|
||||
thrown(AbortRunException)
|
||||
|
||||
when:
|
||||
client.traceProgress([:], '1234', '5678')
|
||||
then:
|
||||
thrown(AbortRunException)
|
||||
|
||||
when:
|
||||
client.traceComplete([:], '1234', '5678')
|
||||
then:
|
||||
notThrown(AbortRunException)
|
||||
|
||||
when:
|
||||
client.traceHeartbeat([:], '1234', '5678')
|
||||
then:
|
||||
notThrown(AbortRunException)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin
|
||||
|
||||
import nextflow.util.Duration
|
||||
import spock.lang.Specification
|
||||
|
||||
/**
|
||||
* Unit tests for TowerConfig
|
||||
*/
|
||||
class TowerConfigTest extends Specification {
|
||||
|
||||
def 'should use default endpoint when not specified'() {
|
||||
when:
|
||||
def config = new TowerConfig([:], [TOWER_API_ENDPOINT: 'https://example.com'])
|
||||
then:
|
||||
config.endpoint == 'https://example.com'
|
||||
|
||||
when:
|
||||
config = new TowerConfig([:], [:])
|
||||
then:
|
||||
config.endpoint == 'https://api.cloud.seqera.io'
|
||||
}
|
||||
|
||||
def 'should use default timeout values when not specified'() {
|
||||
when:
|
||||
def config = new TowerConfig([:], [:])
|
||||
|
||||
then:
|
||||
config.httpConnectTimeout == Duration.of('60s')
|
||||
config.httpReadTimeout == Duration.of('60s')
|
||||
}
|
||||
|
||||
def 'should use provided connect timeout when specified'() {
|
||||
when:
|
||||
def config = new TowerConfig([httpConnectTimeout: Duration.of('30s')], [:])
|
||||
|
||||
then:
|
||||
config.httpConnectTimeout == Duration.of('30s')
|
||||
config.httpReadTimeout == Duration.of('60s')
|
||||
}
|
||||
|
||||
def 'should use provided read timeout when specified'() {
|
||||
when:
|
||||
def config = new TowerConfig([httpReadTimeout: Duration.of('120s')], [:])
|
||||
|
||||
then:
|
||||
config.httpConnectTimeout == Duration.of('60s')
|
||||
config.httpReadTimeout == Duration.of('120s')
|
||||
}
|
||||
|
||||
def 'should parse timeout from string value'() {
|
||||
when:
|
||||
def config = new TowerConfig([httpConnectTimeout: '5s', httpReadTimeout: '2m'], [:])
|
||||
|
||||
then:
|
||||
config.httpConnectTimeout == Duration.of('5s')
|
||||
config.httpReadTimeout == Duration.of('2m')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin
|
||||
|
||||
import nextflow.Session
|
||||
import spock.lang.Specification
|
||||
import spock.lang.Unroll
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class TowerFactoryTest extends Specification {
|
||||
|
||||
def 'should create a tower observer' () {
|
||||
given:
|
||||
def factory = new TowerFactory(env: [TOWER_ACCESS_TOKEN: '123'])
|
||||
|
||||
when:
|
||||
def session = Mock(Session) { getConfig() >> [tower: [enabled: true]] }
|
||||
def observer = factory.create(session)[0] as TowerObserver
|
||||
then:
|
||||
observer.@client.endpoint == TowerClient.DEF_ENDPOINT_URL
|
||||
|
||||
when:
|
||||
session = Mock(Session) { getConfig() >> [tower: [enabled: true, endpoint:'http://foo.com/api', accessToken: 'xyz']] }
|
||||
observer = factory.create(session)[0] as TowerObserver
|
||||
then:
|
||||
observer.@client.endpoint == 'http://foo.com/api'
|
||||
}
|
||||
|
||||
def 'should not create a tower observer' () {
|
||||
given:
|
||||
def session = Mock(Session)
|
||||
def factory = new TowerFactory()
|
||||
|
||||
when:
|
||||
def result = factory.create(session)
|
||||
then:
|
||||
session.getConfig() >> [:]
|
||||
then:
|
||||
result == []
|
||||
}
|
||||
|
||||
def 'should create with workspace id'() {
|
||||
//
|
||||
// the workspace id is taken from the env
|
||||
//
|
||||
when:
|
||||
def session = Mock(Session) { getConfig() >> [tower: [enabled: true, accessToken: 'xyz']] }
|
||||
def factory = new TowerFactory(env: [TOWER_WORKSPACE_ID: '100'])
|
||||
def observer = (TowerObserver) factory.create(session)[0]
|
||||
then:
|
||||
observer.getWorkspaceId() == '100'
|
||||
|
||||
//
|
||||
// the workspace id is taken from the config
|
||||
//
|
||||
when:
|
||||
session = Mock(Session) { getConfig() >> [tower: [enabled: true, workspaceId: '200', accessToken: 'xyz']] }
|
||||
factory = new TowerFactory(env: [:])
|
||||
observer = (TowerObserver) factory.create(session)[0]
|
||||
then:
|
||||
observer.getWorkspaceId() == '200'
|
||||
|
||||
//
|
||||
// the workspace id is set both in the config and the env
|
||||
// the config has the priority
|
||||
//
|
||||
when:
|
||||
session = Mock(Session) { getConfig() >> [tower: [enabled: true, workspaceId: '200', accessToken: 'xyz']] }
|
||||
factory = new TowerFactory(env: [TOWER_WORKSPACE_ID: '100'])
|
||||
observer = (TowerObserver) factory.create(session)[0]
|
||||
then:
|
||||
observer.getWorkspaceId() == '200'
|
||||
|
||||
//
|
||||
// when TOWER_WORKFLOW_ID is set is a tower launch
|
||||
// then the workspace id is only taken from the env
|
||||
//
|
||||
when:
|
||||
session = Mock(Session) { getConfig() >> [tower: [enabled: true, workspaceId: '200', accessToken: 'xyz']] }
|
||||
factory = new TowerFactory(env: [TOWER_WORKSPACE_ID: '100', TOWER_WORKFLOW_ID: '111222333', TOWER_ACCESS_TOKEN: 'xyz'])
|
||||
observer = (TowerObserver) factory.create(session)[0]
|
||||
then:
|
||||
observer.getWorkspaceId() == '100'
|
||||
|
||||
//
|
||||
// when enabled is false but `TOWER_WORKFLOW_ID` is provided
|
||||
// then the observer should be created
|
||||
//
|
||||
when:
|
||||
session = Mock(Session) { getConfig() >> [tower: [enabled: false]]}
|
||||
factory = new TowerFactory(env: [TOWER_WORKSPACE_ID: '100', TOWER_WORKFLOW_ID: '111222333', TOWER_ACCESS_TOKEN: 'xyz'])
|
||||
observer = (TowerObserver) factory.create(session)[0]
|
||||
then:
|
||||
observer.getWorkspaceId() == '100'
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should create tower http auth provider' () {
|
||||
given:
|
||||
def factory = new TowerFactory()
|
||||
and:
|
||||
def provider = factory.provider('https://tower.nf', 'xyz123')
|
||||
and:
|
||||
def conn = Spy(HttpURLConnection) {
|
||||
getURL() >> new URL(URL_STR)
|
||||
}
|
||||
|
||||
expect:
|
||||
provider.authorize(conn) == EXPECTED
|
||||
and:
|
||||
conn.getRequestProperty('Authorization') == AUTH
|
||||
|
||||
where:
|
||||
URL_STR | EXPECTED | AUTH
|
||||
'http://foo.com' | false | null
|
||||
'https://tower.nf/' | true | 'Bearer xyz123'
|
||||
'https://tower.nf/this/that' | true | 'Bearer xyz123'
|
||||
'HTTPS://TOWER.NF/THIS/THAT' | true | 'Bearer xyz123'
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,576 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin
|
||||
|
||||
import static com.github.tomakehurst.wiremock.client.WireMock.*
|
||||
|
||||
import java.time.Instant
|
||||
import java.time.temporal.ChronoUnit
|
||||
|
||||
import com.github.tomakehurst.wiremock.WireMockServer
|
||||
import com.github.tomakehurst.wiremock.client.WireMock
|
||||
import com.github.tomakehurst.wiremock.stubbing.Scenario
|
||||
import com.google.gson.GsonBuilder
|
||||
import io.seqera.tower.plugin.exception.UnauthorizedException
|
||||
import nextflow.Global
|
||||
import nextflow.Session
|
||||
import nextflow.SysEnv
|
||||
import nextflow.exception.AbortRunException
|
||||
import nextflow.script.WorkflowMetadata
|
||||
import nextflow.serde.gson.InstantAdapter
|
||||
import spock.lang.Shared
|
||||
import spock.lang.Specification
|
||||
/**
|
||||
* Test cases for the TowerFusionEnv class.
|
||||
*
|
||||
* @author Alberto Miranda <alberto.miranda@seqera.io>
|
||||
*/
|
||||
class TowerFusionEnvTest extends Specification {
|
||||
|
||||
@Shared
|
||||
WireMockServer wireMockServer
|
||||
|
||||
def setupSpec() {
|
||||
wireMockServer = new WireMockServer(0)
|
||||
wireMockServer.start()
|
||||
}
|
||||
|
||||
def cleanupSpec() {
|
||||
wireMockServer.stop()
|
||||
}
|
||||
|
||||
def setup() {
|
||||
wireMockServer.resetAll()
|
||||
SysEnv.push([:]) // <-- ensure the system host env does not interfere
|
||||
}
|
||||
|
||||
def cleanup() {
|
||||
SysEnv.pop() // <-- restore the system host env
|
||||
}
|
||||
|
||||
static String toJson(Object obj) {
|
||||
new GsonBuilder()
|
||||
.registerTypeAdapter(Instant, new InstantAdapter())
|
||||
.create()
|
||||
.toJson(obj)
|
||||
}
|
||||
|
||||
def 'should return the endpoint from the config'() {
|
||||
given: 'a session'
|
||||
SysEnv.push(['TOWER_API_ENDPOINT': 'https://tower.nf', 'TOWER_ACCESS_TOKEN': 'abc123'])
|
||||
Global.session = Mock(Session) {
|
||||
config >> [
|
||||
tower: [
|
||||
endpoint: 'https://tower.nf'
|
||||
]
|
||||
]
|
||||
}
|
||||
|
||||
when: 'the provider is created'
|
||||
def provider = new TowerFusionToken()
|
||||
|
||||
then: 'the endpoint has the expected value'
|
||||
provider.endpoint == 'https://tower.nf'
|
||||
cleanup:
|
||||
SysEnv.pop()
|
||||
}
|
||||
|
||||
def 'should return the endpoint from the environment'() {
|
||||
setup:
|
||||
SysEnv.push(['TOWER_API_ENDPOINT': 'https://tower.nf', 'TOWER_ACCESS_TOKEN': 'abc123'])
|
||||
Global.session = Mock(Session) {
|
||||
config >> [:]
|
||||
}
|
||||
|
||||
when: 'the provider is created'
|
||||
def provider = new TowerFusionToken()
|
||||
|
||||
then: 'the endpoint has the expected value'
|
||||
provider.endpoint == 'https://tower.nf'
|
||||
|
||||
cleanup:
|
||||
SysEnv.pop()
|
||||
}
|
||||
|
||||
def 'should return the default endpoint'() {
|
||||
SysEnv.push(['TOWER_ACCESS_TOKEN': 'abc123'])
|
||||
when: 'session config is empty'
|
||||
Global.session = Mock(Session) {
|
||||
config >> [
|
||||
tower: [:]
|
||||
]
|
||||
}
|
||||
def provider = new TowerFusionToken()
|
||||
|
||||
then: 'the endpoint has the expected value'
|
||||
provider.endpoint == TowerClient.DEF_ENDPOINT_URL
|
||||
|
||||
when: 'session config is null'
|
||||
Global.session = Mock(Session) {
|
||||
config >> null
|
||||
}
|
||||
|
||||
then: 'the endpoint has the expected value'
|
||||
provider.endpoint == TowerClient.DEF_ENDPOINT_URL
|
||||
|
||||
when: 'session config is missing'
|
||||
Global.session = Mock(Session) {
|
||||
config >> [:]
|
||||
}
|
||||
|
||||
then: 'the endpoint has the expected value'
|
||||
provider.endpoint == TowerClient.DEF_ENDPOINT_URL
|
||||
|
||||
when: 'session.config.tower.endpoint is not defined'
|
||||
Global.session = Mock(Session) {
|
||||
config >> [
|
||||
tower: [:]
|
||||
]
|
||||
}
|
||||
|
||||
then: 'the endpoint has the expected value'
|
||||
provider.endpoint == TowerClient.DEF_ENDPOINT_URL
|
||||
|
||||
when: 'session.config.tower.endpoint is null'
|
||||
Global.session = Mock(Session) {
|
||||
config >> [
|
||||
tower: [
|
||||
endpoint: null
|
||||
]
|
||||
]
|
||||
}
|
||||
|
||||
then: 'the endpoint has the expected value'
|
||||
provider.endpoint == TowerClient.DEF_ENDPOINT_URL
|
||||
|
||||
when: 'session.config.tower.endpoint is empty'
|
||||
Global.session = Mock(Session) {
|
||||
config >> [
|
||||
tower: [
|
||||
endpoint: ''
|
||||
]
|
||||
]
|
||||
}
|
||||
provider = new TowerFusionToken()
|
||||
|
||||
then: 'the endpoint has the expected value'
|
||||
provider.endpoint == TowerClient.DEF_ENDPOINT_URL
|
||||
|
||||
when: 'session.config.tower.endpoint is defined as "-"'
|
||||
Global.session = Mock(Session) {
|
||||
config >> [
|
||||
tower: [
|
||||
endpoint: '-'
|
||||
]
|
||||
]
|
||||
}
|
||||
provider = new TowerFusionToken()
|
||||
|
||||
then: 'the endpoint has the expected value'
|
||||
provider.endpoint == TowerClient.DEF_ENDPOINT_URL
|
||||
|
||||
cleanup:
|
||||
SysEnv.pop()
|
||||
}
|
||||
|
||||
def 'should return the access token from the config'() {
|
||||
given: 'a session'
|
||||
Global.session = Mock(Session) {
|
||||
config >> [
|
||||
tower: [
|
||||
accessToken: 'abc123'
|
||||
]
|
||||
]
|
||||
}
|
||||
|
||||
when: 'the provider is created'
|
||||
def provider = new TowerFusionToken()
|
||||
|
||||
then: 'the access token has the expected value'
|
||||
provider.accessToken == 'abc123'
|
||||
}
|
||||
|
||||
def 'should return the access token from the environment'() {
|
||||
setup:
|
||||
Global.session = Mock(Session) {
|
||||
config >> [:]
|
||||
}
|
||||
SysEnv.push(['TOWER_ACCESS_TOKEN': 'abc123'])
|
||||
|
||||
when: 'the provider is created'
|
||||
def provider = new TowerFusionToken()
|
||||
|
||||
then: 'the access token has the expected value'
|
||||
provider.accessToken == 'abc123'
|
||||
|
||||
cleanup:
|
||||
SysEnv.pop()
|
||||
}
|
||||
|
||||
def 'should prefer the access token from the config'() {
|
||||
setup:
|
||||
Global.session = Mock(Session) {
|
||||
config >> [
|
||||
tower: [
|
||||
accessToken: 'abc123'
|
||||
]
|
||||
]
|
||||
}
|
||||
SysEnv.push(['TOWER_ACCESS_TOKEN': 'xyz789'])
|
||||
|
||||
when: 'the provider is created'
|
||||
def provider = new TowerFusionToken()
|
||||
|
||||
then: 'the access token has the expected value'
|
||||
provider.accessToken == 'abc123'
|
||||
|
||||
cleanup:
|
||||
SysEnv.pop()
|
||||
}
|
||||
|
||||
def 'should prefer the access token from the config despite being null'() {
|
||||
setup:
|
||||
Global.session = Mock(Session) {
|
||||
config >> [
|
||||
tower: [
|
||||
accessToken: null
|
||||
]
|
||||
]
|
||||
}
|
||||
SysEnv.push(['TOWER_ACCESS_TOKEN': 'xyz789'])
|
||||
|
||||
when: 'the provider is created'
|
||||
def provider = new TowerFusionToken()
|
||||
|
||||
then: 'the access token has the expected value'
|
||||
def e = thrown(AbortRunException)
|
||||
e.message.contains("Missing Seqera Platform access token")
|
||||
|
||||
cleanup:
|
||||
SysEnv.pop()
|
||||
}
|
||||
|
||||
def 'should prefer the access token from the environment if TOWER_WORKFLOW_ID is set'() {
|
||||
setup:
|
||||
Global.session = Mock(Session) {
|
||||
config >> [
|
||||
tower: [
|
||||
accessToken: 'abc123'
|
||||
]
|
||||
]
|
||||
}
|
||||
SysEnv.push(['TOWER_ACCESS_TOKEN' : 'xyz789', 'TOWER_WORKFLOW_ID': '123'])
|
||||
|
||||
when: 'the provider is created'
|
||||
def provider = new TowerFusionToken()
|
||||
|
||||
then: 'the access token has the expected value'
|
||||
provider.accessToken == 'xyz789'
|
||||
|
||||
cleanup:
|
||||
SysEnv.pop()
|
||||
}
|
||||
|
||||
def 'should get a license token with config'() {
|
||||
given:
|
||||
def config = [
|
||||
enabled : true,
|
||||
endpoint : wireMockServer.baseUrl(),
|
||||
accessToken: 'eyJ0aWQiOiAxMTkxN30uNWQ5MGFmYWU2YjhhNmFmY2FlNjVkMTQ4ZDFhM2ZlNzlmMmNjN2I4Mw==',
|
||||
workspaceId: '67890'
|
||||
]
|
||||
def session = Mock(Session)
|
||||
def meta = new WorkflowMetadata(
|
||||
session: session,
|
||||
projectName: 'the-project-name',
|
||||
repository: 'git://repo.com/foo')
|
||||
session.getConfig() >> [ tower: config ]
|
||||
session.getUniqueId() >> UUID.randomUUID()
|
||||
session.getWorkflowMetadata() >> meta
|
||||
def PRODUCT = 'some-product'
|
||||
def VERSION = 'some-version'
|
||||
and:
|
||||
Global.session = session
|
||||
def provider = new TowerFusionToken()
|
||||
and: 'a mock endpoint at flow create'
|
||||
wireMockServer.stubFor(
|
||||
WireMock.post(urlEqualTo("/trace/create?workspaceId=${config.workspaceId}"))
|
||||
.willReturn(
|
||||
WireMock.aResponse()
|
||||
.withStatus(200)
|
||||
.withBody('{"message": "", "workflowId": "1234"}')
|
||||
)
|
||||
)
|
||||
and:
|
||||
def observer = new TowerFactory().create(session)[0]
|
||||
observer.onFlowCreate(session)
|
||||
|
||||
and: 'a mock endpoint returning a valid token'
|
||||
final now = Instant.now()
|
||||
final expirationDate = toJson(now.plus(1, ChronoUnit.DAYS))
|
||||
wireMockServer.stubFor(
|
||||
WireMock.post(urlEqualTo("/license/token/"))
|
||||
.withHeader('Authorization', equalTo("Bearer ${config.accessToken}"))
|
||||
.withRequestBody(matchingJsonPath('$.product', equalTo("some-product")))
|
||||
.withRequestBody(matchingJsonPath('$.version', equalTo("some-version")))
|
||||
.withRequestBody(matchingJsonPath('$.workspaceId', equalTo("67890")))
|
||||
.willReturn(
|
||||
WireMock.aResponse()
|
||||
.withStatus(200)
|
||||
.withHeader('Content-Type', 'application/json')
|
||||
.withBody('{"signedToken":"xyz789", "expiresAt":' + expirationDate + '}')
|
||||
)
|
||||
)
|
||||
|
||||
when: 'a license token is requested'
|
||||
final token = provider.getLicenseToken(PRODUCT, VERSION)
|
||||
|
||||
then: 'the token has the expected value'
|
||||
token == 'xyz789'
|
||||
|
||||
and: 'the request is correct'
|
||||
wireMockServer.verify(1, WireMock.postRequestedFor(WireMock.urlEqualTo("/license/token/"))
|
||||
.withHeader('Authorization', WireMock.equalTo("Bearer ${config.accessToken}")))
|
||||
}
|
||||
|
||||
def 'should get a license token with environment'() {
|
||||
given:
|
||||
def accessToken = 'eyJ0aWQiOiAxMTkxN30uNWQ5MGFmYWU2YjhhNmFmY2FlNjVkMTQ4ZDFhM2ZlNzlmMmNjN2I4Mw=='
|
||||
def workspaceId = '67890'
|
||||
SysEnv.push([
|
||||
TOWER_WORKFLOW_ID: '12345',
|
||||
TOWER_ACCESS_TOKEN: accessToken,
|
||||
TOWER_WORKSPACE_ID: workspaceId,
|
||||
TOWER_API_ENDPOINT: wireMockServer.baseUrl()
|
||||
])
|
||||
def session = Mock(Session)
|
||||
def meta = new WorkflowMetadata(
|
||||
session: session,
|
||||
projectName: 'the-project-name',
|
||||
repository: 'git://repo.com/foo')
|
||||
session.getConfig() >> [:]
|
||||
session.getUniqueId() >> UUID.randomUUID()
|
||||
session.getWorkflowMetadata() >> meta
|
||||
def PRODUCT = 'some-product'
|
||||
def VERSION = 'some-version'
|
||||
and:
|
||||
Global.session = session
|
||||
def provider = new TowerFusionToken()
|
||||
and: 'a mock endpoint at flow create'
|
||||
wireMockServer.stubFor(
|
||||
WireMock.post(urlEqualTo("/trace/create?workspaceId=${workspaceId}"))
|
||||
.willReturn(
|
||||
WireMock.aResponse()
|
||||
.withStatus(200)
|
||||
.withBody('{"message": "", "workflowId": "1234"}')
|
||||
)
|
||||
)
|
||||
and:
|
||||
def client = new TowerFactory().create(session)[0]
|
||||
client.onFlowCreate(session)
|
||||
|
||||
and: 'a mock endpoint returning a valid token'
|
||||
final now = Instant.now()
|
||||
final expirationDate = toJson(now.plus(1, ChronoUnit.DAYS))
|
||||
wireMockServer.stubFor(
|
||||
WireMock.post(urlEqualTo("/license/token/"))
|
||||
.withHeader('Authorization', equalTo("Bearer $accessToken"))
|
||||
.withRequestBody(matchingJsonPath('$.product', equalTo("some-product")))
|
||||
.withRequestBody(matchingJsonPath('$.version', equalTo("some-version")))
|
||||
.withRequestBody(matchingJsonPath('$.workspaceId', equalTo("${workspaceId}")))
|
||||
.willReturn(
|
||||
WireMock.aResponse()
|
||||
.withStatus(200)
|
||||
.withHeader('Content-Type', 'application/json')
|
||||
.withBody('{"signedToken":"xyz789", "expiresAt":' + expirationDate + '}')
|
||||
)
|
||||
)
|
||||
|
||||
when: 'a license token is requested'
|
||||
final token = provider.getLicenseToken(PRODUCT, VERSION)
|
||||
|
||||
then: 'the token has the expected value'
|
||||
token == 'xyz789'
|
||||
|
||||
and: 'the request is correct'
|
||||
wireMockServer.verify(1, WireMock.postRequestedFor(WireMock.urlEqualTo("/license/token/"))
|
||||
.withHeader('Authorization', WireMock.equalTo("Bearer ${accessToken}")))
|
||||
|
||||
cleanup:
|
||||
SysEnv.pop()
|
||||
}
|
||||
|
||||
def 'should refresh the auth token on 401 and retry the request'() {
|
||||
given:
|
||||
def accessToken = 'eyJ0aWQiOiAxMTkxN30uNWQ5MGFmYWU2YjhhNmFmY2FlNjVkMTQ4ZDFhM2ZlNzlmMmNjN2I4Mw=='
|
||||
def workspaceId = '67890'
|
||||
SysEnv.push([
|
||||
TOWER_WORKFLOW_ID: '12345',
|
||||
TOWER_ACCESS_TOKEN: accessToken,
|
||||
TOWER_REFRESH_TOKEN: 'xyz-refresh',
|
||||
TOWER_WORKSPACE_ID: workspaceId,
|
||||
TOWER_API_ENDPOINT: wireMockServer.baseUrl()
|
||||
])
|
||||
def session = Mock(Session)
|
||||
def meta = new WorkflowMetadata(
|
||||
session: session,
|
||||
projectName: 'the-project-name',
|
||||
repository: 'git://repo.com/foo')
|
||||
session.getConfig() >> [:]
|
||||
session.getUniqueId() >> UUID.randomUUID()
|
||||
session.getWorkflowMetadata() >> meta
|
||||
def PRODUCT = 'some-product'
|
||||
def VERSION = 'some-version'
|
||||
and:
|
||||
Global.session = session
|
||||
def provider = new TowerFusionToken()
|
||||
and: 'a mock endpoint at flow create'
|
||||
wireMockServer.stubFor(
|
||||
WireMock.post(urlEqualTo("/trace/create?workspaceId=${workspaceId}"))
|
||||
.willReturn(
|
||||
WireMock.aResponse()
|
||||
.withStatus(200)
|
||||
.withBody('{"message": "", "workflowId": "1234"}')
|
||||
)
|
||||
)
|
||||
and:
|
||||
def client = new TowerFactory().create(session)[0]
|
||||
client.onFlowCreate(session)
|
||||
|
||||
and: 'prepare stubs'
|
||||
|
||||
final now = Instant.now()
|
||||
final expirationDate = toJson(now.plus(1, ChronoUnit.DAYS))
|
||||
|
||||
// 1️⃣ First attempt: /license/token/ fails with 401
|
||||
wireMockServer.stubFor(
|
||||
WireMock.post(urlEqualTo("/license/token/"))
|
||||
.withHeader('Authorization', equalTo("Bearer $accessToken"))
|
||||
.inScenario("Refresh flow")
|
||||
.whenScenarioStateIs(Scenario.STARTED)
|
||||
.willReturn(WireMock.aResponse().withStatus(401))
|
||||
.willSetStateTo("Token Refreshed")
|
||||
)
|
||||
|
||||
// 2️⃣ Refresh token call
|
||||
wireMockServer.stubFor(
|
||||
WireMock.post(urlEqualTo("/oauth/access_token"))
|
||||
.withHeader('Content-Type', equalTo('application/x-www-form-urlencoded'))
|
||||
.withRequestBody(containing('grant_type=refresh_token'))
|
||||
.withRequestBody(containing('refresh_token=xyz-refresh'))
|
||||
.willReturn(
|
||||
WireMock.aResponse()
|
||||
.withStatus(200)
|
||||
.withHeader('Set-Cookie', 'JWT=new-abc-token; Path=/; HttpOnly')
|
||||
.withHeader('Set-Cookie', 'JWT_REFRESH_TOKEN=new-refresh-456; Path=/; HttpOnly')
|
||||
.withBody('{"token_type":"Bearer"}')
|
||||
)
|
||||
)
|
||||
|
||||
// 3️⃣ Retry: /license/token/ succeeds
|
||||
wireMockServer.stubFor(
|
||||
WireMock.post(urlEqualTo("/license/token/"))
|
||||
.withHeader('Authorization', equalTo('Bearer new-abc-token'))
|
||||
.inScenario("Refresh flow")
|
||||
.whenScenarioStateIs("Token Refreshed")
|
||||
.willReturn(
|
||||
WireMock.aResponse()
|
||||
.withStatus(200)
|
||||
.withHeader('Content-Type', 'application/json')
|
||||
.withBody('{"signedToken":"xyz789", "expiresAt":' + expirationDate + '}')
|
||||
)
|
||||
)
|
||||
|
||||
when:
|
||||
final token = provider.getLicenseToken(PRODUCT, VERSION)
|
||||
|
||||
then:
|
||||
token == 'xyz789'
|
||||
|
||||
and: 'verify that refresh endpoint was called'
|
||||
wireMockServer.verify(1, WireMock.postRequestedFor(WireMock.urlEqualTo("/oauth/access_token")))
|
||||
|
||||
and: 'verify both requests to license endpoint'
|
||||
wireMockServer.verify(2, WireMock.postRequestedFor(urlEqualTo("/license/token/")))
|
||||
|
||||
cleanup:
|
||||
SysEnv.pop()
|
||||
}
|
||||
|
||||
def 'should throw UnauthorizedException if getting a token fails with 401'() {
|
||||
given: 'a TowerFusionEnv provider'
|
||||
def config = [
|
||||
enabled : true,
|
||||
endpoint : wireMockServer.baseUrl(),
|
||||
accessToken: 'eyJ0aWQiOiAxMTkxN30uNWQ5MGFmYWU2YjhhNmFmY2FlNjVkMTQ4ZDFhM2ZlNzlmMmNjN2I4Mw==',
|
||||
workspaceId: '67890'
|
||||
]
|
||||
def session = Mock(Session)
|
||||
def meta = new WorkflowMetadata(
|
||||
session: session,
|
||||
projectName: 'the-project-name',
|
||||
repository: 'git://repo.com/foo')
|
||||
session.getConfig() >> [ tower: config ]
|
||||
session.getUniqueId() >> UUID.randomUUID()
|
||||
session.getWorkflowMetadata() >> meta
|
||||
and:
|
||||
Global.session = session
|
||||
def provider = new TowerFusionToken()
|
||||
and: 'a mock endpoint at flow create'
|
||||
wireMockServer.stubFor(
|
||||
WireMock.post(urlEqualTo("/trace/create?workspaceId=${config.workspaceId}"))
|
||||
.willReturn(
|
||||
WireMock.aResponse()
|
||||
.withStatus(200)
|
||||
.withBody('{"message": "", "workflowId": "1234"}')
|
||||
)
|
||||
)
|
||||
and:
|
||||
def client = new TowerFactory().create(session)[0]
|
||||
client.onFlowCreate(session)
|
||||
and: 'a mock endpoint returning an error'
|
||||
wireMockServer.stubFor(
|
||||
WireMock.post(WireMock.urlEqualTo("/license/token/"))
|
||||
.withHeader('Authorization', WireMock.equalTo("Bearer ${config.accessToken}"))
|
||||
.willReturn(
|
||||
WireMock.aResponse()
|
||||
.withStatus(401)
|
||||
.withHeader('Content-Type', 'application/json')
|
||||
.withBody('{"error":"Unauthorized"}')
|
||||
)
|
||||
)
|
||||
|
||||
when: 'a license token is requested'
|
||||
provider.getLicenseToken('some-product', 'some-version')
|
||||
|
||||
then: 'an exception is thrown'
|
||||
thrown(UnauthorizedException)
|
||||
}
|
||||
|
||||
def 'should deserialize response' () {
|
||||
given:
|
||||
def ts = Instant.ofEpochSecond(1738788914)
|
||||
def json = '{"signedToken":"foo","expiresAt":"2025-02-05T20:55:14Z"}'
|
||||
|
||||
when:
|
||||
def resp = TowerFusionToken.parseLicenseTokenResponse(json)
|
||||
then:
|
||||
resp.signedToken == 'foo'
|
||||
resp.expiresAt == ts
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin
|
||||
|
||||
import java.time.Instant
|
||||
import java.time.ZoneOffset
|
||||
|
||||
import groovy.json.JsonGenerator
|
||||
import groovy.json.JsonSlurper
|
||||
import nextflow.container.resolver.ContainerMeta
|
||||
import nextflow.trace.ProgressRecord
|
||||
import nextflow.trace.WorkflowStats
|
||||
import spock.lang.Specification
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class TowerJsonGeneratorTest extends Specification {
|
||||
|
||||
def 'should chomp too long values' () {
|
||||
given:
|
||||
def scheme = [foo: 5, 'bar.one': 5]
|
||||
def gen = new TowerJsonGenerator(new JsonGenerator.Options(), scheme)
|
||||
|
||||
when:
|
||||
def x = gen.toJson( [foo: "Hola", bar: 'mundo'] )
|
||||
then:
|
||||
x == '{"foo":"Hola","bar":"mundo"}'
|
||||
|
||||
when:
|
||||
x = gen.toJson( [foo: "Hello world"] )
|
||||
then:
|
||||
x == '{"foo":"Hello"}'
|
||||
|
||||
when:
|
||||
x = gen.toJson( [bar: [one: "Hello world", two: "Hola mundo"]] )
|
||||
then:
|
||||
x == '{"bar":{"one":"Hello","two":"Hola mundo"}}'
|
||||
}
|
||||
|
||||
def 'should normalise gitmodules attribute' () {
|
||||
given:
|
||||
def scheme = ['workflow.manifest.gitmodules': 10]
|
||||
def gen = new TowerJsonGenerator(new JsonGenerator.Options(), scheme)
|
||||
|
||||
when:
|
||||
def json = gen.toJson( [workflow: [manifest: [gitmodules: ['a','b','c']]]] )
|
||||
then:
|
||||
json == '{"workflow":{"manifest":{"gitmodules":"a,b,c"}}}'
|
||||
|
||||
when:
|
||||
json = gen.toJson( [workflow: [manifest: [gitmodules: 'abc']]] )
|
||||
then:
|
||||
json == '{"workflow":{"manifest":{"gitmodules":"abc"}}}'
|
||||
|
||||
when:
|
||||
json = gen.toJson( [workflow: [manifest: [gitmodules: '123456789012345']]] )
|
||||
then:
|
||||
json == '{"workflow":{"manifest":{"gitmodules":"1234567890"}}}'
|
||||
}
|
||||
|
||||
def 'should serialise progress records' () {
|
||||
given:
|
||||
def gen = new TowerJsonGenerator(new JsonGenerator.Options(), [:])
|
||||
and:
|
||||
def rec1 = new ProgressRecord(1, 'foo')
|
||||
|
||||
and:
|
||||
def rec2 = new ProgressRecord(2, 'bar')
|
||||
rec2.pending = 1
|
||||
rec2.submitted = 2
|
||||
rec2.running = 3
|
||||
rec2.succeeded = 4
|
||||
rec2.failed = 5
|
||||
rec2.aborted = 6
|
||||
rec2.stored = 7
|
||||
rec2.ignored = 8
|
||||
rec2.retries = 9
|
||||
rec2.cached = 10
|
||||
rec2.loadCpus = 11
|
||||
rec2.loadMemory = 12
|
||||
rec2.peakRunning = 13
|
||||
rec2.peakCpus = 14
|
||||
rec2.peakMemory = 15
|
||||
rec2.terminated = true
|
||||
|
||||
when:
|
||||
def json = gen.toJson([progress: [rec1, rec2]])
|
||||
then:
|
||||
def copy = (Map)new JsonSlurper().parseText(json)
|
||||
copy.size() == 1
|
||||
|
||||
and:
|
||||
def progress = (List<Map>)copy.progress
|
||||
progress.size() == 2
|
||||
|
||||
and:
|
||||
progress.get(0) == [
|
||||
index:1,
|
||||
name: 'foo',
|
||||
workDir: null,
|
||||
pending: 0,
|
||||
submitted: 0,
|
||||
running: 0,
|
||||
succeeded: 0,
|
||||
failed: 0,
|
||||
aborted: 0,
|
||||
stored: 0,
|
||||
ignored: 0,
|
||||
retries: 0,
|
||||
cached: 0,
|
||||
loadCpus: 0,
|
||||
loadMemory: 0,
|
||||
peakCpus: 0,
|
||||
peakMemory: 0,
|
||||
peakRunning: 0,
|
||||
terminated: false]
|
||||
and:
|
||||
progress[1] == [
|
||||
index:2,
|
||||
name: 'bar',
|
||||
workDir: null,
|
||||
pending: 1,
|
||||
submitted: 2,
|
||||
running: 3,
|
||||
succeeded: 4,
|
||||
failed: 5,
|
||||
aborted: 6,
|
||||
stored: 7,
|
||||
ignored: 8,
|
||||
retries: 9,
|
||||
cached: 10,
|
||||
loadCpus: 11,
|
||||
loadMemory: 12,
|
||||
peakRunning: 13,
|
||||
peakCpus: 14,
|
||||
peakMemory: 15,
|
||||
terminated: true]
|
||||
|
||||
}
|
||||
|
||||
|
||||
def 'should serialise workflow stats' () {
|
||||
given:
|
||||
def gen = new TowerJsonGenerator(new JsonGenerator.Options(), [:])
|
||||
and:
|
||||
def rec1 = new ProgressRecord(1, 'foo')
|
||||
def rec2 = new ProgressRecord(2, 'bar')
|
||||
rec2.pending = 1
|
||||
rec2.submitted = 2
|
||||
rec2.running = 3
|
||||
rec2.succeeded = 4
|
||||
rec2.failed = 5
|
||||
rec2.aborted = 6
|
||||
rec2.stored = 7
|
||||
rec2.ignored = 8
|
||||
rec2.retries = 9
|
||||
rec2.cached = 10
|
||||
rec2.loadCpus = 11
|
||||
rec2.loadMemory = 12
|
||||
rec2.peakRunning = 13
|
||||
rec2.peakCpus = 14
|
||||
rec2.peakMemory = 15
|
||||
rec2.terminated = true
|
||||
and:
|
||||
def stats = new WorkflowStats(
|
||||
succeededCount: 1,
|
||||
cachedCount: 2,
|
||||
failedCount: 3,
|
||||
ignoredCount: 4,
|
||||
pendingCount: 5,
|
||||
submittedCount: 6,
|
||||
runningCount: 7,
|
||||
retriesCount: 8,
|
||||
abortedCount: 9,
|
||||
records: [1:rec1, 2:rec2])
|
||||
|
||||
when:
|
||||
def json = gen.toJson(new WorkflowProgress(stats))
|
||||
then:
|
||||
def copy = (Map)new JsonSlurper().parseText(json)
|
||||
copy.succeeded == 1
|
||||
copy.cached == 2
|
||||
copy.failed == 3
|
||||
copy.ignored == 4
|
||||
copy.pending == 5
|
||||
copy.submitted == 6
|
||||
copy.running == 7
|
||||
copy.retries == 8
|
||||
copy.aborted == 9
|
||||
and:
|
||||
(copy.processes as List).size() == 2
|
||||
and:
|
||||
with(copy.processes[0] as Map) {
|
||||
index == 1
|
||||
name == 'foo'
|
||||
pending == 0
|
||||
submitted == 0
|
||||
running == 0
|
||||
succeeded == 0
|
||||
failed == 0
|
||||
aborted == 0
|
||||
stored == 0
|
||||
ignored == 0
|
||||
retries == 0
|
||||
cached == 0
|
||||
loadCpus == 0
|
||||
loadMemory == 0
|
||||
peakCpus == 0
|
||||
peakMemory == 0
|
||||
peakRunning == 0
|
||||
terminated == false
|
||||
}
|
||||
and:
|
||||
with(copy.processes[1] as Map) {
|
||||
index == 2
|
||||
name == 'bar'
|
||||
pending ==1
|
||||
submitted == 2
|
||||
running == 3
|
||||
succeeded == 4
|
||||
failed == 5
|
||||
aborted == 6
|
||||
stored == 7
|
||||
ignored == 8
|
||||
retries == 9
|
||||
cached == 10
|
||||
loadCpus == 11
|
||||
loadMemory == 12
|
||||
peakRunning == 13
|
||||
peakCpus == 14
|
||||
peakMemory == 15
|
||||
terminated == true
|
||||
}
|
||||
}
|
||||
|
||||
def 'should serialise container meta' () {
|
||||
given:
|
||||
def gen = TowerJsonGenerator.create([:])
|
||||
and:
|
||||
def ts = Instant.ofEpochSecond(1742421070).atOffset(ZoneOffset.ofHours(2))
|
||||
def c1 = new ContainerMeta(
|
||||
requestId:'r-1',
|
||||
requestTime:ts,
|
||||
buildId: 'bd-2',
|
||||
scanId: 'sc-3',
|
||||
mirrorId: 'mr-4',
|
||||
cached: false,
|
||||
freeze: true,
|
||||
sourceImage: 'debian:latest',
|
||||
targetImage: 'wave/debian')
|
||||
|
||||
when:
|
||||
def json = gen.toJson([containers: [c1]])
|
||||
then:
|
||||
json == '{"containers":[{"requestId":"r-1","sourceImage":"debian:latest","targetImage":"wave/debian","buildId":"bd-2","mirrorId":"mr-4","scanId":"sc-3","cached":false,"freeze":true,"requestTime":"2025-03-19T23:51:10+02:00"}]}'
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,591 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.time.Instant
|
||||
import java.time.OffsetDateTime
|
||||
import java.time.ZoneId
|
||||
|
||||
import nextflow.Session
|
||||
import nextflow.SysEnv
|
||||
import nextflow.cloud.types.CloudMachineInfo
|
||||
import nextflow.cloud.types.PriceModel
|
||||
import nextflow.container.DockerConfig
|
||||
import nextflow.container.resolver.ContainerMeta
|
||||
import nextflow.exception.AbortRunException
|
||||
import nextflow.script.PlatformMetadata
|
||||
import nextflow.script.ScriptBinding
|
||||
import nextflow.script.WorkflowMetadata
|
||||
import nextflow.trace.TraceRecord
|
||||
import nextflow.trace.WorkflowStats
|
||||
import nextflow.trace.WorkflowStatsObserver
|
||||
import nextflow.util.ProcessHelper
|
||||
import spock.lang.Specification
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class TowerObserverTest extends Specification {
|
||||
|
||||
protected boolean aroundNow(value) {
|
||||
def now = Instant.now().toEpochMilli()
|
||||
value > now-1_000 && value <= now
|
||||
}
|
||||
|
||||
private TowerObserver newObserver(Session session, Map env = [:]) {
|
||||
def client = Mock(TowerClient)
|
||||
def observer = new TowerObserver(session, client, null, env)
|
||||
observer.@reports = Mock(TowerReports)
|
||||
return observer
|
||||
}
|
||||
|
||||
def 'should create message map' () {
|
||||
given:
|
||||
def session = Mock(Session)
|
||||
def params = new ScriptBinding.ParamsMap(x: "hello")
|
||||
def meta = Mock(WorkflowMetadata)
|
||||
def observer = Spy(newObserver(session))
|
||||
observer.@workflowId = '12ef'
|
||||
|
||||
when:
|
||||
def map = observer.makeCompleteReq(session)
|
||||
then:
|
||||
1 * session.getWorkflowMetadata() >> meta
|
||||
1 * session.getParams() >> params
|
||||
1 * meta.toMap() >> [foo:1, bar:2, container: [p1: 'c1', p2: 'c2']]
|
||||
1 * observer.getMetricsList() >> [[process:'foo', cpu: [min: 1, max:5], time: [min: 6, max: 9]]]
|
||||
1 * observer.getWorkflowProgress(false) >> new WorkflowProgress()
|
||||
1 * observer.getOutFile() >> 'bar.out'
|
||||
1 * observer.getLogFile() >> 'foo.out'
|
||||
1 * observer.getOperationId() >> 'op-12345'
|
||||
then:
|
||||
map.workflow.foo == 1
|
||||
map.workflow.bar == 2
|
||||
map.workflow.id == '12ef'
|
||||
map.workflow.params == [x: 'hello']
|
||||
map.workflow.container == null
|
||||
map.metrics == [[process:'foo', cpu: [min: 1, max:5], time: [min: 6, max: 9]]]
|
||||
map.progress == new WorkflowProgress()
|
||||
and:
|
||||
aroundNow(map.instant)
|
||||
and:
|
||||
map.workflow.outFile == 'bar.out'
|
||||
map.workflow.logFile == 'foo.out'
|
||||
map.workflow.operationId == 'op-12345'
|
||||
}
|
||||
|
||||
def 'should capitalise underscores' () {
|
||||
given:
|
||||
def tower = new TowerObserver(Mock(Session), Mock(TowerClient), "ws1234", [:] )
|
||||
|
||||
expect:
|
||||
tower.underscoreToCamelCase(STR) == EXPECTED
|
||||
where:
|
||||
STR | EXPECTED
|
||||
'abc' | 'abc'
|
||||
'a_b_c' | 'aBC'
|
||||
'foo__bar' | 'fooBar'
|
||||
}
|
||||
|
||||
def 'should post task records' () {
|
||||
given:
|
||||
def session = Mock(Session)
|
||||
def PROGRESS = Mock(WorkflowProgress) { getRunning()>>1; getSucceeded()>>2; getFailed()>>3 }
|
||||
def observer = Spy(newObserver(session))
|
||||
observer.@workflowId = 'xyz-123'
|
||||
|
||||
def nowTs = System.currentTimeMillis()
|
||||
def submitTs = nowTs-2000
|
||||
def startTs = nowTs-1000
|
||||
|
||||
def trace = new TraceRecord([
|
||||
taskId: 10,
|
||||
process: 'foo',
|
||||
workdir: "/work/dir",
|
||||
cpus: 1,
|
||||
submit: submitTs,
|
||||
start: startTs,
|
||||
complete: nowTs ])
|
||||
trace.executorName= 'batch'
|
||||
trace.machineInfo = new CloudMachineInfo('m4.large', 'eu-west-1b', PriceModel.spot)
|
||||
trace.containerMeta = new ContainerMeta(requestId: '12345', sourceImage: 'ubuntu:latest', targetImage: 'wave.io/12345/ubuntu:latest')
|
||||
|
||||
when:
|
||||
def req = observer.makeTasksReq([trace])
|
||||
then:
|
||||
observer.getWorkflowProgress(true) >> PROGRESS
|
||||
and:
|
||||
req.tasks[0].taskId == 10
|
||||
req.tasks[0].process == 'foo'
|
||||
req.tasks[0].workdir == "/work/dir"
|
||||
req.tasks[0].cpus == 1
|
||||
req.tasks[0].submit == OffsetDateTime.ofInstant(Instant.ofEpochMilli(submitTs), ZoneId.systemDefault())
|
||||
req.tasks[0].start == OffsetDateTime.ofInstant(Instant.ofEpochMilli(startTs), ZoneId.systemDefault())
|
||||
req.tasks[0].executor == 'batch'
|
||||
req.tasks[0].machineType == 'm4.large'
|
||||
req.tasks[0].cloudZone == 'eu-west-1b'
|
||||
req.tasks[0].priceModel == 'spot'
|
||||
and:
|
||||
req.progress.running == 1
|
||||
req.progress.succeeded == 2
|
||||
req.progress.failed == 3
|
||||
and:
|
||||
req.containers[0].requestId == '12345'
|
||||
req.containers[0].sourceImage == 'ubuntu:latest'
|
||||
req.containers[0].targetImage == 'wave.io/12345/ubuntu:latest'
|
||||
and:
|
||||
aroundNow(req.instant)
|
||||
}
|
||||
|
||||
static now_millis = System.currentTimeMillis()
|
||||
static now_instant = OffsetDateTime.ofInstant(Instant.ofEpochMilli(now_millis), ZoneId.systemDefault())
|
||||
|
||||
def 'should fix field types' () {
|
||||
|
||||
expect:
|
||||
TowerObserver.fixTaskField(FIELD, VALUE) == EXPECTED
|
||||
|
||||
where:
|
||||
FIELD | VALUE | EXPECTED
|
||||
'foo' | 'hola' | 'hola'
|
||||
'submit' | now_millis | now_instant
|
||||
'start' | now_millis | now_instant
|
||||
'complete' | now_millis | now_instant
|
||||
'complete' | 0 | null
|
||||
}
|
||||
|
||||
def 'should create workflow json' () {
|
||||
|
||||
given:
|
||||
def sessionId = UUID.randomUUID()
|
||||
def dir = Files.createTempDirectory('test')
|
||||
def session = Mock(Session)
|
||||
session.getUniqueId() >> sessionId
|
||||
session.getRunName() >> 'foo'
|
||||
session.config >> [:]
|
||||
session.containerConfig >> new DockerConfig([:])
|
||||
session.getParams() >> new ScriptBinding.ParamsMap([foo:'Hello', bar:'World'])
|
||||
|
||||
def meta = new WorkflowMetadata(
|
||||
session: session,
|
||||
projectName: 'the-project-name',
|
||||
repository: 'git://repo.com/foo' )
|
||||
session.getWorkflowMetadata() >> meta
|
||||
session.getStatsObserver() >> Mock(WorkflowStatsObserver) { getStats() >> new WorkflowStats() }
|
||||
|
||||
def observer = Spy(newObserver(session, ENV))
|
||||
observer.getOperationId() >> 'op-112233'
|
||||
observer.getLogFile() >> 'log.file'
|
||||
observer.getOutFile() >> 'out.file'
|
||||
|
||||
when:
|
||||
def req1 = observer.makeCreateReq(session)
|
||||
then:
|
||||
req1.sessionId == sessionId.toString()
|
||||
req1.runName == 'foo'
|
||||
req1.projectName == 'the-project-name'
|
||||
req1.repository == 'git://repo.com/foo'
|
||||
req1.workflowId == WORKFLOW_ID
|
||||
and:
|
||||
aroundNow(req1.instant)
|
||||
|
||||
when:
|
||||
def req = observer.makeBeginReq(session)
|
||||
then:
|
||||
observer.getWorkflowId() >> '12345'
|
||||
and:
|
||||
req.workflow.id == '12345'
|
||||
req.workflow.params == [foo:'Hello', bar:'World']
|
||||
req.workflow.outFile == 'out.file'
|
||||
req.workflow.logFile == 'log.file'
|
||||
req.workflow.operationId == 'op-112233'
|
||||
and:
|
||||
req.towerLaunch == TOWER_LAUNCH
|
||||
and:
|
||||
aroundNow(req.instant)
|
||||
|
||||
cleanup:
|
||||
dir?.deleteDir()
|
||||
|
||||
where:
|
||||
ENV | WORKFLOW_ID | TOWER_LAUNCH
|
||||
[:] | null | false
|
||||
[TOWER_WORKFLOW_ID: '1234'] | '1234' | true
|
||||
|
||||
}
|
||||
|
||||
def 'should convert map' () {
|
||||
given:
|
||||
def tower = new TowerObserver(Mock(Session), Mock(TowerClient), "ws1234", [:] )
|
||||
|
||||
expect:
|
||||
tower.mapToString(null) == null
|
||||
tower.mapToString('ciao') == 'ciao'
|
||||
tower.mapToString([:]) == null
|
||||
tower.mapToString([p:'foo', q:'bar']) == null
|
||||
}
|
||||
|
||||
def 'should create init request' () {
|
||||
given:
|
||||
def uuid = UUID.randomUUID()
|
||||
def meta = Mock(WorkflowMetadata) {
|
||||
getProjectName() >> 'the-project-name'
|
||||
getRepository() >> 'git://repo.com/foo'
|
||||
}
|
||||
def session = Mock(Session) {
|
||||
getUniqueId() >> uuid
|
||||
getRunName() >> 'foo_bar'
|
||||
getWorkflowMetadata() >> meta
|
||||
}
|
||||
def observer = newObserver(session, [TOWER_WORKFLOW_ID: 'x123'])
|
||||
|
||||
when:
|
||||
def req = observer.makeCreateReq(session)
|
||||
then:
|
||||
req.sessionId == uuid.toString()
|
||||
req.runName == 'foo_bar'
|
||||
req.projectName == 'the-project-name'
|
||||
req.repository == 'git://repo.com/foo'
|
||||
req.workflowId == 'x123'
|
||||
and:
|
||||
aroundNow(req.instant)
|
||||
|
||||
and:
|
||||
observer.towerLaunch
|
||||
}
|
||||
|
||||
def 'should post create request' () {
|
||||
given:
|
||||
def uuid = UUID.randomUUID()
|
||||
def platform = new PlatformMetadata()
|
||||
def meta = Mock(WorkflowMetadata) {
|
||||
getProjectName() >> 'the-project-name'
|
||||
getRepository() >> 'git://repo.com/foo'
|
||||
getPlatform() >> platform
|
||||
}
|
||||
def session = Mock(Session) {
|
||||
getUniqueId() >> uuid
|
||||
getRunName() >> 'foo_bar'
|
||||
getWorkflowMetadata() >> meta
|
||||
}
|
||||
def towerClient = Mock(TowerClient)
|
||||
def observer = Spy(new TowerObserver(session, towerClient, null, [:]))
|
||||
observer.@reports = Mock(TowerReports)
|
||||
|
||||
when:
|
||||
observer.onFlowCreate(session)
|
||||
then:
|
||||
1 * observer.makeCreateReq(session) >> [runName: 'foo']
|
||||
1 * towerClient.traceCreate([runName: 'foo'], null) >> [workflowId: 'xyz123', watchUrl: 'https://cloud.seqera.io/watch/xyz123']
|
||||
and:
|
||||
observer.runName == 'foo_bar'
|
||||
observer.runId == uuid.toString()
|
||||
and:
|
||||
observer.workflowId == 'xyz123'
|
||||
observer.@watchUrl == 'https://cloud.seqera.io/watch/xyz123'
|
||||
!observer.towerLaunch
|
||||
and:
|
||||
platform.workflowId == 'xyz123'
|
||||
platform.workflowUrl == 'https://cloud.seqera.io/watch/xyz123'
|
||||
|
||||
}
|
||||
|
||||
def 'should set workflowUrl on platform metadata during onFlowBegin' () {
|
||||
given:
|
||||
def platform = new PlatformMetadata()
|
||||
def meta = Mock(WorkflowMetadata) {
|
||||
getPlatform() >> platform
|
||||
}
|
||||
def session = Mock(Session) {
|
||||
getWorkflowMetadata() >> meta
|
||||
}
|
||||
def towerClient = Mock(TowerClient)
|
||||
def observer = Spy(new TowerObserver(session, towerClient, null, [:]))
|
||||
observer.@reports = Mock(TowerReports)
|
||||
observer.@workflowId = 'abc123'
|
||||
|
||||
when:
|
||||
observer.onFlowBegin()
|
||||
then:
|
||||
1 * observer.makeBeginReq(session) >> [foo: 'bar']
|
||||
1 * towerClient.traceBegin([foo: 'bar'], null, 'abc123') >> [watchUrl: 'https://cloud.seqera.io/watch/abc123']
|
||||
and:
|
||||
observer.@watchUrl == 'https://cloud.seqera.io/watch/abc123'
|
||||
platform.workflowUrl == 'https://cloud.seqera.io/watch/abc123'
|
||||
|
||||
cleanup:
|
||||
observer.@sender?.interrupt()
|
||||
}
|
||||
|
||||
def 'should fetch workflow meta' () {
|
||||
given:
|
||||
def session = Mock(Session)
|
||||
def observer = newObserver(session, ENV)
|
||||
|
||||
expect:
|
||||
observer.getOperationId() == OP_ID
|
||||
observer.getLogFile() == LOG_FILE
|
||||
observer.getOutFile() == OUT_FILE
|
||||
|
||||
where:
|
||||
OP_ID | OUT_FILE | LOG_FILE | ENV
|
||||
null | null | null | [:]
|
||||
"local-platform::${ProcessHelper.selfPid()}" | null | null | [TOWER_ALLOW_NEXTFLOW_LOGS:'true']
|
||||
'aws-batch::1234z' | 'xyz.out' | 'hola.log' | [TOWER_ALLOW_NEXTFLOW_LOGS:'true', AWS_BATCH_JOB_ID: '1234z', NXF_OUT_FILE: 'xyz.out', NXF_LOG_FILE: 'hola.log']
|
||||
}
|
||||
|
||||
def 'should deduplicate containers' () {
|
||||
given:
|
||||
def session = Mock(Session)
|
||||
def observer = newObserver(session)
|
||||
and:
|
||||
def c1 = new ContainerMeta(requestId: '12345', sourceImage: 'ubuntu:latest', targetImage: 'wave.io/12345/ubuntu:latest')
|
||||
def c2 = new ContainerMeta(requestId: '54321', sourceImage: 'ubuntu:latest', targetImage: 'wave.io/54321/ubuntu:latest')
|
||||
and:
|
||||
def trace1 = new TraceRecord(
|
||||
taskId: 1,
|
||||
process: 'foo',
|
||||
workdir: "/work/dir",
|
||||
cpus: 1,
|
||||
submit: System.currentTimeMillis(),
|
||||
start: System.currentTimeMillis(),
|
||||
complete: System.currentTimeMillis())
|
||||
trace1.containerMeta = c1
|
||||
and:
|
||||
def trace2 = new TraceRecord(
|
||||
taskId: 2,
|
||||
process: 'foo',
|
||||
workdir: "/work/dir",
|
||||
cpus: 1,
|
||||
submit: System.currentTimeMillis(),
|
||||
start: System.currentTimeMillis(),
|
||||
complete: System.currentTimeMillis())
|
||||
trace2.containerMeta = c2
|
||||
and:
|
||||
def trace3 = new TraceRecord(
|
||||
taskId: 3,
|
||||
process: 'foo',
|
||||
workdir: "/work/dir",
|
||||
cpus: 1,
|
||||
submit: System.currentTimeMillis(),
|
||||
start: System.currentTimeMillis(),
|
||||
complete: System.currentTimeMillis())
|
||||
trace3.containerMeta = c2
|
||||
|
||||
expect:
|
||||
observer.getNewContainers([trace1]) == [c1]
|
||||
and:
|
||||
observer.getNewContainers([trace1]) == []
|
||||
and:
|
||||
observer.getNewContainers([trace1, trace2, trace3]) == [c2]
|
||||
}
|
||||
|
||||
def 'should not send complete request when onFlowBegin was not invoked' () {
|
||||
given:
|
||||
def session = Mock(Session)
|
||||
def towerClient = Mock(TowerClient)
|
||||
def observer = Spy(new TowerObserver(session, towerClient, null, [:]))
|
||||
def reports = Mock(TowerReports)
|
||||
observer.@reports = reports
|
||||
observer.@workflowId = 'xyz-123'
|
||||
observer.@sender = null
|
||||
|
||||
when:
|
||||
observer.onFlowComplete()
|
||||
|
||||
then:
|
||||
1 * reports.publishRuntimeReports()
|
||||
1 * reports.flowComplete()
|
||||
0 * towerClient.traceComplete(_, _, _)
|
||||
}
|
||||
|
||||
def 'should apply platform metadata from trace create response'() {
|
||||
given:
|
||||
def metadata = new WorkflowMetadata()
|
||||
def session = Mock(Session) {
|
||||
getWorkflowMetadata() >> metadata
|
||||
}
|
||||
def observer = new TowerObserver(session, Mock(TowerClient), '1234', SysEnv.get())
|
||||
|
||||
def responseMetadata = [
|
||||
userId: 39,
|
||||
userName: 'user',
|
||||
userOrganization: 'ACME Inc.',
|
||||
workspaceId: 1234,
|
||||
workspaceName: 'Workspace-Name',
|
||||
workspaceFullName: 'Full Workspace Name',
|
||||
orgName: 'ACME Inc.',
|
||||
computeEnvId: 'ce1234',
|
||||
computeEnvName: 'ce-test',
|
||||
computeEnvPlatform: 'aws-batch',
|
||||
pipelineName: 'test-pipeline',
|
||||
pipelineId: 'pipe1234',
|
||||
revision: 'v1.1',
|
||||
commitId: 'abcd12345'
|
||||
]
|
||||
|
||||
when:
|
||||
observer.applyPlatformMetadata(responseMetadata)
|
||||
|
||||
then:
|
||||
metadata.platform.user.id == '39'
|
||||
metadata.platform.user.userName == 'user'
|
||||
metadata.platform.user.organization == 'ACME Inc.'
|
||||
metadata.platform.workspace.id == '1234'
|
||||
metadata.platform.workspace.name == 'Workspace-Name'
|
||||
metadata.platform.workspace.fullName == 'Full Workspace Name'
|
||||
metadata.platform.workspace.organization == 'ACME Inc.'
|
||||
metadata.platform.computeEnv.id == 'ce1234'
|
||||
metadata.platform.computeEnv.name == 'ce-test'
|
||||
metadata.platform.computeEnv.platform == 'aws-batch'
|
||||
metadata.platform.pipeline.id == 'pipe1234'
|
||||
metadata.platform.pipeline.name == 'test-pipeline'
|
||||
metadata.platform.pipeline.revision == 'v1.1'
|
||||
metadata.platform.pipeline.commitId == 'abcd12345'
|
||||
}
|
||||
|
||||
def 'should include numSpotInterruptions in task map'() {
|
||||
given:
|
||||
def session = Mock(Session)
|
||||
def observer = Spy(newObserver(session))
|
||||
observer.getWorkflowProgress(true) >> new WorkflowProgress()
|
||||
|
||||
def now = System.currentTimeMillis()
|
||||
def trace = new TraceRecord([
|
||||
taskId: 42,
|
||||
process: 'foo',
|
||||
workdir: "/work/dir",
|
||||
cpus: 1,
|
||||
submit: now-2000,
|
||||
start: now-1000,
|
||||
complete: now
|
||||
])
|
||||
trace.setNumSpotInterruptions(3)
|
||||
|
||||
when:
|
||||
def req = observer.makeTasksReq([trace])
|
||||
|
||||
then:
|
||||
req.tasks.size() == 1
|
||||
req.tasks[0].numSpotInterruptions == 3
|
||||
}
|
||||
|
||||
def 'should include logStreamId in task map'() {
|
||||
given:
|
||||
def session = Mock(Session)
|
||||
def observer = Spy(newObserver(session))
|
||||
observer.getWorkflowProgress(true) >> new WorkflowProgress()
|
||||
|
||||
def now = System.currentTimeMillis()
|
||||
def trace = new TraceRecord([
|
||||
taskId: 42,
|
||||
process: 'foo',
|
||||
workdir: "/work/dir",
|
||||
cpus: 1,
|
||||
submit: now-2000,
|
||||
start: now-1000,
|
||||
complete: now
|
||||
])
|
||||
trace.setLogStreamId('arn:aws:logs:us-east-1:123456789:log-group:/ecs/task:log-stream:abc123')
|
||||
|
||||
when:
|
||||
def req = observer.makeTasksReq([trace])
|
||||
|
||||
then:
|
||||
req.tasks.size() == 1
|
||||
req.tasks[0].logStreamId == 'arn:aws:logs:us-east-1:123456789:log-group:/ecs/task:log-stream:abc123'
|
||||
}
|
||||
|
||||
def 'should include resourceAllocation in task map'() {
|
||||
given:
|
||||
def session = Mock(Session)
|
||||
def observer = Spy(newObserver(session))
|
||||
observer.getWorkflowProgress(true) >> new WorkflowProgress()
|
||||
|
||||
def now = System.currentTimeMillis()
|
||||
def trace = new TraceRecord([
|
||||
taskId: 42,
|
||||
process: 'foo',
|
||||
workdir: "/work/dir",
|
||||
cpus: 1,
|
||||
submit: now-2000,
|
||||
start: now-1000,
|
||||
complete: now,
|
||||
accelerator: 2,
|
||||
acceleratorType: 'v100'
|
||||
])
|
||||
trace.setResourceAllocation([cpuShares: 2048, memoryMiB: 4096, time: '1h'])
|
||||
|
||||
when:
|
||||
def req = observer.makeTasksReq([trace])
|
||||
|
||||
then:
|
||||
req.tasks.size() == 1
|
||||
req.tasks[0].accelerator == 2
|
||||
req.tasks[0].acceleratorType == 'v100'
|
||||
req.tasks[0].resourceAllocation == [cpuShares: 2048, memoryMiB: 4096, time: '1h']
|
||||
}
|
||||
|
||||
def 'should include gpuMetrics in task map'() {
|
||||
given:
|
||||
def session = Mock(Session)
|
||||
def observer = Spy(newObserver(session))
|
||||
observer.getWorkflowProgress(true) >> new WorkflowProgress()
|
||||
|
||||
def now = System.currentTimeMillis()
|
||||
def trace = new TraceRecord([
|
||||
taskId: 42,
|
||||
process: 'foo',
|
||||
workdir: "/work/dir",
|
||||
cpus: 1,
|
||||
submit: now-2000,
|
||||
start: now-1000,
|
||||
complete: now
|
||||
])
|
||||
trace.setGpuMetrics([name: 'Tesla T4', mem: 15360, driver: '580.126.09', active_time: 651030, pct: 75, peak: 100])
|
||||
|
||||
when:
|
||||
def req = observer.makeTasksReq([trace])
|
||||
|
||||
then:
|
||||
req.tasks.size() == 1
|
||||
req.tasks[0].gpuMetrics.name == 'Tesla T4'
|
||||
req.tasks[0].gpuMetrics.mem == 15360
|
||||
req.tasks[0].gpuMetrics.pct == 75
|
||||
req.tasks[0].gpuMetrics.peak == 100
|
||||
}
|
||||
|
||||
def 'should throw AbortRunException if workflow id is not found'() {
|
||||
given:
|
||||
def session = Mock(Session){
|
||||
getUniqueId() >> UUID.randomUUID()
|
||||
getWorkflowMetadata() >> Mock(WorkflowMetadata)
|
||||
}
|
||||
def client = Mock(TowerClient){
|
||||
traceCreate(_,_) >> [:]
|
||||
}
|
||||
def observer = new TowerObserver(session, client, null, [:])
|
||||
|
||||
when:
|
||||
observer.onFlowCreate(session)
|
||||
then:
|
||||
thrown(AbortRunException)
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin
|
||||
|
||||
import nextflow.Session
|
||||
import spock.lang.Specification
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Jordi Deu-Pons <jordi@seqera.io>
|
||||
*/
|
||||
class TowerReportsTest extends Specification {
|
||||
|
||||
def 'should convert to glob pattern'() {
|
||||
|
||||
expect:
|
||||
TowerReports.convertToGlobPattern(KEY) == EXPECTED
|
||||
|
||||
where:
|
||||
KEY | EXPECTED
|
||||
"multiqc.html" | "glob:**/multiqc.html"
|
||||
"**/multiqc.html" | "glob:**/multiqc.html"
|
||||
"*/multiqc.html" | "glob:**/*/multiqc.html"
|
||||
"reports/*.html" | "glob:**/reports/*.html"
|
||||
|
||||
}
|
||||
|
||||
def 'should load reports from tower yml file'() {
|
||||
|
||||
given:
|
||||
def launchDir = File.createTempDir()
|
||||
def workflowId = "1khBUM1SUioskd"
|
||||
def config = new File(launchDir, "nf-${workflowId}-tower.yml")
|
||||
config.text = """
|
||||
reports:
|
||||
multiqc_report.html:
|
||||
display: "MultiQC HTML report"
|
||||
deseq2.plots.pdf:
|
||||
display: "All samples STAR Salmon DESeq2 QC PDF plots"
|
||||
salmon.merged.gene_counts.tsv:
|
||||
display: "All samples STAR Salmon merged gene raw counts"
|
||||
"*.merged.gene_counts.tsv":
|
||||
display: "All samples STAR Salmon merged gene raw counts"
|
||||
""".stripIndent()
|
||||
|
||||
when:
|
||||
def reports = new TowerReports(Mock(Session))
|
||||
def entries = reports.parseReportEntries(launchDir.toPath(), workflowId)
|
||||
|
||||
then:
|
||||
entries == [
|
||||
"multiqc_report.html" : ["display": "MultiQC HTML report"],
|
||||
"deseq2.plots.pdf" : ["display": "All samples STAR Salmon DESeq2 QC PDF plots"],
|
||||
"salmon.merged.gene_counts.tsv": ["display": "All samples STAR Salmon merged gene raw counts"],
|
||||
"*.merged.gene_counts.tsv" : ["display": "All samples STAR Salmon merged gene raw counts"]
|
||||
].collect()
|
||||
|
||||
|
||||
}
|
||||
|
||||
def 'should generate reports file'() {
|
||||
|
||||
given: 'a launch directory with a tower yaml file'
|
||||
def launchDir = File.createTempDir()
|
||||
def workflowId = "1khBUM1SUioskd"
|
||||
def config = new File(launchDir, "nf-${workflowId}-tower.yml")
|
||||
config.text = """
|
||||
reports:
|
||||
multiqc_report.html:
|
||||
display: "MultiQC HTML report"
|
||||
deseq2.plots.pdf:
|
||||
display: "All samples STAR Salmon DESeq2 QC PDF plots"
|
||||
mimeType: "application/pdf"
|
||||
"*.merged.gene_counts.tsv":
|
||||
display: "All samples STAR Salmon merged gene raw counts"
|
||||
""".stripIndent()
|
||||
|
||||
and: 'a tower reports instance'
|
||||
def session = new Session()
|
||||
TowerReports reports = Spy(TowerReports, constructorArgs: [session])
|
||||
reports.launchDir >> launchDir.toPath()
|
||||
|
||||
and: 'some reports'
|
||||
def repo1 = new File(launchDir, "multiqc_report.html")
|
||||
repo1.text = "html"
|
||||
def repo2 = new File(launchDir, "deseq2.plots.pdf")
|
||||
repo2.text = "pdf"
|
||||
def repo3 = new File(launchDir, "salmon.merged.gene_counts.tsv")
|
||||
repo3.text = "tsv"
|
||||
|
||||
when: 'a workflow runs'
|
||||
reports.flowCreate(workflowId)
|
||||
reports.filePublish(repo1.toPath())
|
||||
reports.filePublish(repo2.toPath())
|
||||
reports.filePublish(repo3.toPath())
|
||||
reports.flowComplete()
|
||||
|
||||
then:
|
||||
def result = new File(launchDir, "nf-${workflowId}-reports.tsv")
|
||||
result.text == "key\tpath\tsize\tdisplay\tmime_type\n" +
|
||||
"multiqc_report.html\t${repo1.toPath().toUriString()}\t4\tMultiQC HTML report\t\n" +
|
||||
"deseq2.plots.pdf\t${repo2.toPath().toUriString()}\t3\tAll samples STAR Salmon DESeq2 QC PDF plots\tapplication/pdf\n" +
|
||||
"*.merged.gene_counts.tsv\t${repo3.toPath().toUriString()}\t3\tAll samples STAR Salmon merged gene raw counts\t\n"
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin
|
||||
|
||||
import nextflow.util.Duration
|
||||
import nextflow.util.RetryConfig
|
||||
import spock.lang.Specification
|
||||
|
||||
/**
|
||||
* Unit tests for TowerRetryPolicy
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class TowerRetryPolicyTest extends Specification {
|
||||
|
||||
def 'should validate default values of tower retry policy'() {
|
||||
when:
|
||||
def policy = new TowerRetryPolicy([:])
|
||||
|
||||
then:
|
||||
policy.delay == RetryConfig.DEFAULT_DELAY
|
||||
policy.maxDelay == RetryConfig.DEFAULT_MAX_DELAY
|
||||
policy.maxAttempts == RetryConfig.DEFAULT_MAX_ATTEMPTS
|
||||
policy.jitter == RetryConfig.DEFAULT_JITTER
|
||||
policy.multiplier == RetryConfig.DEFAULT_MULTIPLIER
|
||||
}
|
||||
|
||||
def 'should use provided values when specified'() {
|
||||
when:
|
||||
def customOptions = [
|
||||
delay: '1s' as nextflow.util.Duration,
|
||||
maxDelay: '60s' as nextflow.util.Duration,
|
||||
maxAttempts: 3,
|
||||
jitter: 0.5,
|
||||
multiplier: 1.5
|
||||
]
|
||||
def policy = new TowerRetryPolicy(customOptions)
|
||||
|
||||
then:
|
||||
policy.delay == customOptions.delay
|
||||
policy.maxDelay == customOptions.maxDelay
|
||||
policy.maxAttempts == 3
|
||||
policy.jitter == 0.5d
|
||||
policy.multiplier == 1.5d
|
||||
}
|
||||
|
||||
def 'should use provided values when specified'() {
|
||||
when:
|
||||
def policy = new TowerRetryPolicy([:], [backOffDelay: 500, maxRetries: 100, backOffBase: 5])
|
||||
|
||||
then:
|
||||
policy.delay == Duration.of('500ms')
|
||||
policy.maxAttempts == 100
|
||||
policy.multiplier == 5
|
||||
and:
|
||||
policy.maxDelay == RetryConfig.DEFAULT_MAX_DELAY
|
||||
policy.jitter == RetryConfig.DEFAULT_JITTER
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,217 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin.dataset
|
||||
|
||||
import java.nio.file.AccessDeniedException
|
||||
import java.nio.file.NoSuchFileException
|
||||
|
||||
import groovy.json.JsonOutput
|
||||
import io.seqera.tower.plugin.TowerClient
|
||||
import io.seqera.tower.plugin.exception.ForbiddenException
|
||||
import io.seqera.tower.plugin.exception.NotFoundException
|
||||
import io.seqera.tower.plugin.exception.UnauthorizedException
|
||||
import nextflow.exception.AbortOperationException
|
||||
import spock.lang.Specification
|
||||
|
||||
/**
|
||||
* Tests for {@link SeqeraDatasetClient} using a mock {@link TowerClient}.
|
||||
*/
|
||||
class SeqeraDatasetClientTest extends Specification {
|
||||
|
||||
private TowerClient mockTower(String endpoint = 'https://api.example.com') {
|
||||
def tc = Mock(TowerClient)
|
||||
tc.endpoint >> endpoint
|
||||
return tc
|
||||
}
|
||||
private TowerClient spyTower(String endpoint = 'https://api.example.com') {
|
||||
def tc = Spy(TowerClient)
|
||||
tc.@endpoint = endpoint
|
||||
return tc
|
||||
}
|
||||
|
||||
private static TowerClient.Response ok(String body) {
|
||||
new TowerClient.Response(200, body)
|
||||
}
|
||||
|
||||
private static TowerClient.Response error(int code) {
|
||||
new TowerClient.Response(code, "error $code")
|
||||
}
|
||||
|
||||
// ---- listUserWorkspacesAndOrgs ----
|
||||
|
||||
def "listUserWorkspacesAndOrgs returns parsed DTOs"() {
|
||||
given:
|
||||
def body = JsonOutput.toJson([orgsAndWorkspaces: [
|
||||
[orgId: 1, orgName: 'acme', workspaceId: 10, workspaceName: 'research', workspaceFullName: 'acme/research']
|
||||
]])
|
||||
def tc = spyTower()
|
||||
tc.sendApiRequest('https://api.example.com/user/42/workspaces') >> ok(body)
|
||||
def client = new SeqeraDatasetClient(tc)
|
||||
|
||||
when:
|
||||
def list = client.listUserWorkspacesAndOrgs(42L)
|
||||
|
||||
then:
|
||||
list.size() == 1
|
||||
list[0].orgName == 'acme'
|
||||
list[0].workspaceId == 10L
|
||||
list[0].workspaceName == 'research'
|
||||
}
|
||||
|
||||
// ---- listDatasets ----
|
||||
|
||||
def "listDatasets returns parsed DatasetDto list"() {
|
||||
given:
|
||||
def body = JsonOutput.toJson([datasets: [
|
||||
[id: 'ds-1', name: 'samples', version: 2, mediaType: 'text/csv',
|
||||
dateCreated: '2024-01-01T00:00:00Z', lastUpdated: '2024-01-02T00:00:00Z']
|
||||
], totalSize: 1])
|
||||
def tc = mockTower()
|
||||
tc.sendApiRequest('https://api.example.com/datasets?workspaceId=99') >> ok(body)
|
||||
def client = new SeqeraDatasetClient(tc)
|
||||
|
||||
when:
|
||||
def list = client.listDatasets(99L)
|
||||
|
||||
then:
|
||||
list.size() == 1
|
||||
list[0].id == 'ds-1'
|
||||
list[0].name == 'samples'
|
||||
list[0].version == 2L
|
||||
}
|
||||
|
||||
def "listDatasets returns empty list when no datasets"() {
|
||||
given:
|
||||
def tc = mockTower()
|
||||
tc.sendApiRequest('https://api.example.com/datasets?workspaceId=99') >>
|
||||
ok(JsonOutput.toJson([datasets: [], totalSize: 0]))
|
||||
def client = new SeqeraDatasetClient(tc)
|
||||
|
||||
when:
|
||||
def list = client.listDatasets(99L)
|
||||
|
||||
then:
|
||||
list.isEmpty()
|
||||
}
|
||||
|
||||
// ---- listVersions ----
|
||||
|
||||
def "listVersions returns parsed DatasetVersionDto list"() {
|
||||
given:
|
||||
def body = JsonOutput.toJson([versions: [
|
||||
[datasetId: 'ds-1', version: 1, fileName: 'samples.csv',
|
||||
mediaType: 'text/csv', hasHeader: true, dateCreated: '2024-01-01T00:00:00Z', disabled: false]
|
||||
]])
|
||||
def tc = mockTower()
|
||||
tc.sendApiRequest('https://api.example.com/datasets/ds-1/versions?workspaceId=1234') >> ok(body)
|
||||
def client = new SeqeraDatasetClient(tc)
|
||||
|
||||
when:
|
||||
def list = client.listVersions('ds-1', 1234)
|
||||
|
||||
then:
|
||||
list.size() == 1
|
||||
list[0].version == 1L
|
||||
list[0].fileName == 'samples.csv'
|
||||
list[0].hasHeader
|
||||
!list[0].disabled
|
||||
}
|
||||
|
||||
// ---- downloadDataset ----
|
||||
|
||||
def "downloadDataset returns InputStream with correct content"() {
|
||||
given:
|
||||
def content = 'col1,col2\n1,2\n'
|
||||
def tc = mockTower()
|
||||
tc.sendStreamingRequest('https://api.example.com/datasets/ds-1/v/1/n/samples.csv?workspaceId=1234') >> new ByteArrayInputStream(content.getBytes('UTF-8'))
|
||||
def client = new SeqeraDatasetClient(tc)
|
||||
|
||||
when:
|
||||
def stream = client.downloadDataset('ds-1', '1', 'samples.csv', 1234)
|
||||
|
||||
then:
|
||||
stream.text == content
|
||||
}
|
||||
|
||||
def "downloadDataset URL-encodes the filename"() {
|
||||
given:
|
||||
def tc = mockTower()
|
||||
def client = new SeqeraDatasetClient(tc)
|
||||
|
||||
when:
|
||||
client.downloadDataset('ds-1', '1', 'my file.csv',1234)
|
||||
|
||||
then:
|
||||
1 * tc.sendStreamingRequest('https://api.example.com/datasets/ds-1/v/1/n/my%20file.csv?workspaceId=1234') >> new ByteArrayInputStream('data'.getBytes('UTF-8'))
|
||||
}
|
||||
|
||||
def "downloadDataset throws NoSuchFileException on 404"() {
|
||||
given:
|
||||
def tc = mockTower()
|
||||
tc.sendStreamingRequest(_) >> { throw new NotFoundException("not found") }
|
||||
def client = new SeqeraDatasetClient(tc)
|
||||
|
||||
when:
|
||||
client.downloadDataset('ds-missing', '1', 'file.csv', 1234)
|
||||
|
||||
then:
|
||||
thrown(NoSuchFileException)
|
||||
}
|
||||
|
||||
def "downloadDataset throws AccessDeniedException on 403"() {
|
||||
given:
|
||||
def tc = mockTower()
|
||||
tc.sendStreamingRequest(_) >> { throw new ForbiddenException("forbidden") }
|
||||
def client = new SeqeraDatasetClient(tc)
|
||||
|
||||
when:
|
||||
client.downloadDataset('ds-1', '1', 'file.csv', 1234)
|
||||
|
||||
then:
|
||||
thrown(AccessDeniedException)
|
||||
}
|
||||
|
||||
def "downloadDataset throws AbortOperationException on 401"() {
|
||||
given:
|
||||
def tc = mockTower()
|
||||
tc.sendStreamingRequest(_) >> { throw new UnauthorizedException("unauthorized") }
|
||||
def client = new SeqeraDatasetClient(tc)
|
||||
|
||||
when:
|
||||
client.downloadDataset('ds-1', '1', 'file.csv', 1234)
|
||||
|
||||
then:
|
||||
thrown(AbortOperationException)
|
||||
}
|
||||
|
||||
// ---- createDataset ----
|
||||
|
||||
def "createDataset posts and returns created dataset"() {
|
||||
given:
|
||||
def responseBody = JsonOutput.toJson([dataset: [id: 'ds-new', name: 'results']])
|
||||
def tc = mockTower()
|
||||
tc.sendApiRequest('https://api.example.com/datasets?workspaceId=10', [name: 'results'], 'POST') >> ok(responseBody)
|
||||
def client = new SeqeraDatasetClient(tc)
|
||||
|
||||
when:
|
||||
def dto = client.createDataset(10L, 'results')
|
||||
|
||||
then:
|
||||
dto.id == 'ds-new'
|
||||
dto.name == 'results'
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin.fs
|
||||
|
||||
import java.nio.ByteBuffer
|
||||
|
||||
import spock.lang.Specification
|
||||
|
||||
class DatasetInputStreamTest extends Specification {
|
||||
|
||||
def 'should read bytes into buffer'() {
|
||||
given:
|
||||
def data = 'hello world'.bytes
|
||||
def channel = new DatasetInputStream(new ByteArrayInputStream(data))
|
||||
def buf = ByteBuffer.allocate(data.length)
|
||||
|
||||
when:
|
||||
def n = channel.read(buf)
|
||||
|
||||
then:
|
||||
n == data.length
|
||||
buf.array() == data
|
||||
}
|
||||
|
||||
def 'should advance position after read'() {
|
||||
given:
|
||||
def data = 'abcdef'.bytes
|
||||
def channel = new DatasetInputStream(new ByteArrayInputStream(data))
|
||||
|
||||
when:
|
||||
channel.read(ByteBuffer.allocate(3))
|
||||
|
||||
then:
|
||||
channel.position() == 3
|
||||
|
||||
when:
|
||||
channel.read(ByteBuffer.allocate(3))
|
||||
|
||||
then:
|
||||
channel.position() == 6
|
||||
}
|
||||
|
||||
def 'should return -1 at end of stream'() {
|
||||
given:
|
||||
def channel = new DatasetInputStream(new ByteArrayInputStream(new byte[0]))
|
||||
|
||||
when:
|
||||
def n = channel.read(ByteBuffer.allocate(8))
|
||||
|
||||
then:
|
||||
n == -1
|
||||
channel.position() == 0
|
||||
}
|
||||
|
||||
def 'should read partial buffer when stream has fewer bytes'() {
|
||||
given:
|
||||
def data = 'hi'.bytes
|
||||
def channel = new DatasetInputStream(new ByteArrayInputStream(data))
|
||||
def buf = ByteBuffer.allocate(100)
|
||||
|
||||
when:
|
||||
def n = channel.read(buf)
|
||||
|
||||
then:
|
||||
n == 2
|
||||
channel.position() == 2
|
||||
}
|
||||
|
||||
def 'should be open initially and closed after close()'() {
|
||||
given:
|
||||
def channel = new DatasetInputStream(new ByteArrayInputStream(new byte[0]))
|
||||
|
||||
expect:
|
||||
channel.isOpen()
|
||||
|
||||
when:
|
||||
channel.close()
|
||||
|
||||
then:
|
||||
!channel.isOpen()
|
||||
}
|
||||
|
||||
def 'should close underlying stream on close()'() {
|
||||
given:
|
||||
def stream = Mock(InputStream)
|
||||
def channel = new DatasetInputStream(stream)
|
||||
|
||||
when:
|
||||
channel.close()
|
||||
|
||||
then:
|
||||
1 * stream.close()
|
||||
!channel.isOpen()
|
||||
}
|
||||
|
||||
def 'should throw on size'() {
|
||||
when:
|
||||
new DatasetInputStream(new ByteArrayInputStream(new byte[0])).size()
|
||||
|
||||
then:
|
||||
thrown(UnsupportedOperationException)
|
||||
}
|
||||
|
||||
def 'should throw on write'() {
|
||||
when:
|
||||
new DatasetInputStream(new ByteArrayInputStream(new byte[0])).write(ByteBuffer.allocate(1))
|
||||
|
||||
then:
|
||||
thrown(UnsupportedOperationException)
|
||||
}
|
||||
|
||||
def 'should throw on seek'() {
|
||||
when:
|
||||
new DatasetInputStream(new ByteArrayInputStream(new byte[0])).position(0L)
|
||||
|
||||
then:
|
||||
thrown(UnsupportedOperationException)
|
||||
}
|
||||
|
||||
def 'should throw on truncate'() {
|
||||
when:
|
||||
new DatasetInputStream(new ByteArrayInputStream(new byte[0])).truncate(0L)
|
||||
|
||||
then:
|
||||
thrown(UnsupportedOperationException)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,411 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin.fs
|
||||
|
||||
import java.nio.file.AccessDeniedException
|
||||
import java.nio.file.DirectoryStream
|
||||
import java.nio.file.FileSystemAlreadyExistsException
|
||||
import java.nio.file.InvalidPathException
|
||||
import java.nio.file.NoSuchFileException
|
||||
import java.nio.file.attribute.BasicFileAttributes
|
||||
|
||||
import groovy.json.JsonOutput
|
||||
import io.seqera.tower.plugin.TowerClient
|
||||
import io.seqera.tower.plugin.dataset.SeqeraDatasetClient
|
||||
import nextflow.exception.AbortOperationException
|
||||
import spock.lang.Specification
|
||||
|
||||
/**
|
||||
* Tests for {@link SeqeraFileSystemProvider} using a mock {@link TowerClient}.
|
||||
*/
|
||||
class SeqeraFileSystemProviderTest extends Specification {
|
||||
|
||||
private static final String ENDPOINT = 'https://api.example.com'
|
||||
|
||||
private TowerClient spyTower() {
|
||||
def tc = Spy(TowerClient)
|
||||
tc.@endpoint = ENDPOINT
|
||||
return tc
|
||||
}
|
||||
|
||||
private static TowerClient.Response ok(String body) {
|
||||
new TowerClient.Response(200, body)
|
||||
}
|
||||
|
||||
private static TowerClient.Response error(int code) {
|
||||
new TowerClient.Response(code, "error $code")
|
||||
}
|
||||
|
||||
private SeqeraFileSystem buildFs(TowerClient tc) {
|
||||
final client = new SeqeraDatasetClient(tc)
|
||||
final provider = new SeqeraFileSystemProvider()
|
||||
return new SeqeraFileSystem(provider, client)
|
||||
}
|
||||
|
||||
private static String userInfoJson() {
|
||||
JsonOutput.toJson([user: [id: 42L, userName: 'testuser']])
|
||||
}
|
||||
|
||||
private static String workspacesJson() {
|
||||
JsonOutput.toJson([orgsAndWorkspaces: [
|
||||
[orgId: 1L, orgName: 'acme', workspaceId: 10L, workspaceName: 'research', workspaceFullName: 'acme/research']
|
||||
]])
|
||||
}
|
||||
|
||||
private static String datasetsJson() {
|
||||
JsonOutput.toJson([datasets: [
|
||||
[id: 'ds-1', name: 'samples', version: 2L, mediaType: 'text/csv',
|
||||
workspaceId: 10L,
|
||||
dateCreated: '2024-01-01T00:00:00Z', lastUpdated: '2024-01-02T00:00:00Z']
|
||||
], totalSize: 1])
|
||||
}
|
||||
|
||||
private static String versionsJson() {
|
||||
JsonOutput.toJson([versions: [
|
||||
[datasetId: 'ds-1', version: 1L, fileName: 'samples.csv',
|
||||
mediaType: 'text/csv', hasHeader: true, dateCreated: '2024-01-01T00:00:00Z', disabled: false],
|
||||
[datasetId: 'ds-1', version: 2L, fileName: 'samples_v2.csv',
|
||||
mediaType: 'text/csv', hasHeader: true, dateCreated: '2024-01-02T00:00:00Z', disabled: false]
|
||||
]])
|
||||
}
|
||||
|
||||
// ---- newInputStream - latest version ----
|
||||
|
||||
def "newInputStream resolves latest version and downloads correct content"() {
|
||||
given:
|
||||
def tc = spyTower()
|
||||
tc.sendApiRequest("${ENDPOINT}/user-info") >> ok(userInfoJson())
|
||||
tc.sendApiRequest("${ENDPOINT}/user/42/workspaces") >> ok(workspacesJson())
|
||||
tc.sendApiRequest("${ENDPOINT}/datasets?workspaceId=10") >> ok(datasetsJson())
|
||||
tc.sendApiRequest("${ENDPOINT}/datasets/ds-1/versions?workspaceId=10") >> ok(versionsJson())
|
||||
final csvContent = 'col1,col2\n1,2\n3,4\n'
|
||||
tc.sendStreamingRequest("${ENDPOINT}/datasets/ds-1/v/2/n/samples_v2.csv?workspaceId=10") >> new ByteArrayInputStream(csvContent.getBytes('UTF-8'))
|
||||
|
||||
final fs = buildFs(tc)
|
||||
final path = new SeqeraPath(fs, 'seqera://acme/research/datasets/samples')
|
||||
|
||||
when:
|
||||
final text = fs.provider().newInputStream(path).text
|
||||
|
||||
then:
|
||||
text == csvContent
|
||||
}
|
||||
|
||||
// ---- newInputStream - pinned version ----
|
||||
|
||||
def "newInputStream uses pinned version when @ver suffix given"() {
|
||||
given:
|
||||
def tc = spyTower()
|
||||
tc.sendApiRequest("${ENDPOINT}/user-info") >> ok(userInfoJson())
|
||||
tc.sendApiRequest("${ENDPOINT}/user/42/workspaces") >> ok(workspacesJson())
|
||||
tc.sendApiRequest("${ENDPOINT}/datasets?workspaceId=10") >> ok(datasetsJson())
|
||||
tc.sendApiRequest("${ENDPOINT}/datasets/ds-1/versions?workspaceId=10") >> ok(versionsJson())
|
||||
final csvContent = 'col1,col2\n1,2\n'
|
||||
tc.sendStreamingRequest("${ENDPOINT}/datasets/ds-1/v/1/n/samples.csv?workspaceId=10") >> new ByteArrayInputStream(csvContent.getBytes('UTF-8'))
|
||||
|
||||
final fs = buildFs(tc)
|
||||
final path = new SeqeraPath(fs, 'seqera://acme/research/datasets/samples@1')
|
||||
|
||||
when:
|
||||
final text = fs.provider().newInputStream(path).text
|
||||
|
||||
then:
|
||||
text == csvContent
|
||||
}
|
||||
|
||||
// ---- newInputStream - missing dataset ----
|
||||
|
||||
def "newInputStream throws NoSuchFileException for unknown dataset"() {
|
||||
given:
|
||||
def tc = spyTower()
|
||||
tc.sendApiRequest("${ENDPOINT}/user-info") >> ok(userInfoJson())
|
||||
tc.sendApiRequest("${ENDPOINT}/user/42/workspaces") >> ok(workspacesJson())
|
||||
tc.sendApiRequest("${ENDPOINT}/datasets?workspaceId=10") >>
|
||||
ok(JsonOutput.toJson([datasets: [], totalSize: 0]))
|
||||
|
||||
final fs = buildFs(tc)
|
||||
final path = new SeqeraPath(fs, 'seqera://acme/research/datasets/missing-dataset')
|
||||
|
||||
when:
|
||||
fs.provider().newInputStream(path)
|
||||
|
||||
then:
|
||||
thrown(NoSuchFileException)
|
||||
}
|
||||
|
||||
// ---- newInputStream - pinned version not found ----
|
||||
|
||||
def "newInputStream throws NoSuchFileException for unknown pinned version"() {
|
||||
given:
|
||||
def tc = spyTower()
|
||||
tc.sendApiRequest("${ENDPOINT}/user-info") >> ok(userInfoJson())
|
||||
tc.sendApiRequest("${ENDPOINT}/user/42/workspaces") >> ok(workspacesJson())
|
||||
tc.sendApiRequest("${ENDPOINT}/datasets?workspaceId=10") >> ok(datasetsJson())
|
||||
tc.sendApiRequest("${ENDPOINT}/datasets/ds-1/versions?workspaceId=10") >> ok(versionsJson())
|
||||
|
||||
final fs = buildFs(tc)
|
||||
final path = new SeqeraPath(fs, 'seqera://acme/research/datasets/samples@99')
|
||||
|
||||
when:
|
||||
fs.provider().newInputStream(path)
|
||||
|
||||
then:
|
||||
thrown(NoSuchFileException)
|
||||
}
|
||||
|
||||
// ---- readAttributes ----
|
||||
|
||||
def "readAttributes returns directory attributes for depth < 4"() {
|
||||
given:
|
||||
def tc = spyTower()
|
||||
tc.sendApiRequest("${ENDPOINT}/user-info") >> ok(userInfoJson())
|
||||
tc.sendApiRequest("${ENDPOINT}/user/42/workspaces") >> ok(workspacesJson())
|
||||
|
||||
final fs = buildFs(tc)
|
||||
final path = new SeqeraPath(fs, 'seqera://acme/research')
|
||||
|
||||
when:
|
||||
final attrs = fs.provider().readAttributes(path, java.nio.file.attribute.BasicFileAttributes)
|
||||
|
||||
then:
|
||||
attrs.isDirectory()
|
||||
!attrs.isRegularFile()
|
||||
}
|
||||
|
||||
def "readAttributes returns file attributes for dataset path"() {
|
||||
given:
|
||||
def tc = spyTower()
|
||||
tc.sendApiRequest("${ENDPOINT}/user-info") >> ok(userInfoJson())
|
||||
tc.sendApiRequest("${ENDPOINT}/user/42/workspaces") >> ok(workspacesJson())
|
||||
tc.sendApiRequest("${ENDPOINT}/datasets?workspaceId=10") >> ok(datasetsJson())
|
||||
|
||||
final fs = buildFs(tc)
|
||||
final path = new SeqeraPath(fs, 'seqera://acme/research/datasets/samples')
|
||||
|
||||
when:
|
||||
final attrs = fs.provider().readAttributes(path, BasicFileAttributes)
|
||||
|
||||
then:
|
||||
!attrs.isDirectory()
|
||||
attrs.isRegularFile()
|
||||
}
|
||||
|
||||
// ---- newDirectoryStream (T023) ----
|
||||
|
||||
def "newDirectoryStream on root returns org names"() {
|
||||
given:
|
||||
def tc = spyTower()
|
||||
tc.sendApiRequest("${ENDPOINT}/user-info") >> ok(userInfoJson())
|
||||
tc.sendApiRequest("${ENDPOINT}/user/42/workspaces") >> ok(workspacesJson())
|
||||
|
||||
final fs = buildFs(tc)
|
||||
final root = new SeqeraPath(fs, 'seqera://')
|
||||
|
||||
when:
|
||||
def entries = fs.provider().newDirectoryStream(root, null).toList()
|
||||
|
||||
then:
|
||||
entries.size() == 1
|
||||
entries[0].toString() == 'seqera://acme'
|
||||
}
|
||||
|
||||
def "newDirectoryStream on org returns workspace names"() {
|
||||
given:
|
||||
def tc = spyTower()
|
||||
tc.sendApiRequest("${ENDPOINT}/user-info") >> ok(userInfoJson())
|
||||
tc.sendApiRequest("${ENDPOINT}/user/42/workspaces") >> ok(workspacesJson())
|
||||
|
||||
final fs = buildFs(tc)
|
||||
final orgPath = new SeqeraPath(fs, 'seqera://acme')
|
||||
|
||||
when:
|
||||
def entries = fs.provider().newDirectoryStream(orgPath, null).toList()
|
||||
|
||||
then:
|
||||
entries.size() == 1
|
||||
entries[0].toString() == 'seqera://acme/research'
|
||||
}
|
||||
|
||||
def "newDirectoryStream on workspace returns datasets resource type"() {
|
||||
given:
|
||||
def tc = spyTower()
|
||||
final fs = buildFs(tc)
|
||||
final wsPath = new SeqeraPath(fs, 'seqera://acme/research')
|
||||
|
||||
when:
|
||||
def entries = fs.provider().newDirectoryStream(wsPath, null).toList()
|
||||
|
||||
then:
|
||||
entries.size() == 1
|
||||
entries[0].toString() == 'seqera://acme/research/datasets'
|
||||
}
|
||||
|
||||
def "newDirectoryStream on datasets dir returns dataset names"() {
|
||||
given:
|
||||
def tc = spyTower()
|
||||
tc.sendApiRequest("${ENDPOINT}/user-info") >> ok(userInfoJson())
|
||||
tc.sendApiRequest("${ENDPOINT}/user/42/workspaces") >> ok(workspacesJson())
|
||||
tc.sendApiRequest("${ENDPOINT}/datasets?workspaceId=10") >> ok(datasetsJson())
|
||||
|
||||
final fs = buildFs(tc)
|
||||
final dsDir = new SeqeraPath(fs, 'seqera://acme/research/datasets')
|
||||
|
||||
when:
|
||||
def entries = fs.provider().newDirectoryStream(dsDir, null).toList()
|
||||
|
||||
then:
|
||||
entries.size() == 1
|
||||
entries[0].toString() == 'seqera://acme/research/datasets/samples'
|
||||
}
|
||||
|
||||
def "newDirectoryStream on datasets dir with empty workspace returns empty stream"() {
|
||||
given:
|
||||
def tc = spyTower()
|
||||
tc.sendApiRequest("${ENDPOINT}/user-info") >> ok(userInfoJson())
|
||||
tc.sendApiRequest("${ENDPOINT}/user/42/workspaces") >> ok(workspacesJson())
|
||||
tc.sendApiRequest("${ENDPOINT}/datasets?workspaceId=10") >>
|
||||
ok(JsonOutput.toJson([datasets: [], totalSize: 0]))
|
||||
|
||||
final fs = buildFs(tc)
|
||||
final dsDir = new SeqeraPath(fs, 'seqera://acme/research/datasets')
|
||||
|
||||
when:
|
||||
def entries = fs.provider().newDirectoryStream(dsDir, null).toList()
|
||||
|
||||
then:
|
||||
entries.isEmpty()
|
||||
}
|
||||
|
||||
def "newDirectoryStream filter is applied to entries"() {
|
||||
given:
|
||||
def tc = spyTower()
|
||||
tc.sendApiRequest("${ENDPOINT}/user-info") >> ok(userInfoJson())
|
||||
tc.sendApiRequest("${ENDPOINT}/user/42/workspaces") >> ok(workspacesJson())
|
||||
tc.sendApiRequest("${ENDPOINT}/datasets?workspaceId=10") >> ok(JsonOutput.toJson([datasets: [
|
||||
[id: 'ds-1', name: 'samples', version: 1L, mediaType: 'text/csv', workspaceId: 10L,
|
||||
dateCreated: '2024-01-01T00:00:00Z', lastUpdated: '2024-01-02T00:00:00Z'],
|
||||
[id: 'ds-2', name: 'results', version: 1L, mediaType: 'text/csv', workspaceId: 10L,
|
||||
dateCreated: '2024-01-01T00:00:00Z', lastUpdated: '2024-01-02T00:00:00Z']
|
||||
], totalSize: 2]))
|
||||
|
||||
final fs = buildFs(tc)
|
||||
final dsDir = new SeqeraPath(fs, 'seqera://acme/research/datasets')
|
||||
final filter = { java.nio.file.Path p -> p.toString().contains('results') } as DirectoryStream.Filter
|
||||
|
||||
when:
|
||||
def entries = fs.provider().newDirectoryStream(dsDir, filter).toList()
|
||||
|
||||
then:
|
||||
entries.size() == 1
|
||||
entries[0].toString() == 'seqera://acme/research/datasets/results'
|
||||
}
|
||||
|
||||
// ---- error scenarios (T028) ----
|
||||
|
||||
def "readAttributes throws NoSuchFileException for unknown org"() {
|
||||
given:
|
||||
def tc = spyTower()
|
||||
tc.sendApiRequest("${ENDPOINT}/user-info") >> ok(userInfoJson())
|
||||
tc.sendApiRequest("${ENDPOINT}/user/42/workspaces") >> ok(workspacesJson())
|
||||
|
||||
final fs = buildFs(tc)
|
||||
final path = new SeqeraPath(fs, 'seqera://unknown-org/research')
|
||||
|
||||
when:
|
||||
fs.provider().readAttributes(path, BasicFileAttributes)
|
||||
|
||||
then:
|
||||
thrown(NoSuchFileException)
|
||||
}
|
||||
|
||||
def "newInputStream throws NoSuchFileException containing dataset name for missing dataset"() {
|
||||
given:
|
||||
def tc = spyTower()
|
||||
tc.sendApiRequest("${ENDPOINT}/user-info") >> ok(userInfoJson())
|
||||
tc.sendApiRequest("${ENDPOINT}/user/42/workspaces") >> ok(workspacesJson())
|
||||
tc.sendApiRequest("${ENDPOINT}/datasets?workspaceId=10") >>
|
||||
ok(JsonOutput.toJson([datasets: [], totalSize: 0]))
|
||||
|
||||
final fs = buildFs(tc)
|
||||
final path = new SeqeraPath(fs, 'seqera://acme/research/datasets/missing-dataset')
|
||||
|
||||
when:
|
||||
fs.provider().newInputStream(path)
|
||||
|
||||
then:
|
||||
def e = thrown(NoSuchFileException)
|
||||
e.file?.contains('missing-dataset')
|
||||
}
|
||||
|
||||
def "getUserInfo 401 propagates as AbortOperationException"() {
|
||||
given:
|
||||
def tc = spyTower()
|
||||
tc.sendApiRequest("${ENDPOINT}/user-info") >> new TowerClient.Response(401, 'Unauthorized')
|
||||
|
||||
final fs = buildFs(tc)
|
||||
final path = new SeqeraPath(fs, 'seqera://acme/research/datasets/samples')
|
||||
|
||||
when:
|
||||
fs.provider().newInputStream(path)
|
||||
|
||||
then:
|
||||
thrown(AbortOperationException)
|
||||
}
|
||||
|
||||
def "getUserInfo 403 propagates as AccessDeniedException"() {
|
||||
given:
|
||||
def tc = spyTower()
|
||||
tc.sendApiRequest("${ENDPOINT}/user-info") >> new TowerClient.Response(403, 'Forbidden')
|
||||
|
||||
final fs = buildFs(tc)
|
||||
final path = new SeqeraPath(fs, 'seqera://acme/research/datasets/samples')
|
||||
|
||||
when:
|
||||
fs.provider().newInputStream(path)
|
||||
|
||||
then:
|
||||
thrown(AccessDeniedException)
|
||||
}
|
||||
|
||||
def "SeqeraPath constructor throws InvalidPathException for path with empty workspace segment"() {
|
||||
given:
|
||||
def tc = spyTower()
|
||||
final fs = buildFs(tc)
|
||||
|
||||
when:
|
||||
new SeqeraPath(fs, 'seqera://acme//datasets/samples')
|
||||
|
||||
then:
|
||||
thrown(InvalidPathException)
|
||||
}
|
||||
|
||||
// ---- newFileSystem contract ----
|
||||
|
||||
def "newFileSystem throws FileSystemAlreadyExistsException when filesystem exists"() {
|
||||
given: 'a provider with an existing filesystem'
|
||||
def tc = spyTower()
|
||||
def provider = new SeqeraFileSystemProvider()
|
||||
def fs = new SeqeraFileSystem(provider, new SeqeraDatasetClient(tc))
|
||||
provider.@fileSystem = fs
|
||||
|
||||
when:
|
||||
provider.newFileSystem(new URI('seqera://test'), [:])
|
||||
|
||||
then:
|
||||
thrown(FileSystemAlreadyExistsException)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin.fs
|
||||
|
||||
import java.nio.file.NoSuchFileException
|
||||
|
||||
import groovy.json.JsonOutput
|
||||
import io.seqera.tower.plugin.TowerClient
|
||||
import io.seqera.tower.plugin.dataset.SeqeraDatasetClient
|
||||
import spock.lang.Specification
|
||||
|
||||
/**
|
||||
* Tests for {@link SeqeraFileSystem} caching and workspace resolution using a mock {@link TowerClient}.
|
||||
*/
|
||||
class SeqeraFileSystemTest extends Specification {
|
||||
|
||||
private static final String ENDPOINT = 'https://api.example.com'
|
||||
|
||||
private TowerClient spyTower() {
|
||||
def tc = Spy(TowerClient)
|
||||
tc.@endpoint = ENDPOINT
|
||||
return tc
|
||||
}
|
||||
|
||||
private static TowerClient.Response ok(String body) {
|
||||
new TowerClient.Response(200, body)
|
||||
}
|
||||
|
||||
private static String userInfoJson() {
|
||||
JsonOutput.toJson([user: [id: 42L, userName: 'testuser']])
|
||||
}
|
||||
|
||||
private static String workspacesJson() {
|
||||
JsonOutput.toJson([orgsAndWorkspaces: [
|
||||
[orgId: 1L, orgName: 'acme', workspaceId: 10L, workspaceName: 'research', workspaceFullName: 'acme/research'],
|
||||
[orgId: 1L, orgName: 'acme', workspaceId: 20L, workspaceName: 'dev', workspaceFullName: 'acme/dev'],
|
||||
[orgId: 2L, orgName: 'other', workspaceId: 30L, workspaceName: 'ws', workspaceFullName: 'other/ws']
|
||||
]])
|
||||
}
|
||||
|
||||
private SeqeraFileSystem buildFs(TowerClient tc) {
|
||||
new SeqeraFileSystem(new SeqeraFileSystemProvider(), new SeqeraDatasetClient(tc))
|
||||
}
|
||||
|
||||
// ---- cache loading ----
|
||||
|
||||
def "loadOrgWorkspaceCache is called only once across multiple invocations"() {
|
||||
given:
|
||||
def tc = spyTower()
|
||||
final fs = buildFs(tc)
|
||||
|
||||
when:
|
||||
fs.loadOrgWorkspaceCache()
|
||||
fs.loadOrgWorkspaceCache()
|
||||
|
||||
then:
|
||||
1 * tc.sendApiRequest("${ENDPOINT}/user-info") >> ok(userInfoJson())
|
||||
1 * tc.sendApiRequest("${ENDPOINT}/user/42/workspaces") >> ok(workspacesJson())
|
||||
}
|
||||
|
||||
def "listOrgNames returns distinct org names from cache"() {
|
||||
given:
|
||||
def tc = spyTower()
|
||||
tc.sendApiRequest("${ENDPOINT}/user-info") >> ok(userInfoJson())
|
||||
tc.sendApiRequest("${ENDPOINT}/user/42/workspaces") >> ok(workspacesJson())
|
||||
final fs = buildFs(tc)
|
||||
|
||||
when:
|
||||
def orgs = fs.listOrgNames()
|
||||
|
||||
then:
|
||||
orgs.size() == 2
|
||||
orgs.contains('acme')
|
||||
orgs.contains('other')
|
||||
}
|
||||
|
||||
def "listWorkspaceNames returns workspace names for the given org"() {
|
||||
given:
|
||||
def tc = spyTower()
|
||||
tc.sendApiRequest("${ENDPOINT}/user-info") >> ok(userInfoJson())
|
||||
tc.sendApiRequest("${ENDPOINT}/user/42/workspaces") >> ok(workspacesJson())
|
||||
final fs = buildFs(tc)
|
||||
|
||||
when:
|
||||
def names = fs.listWorkspaceNames('acme')
|
||||
|
||||
then:
|
||||
names.size() == 2
|
||||
names.containsAll(['research', 'dev'])
|
||||
}
|
||||
|
||||
// ---- resolveWorkspaceId ----
|
||||
|
||||
def "resolveWorkspaceId returns correct ID for known org and workspace"() {
|
||||
given:
|
||||
def tc = spyTower()
|
||||
tc.sendApiRequest("${ENDPOINT}/user-info") >> ok(userInfoJson())
|
||||
tc.sendApiRequest("${ENDPOINT}/user/42/workspaces") >> ok(workspacesJson())
|
||||
final fs = buildFs(tc)
|
||||
|
||||
when:
|
||||
def id = fs.resolveWorkspaceId('acme', 'research')
|
||||
|
||||
then:
|
||||
id == 10L
|
||||
}
|
||||
|
||||
def "resolveWorkspaceId throws NoSuchFileException for unknown org"() {
|
||||
given:
|
||||
def tc = spyTower()
|
||||
tc.sendApiRequest("${ENDPOINT}/user-info") >> ok(userInfoJson())
|
||||
tc.sendApiRequest("${ENDPOINT}/user/42/workspaces") >> ok(workspacesJson())
|
||||
final fs = buildFs(tc)
|
||||
|
||||
when:
|
||||
fs.resolveWorkspaceId('unknown-org', 'research')
|
||||
|
||||
then:
|
||||
thrown(NoSuchFileException)
|
||||
}
|
||||
|
||||
def "resolveWorkspaceId throws NoSuchFileException for unknown workspace within known org"() {
|
||||
given:
|
||||
def tc = spyTower()
|
||||
tc.sendApiRequest("${ENDPOINT}/user-info") >> ok(userInfoJson())
|
||||
tc.sendApiRequest("${ENDPOINT}/user/42/workspaces") >> ok(workspacesJson())
|
||||
final fs = buildFs(tc)
|
||||
|
||||
when:
|
||||
fs.resolveWorkspaceId('acme', 'no-such-ws')
|
||||
|
||||
then:
|
||||
thrown(NoSuchFileException)
|
||||
}
|
||||
|
||||
// ---- dataset cache ----
|
||||
|
||||
def "resolveDatasets populates cache and returns datasets"() {
|
||||
given:
|
||||
def tc = spyTower()
|
||||
tc.sendApiRequest("${ENDPOINT}/datasets?workspaceId=10") >>
|
||||
ok(JsonOutput.toJson([datasets: [
|
||||
[id: 'ds-1', name: 'samples', version: 1L, mediaType: 'text/csv',
|
||||
dateCreated: '2024-01-01T00:00:00Z', lastUpdated: '2024-01-02T00:00:00Z']
|
||||
], totalSize: 1]))
|
||||
final fs = buildFs(tc)
|
||||
|
||||
when:
|
||||
def datasets = fs.resolveDatasets(10L)
|
||||
|
||||
then:
|
||||
datasets.size() == 1
|
||||
datasets[0].name == 'samples'
|
||||
}
|
||||
|
||||
def "resolveDatasets returns cached result on second call without extra API request"() {
|
||||
given:
|
||||
def tc = spyTower()
|
||||
final datasetsJson = JsonOutput.toJson([datasets: [
|
||||
[id: 'ds-1', name: 'samples', version: 1L, mediaType: 'text/csv',
|
||||
dateCreated: '2024-01-01T00:00:00Z', lastUpdated: '2024-01-02T00:00:00Z']
|
||||
], totalSize: 1])
|
||||
final fs = buildFs(tc)
|
||||
|
||||
when:
|
||||
fs.resolveDatasets(10L)
|
||||
fs.resolveDatasets(10L)
|
||||
|
||||
then:
|
||||
1 * tc.sendApiRequest("${ENDPOINT}/datasets?workspaceId=10") >> ok(datasetsJson)
|
||||
}
|
||||
|
||||
def "invalidateDatasetCache forces re-fetch on next resolveDatasets call"() {
|
||||
given:
|
||||
def tc = spyTower()
|
||||
final datasetsJson = JsonOutput.toJson([datasets: [
|
||||
[id: 'ds-1', name: 'samples', version: 1L, mediaType: 'text/csv',
|
||||
dateCreated: '2024-01-01T00:00:00Z', lastUpdated: '2024-01-02T00:00:00Z']
|
||||
], totalSize: 1])
|
||||
final fs = buildFs(tc)
|
||||
|
||||
when:
|
||||
fs.resolveDatasets(10L)
|
||||
fs.invalidateDatasetCache(10L)
|
||||
fs.resolveDatasets(10L)
|
||||
|
||||
then:
|
||||
2 * tc.sendApiRequest("${ENDPOINT}/datasets?workspaceId=10") >> ok(datasetsJson)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,521 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin.fs
|
||||
|
||||
import io.seqera.tower.plugin.dataset.SeqeraDatasetClient
|
||||
import spock.lang.Specification
|
||||
|
||||
/**
|
||||
* Unit tests for {@link SeqeraPath}.
|
||||
*/
|
||||
class SeqeraPathTest extends Specification {
|
||||
|
||||
private SeqeraFileSystem mockFs() {
|
||||
def provider = new SeqeraFileSystemProvider()
|
||||
def client = Mock(SeqeraDatasetClient)
|
||||
return new SeqeraFileSystem(provider, client)
|
||||
}
|
||||
|
||||
def "depth 0 - root path"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
def path = new SeqeraPath(fs, 'seqera://')
|
||||
|
||||
expect:
|
||||
path.depth() == 0
|
||||
path.isDirectory()
|
||||
!path.isRegularFile()
|
||||
path.org == null
|
||||
path.workspace == null
|
||||
}
|
||||
|
||||
def "depth 1 - org path"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
def path = new SeqeraPath(fs, 'seqera://acme')
|
||||
|
||||
expect:
|
||||
path.depth() == 1
|
||||
path.isDirectory()
|
||||
!path.isRegularFile()
|
||||
path.org == 'acme'
|
||||
path.workspace == null
|
||||
}
|
||||
|
||||
def "depth 2 - workspace path"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
def path = new SeqeraPath(fs, 'seqera://acme/research')
|
||||
|
||||
expect:
|
||||
path.depth() == 2
|
||||
path.isDirectory()
|
||||
path.org == 'acme'
|
||||
path.workspace == 'research'
|
||||
path.resourceType == null
|
||||
}
|
||||
|
||||
def "depth 3 - resource type path"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
def path = new SeqeraPath(fs, 'seqera://acme/research/datasets')
|
||||
|
||||
expect:
|
||||
path.depth() == 3
|
||||
path.isDirectory()
|
||||
path.org == 'acme'
|
||||
path.workspace == 'research'
|
||||
path.resourceType == 'datasets'
|
||||
path.datasetName == null
|
||||
}
|
||||
|
||||
def "depth 4 - dataset file path"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
def path = new SeqeraPath(fs, 'seqera://acme/research/datasets/samples')
|
||||
|
||||
expect:
|
||||
path.depth() == 4
|
||||
!path.isDirectory()
|
||||
path.isRegularFile()
|
||||
path.org == 'acme'
|
||||
path.workspace == 'research'
|
||||
path.resourceType == 'datasets'
|
||||
path.datasetName == 'samples'
|
||||
path.version == null
|
||||
}
|
||||
|
||||
def "depth 4 - dataset with pinned version"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
def path = new SeqeraPath(fs, 'seqera://acme/research/datasets/samples@2')
|
||||
|
||||
expect:
|
||||
path.depth() == 4
|
||||
path.datasetName == 'samples'
|
||||
path.version == '2'
|
||||
}
|
||||
|
||||
def "toUri round-trip - no version"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
def uri = 'seqera://acme/research/datasets/samples'
|
||||
def path = new SeqeraPath(fs, uri)
|
||||
|
||||
expect:
|
||||
path.toUri().toString() == uri
|
||||
path.toString() == uri
|
||||
}
|
||||
|
||||
def "toUri round-trip - with version"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
def uri = 'seqera://acme/research/datasets/samples@2'
|
||||
def path = new SeqeraPath(fs, uri)
|
||||
|
||||
expect:
|
||||
path.toUri().toString() == uri
|
||||
}
|
||||
|
||||
def "getParent - depth 4 returns depth 3"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
def path = new SeqeraPath(fs, 'seqera://acme/research/datasets/samples')
|
||||
|
||||
when:
|
||||
def parent = path.getParent()
|
||||
|
||||
then:
|
||||
parent.toString() == 'seqera://acme/research/datasets'
|
||||
(parent as SeqeraPath).depth() == 3
|
||||
}
|
||||
|
||||
def "getParent - depth 3 returns depth 2"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
def path = new SeqeraPath(fs, 'seqera://acme/research/datasets')
|
||||
|
||||
expect:
|
||||
path.getParent().toString() == 'seqera://acme/research'
|
||||
}
|
||||
|
||||
def "getParent - depth 1 returns depth 0 root"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
def path = new SeqeraPath(fs, 'seqera://acme')
|
||||
|
||||
expect:
|
||||
path.getParent().toString() == 'seqera://'
|
||||
}
|
||||
|
||||
def "getParent - root returns null"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
def path = new SeqeraPath(fs, 'seqera://')
|
||||
|
||||
expect:
|
||||
path.getParent() == null
|
||||
}
|
||||
|
||||
def "resolve - appends segment to workspace"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
def path = new SeqeraPath(fs, 'seqera://acme/research')
|
||||
|
||||
when:
|
||||
def resolved = path.resolve('datasets')
|
||||
|
||||
then:
|
||||
resolved.toString() == 'seqera://acme/research/datasets'
|
||||
(resolved as SeqeraPath).depth() == 3
|
||||
}
|
||||
|
||||
def "resolve - appends dataset name to resource type"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
def path = new SeqeraPath(fs, 'seqera://acme/research/datasets')
|
||||
|
||||
when:
|
||||
def resolved = path.resolve('my-dataset')
|
||||
|
||||
then:
|
||||
resolved.toString() == 'seqera://acme/research/datasets/my-dataset'
|
||||
}
|
||||
|
||||
def "resolve - dataset name with version"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
def path = new SeqeraPath(fs, 'seqera://acme/research/datasets')
|
||||
|
||||
when:
|
||||
def resolved = path.resolve('samples@3')
|
||||
|
||||
then:
|
||||
(resolved as SeqeraPath).datasetName == 'samples'
|
||||
(resolved as SeqeraPath).version == '3'
|
||||
}
|
||||
|
||||
def "equality and hashCode"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
def p1 = new SeqeraPath(fs, 'seqera://acme/research/datasets/samples')
|
||||
def p2 = new SeqeraPath(fs, 'seqera://acme/research/datasets/samples')
|
||||
def p3 = new SeqeraPath(fs, 'seqera://acme/research/datasets/other')
|
||||
|
||||
expect:
|
||||
p1 == p2
|
||||
p1.hashCode() == p2.hashCode()
|
||||
p1 != p3
|
||||
}
|
||||
|
||||
def "isAbsolute always true"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
|
||||
expect:
|
||||
new SeqeraPath(fs, 'seqera://acme').isAbsolute()
|
||||
new SeqeraPath(fs, 'seqera://').isAbsolute()
|
||||
}
|
||||
|
||||
def "getNameCount equals depth"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
|
||||
expect:
|
||||
new SeqeraPath(fs, 'seqera://').nameCount == 0
|
||||
new SeqeraPath(fs, 'seqera://acme').nameCount == 1
|
||||
new SeqeraPath(fs, 'seqera://acme/research/datasets/samples').nameCount == 4
|
||||
}
|
||||
|
||||
// ---- relativize ----
|
||||
|
||||
def "relativize returns correct relative path string"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
|
||||
expect:
|
||||
new SeqeraPath(fs, base).relativize(new SeqeraPath(fs, other)).toString() == expected
|
||||
|
||||
where:
|
||||
base | other | expected
|
||||
'seqera://acme' | 'seqera://acme/research' | 'research'
|
||||
'seqera://acme/research' | 'seqera://acme/research/datasets' | 'datasets'
|
||||
'seqera://acme/research' | 'seqera://acme/research/datasets/samples' | 'datasets/samples'
|
||||
'seqera://acme/research/datasets' | 'seqera://acme/research/datasets/samples' | 'samples'
|
||||
'seqera://acme/research/datasets/samples' | 'seqera://acme/research/datasets/samples' | ''
|
||||
}
|
||||
|
||||
def "relativize result round-trips through resolve"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
def base = new SeqeraPath(fs, 'seqera://acme/research')
|
||||
def target = new SeqeraPath(fs, 'seqera://acme/research/datasets/samples')
|
||||
|
||||
when:
|
||||
def rel = base.relativize(target)
|
||||
def restored = base.resolve(rel)
|
||||
|
||||
then:
|
||||
rel.toString() == 'datasets/samples'
|
||||
!rel.isAbsolute()
|
||||
restored == target
|
||||
}
|
||||
|
||||
def "relativize produces '..' segments for upward traversal"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
|
||||
expect:
|
||||
new SeqeraPath(fs, base).relativize(new SeqeraPath(fs, other)).toString() == expected
|
||||
|
||||
where:
|
||||
base | other | expected
|
||||
'seqera://acme/research' | 'seqera://acme/dev' | '../dev'
|
||||
'seqera://acme/research/datasets' | 'seqera://acme/dev' | '../../dev'
|
||||
'seqera://acme' | 'seqera://other' | '../other'
|
||||
'seqera://acme/ws1' | 'seqera://acme/ws2' | '../ws2'
|
||||
'seqera://acme/research/datasets/samples' | 'seqera://acme/research/datasets/other' | '../other'
|
||||
}
|
||||
|
||||
// ---- multi-segment resolve ----
|
||||
|
||||
def "resolve with multi-segment string builds correct path"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
def base = new SeqeraPath(fs, 'seqera://acme/research')
|
||||
|
||||
expect:
|
||||
base.resolve('datasets/samples').toString() == 'seqera://acme/research/datasets/samples'
|
||||
base.resolve('datasets').toString() == 'seqera://acme/research/datasets'
|
||||
}
|
||||
|
||||
def "resolve with absolute seqera URI returns that URI"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
def base = new SeqeraPath(fs, 'seqera://acme/research')
|
||||
def absolute = 'seqera://other/ws/datasets/report'
|
||||
|
||||
expect:
|
||||
base.resolve(absolute).toString() == absolute
|
||||
}
|
||||
|
||||
def "isAbsolute is false for relative paths produced by relativize"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
def rel = new SeqeraPath(fs, 'seqera://acme').relativize(new SeqeraPath(fs, 'seqera://acme/research'))
|
||||
|
||||
expect:
|
||||
!rel.isAbsolute()
|
||||
rel.toString() == 'research'
|
||||
}
|
||||
|
||||
// ---- getFileName ----
|
||||
|
||||
def "getFileName returns relative path for each depth"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
|
||||
expect:
|
||||
new SeqeraPath(fs, 'seqera://').getFileName() == null
|
||||
new SeqeraPath(fs, 'seqera://acme').getFileName().toString() == 'acme'
|
||||
!new SeqeraPath(fs, 'seqera://acme').getFileName().isAbsolute()
|
||||
new SeqeraPath(fs, 'seqera://acme/research').getFileName().toString() == 'research'
|
||||
new SeqeraPath(fs, 'seqera://acme/research/datasets').getFileName().toString() == 'datasets'
|
||||
new SeqeraPath(fs, 'seqera://acme/research/datasets/samples').getFileName().toString() == 'samples'
|
||||
new SeqeraPath(fs, 'seqera://acme/research/datasets/samples@2').getFileName().toString() == 'samples@2'
|
||||
}
|
||||
|
||||
def "getFileName is not absolute (uses relative constructor)"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
def name = new SeqeraPath(fs, 'seqera://acme/research').getFileName()
|
||||
|
||||
expect:
|
||||
!name.isAbsolute()
|
||||
name.toString() == 'research'
|
||||
name.getFileSystem() == null
|
||||
}
|
||||
|
||||
// ---- asUri ----
|
||||
|
||||
def "asUri - valid full path round-trips"() {
|
||||
expect:
|
||||
SeqeraPath.asUri('seqera://acme/research/datasets/samples').toString() == 'seqera://acme/research/datasets/samples'
|
||||
SeqeraPath.asUri('seqera://acme/research').toString() == 'seqera://acme/research'
|
||||
}
|
||||
|
||||
def "asUri - empty path returns root URI"() {
|
||||
expect:
|
||||
SeqeraPath.asUri('seqera://').toString() == 'seqera:///'
|
||||
}
|
||||
|
||||
def "asUri - path starting with dot has dot stripped"() {
|
||||
expect:
|
||||
// seqera://. → strips dot → seqera:// → hits empty-path case → seqera:///
|
||||
SeqeraPath.asUri('seqera://.').toString() == 'seqera:///'
|
||||
// seqera://./foo/bar → strips dot only (substring from index 10) → seqera:///foo/bar
|
||||
SeqeraPath.asUri('seqera://./foo/bar').toString() == 'seqera://foo/bar'
|
||||
}
|
||||
|
||||
def "asUri - triple slash path throws IllegalArgumentException"() {
|
||||
when:
|
||||
SeqeraPath.asUri('seqera:///something')
|
||||
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
}
|
||||
|
||||
def "asUri - missing protocol prefix throws IllegalArgumentException"() {
|
||||
when:
|
||||
SeqeraPath.asUri('s3://bucket/key')
|
||||
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
}
|
||||
|
||||
def "asUri - null or empty throws IllegalArgumentException"() {
|
||||
when:
|
||||
SeqeraPath.asUri(null)
|
||||
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
|
||||
when:
|
||||
SeqeraPath.asUri('')
|
||||
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
}
|
||||
|
||||
// ---- startsWith ----
|
||||
|
||||
def "startsWith - same path returns true"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
def path = new SeqeraPath(fs, 'seqera://acme/research/datasets/samples')
|
||||
|
||||
expect:
|
||||
path.startsWith(new SeqeraPath(fs, 'seqera://acme/research/datasets/samples'))
|
||||
}
|
||||
|
||||
def "startsWith - prefix path returns true"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
def path = new SeqeraPath(fs, 'seqera://acme/research/datasets/samples')
|
||||
|
||||
expect:
|
||||
path.startsWith(new SeqeraPath(fs, 'seqera://acme'))
|
||||
path.startsWith(new SeqeraPath(fs, 'seqera://acme/research'))
|
||||
path.startsWith(new SeqeraPath(fs, 'seqera://'))
|
||||
}
|
||||
|
||||
def "startsWith - component-wise not substring"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
def path = new SeqeraPath(fs, 'seqera://acme-corp/research/datasets/samples')
|
||||
|
||||
expect: 'acme is a substring prefix of acme-corp but not a component prefix'
|
||||
!path.startsWith(new SeqeraPath(fs, 'seqera://acme'))
|
||||
}
|
||||
|
||||
def "startsWith - longer path returns false"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
def path = new SeqeraPath(fs, 'seqera://acme')
|
||||
|
||||
expect:
|
||||
!path.startsWith(new SeqeraPath(fs, 'seqera://acme/research'))
|
||||
}
|
||||
|
||||
def "startsWith with string"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
def path = new SeqeraPath(fs, 'seqera://acme/research/datasets/samples')
|
||||
|
||||
expect:
|
||||
path.startsWith('seqera://acme')
|
||||
!path.startsWith('seqera://acm')
|
||||
}
|
||||
|
||||
// ---- endsWith ----
|
||||
|
||||
def "endsWith - absolute path requires exact match"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
def path = new SeqeraPath(fs, 'seqera://acme/research/datasets/samples')
|
||||
|
||||
expect:
|
||||
path.endsWith(new SeqeraPath(fs, 'seqera://acme/research/datasets/samples'))
|
||||
!path.endsWith(new SeqeraPath(fs, 'seqera://acme/research'))
|
||||
}
|
||||
|
||||
def "endsWith - relative path matches trailing components"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
def path = new SeqeraPath(fs, 'seqera://acme/research/datasets/samples')
|
||||
|
||||
expect:
|
||||
path.endsWith(new SeqeraPath('samples'))
|
||||
path.endsWith(new SeqeraPath('datasets/samples'))
|
||||
!path.endsWith(new SeqeraPath('other'))
|
||||
}
|
||||
|
||||
def "endsWith - component-wise not substring"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
def path = new SeqeraPath(fs, 'seqera://acme/research/datasets/my-samples')
|
||||
|
||||
expect: 'samples is a substring suffix of my-samples but not a component match'
|
||||
!path.endsWith(new SeqeraPath('samples'))
|
||||
}
|
||||
|
||||
// ---- iterator ----
|
||||
|
||||
def "iterator returns relative name components"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
def path = new SeqeraPath(fs, 'seqera://acme/research/datasets/samples')
|
||||
|
||||
when:
|
||||
def parts = path.iterator().collect { it.toString() }
|
||||
|
||||
then:
|
||||
parts == ['acme', 'research', 'datasets', 'samples']
|
||||
path.iterator().every { !it.isAbsolute() }
|
||||
}
|
||||
|
||||
def "iterator on root returns empty"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
def path = new SeqeraPath(fs, 'seqera://')
|
||||
|
||||
expect:
|
||||
!path.iterator().hasNext()
|
||||
}
|
||||
|
||||
def "iterator on org returns single element"() {
|
||||
given:
|
||||
def fs = mockFs()
|
||||
def path = new SeqeraPath(fs, 'seqera://acme')
|
||||
|
||||
when:
|
||||
def parts = path.iterator().collect { it.toString() }
|
||||
|
||||
then:
|
||||
parts == ['acme']
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,932 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.seqera.tower.plugin.launch
|
||||
|
||||
import io.seqera.http.HxClient
|
||||
import io.seqera.tower.plugin.TowerClient
|
||||
import nextflow.cli.CmdLaunch
|
||||
import nextflow.exception.AbortOperationException
|
||||
import org.junit.Rule
|
||||
import spock.lang.Specification
|
||||
import spock.lang.TempDir
|
||||
import test.OutputCapture
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
|
||||
/**
|
||||
* Test LaunchCommandImpl functionality
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
class LaunchCommandImplTest extends Specification {
|
||||
|
||||
@Rule
|
||||
OutputCapture capture = new OutputCapture()
|
||||
|
||||
@TempDir
|
||||
Path tempDir
|
||||
|
||||
// ===== Pipeline Validation Tests =====
|
||||
|
||||
def 'should reject local file path'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
|
||||
when:
|
||||
cmd.validateAndResolvePipeline('/path/to/local/workflow')
|
||||
|
||||
then:
|
||||
def ex = thrown(AbortOperationException)
|
||||
ex.message.contains('Local file paths are not supported')
|
||||
}
|
||||
|
||||
def 'should reject relative file path with ./'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
|
||||
when:
|
||||
cmd.validateAndResolvePipeline('./local/workflow')
|
||||
|
||||
then:
|
||||
def ex = thrown(AbortOperationException)
|
||||
ex.message.contains('Local file paths are not supported')
|
||||
}
|
||||
|
||||
def 'should reject relative file path with ../'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
|
||||
when:
|
||||
cmd.validateAndResolvePipeline('../local/workflow')
|
||||
|
||||
then:
|
||||
def ex = thrown(AbortOperationException)
|
||||
ex.message.contains('Local file paths are not supported')
|
||||
}
|
||||
|
||||
def 'should accept remote repository URL'() {
|
||||
given:
|
||||
def cmd = Spy(LaunchCommandImpl)
|
||||
|
||||
and:
|
||||
cmd.resolvePipelineUrl(_) >> 'https://github.com/org/repo'
|
||||
|
||||
when:
|
||||
def result = cmd.validateAndResolvePipeline('https://github.com/org/repo')
|
||||
|
||||
then:
|
||||
result == 'https://github.com/org/repo'
|
||||
}
|
||||
|
||||
def 'should accept github short name'() {
|
||||
given:
|
||||
def cmd = Spy(LaunchCommandImpl)
|
||||
|
||||
and:
|
||||
cmd.resolvePipelineUrl(_) >> 'https://github.com/nf-core/rnaseq'
|
||||
|
||||
when:
|
||||
def result = cmd.validateAndResolvePipeline('nf-core/rnaseq')
|
||||
|
||||
then:
|
||||
result == 'https://github.com/nf-core/rnaseq'
|
||||
}
|
||||
|
||||
def 'should identify local path correctly'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
|
||||
expect:
|
||||
cmd.isLocalPath('/absolute/path') == true
|
||||
cmd.isLocalPath('./relative/path') == true
|
||||
cmd.isLocalPath('../parent/path') == true
|
||||
cmd.isLocalPath('C:\\windows\\path') == true
|
||||
cmd.isLocalPath('remote/repo') == false
|
||||
cmd.isLocalPath('https://github.com/org/repo') == false
|
||||
}
|
||||
|
||||
// ===== Parameter Parsing Tests =====
|
||||
|
||||
def 'should parse simple parameters'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
def params = ['input': 'data.csv', 'output': 'results/']
|
||||
|
||||
when:
|
||||
def paramsText = cmd.buildParamsText(params, null)
|
||||
|
||||
then:
|
||||
paramsText != null
|
||||
paramsText.contains('"input"')
|
||||
paramsText.contains('data.csv')
|
||||
paramsText.contains('"output"')
|
||||
paramsText.contains('results/')
|
||||
}
|
||||
|
||||
def 'should parse nested parameters with dot notation'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
def params = ['genome.fasta': 'hg38.fa', 'genome.index': 'hg38.idx']
|
||||
|
||||
when:
|
||||
def paramsText = cmd.buildParamsText(params, null)
|
||||
|
||||
then:
|
||||
paramsText != null
|
||||
paramsText.contains('"genome"')
|
||||
paramsText.contains('"fasta"')
|
||||
paramsText.contains('hg38.fa')
|
||||
paramsText.contains('"index"')
|
||||
paramsText.contains('hg38.idx')
|
||||
}
|
||||
|
||||
def 'should parse boolean parameters'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
def params = ['verbose': 'true', 'skip': 'false']
|
||||
|
||||
when:
|
||||
def paramsText = cmd.buildParamsText(params, null)
|
||||
|
||||
then:
|
||||
paramsText != null
|
||||
paramsText.contains('"verbose":true')
|
||||
paramsText.contains('"skip":false')
|
||||
}
|
||||
|
||||
def 'should parse numeric parameters'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
def params = ['threads': '8', 'memory': '16.5', 'size': '1000000']
|
||||
|
||||
when:
|
||||
def paramsText = cmd.buildParamsText(params, null)
|
||||
|
||||
then:
|
||||
paramsText != null
|
||||
paramsText.contains('"threads":8')
|
||||
paramsText.contains('"memory":16.5')
|
||||
paramsText.contains('"size":1000000')
|
||||
}
|
||||
|
||||
def 'should parse parameters from JSON file'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
def paramsFile = tempDir.resolve('params.json')
|
||||
Files.writeString(paramsFile, '{"input": "data.csv", "output": "results/"}')
|
||||
|
||||
when:
|
||||
def paramsText = cmd.buildParamsText([:], paramsFile.toString())
|
||||
|
||||
then:
|
||||
paramsText != null
|
||||
paramsText.contains('"input"')
|
||||
paramsText.contains('data.csv')
|
||||
}
|
||||
|
||||
def 'should parse parameters from YAML file'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
def paramsFile = tempDir.resolve('params.yml')
|
||||
Files.writeString(paramsFile, 'input: data.csv\noutput: results/')
|
||||
|
||||
when:
|
||||
def paramsText = cmd.buildParamsText([:], paramsFile.toString())
|
||||
|
||||
then:
|
||||
paramsText != null
|
||||
paramsText.contains('"input"')
|
||||
paramsText.contains('data.csv')
|
||||
}
|
||||
|
||||
def 'should merge CLI params with params file, CLI taking precedence'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
def paramsFile = tempDir.resolve('params.json')
|
||||
Files.writeString(paramsFile, '{"input": "file.csv", "output": "old/"}')
|
||||
def params = ['output': 'new/']
|
||||
|
||||
when:
|
||||
def paramsText = cmd.buildParamsText(params, paramsFile.toString())
|
||||
|
||||
then:
|
||||
paramsText != null
|
||||
paramsText.contains('"input"')
|
||||
paramsText.contains('file.csv')
|
||||
paramsText.contains('"output"')
|
||||
paramsText.contains('new/')
|
||||
}
|
||||
|
||||
def 'should reject invalid params file extension'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
def paramsFile = tempDir.resolve('params.txt')
|
||||
Files.writeString(paramsFile, 'input: data.csv')
|
||||
|
||||
when:
|
||||
cmd.buildParamsText([:], paramsFile.toString())
|
||||
|
||||
then:
|
||||
thrown(AbortOperationException)
|
||||
}
|
||||
|
||||
def 'should handle missing params file'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
|
||||
when:
|
||||
cmd.buildParamsText([:], '/nonexistent/params.json')
|
||||
|
||||
then:
|
||||
thrown(AbortOperationException)
|
||||
}
|
||||
|
||||
def 'should return null when no parameters provided'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
|
||||
when:
|
||||
def paramsText = cmd.buildParamsText([:], null)
|
||||
|
||||
then:
|
||||
paramsText == null
|
||||
}
|
||||
|
||||
def 'should convert kebab-case to camelCase in parameter names'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
def params = ['max-memory': '16GB', 'output-dir': 'results/']
|
||||
|
||||
when:
|
||||
def paramsText = cmd.buildParamsText(params, null)
|
||||
|
||||
then:
|
||||
paramsText != null
|
||||
paramsText.contains('"maxMemory"')
|
||||
paramsText.contains('"outputDir"')
|
||||
}
|
||||
|
||||
def 'should handle escaped dots in parameter names'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
def params = ['file\\.name': 'test.csv']
|
||||
|
||||
when:
|
||||
def paramsText = cmd.buildParamsText(params, null)
|
||||
|
||||
then:
|
||||
paramsText != null
|
||||
paramsText.contains('"file.name"')
|
||||
}
|
||||
|
||||
// ===== Config File Tests =====
|
||||
|
||||
def 'should read config file content'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
def configFile = tempDir.resolve('test.config')
|
||||
Files.writeString(configFile, 'process.cpus = 8\nprocess.memory = "16 GB"')
|
||||
|
||||
when:
|
||||
def configText = cmd.buildConfigText([configFile.toString()])
|
||||
|
||||
then:
|
||||
configText != null
|
||||
configText.contains('process.cpus = 8')
|
||||
configText.contains('process.memory = "16 GB"')
|
||||
}
|
||||
|
||||
def 'should concatenate multiple config files'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
def config1 = tempDir.resolve('config1.config')
|
||||
def config2 = tempDir.resolve('config2.config')
|
||||
Files.writeString(config1, 'process.cpus = 8')
|
||||
Files.writeString(config2, 'process.memory = "16 GB"')
|
||||
|
||||
when:
|
||||
def configText = cmd.buildConfigText([config1.toString(), config2.toString()])
|
||||
|
||||
then:
|
||||
configText != null
|
||||
configText.contains('process.cpus = 8')
|
||||
configText.contains('process.memory = "16 GB"')
|
||||
}
|
||||
|
||||
def 'should return null when no config files provided'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
|
||||
when:
|
||||
def configText = cmd.buildConfigText(null)
|
||||
|
||||
then:
|
||||
configText == null
|
||||
|
||||
when:
|
||||
configText = cmd.buildConfigText([])
|
||||
|
||||
then:
|
||||
configText == null
|
||||
}
|
||||
|
||||
def 'should throw exception for missing config file'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
|
||||
when:
|
||||
cmd.buildConfigText(['/nonexistent/config.config'])
|
||||
|
||||
then:
|
||||
thrown(AbortOperationException)
|
||||
}
|
||||
|
||||
// ===== Launch Context Tests =====
|
||||
|
||||
def 'should throw error when no access token configured'() {
|
||||
given:
|
||||
def cmd = Spy(LaunchCommandImpl)
|
||||
cmd.readConfig() >> [:]
|
||||
def options = new CmdLaunch.LaunchOptions(pipeline: 'nf-core/rnaseq')
|
||||
|
||||
when:
|
||||
cmd.initializeLaunchContext(options)
|
||||
|
||||
then:
|
||||
def ex = thrown(AbortOperationException)
|
||||
ex.message.contains('No authentication found')
|
||||
ex.message.contains('nextflow auth login')
|
||||
}
|
||||
|
||||
def 'should initialize context with valid config'() {
|
||||
given:
|
||||
def config = [
|
||||
'tower.accessToken': 'test-token',
|
||||
'tower.endpoint': 'https://api.cloud.seqera.io'
|
||||
]
|
||||
def client = Mock(TowerClient){
|
||||
getUserInfo() >> [name: 'testuser', id: '123']
|
||||
getUserWorkspaceDetails(_,_) >> null
|
||||
}
|
||||
def cmd = Spy(new LaunchCommandImpl())
|
||||
cmd.createTowerClient(_,_) >> client
|
||||
cmd.readConfig() >> config
|
||||
cmd.resolveWorkspaceId(_, _, _, _) >> null
|
||||
cmd.resolveComputeEnvironment(_,_, _, _, _) >> [id: 'ce-123', name: 'test-ce', workDir: 's3://bucket/work']
|
||||
|
||||
def options = new CmdLaunch.LaunchOptions(pipeline: 'nf-core/rnaseq')
|
||||
|
||||
when:
|
||||
def context = cmd.initializeLaunchContext(options)
|
||||
|
||||
then:
|
||||
context.accessToken == 'test-token'
|
||||
context.apiEndpoint == 'https://api.cloud.seqera.io'
|
||||
context.userName == 'testuser'
|
||||
context.computeEnvId == 'ce-123'
|
||||
context.computeEnvName == 'test-ce'
|
||||
context.workDir == 's3://bucket/work'
|
||||
}
|
||||
|
||||
def 'should use default endpoint when not configured'() {
|
||||
given:
|
||||
def config = ['tower.accessToken': 'test-token']
|
||||
def client = Mock(TowerClient){
|
||||
getUserInfo() >> [name: 'testuser', id: '123']
|
||||
getUserWorkspaceDetails(_, _) >> null
|
||||
}
|
||||
def cmd = Spy(new LaunchCommandImpl())
|
||||
cmd.createTowerClient(_,_) >> client
|
||||
cmd.readConfig() >> config
|
||||
cmd.resolveWorkspaceId(_, _, _, _) >> null
|
||||
cmd.resolveComputeEnvironment(_,_, _, _, _) >> [id: 'ce-123', name: 'test-ce', workDir: 's3://bucket/work']
|
||||
|
||||
def options = new CmdLaunch.LaunchOptions(pipeline: 'nf-core/rnaseq')
|
||||
|
||||
when:
|
||||
def context = cmd.initializeLaunchContext(options)
|
||||
|
||||
then:
|
||||
context.apiEndpoint == 'https://api.cloud.seqera.io'
|
||||
}
|
||||
|
||||
def 'should resolve workspace details when workspace ID provided'() {
|
||||
given:
|
||||
|
||||
def config = ['tower.accessToken': 'test-token', 'tower.workspaceId': 12345]
|
||||
def client = Mock(TowerClient){
|
||||
getUserInfo() >> [name: 'testuser', id: '123']
|
||||
getUserWorkspaceDetails(_, _) >> [orgName: 'TestOrg', workspaceName: 'TestWS']
|
||||
}
|
||||
def cmd = Spy(new LaunchCommandImpl())
|
||||
cmd.createTowerClient(_,_) >> client
|
||||
cmd.readConfig() >> config
|
||||
cmd.resolveWorkspaceId(_, _, _, _) >> 12345L
|
||||
cmd.resolveComputeEnvironment(_, _, _, _, _) >> [id: 'ce-123', name: 'test-ce', workDir: 's3://bucket/work']
|
||||
|
||||
def options = new CmdLaunch.LaunchOptions(pipeline: 'nf-core/rnaseq')
|
||||
|
||||
when:
|
||||
def context = cmd.initializeLaunchContext(options)
|
||||
|
||||
then:
|
||||
context.workspaceId == 12345L
|
||||
context.orgName == 'TestOrg'
|
||||
context.workspaceName == 'TestWS'
|
||||
}
|
||||
|
||||
// ===== Compute Environment Tests =====
|
||||
|
||||
def 'should find compute environment by name'() {
|
||||
given:
|
||||
def cmd = Spy(LaunchCommandImpl)
|
||||
def computeEnvs = [
|
||||
[id: 'ce-1', name: 'primary-ce', primary: true],
|
||||
[id: 'ce-2', name: 'secondary-ce', primary: false]
|
||||
]
|
||||
cmd.listComputeEnvironments(_, _) >> computeEnvs
|
||||
|
||||
when:
|
||||
def result = cmd.findComputeEnv(Mock(TowerClient), 'secondary-ce', null)
|
||||
|
||||
then:
|
||||
result.id == 'ce-2'
|
||||
result.name == 'secondary-ce'
|
||||
}
|
||||
|
||||
def 'should find primary compute environment when name not provided'() {
|
||||
given:
|
||||
def cmd = Spy(LaunchCommandImpl)
|
||||
def computeEnvs = [
|
||||
[id: 'ce-1', name: 'primary-ce', primary: true],
|
||||
[id: 'ce-2', name: 'secondary-ce', primary: false]
|
||||
]
|
||||
cmd.listComputeEnvironments(_, _) >> computeEnvs
|
||||
|
||||
when:
|
||||
def result = cmd.findComputeEnv( Mock(TowerClient) ,null, null)
|
||||
|
||||
then:
|
||||
result.id == 'ce-1'
|
||||
result.name == 'primary-ce'
|
||||
result.primary == true
|
||||
}
|
||||
|
||||
def 'should return null when compute environment not found'() {
|
||||
given:
|
||||
def cmd = Spy(LaunchCommandImpl)
|
||||
cmd.listComputeEnvironments(_, _) >> []
|
||||
|
||||
when:
|
||||
def result = cmd.findComputeEnv(Mock(TowerClient), 'nonexistent', null)
|
||||
|
||||
then:
|
||||
result == null
|
||||
}
|
||||
|
||||
def 'should throw error when compute environment not found'() {
|
||||
given:
|
||||
def cmd = Spy(LaunchCommandImpl)
|
||||
// Mock findComputeEnv to return null (not found)
|
||||
cmd.findComputeEnv(_,'nonexistent', null) >> null
|
||||
|
||||
when:
|
||||
cmd.resolveComputeEnvironment(null, 'nonexistent', null, 'token', 'https://api.cloud.seqera.io')
|
||||
|
||||
then:
|
||||
def ex = thrown(AbortOperationException)
|
||||
ex.message.contains('Compute environment \'nonexistent\' not found')
|
||||
}
|
||||
|
||||
def 'should throw error when no primary compute environment'() {
|
||||
given:
|
||||
def cmd = Spy(LaunchCommandImpl)
|
||||
// Mock findComputeEnv to return null (no primary found)
|
||||
cmd.createTowerClient(_,_) >> Mock(TowerClient)
|
||||
cmd.findComputeEnv(_ , null, null) >> null
|
||||
|
||||
when:
|
||||
cmd.resolveComputeEnvironment(null, null, null, 'token', 'https://api.cloud.seqera.io')
|
||||
|
||||
then:
|
||||
def ex = thrown(AbortOperationException)
|
||||
ex.message.contains('No primary compute environment found')
|
||||
}
|
||||
|
||||
// ===== Work Directory Tests =====
|
||||
|
||||
def 'should use CLI work dir when provided'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
def computeEnvInfo = [workDir: 's3://default/work']
|
||||
|
||||
when:
|
||||
def workDir = cmd.resolveWorkDirectory('s3://custom/work', computeEnvInfo)
|
||||
|
||||
then:
|
||||
workDir == 's3://custom/work'
|
||||
}
|
||||
|
||||
def 'should use compute environment work dir when CLI not provided'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
def computeEnvInfo = [workDir: 's3://default/work']
|
||||
|
||||
when:
|
||||
def workDir = cmd.resolveWorkDirectory(null, computeEnvInfo)
|
||||
|
||||
then:
|
||||
workDir == 's3://default/work'
|
||||
}
|
||||
|
||||
def 'should throw error when no work dir available'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
def computeEnvInfo = [:]
|
||||
|
||||
when:
|
||||
cmd.resolveWorkDirectory(null, computeEnvInfo)
|
||||
|
||||
then:
|
||||
def ex = thrown(AbortOperationException)
|
||||
ex.message.contains('Work directory is required')
|
||||
}
|
||||
|
||||
// ===== Launch Request Building Tests =====
|
||||
|
||||
def 'should build basic launch request'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
def options = new CmdLaunch.LaunchOptions(pipeline: 'nf-core/rnaseq')
|
||||
def context = new LaunchCommandImpl.LaunchContext(
|
||||
computeEnvId: 'ce-123',
|
||||
workDir: 's3://bucket/work'
|
||||
)
|
||||
|
||||
when:
|
||||
def request = cmd.buildLaunchRequestPayload(options, context, 'https://github.com/nf-core/rnaseq', null, null)
|
||||
|
||||
then:
|
||||
request.launch.computeEnvId == 'ce-123'
|
||||
request.launch.workDir == 's3://bucket/work'
|
||||
request.launch.pipeline == 'https://github.com/nf-core/rnaseq'
|
||||
request.launch.resume == false
|
||||
request.launch.pullLatest == false
|
||||
request.launch.stubRun == false
|
||||
}
|
||||
|
||||
def 'should include optional parameters in launch request'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
def options = new CmdLaunch.LaunchOptions(
|
||||
pipeline: 'nf-core/rnaseq',
|
||||
runName: 'test-run',
|
||||
revision: 'main',
|
||||
profile: 'test',
|
||||
resume: 'session-id',
|
||||
latest: true,
|
||||
stubRun: true,
|
||||
mainScript: 'main.nf',
|
||||
entryName: 'workflow1'
|
||||
)
|
||||
def context = new LaunchCommandImpl.LaunchContext(
|
||||
computeEnvId: 'ce-123',
|
||||
workDir: 's3://bucket/work'
|
||||
)
|
||||
|
||||
when:
|
||||
def request = cmd.buildLaunchRequestPayload(options, context, 'https://github.com/nf-core/rnaseq',
|
||||
'{"input":"data.csv"}', 'process.cpus = 8')
|
||||
|
||||
then:
|
||||
request.launch.runName == 'test-run'
|
||||
request.launch.revision == 'main'
|
||||
request.launch.configProfiles == 'test'
|
||||
request.launch.resume == true
|
||||
request.launch.pullLatest == true
|
||||
request.launch.stubRun == true
|
||||
request.launch.mainScript == 'main.nf'
|
||||
request.launch.entryName == 'workflow1'
|
||||
request.launch.paramsText == '{"input":"data.csv"}'
|
||||
request.launch.configText == 'process.cpus = 8'
|
||||
}
|
||||
|
||||
def 'should include workspace and user secrets in launch request'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
def options = new CmdLaunch.LaunchOptions(
|
||||
pipeline: 'nf-core/rnaseq',
|
||||
userSecrets: ['MY_USER_SECRET'],
|
||||
workspaceSecrets: ['DRAGEN_USERNAME', 'DRAGEN_PASSWORD']
|
||||
)
|
||||
def context = new LaunchCommandImpl.LaunchContext(
|
||||
computeEnvId: 'ce-123',
|
||||
workDir: 's3://bucket/work'
|
||||
)
|
||||
|
||||
when:
|
||||
def request = cmd.buildLaunchRequestPayload(options, context, 'https://github.com/nf-core/rnaseq', null, null)
|
||||
|
||||
then:
|
||||
request.launch.userSecrets == ['MY_USER_SECRET'] as Set
|
||||
request.launch.workspaceSecrets == ['DRAGEN_USERNAME', 'DRAGEN_PASSWORD'] as Set
|
||||
}
|
||||
|
||||
def 'should not include secrets in launch request when none provided'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
def options = new CmdLaunch.LaunchOptions(pipeline: 'nf-core/rnaseq')
|
||||
def context = new LaunchCommandImpl.LaunchContext(
|
||||
computeEnvId: 'ce-123',
|
||||
workDir: 's3://bucket/work'
|
||||
)
|
||||
|
||||
when:
|
||||
def request = cmd.buildLaunchRequestPayload(options, context, 'https://github.com/nf-core/rnaseq', null, null)
|
||||
|
||||
then:
|
||||
!request.launch.containsKey('userSecrets')
|
||||
!request.launch.containsKey('workspaceSecrets')
|
||||
}
|
||||
|
||||
// ===== Workflow Status Tests =====
|
||||
|
||||
def 'should get color for workflow status'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
|
||||
expect:
|
||||
cmd.getColorForStatus('PENDING') == 'yellow'
|
||||
cmd.getColorForStatus('SUBMITTED') == 'yellow'
|
||||
cmd.getColorForStatus('RUNNING') == 'blue'
|
||||
cmd.getColorForStatus('SUCCEEDED') == 'green'
|
||||
cmd.getColorForStatus('FAILED') == 'red'
|
||||
cmd.getColorForStatus('CANCELLED') == 'red'
|
||||
cmd.getColorForStatus('ABORTED') == 'red'
|
||||
cmd.getColorForStatus(null) == 'cyan'
|
||||
}
|
||||
|
||||
def 'should get spinner mode for workflow status'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
|
||||
expect:
|
||||
cmd.getSpinnerMode('PENDING', false) == 'waiting'
|
||||
cmd.getSpinnerMode('SUBMITTED', false) == 'waiting'
|
||||
cmd.getSpinnerMode('RUNNING', false) == 'running'
|
||||
cmd.getSpinnerMode('SUCCEEDED', false) == 'succeeded'
|
||||
cmd.getSpinnerMode('FAILED', false) == 'failed'
|
||||
cmd.getSpinnerMode('CANCELLED', false) == 'failed'
|
||||
cmd.getSpinnerMode('ABORTED', false) == 'failed'
|
||||
cmd.getSpinnerMode(null, false) == 'waiting'
|
||||
}
|
||||
|
||||
def 'should format workflow status'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
|
||||
when:
|
||||
def formatted = cmd.formatWorkflowStatus('RUNNING')
|
||||
|
||||
then:
|
||||
formatted.contains('Workflow status:')
|
||||
formatted.contains('RUNNING')
|
||||
}
|
||||
|
||||
// ===== Workspace Resolution Tests =====
|
||||
|
||||
def 'should use workspace ID from config'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
def config = ['tower.workspaceId': 12345L]
|
||||
|
||||
when:
|
||||
def workspaceId = cmd.resolveWorkspaceId(config, null, 'token', 'endpoint')
|
||||
|
||||
then:
|
||||
workspaceId == 12345L
|
||||
}
|
||||
|
||||
def 'should lookup workspace by name'() {
|
||||
given:
|
||||
def config = [:]
|
||||
def workspaces = [
|
||||
[workspaceId: 111, workspaceName: 'ws1'],
|
||||
[workspaceId: 222, workspaceName: 'ws2']
|
||||
]
|
||||
def client = Mock(TowerClient) {
|
||||
getUserInfo() >> [id: 'user-123']
|
||||
}
|
||||
def cmd = Spy(new LaunchCommandImpl())
|
||||
cmd.createTowerClient(_,_) >> client
|
||||
cmd.listUserWorkspaces(_, _) >> workspaces
|
||||
|
||||
when:
|
||||
def workspaceId = cmd.resolveWorkspaceId(config, 'ws2', 'token', 'endpoint')
|
||||
|
||||
then:
|
||||
workspaceId == 222
|
||||
}
|
||||
|
||||
def 'should throw error when workspace not found by name'() {
|
||||
given:
|
||||
def config = [:]
|
||||
def client = Mock(TowerClient) {
|
||||
getUserInfo() >> [id: 'user-123']
|
||||
}
|
||||
def cmd = Spy(new LaunchCommandImpl())
|
||||
cmd.createTowerClient(_,_) >> client
|
||||
cmd.listUserWorkspaces(_, _, _) >> []
|
||||
|
||||
when:
|
||||
cmd.resolveWorkspaceId(config, 'nonexistent', 'token', 'endpoint')
|
||||
|
||||
then:
|
||||
def ex = thrown(AbortOperationException)
|
||||
ex.message.contains('Workspace \'nonexistent\' not found')
|
||||
}
|
||||
|
||||
def 'should return null when no workspace specified'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
def config = [:]
|
||||
|
||||
when:
|
||||
def workspaceId = cmd.resolveWorkspaceId(config, null, 'token', 'endpoint')
|
||||
|
||||
then:
|
||||
workspaceId == null
|
||||
}
|
||||
|
||||
// ===== Launch Result Tests =====
|
||||
|
||||
def 'should extract launch result with workflow details'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
def response = [workflowId: 'wf-123']
|
||||
def workflowDetails = [
|
||||
workflow: [
|
||||
runName: 'test-run',
|
||||
commitId: 'abc123',
|
||||
revision: 'main'
|
||||
]
|
||||
]
|
||||
def options = new CmdLaunch.LaunchOptions(pipeline: 'nf-core/rnaseq')
|
||||
def context = new LaunchCommandImpl.LaunchContext(
|
||||
apiEndpoint: 'https://api.cloud.seqera.io',
|
||||
userName: 'testuser'
|
||||
)
|
||||
|
||||
when:
|
||||
def result = cmd.extractLaunchResult(response, workflowDetails, options, 'https://github.com/nf-core/rnaseq', context)
|
||||
|
||||
then:
|
||||
result.workflowId == 'wf-123'
|
||||
result.runName == 'test-run'
|
||||
result.commitId == 'abc123'
|
||||
result.revision == 'main'
|
||||
result.repository == 'https://github.com/nf-core/rnaseq'
|
||||
result.trackingUrl.contains('/user/testuser/watch/wf-123/')
|
||||
}
|
||||
|
||||
def 'should extract launch result without workflow details'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
def response = [workflowId: 'wf-123']
|
||||
def options = new CmdLaunch.LaunchOptions(pipeline: 'nf-core/rnaseq', runName: 'custom-run')
|
||||
def context = new LaunchCommandImpl.LaunchContext(
|
||||
apiEndpoint: 'https://api.cloud.seqera.io',
|
||||
userName: 'testuser'
|
||||
)
|
||||
|
||||
when:
|
||||
def result = cmd.extractLaunchResult(response, null, options, 'https://github.com/nf-core/rnaseq', context)
|
||||
|
||||
then:
|
||||
result.workflowId == 'wf-123'
|
||||
result.runName == 'custom-run'
|
||||
result.commitId == 'unknown'
|
||||
}
|
||||
|
||||
def 'should build tracking URL for organization workspace'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
def response = [workflowId: 'wf-123']
|
||||
def options = new CmdLaunch.LaunchOptions(pipeline: 'nf-core/rnaseq')
|
||||
def context = new LaunchCommandImpl.LaunchContext(
|
||||
apiEndpoint: 'https://api.cloud.seqera.io',
|
||||
userName: 'testuser',
|
||||
orgName: 'TestOrg',
|
||||
workspaceName: 'TestWS'
|
||||
)
|
||||
|
||||
when:
|
||||
def result = cmd.extractLaunchResult(response, null, options, 'https://github.com/nf-core/rnaseq', context)
|
||||
|
||||
then:
|
||||
result.trackingUrl == 'https://cloud.seqera.io/orgs/TestOrg/workspaces/TestWS/watch/wf-123/'
|
||||
}
|
||||
|
||||
// ===== Parameter Value Parsing Tests =====
|
||||
|
||||
def 'should parse parameter values correctly'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
|
||||
expect:
|
||||
cmd.parseParamValue('true') == Boolean.TRUE
|
||||
cmd.parseParamValue('false') == Boolean.FALSE
|
||||
cmd.parseParamValue('TRUE') == Boolean.TRUE
|
||||
cmd.parseParamValue('FALSE') == Boolean.FALSE
|
||||
cmd.parseParamValue('42') == 42
|
||||
cmd.parseParamValue('3.14') == 3.14
|
||||
cmd.parseParamValue('text') == 'text'
|
||||
cmd.parseParamValue(null) == null
|
||||
}
|
||||
|
||||
def 'should convert kebab case to camel case'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
|
||||
expect:
|
||||
cmd.kebabToCamelCase('max-memory') == 'maxMemory'
|
||||
cmd.kebabToCamelCase('output-dir') == 'outputDir'
|
||||
cmd.kebabToCamelCase('simple') == 'simple'
|
||||
cmd.kebabToCamelCase('very-long-param-name') == 'veryLongParamName'
|
||||
}
|
||||
|
||||
// ===== Utility Tests =====
|
||||
|
||||
def 'should get web URL from API endpoint'() {
|
||||
given:
|
||||
def cmd = new LaunchCommandImpl()
|
||||
|
||||
expect:
|
||||
cmd.getWebUrlFromApiEndpoint('https://api.cloud.seqera.io') == 'https://cloud.seqera.io'
|
||||
cmd.getWebUrlFromApiEndpoint('https://cloud.seqera.io/api') == 'https://cloud.seqera.io'
|
||||
cmd.getWebUrlFromApiEndpoint('https://custom.example.com') == 'https://custom.example.com'
|
||||
}
|
||||
|
||||
// ===== Data Class Tests =====
|
||||
|
||||
def 'should create LaunchContext with all fields'() {
|
||||
when:
|
||||
def context = new LaunchCommandImpl.LaunchContext(
|
||||
accessToken: 'token',
|
||||
apiEndpoint: 'https://api.cloud.seqera.io',
|
||||
userName: 'testuser',
|
||||
workspaceId: 12345L,
|
||||
orgName: 'TestOrg',
|
||||
workspaceName: 'TestWS',
|
||||
computeEnvId: 'ce-123',
|
||||
computeEnvName: 'test-ce',
|
||||
workDir: 's3://bucket/work'
|
||||
)
|
||||
|
||||
then:
|
||||
context.accessToken == 'token'
|
||||
context.apiEndpoint == 'https://api.cloud.seqera.io'
|
||||
context.userName == 'testuser'
|
||||
context.workspaceId == 12345L
|
||||
context.orgName == 'TestOrg'
|
||||
context.workspaceName == 'TestWS'
|
||||
context.computeEnvId == 'ce-123'
|
||||
context.computeEnvName == 'test-ce'
|
||||
context.workDir == 's3://bucket/work'
|
||||
}
|
||||
|
||||
def 'should create WorkflowLaunchResult with all fields'() {
|
||||
when:
|
||||
def result = new LaunchCommandImpl.WorkflowLaunchResult(
|
||||
workflowId: 'wf-123',
|
||||
runName: 'test-run',
|
||||
commitId: 'abc123',
|
||||
revision: 'main',
|
||||
repository: 'https://github.com/org/repo',
|
||||
trackingUrl: 'https://cloud.seqera.io/watch/wf-123'
|
||||
)
|
||||
|
||||
then:
|
||||
result.workflowId == 'wf-123'
|
||||
result.runName == 'test-run'
|
||||
result.commitId == 'abc123'
|
||||
result.revision == 'main'
|
||||
result.repository == 'https://github.com/org/repo'
|
||||
result.trackingUrl == 'https://cloud.seqera.io/watch/wf-123'
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user