diff --git a/k8s-dvfs/.gitignore b/k8s-dvfs/.gitignore index dbef60b..a12f3e3 100644 --- a/k8s-dvfs/.gitignore +++ b/k8s-dvfs/.gitignore @@ -6,3 +6,5 @@ # Ignore Gradle build output directory build work +lib/groovy-6.0.0-beta-1.jar lib/groovy-6.0.0-beta-1-sources.jar lib/groovy-ant-6.0.0-beta-1.jar lib/groovy-ant-6.0.0-beta-1-sources.jar lib/groovy-astbuilder-6.0.0-beta-1.jar lib/groovy-astbuilder-6.0.0-beta-1-sources.jar lib/groovy-cli-commons-6.0.0-beta-1.jar lib/groovy-cli-commons-6.0.0-beta-1-sources.jar lib/groovy-cli-picocli-6.0.0-beta-1.jar lib/groovy-cli-picocli-6.0.0-beta-1-sources.jar lib/groovy-console-6.0.0-beta-1.jar lib/groovy-console-6.0.0-beta-1-sources.jar lib/groovy-contracts-6.0.0-beta-1.jar lib/groovy-contracts-6.0.0-beta-1-sources.jar lib/groovy-csv-6.0.0-beta-1.jar lib/groovy-csv-6.0.0-beta-1-sources.jar lib/groovy-datetime-6.0.0-beta-1.jar lib/groovy-datetime-6.0.0-beta-1-sources.jar lib/groovy-dateutil-6.0.0-beta-1.jar lib/groovy-dateutil-6.0.0-beta-1-sources.jar lib/groovy-docgenerator-6.0.0-beta-1.jar lib/groovy-docgenerator-6.0.0-beta-1-sources.jar lib/groovy-ginq-6.0.0-beta-1.jar lib/groovy-ginq-6.0.0-beta-1-sources.jar lib/groovy-grape-ivy-6.0.0-beta-1.jar lib/groovy-grape-ivy-6.0.0-beta-1-sources.jar lib/groovy-grape-maven-6.0.0-beta-1.jar lib/groovy-grape-maven-6.0.0-beta-1-sources.jar lib/groovy-groovydoc-6.0.0-beta-1.jar lib/groovy-groovydoc-6.0.0-beta-1-sources.jar lib/groovy-groovysh-6.0.0-beta-1.jar lib/groovy-groovysh-6.0.0-beta-1-sources.jar lib/groovy-http-builder-6.0.0-beta-1.jar lib/groovy-http-builder-6.0.0-beta-1-sources.jar lib/groovy-jmx-6.0.0-beta-1.jar lib/groovy-jmx-6.0.0-beta-1-sources.jar lib/groovy-json-6.0.0-beta-1.jar lib/groovy-json-6.0.0-beta-1-sources.jar lib/groovy-jsr223-6.0.0-beta-1.jar lib/groovy-jsr223-6.0.0-beta-1-sources.jar lib/groovy-macro-6.0.0-beta-1.jar lib/groovy-macro-6.0.0-beta-1-sources.jar lib/groovy-macro-library-6.0.0-beta-1.jar lib/groovy-macro-library-6.0.0-beta-1-sources.jar lib/groovy-markdown-6.0.0-beta-1.jar lib/groovy-markdown-6.0.0-beta-1-sources.jar lib/groovy-nio-6.0.0-beta-1.jar lib/groovy-nio-6.0.0-beta-1-sources.jar lib/groovy-reactor-6.0.0-beta-1.jar lib/groovy-reactor-6.0.0-beta-1-sources.jar lib/groovy-rxjava-6.0.0-beta-1.jar lib/groovy-rxjava-6.0.0-beta-1-sources.jar lib/groovy-servlet-6.0.0-beta-1.jar lib/groovy-servlet-6.0.0-beta-1-sources.jar lib/groovy-sql-6.0.0-beta-1.jar lib/groovy-sql-6.0.0-beta-1-sources.jar lib/groovy-swing-6.0.0-beta-1.jar lib/groovy-swing-6.0.0-beta-1-sources.jar lib/groovy-templates-6.0.0-beta-1.jar lib/groovy-templates-6.0.0-beta-1-sources.jar lib/groovy-test-6.0.0-beta-1.jar lib/groovy-test-6.0.0-beta-1-sources.jar lib/groovy-test-junit5-6.0.0-beta-1.jar lib/groovy-test-junit5-6.0.0-beta-1-sources.jar lib/groovy-test-junit6-6.0.0-beta-1.jar lib/groovy-test-junit6-6.0.0-beta-1-sources.jar lib/groovy-testng-6.0.0-beta-1.jar lib/groovy-testng-6.0.0-beta-1-sources.jar lib/groovy-toml-6.0.0-beta-1.jar lib/groovy-toml-6.0.0-beta-1-sources.jar lib/groovy-typecheckers-6.0.0-beta-1.jar lib/groovy-typecheckers-6.0.0-beta-1-sources.jar lib/groovy-xml-6.0.0-beta-1.jar lib/groovy-xml-6.0.0-beta-1-sources.jar lib/groovy-yaml-6.0.0-beta-1.jar lib/groovy-yaml-6.0.0-beta-1-sources.jar +lib diff --git a/k8s-dvfs/build.gradle b/k8s-dvfs/build.gradle index 1f507d8..4b0035d 100644 --- a/k8s-dvfs/build.gradle +++ b/k8s-dvfs/build.gradle @@ -5,13 +5,46 @@ plugins { version = '0.1.0' nextflowPlugin { - nextflowVersion = '25.10.0' + nextflowVersion = '26.04.0' provider = 'recreational.tech' - className = 'recreationaltech.plugin.K8sDvfsPlugin' + className = 'recreationaltech.plugin.K8sPlugin' + useDefaultDependencies = true + generateSpec = false extensionPoints = [ - 'recreationaltech.plugin.K8sDvfsExtension', - 'recreationaltech.plugin.K8sDvfsFactory' + 'recreationaltech.plugin.K8sConfig', + 'recreationaltech.plugin.K8sExecutor', + 'recreationaltech.plugin.cli.KubeCommandImpl', ] - +} + +sourceSets { + main.java.srcDirs = [] + main.groovy.srcDirs = ['src/main/groovy'] + main.resources.srcDirs = ['src/resources'] + test.groovy.srcDirs = ['src/test/groovy'] + test.java.srcDirs = [] + test.resources.srcDirs = [] +} + +configurations { + // see https://docs.gradle.org/4.1/userguide/dependency_management.html#sub:exclude_transitive_dependencies + runtimeClasspath.exclude group: 'org.slf4j', module: 'slf4j-api' +} + +dependencies { + //compileOnly project(':nextflow') + compileOnly 'org.slf4j:slf4j-api:2.0.17' + compileOnly 'org.pf4j:pf4j:3.14.1' + + api 'org.bouncycastle:bcprov-ext-jdk18on:1.78.1' + api 'org.bouncycastle:bcpkix-jdk18on:1.84' + + //testImplementation(testFixtures(project(":nextflow"))) + testImplementation "org.apache.groovy:groovy:4.0.31" + testImplementation "org.apache.groovy:groovy-nio:4.0.31" +} + +test { + useJUnitPlatform() } diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sConfig.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sConfig.groovy new file mode 100644 index 0000000..794ecf6 --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sConfig.groovy @@ -0,0 +1,527 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin + +import nextflow.config.scopes.Config +import recreationaltech.plugin.client.K8sRetryConfig + +import javax.annotation.Nullable + +import groovy.transform.CompileStatic +import groovy.transform.PackageScope +import groovy.util.logging.Slf4j +import nextflow.BuildInfo +import nextflow.config.spec.ConfigOption +import nextflow.config.spec.ConfigScope +import nextflow.config.spec.ScopeName +import nextflow.container.ContainerHelper +import nextflow.script.dsl.Description +import nextflow.exception.AbortOperationException +import recreationaltech.plugin.client.ClientConfig +import recreationaltech.plugin.client.K8sClient +import recreationaltech.plugin.client.K8sResponseException +import recreationaltech.plugin.model.PodOptions +import recreationaltech.plugin.model.PodSecurityContext +import recreationaltech.plugin.model.PodVolumeClaim +import recreationaltech.plugin.model.ResourceType +import nextflow.util.Duration + +import java.util.concurrent.TimeUnit + +/** + * Model Kubernetes specific settings defined in the nextflow + * configuration file + * + * @author Paolo Di Tommaso + */ +@ScopeName("k8s") +@Description(""" + The `k8s` scope controls the deployment and execution of workflow applications in a Kubernetes cluster. +""") +@Slf4j +@CompileStatic +class K8sConfig implements ConfigScope { + + static final private Map DEFAULT_FUSE_PLUGIN = Map.of('nextflow.io/fuse', 1) + + @ConfigOption + @Description(""" + Automatically mount host paths into the task pods (default: `false`). Only intended for development purposes when using a single node. + """) + final boolean autoMountHostPaths + + @ConfigOption + @Description(""" + Whether to use Kubernetes `Pod` or `Job` resource type to carry out Nextflow tasks (default: `Pod`). + """) + final String computeResourceType + + @ConfigOption + @Description(""" + When `true`, successful pods are automatically deleted (default: `true`). + """) + final private Boolean cleanup + + @ConfigOption + @Description(""" + Map of options for the K8s client. + + If this option is specified, it will be used instead of `.kube/config`. + """) + final Map client + + @ConfigOption + @Description(""" + The interval after which the Kubernetes client configuration is refreshed (default: `50m`). + """) + final Duration clientRefreshInterval + + @ConfigOption + @Description(""" + The Kubernetes [configuration context](https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/) to use. + """) + final String context + + @ConfigOption + @Description(""" + When `true`, set both the pod CPUs `request` and `limit` to the value specified by the `cpus` directive, otherwise set only the `request` (default: `false`). + """) + final boolean cpuLimits + + final K8sDebug debug + + @ConfigOption + @Description(""" + Include the hostname of each task in the execution trace (default: `false`). + """) + final boolean fetchNodeName + + @ConfigOption + @Description(""" + The FUSE device plugin to be used when enabling Fusion in unprivileged mode (default: `['nextflow.io/fuse': 1]`). + """) + final Map fuseDevicePlugin + + @ConfigOption + @Description(""" + The Kubernetes HTTP client request connection timeout e.g. `'60s'`. + """) + final Duration httpConnectTimeout + + @ConfigOption + @Description(""" + The Kubernetes HTTP client request connection read timeout e.g. `'60s'`. + """) + final Duration httpReadTimeout + + @ConfigOption + @Description(""" + The strategy for pulling container images. Can be `IfNotPresent`, `Always`, `Never`. + + [Read more](https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy) + """) + final String imagePullPolicy + + @ConfigOption + @Description(""" + The path where the workflow is launched and the user data is stored (default: `/`). Must be a path in a shared K8s persistent volume. + """) + final String launchDir + + @ConfigOption + @Description(""" + The Kubernetes namespace to use (default: `default`). + """) + final String namespace + + @ConfigOption(types=[List, Map]) + @Description(""" + Additional pod configuration options such as environment variables, config maps, secrets, etc. Allows the same settings as the [pod](https://nextflow.io/docs/latest/process.html#pod) process directive. + """) + final PodOptions pod + + @ConfigOption + @Description(""" + The path where Nextflow projects are downloaded (default: `/projects`). Must be a path in a shared K8s persistent volume. + """) + final String projectDir + + @Deprecated + @ConfigOption + @Description(""" + """) + final String pullPolicy + + final K8sRetryConfig retryPolicy + + @ConfigOption(types=[Integer, String]) + @Description(""" + The user ID to be used to run the containers. Shortcut for the `securityContext` option. + """) + final Object runAsUser + + @ConfigOption + @Description(""" + The [security context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) to use for all pods. + """) + final Map securityContext + + @ConfigOption + @Description(""" + The Kubernetes [service account name](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/) to use. + """) + final String serviceAccount + + @ConfigOption + @Description(""" + The name of the persistent volume claim where the shared work directory is stored. + """) + final String storageClaimName + + @ConfigOption + @Description(""" + The mount path for the persistent volume claim (default: `/workspace`). + """) + final String storageMountPath + + @ConfigOption + @Description(""" + The path in the persistent volume to be mounted (default: `/`). + """) + final String storageSubPath + + @ConfigOption + @Description(""" + """) + final String userName + + @ConfigOption + @Description(""" + The path of the shared work directory (default: `/work`). Must be a path in a shared K8s persistent volume. + """) + final String workDir + + @Description("Node initialization config") + final K8sNodeInitConfig nodeInit + + @ConfigOption + @Description(""" + The image name of the nextflow launcher image + """) + final String nextflowImage + + @ConfigOption + @Description(""" + The run interval of the kubernetes scheduler + """) + final Duration schedulerInterval + + @ConfigOption + @Description(""" + Enables task runtime recording + """) + final boolean recordTaskRuntimes + + @ConfigOption + @Description(""" + The runtime recording file + """) + final String runtimeRecordPath + + @ConfigOption + @Description(""" + Selects a scheduling strategy + """) + final String schedulingStrategy + + @ConfigOption + @Description(""" + Selects a runtime estimator + """) + final String runtimeEstimator + + @ConfigOption + @Description(""" + Noise magnitude used for the noise runtime estimator + """) + final Duration noiseRuntimeEstimatorNoiseMagnitude + + @ConfigOption + @Description(""" + Max. amount of time two runtime estimates can differ to be considered equal. + """) + final Duration runtimeComparisonEpsilon + + @ConfigOption + @Description(""" + Number of saved top runtimes used to classify a task as critical. + """) + final int dvfsSchedulingNumTopRuntimes + + /* required by extension point -- do not remove */ + K8sConfig() { + this(Collections.emptyMap()) + } + + K8sConfig(Map opts) { + autoMountHostPaths = opts.autoMountHostPaths as boolean + cleanup = opts.cleanup as Boolean + client = opts.client as Map + clientRefreshInterval = opts.clientRefreshInterval as Duration ?: Duration.of('50m') + computeResourceType = opts.computeResourceType + context = opts.context + cpuLimits = opts.cpuLimits as boolean + debug = new K8sDebug(opts.debug as Map ?: Collections.emptyMap()) + fetchNodeName = opts.fetchNodeName as boolean + fuseDevicePlugin = parseFuseDevicePlugin(opts.fuseDevicePlugin) + httpConnectTimeout = opts.httpConnectTimeout as Duration + httpReadTimeout = opts.httpReadTimeout as Duration + imagePullPolicy = opts.pullPolicy ?: opts.imagePullPolicy + namespace = opts.namespace + pod = createPodOptions(opts.pod) + retryPolicy = new K8sRetryConfig(opts.retryPolicy as Map ?: Collections.emptyMap()) + runAsUser = opts.runAsUser + securityContext = opts.securityContext as Map + serviceAccount = opts.serviceAccount + storageClaimName = opts.storageClaimName + storageMountPath = opts.storageMountPath ?: '/workspace' + storageSubPath = opts.storageSubPath + userName = opts.userName + nextflowImage = opts.nextflowImage ?: "nextflow/nextflow:${BuildInfo.version}" + schedulerInterval = opts.schedulerInterval as Duration ?: new Duration(10, TimeUnit.SECONDS) + recordTaskRuntimes = opts.recordTaskRuntimes as boolean ?: false + + launchDir = opts.launchDir ?: "${storageMountPath}/${getUserName()}" + projectDir = opts.projectDir ?: "${storageMountPath}/projects" + workDir = opts.workDir ?: "${launchDir}/work" + runtimeRecordPath = opts.runtimeRecordPath as String ?: "${workDir}/runtimes.csv" + + schedulingStrategy = opts.schedulingStrategy as String ?: "Hash" + runtimeEstimator = opts.runtimeEstimator as String ?: "LinearFit" + noiseRuntimeEstimatorNoiseMagnitude = opts.noiseRuntimeEstimatorNoiseMagnitude as Duration ?: new Duration(30, TimeUnit.SECONDS) + runtimeComparisonEpsilon = opts.runtimeComparisonEpsilon as Duration ?: new Duration(10, TimeUnit.SECONDS) + dvfsSchedulingNumTopRuntimes = opts.dvfsSchedulingTopRuntimes as int ?: 3 + + // -- shortcut to pod image pull-policy + if( imagePullPolicy ) + pod.imagePullPolicy = imagePullPolicy + + // -- shortcut to pod volume claim + if( storageClaimName ) { + final volumeClaim = new PodVolumeClaim(storageClaimName, storageMountPath, storageSubPath) + pod.volumeClaims.add(volumeClaim) + } + + // -- shortcut to pod security context + if( runAsUser ) + pod.securityContext = new PodSecurityContext(runAsUser) + else if( securityContext ) + pod.securityContext = new PodSecurityContext(securityContext) + + nodeInit = new K8sNodeInitConfig(opts.nodeInit as Map ?: Collections.emptyMap()) + } + + private PodOptions createPodOptions( value ) { + if( value instanceof List ) + return new PodOptions( value as List ) + + if( value instanceof Map ) + return new PodOptions( [(Map)value] ) + + if( value == null ) + return new PodOptions() + + throw new IllegalArgumentException("Not a valid pod setting: $value") + } + + Map getLabels() { + pod.getLabels() + } + + Map getAnnotations() { + pod.getAnnotations() + } + + boolean getCleanup(boolean defValue=true) { + cleanup == null ? defValue : cleanup + } + + String getUserName() { + userName ?: System.properties.get('user.name') + } + + Map fuseDevicePlugin() { + fuseDevicePlugin + } + + Map parseFuseDevicePlugin(Object value) { + if( value instanceof Map && value.size()==1 ) + return value as Map + if( value != null ) + log.warn1 "Setting 'k8s.fuseDevicePlugin' should be a map containing exactly one entry - offending value: $value" + return DEFAULT_FUSE_PLUGIN + } + + /** + * Whenever the pod should honour the entrypoint defined by the image (default: false) + * + * @return When {@code false} the launcher script is run by using pod `command` attributes which + * overrides the entrypoint point defined by the image. + * + * When {@code true} the launcher is run via the pod `args` attribute, without altering the + * container entrypoint (it does however require to have a bash shell as the image entrypoint) + * + */ + boolean entrypointOverride() { + return ContainerHelper.entrypointOverride() + } + + boolean useJobResource() { ResourceType.Job.name() == computeResourceType } + + String getNextflowImageName() { + return nextflowImage + } + + PodOptions getPodOptions() { + pod + } + + boolean fetchNodeName() { + fetchNodeName + } + + /** + * @return the collection of defined volume claim names + */ + Collection getClaimNames() { + pod.volumeClaims.collect { it.claimName } + } + + Collection getClaimPaths() { + pod.volumeClaims.collect { it.mountPath } + } + + boolean cpuLimitsEnabled() { + cpuLimits + } + + /** + * Find a volume claim name given the mount path + * + * @param path The volume claim mount path + * @return The volume claim name for the given mount path + */ + String findVolumeClaimByPath(String path) { + final result = pod.volumeClaims.find { path.startsWith(it.mountPath) } + return result ? result.claimName : null + } + + ClientConfig getClient() { + final result = client != null + ? clientFromNextflow(client, namespace, serviceAccount) + : clientDiscovery(context, namespace, serviceAccount) + + if( httpConnectTimeout ) + result.httpConnectTimeout = httpConnectTimeout + + if( httpReadTimeout ) + result.httpReadTimeout = httpReadTimeout + + if( retryPolicy ) + result.retryConfig = retryPolicy + + return result + } + + /** + * Get the K8s client config from the declaration made in the Nextflow config file + * + * @param map + * A map representing the clint configuration options define in the nextflow + * config file + * @param namespace + * The K8s namespace to be used. If omitted {@code default} is used. + * @param serviceAccount + * The K8s service account to be used. If omitted {@code default} is used. + * @return + * The Kubernetes {@link ClientConfig} object + */ + @PackageScope ClientConfig clientFromNextflow(Map map, @Nullable String namespace, @Nullable String serviceAccount ) { + ClientConfig.fromNextflowConfig(map,namespace,serviceAccount) + } + + /** + * Discover the K8s client config from the execution environment + * that can be either a `.kube/config` file or service meta file + * when running in a pod. + * + * @param contextName + * The name of the configuration context to be used + * @param namespace + * The Kubernetes namespace to be used + * @param serviceAccount + * The Kubernetes serviceAccount to be used + * @return + * The discovered Kube {@link ClientConfig} object + */ + @PackageScope ClientConfig clientDiscovery(String contextName, String namespace, String serviceAccount) { + ClientConfig.discover(contextName, namespace, serviceAccount) + } + + void checkStorageAndPaths(K8sClient client) { + if( !storageClaimName ) + throw new AbortOperationException("Missing K8s storage volume claim -- The name of a persistence volume claim needs to be provided in the nextflow configuration file") + + log.debug "Kubernetes workDir=$workDir; projectDir=$projectDir; volumeClaims=${getClaimNames()}" + + for( String name : getClaimNames() ) { + try { + client.volumeClaimRead(name) + } + catch (K8sResponseException e) { + if( e.response.code == 404 ) { + throw new AbortOperationException("Unknown volume claim: $name -- make sure a persistent volume claim with the specified name is defined in your K8s cluster") + } + else throw e + } + } + + if( !findVolumeClaimByPath(launchDir) ) + throw new AbortOperationException("Kubernetes `launchDir` must be a path mounted as a persistent volume -- launchDir=$launchDir; volumes=${getClaimPaths().join(', ')}") + + if( !findVolumeClaimByPath(workDir) ) + throw new AbortOperationException("Kubernetes `workDir` must be a path mounted as a persistent volume -- workDir=$workDir; volumes=${getClaimPaths().join(', ')}") + + if( !findVolumeClaimByPath(projectDir) ) + throw new AbortOperationException("Kubernetes `projectDir` must be a path mounted as a persistent volume -- projectDir=$projectDir; volumes=${getClaimPaths().join(', ')}") + + } + + static class K8sDebug implements ConfigScope { + + @ConfigOption + @Description(""" + Save the pod spec for each task to `.command.yaml` in the task directory (default: `false`). + """) + final boolean yaml + + K8sDebug(Map opts) { + yaml = opts.yaml as boolean + } + } +} + diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sDVFSClient.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sDVFSClient.groovy new file mode 100644 index 0000000..ed8d6f0 --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sDVFSClient.groovy @@ -0,0 +1,102 @@ +package recreationaltech.plugin + +import groovy.util.logging.Slf4j + +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse +import java.nio.channels.UnresolvedAddressException + +/** + * The DVFS client uses the deployed DVFS agents to control the operating frequency of the worker nodes. + */ +@Slf4j +class K8sDVFSClient { + private HttpClient httpClient + private Map ipTable + + K8sDVFSClient(String[] nodes, String[] ips) { + assert(nodes.length == ips.length) + this.httpClient = HttpClient.newBuilder().build() + this.ipTable = new HashMap<>() + for (int i = 0; i < nodes.length; i++) { + this.ipTable.put(nodes[i], ips[i]) + } + } + + OptionalLong getNodeCurrentFrequency(String node) { + log.debug("Getting current frequency of node ${node}") + return getNodeFrequency(node, "current") + } + + OptionalLong getNodeMaxFrequency(String node) { + log.debug("Getting max frequency of node ${node}") + return getNodeFrequency(node, "max") + } + + OptionalLong getNodeMinFrequency(String node) { + log.debug("Getting min frequency of node ${node}") + return getNodeFrequency(node, "min") + } + + private OptionalLong getNodeFrequency(String node, String endpoint) { + return getLong(HttpRequest.newBuilder() + .uri(new URI("http://${agentAddress(node)}/cpu/frequency/${endpoint}")) + .GET() + .build()) + } + + boolean setNodeFrequency(String node, int frequency) { + String body = String.valueOf(frequency) + + HttpRequest request = HttpRequest.newBuilder() + .uri(new URI("http://${agentAddress(node)}/cpu/frequency/current")) + .PUT(HttpRequest.BodyPublishers.ofString(body)) + .build() + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.discarding()) + if (response.statusCode() != 200) { + log.error("Request PUT ${request.uri().toString()} returned ${response.statusCode()}") + return false + } + return true + } + + OptionalLong getCPUCount(String node) { + return getLong(HttpRequest.newBuilder() + .uri(new URI("http://${agentAddress(node)}/cpu/count")) + .GET() + .build()) + } + + OptionalLong getMemoryAmount(String node) { + return getLong(HttpRequest.newBuilder() + .uri(new URI("http://${agentAddress(node)}/mem/amount")) + .GET() + .build()) + } + + private OptionalLong getLong(HttpRequest request) { + try { + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()) + if (response.statusCode() != 200) { + log.error("Request GET ${request.uri().toString()} returned ${response.statusCode()}") + return OptionalLong.empty() + } + long parsed = Long.parseLong(response.body()) + return OptionalLong.of(parsed) + } catch (NumberFormatException ex) { + log.error("Unexpected response ${response.body()} - ${ex.message}") + return OptionalLong.empty() + } catch (ConnectException ex) { + log.error("Request failed ${request.uri().toString()}: ${ex.message}") + return OptionalLong.empty() + } catch (UnresolvedAddressException ex) { + log.error("Failed to resolve address ${request.uri().toString()}: ${ex.message}") + return OptionalLong.empty() + } + } + + private String agentAddress(String node) { + return ipTable.get(node) + ":8080" + } +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sDriverLauncher.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sDriverLauncher.groovy new file mode 100644 index 0000000..e4a29de --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sDriverLauncher.groovy @@ -0,0 +1,718 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin + +import groovy.transform.MapConstructor + +import java.lang.reflect.Field +import java.nio.file.NoSuchFileException +import java.nio.file.Path +import java.nio.file.Paths + +import com.beust.jcommander.DynamicParameter +import com.beust.jcommander.Parameter +import com.google.common.hash.Hashing +import groovy.util.logging.Slf4j +import nextflow.cli.CmdKubeRun +import nextflow.cli.CmdRun +import nextflow.config.ConfigBuilder +import nextflow.exception.AbortOperationException +import nextflow.file.FileHelper +import recreationaltech.plugin.client.K8sClient +import recreationaltech.plugin.client.K8sResponseException +import recreationaltech.plugin.model.PodEnv +import recreationaltech.plugin.model.PodMountConfig +import recreationaltech.plugin.model.PodSpecBuilder +import recreationaltech.plugin.model.ResourceType +import nextflow.scm.AssetManager +import nextflow.scm.ProviderConfig +import nextflow.util.ConfigHelper +import nextflow.util.Escape +import org.codehaus.groovy.runtime.MethodClosure +/** + * Configure and submit the execution of pod running the Nextflow main application + * + * @author Paolo Di Tommaso + */ +@Slf4j +@MapConstructor(includeFields = true) +class K8sDriverLauncher { + + /** + * Either a Pod or Job + */ + private ResourceType resourceType = ResourceType.Pod + + /** + * Container image to be used for the Nextflow driver pod + */ + private String headImage + + /** + * Request CPUs to be used for the Nextflow driver pod + */ + private int headCpus + + /** + * Request memory to be used for the Nextflow driver pod + */ + private String headMemory + + /** + * Pre-script to run before nextflow + */ + private String headPreScript + + /** + * Nextflow execution run name + */ + private String runName + + /** + * Workflow project to launch + */ + private String pipelineName + + /** + * Command run options + */ + private CmdKubeRun cmd + + /** + * Kubernetes client + */ + private K8sClient k8sClient + + /** + * Nextflow resolved config object + */ + private ConfigObject config + + /** + * Name of the config map used to propagate the nextflow + * setting in the container + */ + private String configMapName + + /** + * Kubernetes specific config settings + */ + private K8sConfig k8sConfig + + private String paramsFile + + private boolean interactive + + /** + * Runs in background mode + */ + private boolean background + + /** + * Workflow script positional parameters + */ + private List args + + /** + * Plugins to run the workflow + */ + private String plugins + + private K8sNodeInitDeployer initDeployer + + /** + * Launcher entry point. Set-up the environment and create a pod that run the Nextflow + * application (which in turns executed each task as a pod) + * + * @param name Workflow project entry name + * @param args Workflow script positional parameters + */ + void run(String name, List args) { + this.args = args + this.pipelineName = name + this.interactive = name == 'login' + if( background && interactive ) + throw new AbortOperationException("Option -bg conflicts with interactive mode") + this.config = makeConfig(pipelineName) + this.k8sConfig = makeK8sConfig(config.toMap()) + this.k8sClient = makeK8sClient(k8sConfig) + this.k8sConfig.checkStorageAndPaths(k8sClient) + this.initDeployer = new K8sNodeInitDeployer(k8sClient, k8sConfig) + + createK8sConfigMap() + + initDeployer.deploy() + createK8sLauncherPod() + waitPodStart() + // login into container session + if( interactive ) + launchLogin() + // dump pod output + else if( !background ) + printK8sPodOutput() + else + log.debug "Nextflow driver launched in background mode -- pod: $runName" + waitPodEnd() + } + + int shutdown() { + if( background ) + return 0 + // fetch the container exit status + final exitCode = waitPodTermination() + // cleanup the config map if OK + def deleteOnSuccessByDefault = exitCode==0 + if( k8sConfig.getCleanup(deleteOnSuccessByDefault) ) { + deleteConfigMap() + } + // cleanup pre-workflow pods + initDeployer.cleanup() + + return exitCode + } + + protected void waitPodEnd() { + if( background ) + return + final currentState = k8sConfig.useJobResource() ? k8sClient.jobState(runName) : k8sClient.podState(runName) + if (currentState && currentState?.running instanceof Map) { + final name = runName + println "${resourceType} running: $name ... waiting for ${resourceType.lower()} to stop running" + try { + while( true ) { + sleep 10000 + final state = k8sConfig.useJobResource() ? k8sClient.jobState(name) : k8sClient.podState(name) + if ( state && !(state?.running instanceof Map) ) { + println "${resourceType} $name has changed from running state $state" + break + } + } + } + catch( Exception e ) { + log.warn "Caught exception while waiting for ${resourceType.lower()} to stop running" + } + } + } + + protected boolean isWaitTimedOut(long time) { + System.currentTimeMillis()-time > 90_000 + } + + protected int waitPodTermination() { + log.debug "Wait for ${resourceType.lower()} termination name=$runName" + final rnd = new Random() + final time = System.currentTimeMillis() + Map state = null + try { + while( true ) { + sleep rnd.nextInt(500) + state = k8sConfig.useJobResource() ? k8sClient.jobState(runName) : k8sClient.podState(runName) + if( state?.terminated instanceof Map ) + return state.terminated.exitCode as int + + else if( isWaitTimedOut(time) ) + throw new IllegalStateException("Timeout waiting for ${resourceType.lower()} terminated state=$state") + } + } + catch( Exception e ) { + log.warn "Unable to fetch ${resourceType.lower()} exit status -- ${resourceType.lower()}=$runName state=$state" + return 127 + } + } + + protected void deleteConfigMap() { + try { + k8sClient.configDelete(configMapName) + log.debug "Deleted K8s configMap with name: $configMapName" + } + catch ( Exception e ) { + log.warn "Unable to delete configMap: $configMapName", e + } + } + + protected void waitPodStart() { + final name = runName + print "${resourceType} submitted: $name .. waiting to start" + while( true ) { + sleep 1000 + final state = k8sConfig.useJobResource() ? k8sClient.jobState(name) : k8sClient.podState(name) + if( state && !state.containsKey('waiting') ) { + break + } + } + print "\33[2K\r" + println "${resourceType} started: $name" + } + + /** + * Wait for the driver pod creation and prints the log to the + * console standard output + */ + protected void printK8sPodOutput() { + if ( k8sConfig.useJobResource() ) + k8sClient.jobLog(runName, follow:true).eachLine { println it } + else + k8sClient.podLog(runName, follow:true).eachLine { println it } + } + + protected ConfigObject loadConfig( String pipelineName ) { + + // -- load local config if available + final builder = new ConfigBuilder() + .setShowClosures(true) + .setOptions(cmd.launcher.options) + .setProfile(cmd.profile) + .setCmdRun(cmd) + + if( !interactive && !pipelineName.startsWith('/') && !cmd.remoteProfile && !cmd.runRemoteConfig ) { + // -- check and parse project remote config + final pipelineConfig = new AssetManager(pipelineName, cmd.revision, cmd.mainScript, cmd).getConfigFile() + builder.setUserConfigFiles(pipelineConfig) + } + + return builder.buildConfigObject() + } + + protected K8sConfig makeK8sConfig(Map config) { + config.k8s instanceof Map ? new K8sConfig(config.k8s as Map) : new K8sConfig() + } + + protected makeK8sClient( K8sConfig k8sConfig ) { + new K8sClient(k8sConfig.getClient()) + } + + /** + * Retrieve the workflow configuration and merge with the current local one. + * + * @param pipelineName Workflow project name + * @return A {@link Map} modeling the execution configuration settings + */ + protected ConfigObject makeConfig(String pipelineName) { + + def file = new File(pipelineName) + if( !interactive && file.exists() ) { + def message = "The k8s executor cannot run local ${file.directory ? 'project' : 'script'}: $pipelineName" + message += " -- provide the absolute path of a project available in the Kubernetes cluster or the URL of a project hosted in a Git repository" + throw new AbortOperationException(message) + } + + def config = loadConfig(pipelineName) + + // normalize pod entries + def k8s = config.k8s + + if( !k8s.isSet('pod') ) + k8s.pod = [] + else if( k8s.pod instanceof Map ) { + k8s.pod = [ k8s.pod ] + } + else if( !(k8s.pod instanceof List) ) + throw new IllegalArgumentException("Illegal k8s.pod configuratun value: ${k8s.pod}") + + // -- use the volume claims specified in the command line + // to populate the pod config + for( int i=0; i + def name = k as String + def path = v instanceof Map ? v.mountPath : v.toString() + if( !k8s.isSet('storageClaimName') ) { + k8s.storageClaimName = name + k8s.storageMountPath = path + } + else if( !cmd.volMounts ) { + k8s.pod.add( [volumeClaim: name, mountPath: path] ) + } + } + // remove it + k8s.remove('volumeClaims') + } + + // -- set k8s executor + config.process.executor = 'k8s' + + // -- strip default work dir + if( config.workDir == 'work' ) + config.remove('workDir') + + // -- check work dir + if( cmd?.workDir ) + k8s.workDir = cmd.workDir + else if( !k8s.isSet('workDir') && config.workDir ) + k8s.workDir = config.workDir + + if ( plugins ) { + LinkedList plugins = config.plugins ?: [] + plugins.addAll( this.plugins.tokenize(',') ) + config.plugins = plugins + } + + // -- some cleanup + if( !k8s.pod ) + k8s.remove('pod') + + if( !k8s.storageClaimName ) + k8s.remove('storageClaimName') + if( !k8s.storageMountPath ) + k8s.remove('storageMountPath') + + if( !config.libDir ) + config.remove('libDir') + + log.trace "K8s config object:\n${ConfigHelper.toCanonicalString(config).indent(' ')}" + return config + } + + + private Field getField(CmdRun cmd, String name) { + def clazz = cmd.class + while( clazz != CmdRun ) { + clazz = cmd.class.getSuperclass() + } + clazz.getDeclaredField(name) + } + + private void checkUnsupportedOption(String name) { + def field = getField(cmd,name) + if( !field ) { + log.warn "Unknown command-line option to check: $name" + return + } + field.setAccessible(true) + if( field.get(cmd) ) { + def param = field.getAnnotation(Parameter) + def opt = param.names() ? param.names()[0] : "-$name" + abort(opt) + } + } + + private void abort(String opt) { + throw new AbortOperationException("Option `$opt` not supported with Kubernetes deployment") + } + + private void unsupportedCliOptions(MethodClosure... fields) { + unsupportedCliOptions( fields.collect { it.getMethod()} ) + } + + private void unsupportedCliOptions(List names) { + for( String x : names ) { + checkUnsupportedOption(x) + } + } + + private void addOption(List result, MethodClosure m, Closure eval=null) { + def name = m.getMethod() + def field = getField(cmd,name) + field.setAccessible(true) + def val = field.get(cmd) + if( ( eval ? eval(val) : val ) ) { + def param = field.getAnnotation(Parameter) + if( param ) { + result << "${param.names()[0]} ${Escape.wildcards(String.valueOf(val))}" + return + } + + param = field.getAnnotation(DynamicParameter) + if( param && val instanceof Map ) { + val.each { k,v -> + result << "${param.names()[0]}$k ${Escape.wildcards(String.valueOf(v))}" + } + } + } + } + + /** + * @return The nextflow driver command line + */ + protected String getLaunchCli() { + assert cmd + assert pipelineName + + if( interactive ) { + return "tail -f /dev/null" + } + + def result = [] + // -- configure NF command line + result << "nextflow" + + if( cmd.launcher.options.trace ) + result << "-trace ${cmd.launcher.options.trace.join(',')}" + if( cmd.launcher.options.debug ) + result << "-debug ${cmd.launcher.options.debug.join(',')}" + if( cmd.launcher.options.jvmOpts ) + cmd.launcher.options.jvmOpts.each { k,v -> result << "-D$k=$v" } + + result << "run" + result << pipelineName + + if( runName ) + result << '-name' << runName + + addOption(result, cmd.&cacheable, { it==false } ) + addOption(result, cmd.&resume ) + addOption(result, cmd.&poolSize ) + addOption(result, cmd.&pollInterval ) + addOption(result, cmd.&queueSize) + addOption(result, cmd.&revision ) + addOption(result, cmd.&latest ) + addOption(result, cmd.&withTrace ) + addOption(result, cmd.&withTimeline ) + addOption(result, cmd.&withDag ) + addOption(result, cmd.&dumpHashes ) + addOption(result, cmd.&dumpChannels ) + addOption(result, cmd.&env ) + addOption(result, cmd.&process ) + addOption(result, cmd.¶ms ) + addOption(result, cmd.&entryName ) + + if( paramsFile ) { + result << "-params-file $paramsFile" + } + + if ( cmd.runRemoteConfig ) + cmd.runRemoteConfig.forEach { result << "-config $it" } + + if ( cmd.remoteProfile ) + result << "-profile ${cmd.remoteProfile}" + + if( cmd.process?.executor ) + abort('process.executor') + + unsupportedCliOptions( + cmd.&libPath, + cmd.&test, + cmd.&executorOptions, + cmd.&stdin, + cmd.&withSingularity, + cmd.&withApptainer, + cmd.&withDocker, + cmd.&withoutDocker, + cmd.&withMpi, + cmd.&clusterOptions, + cmd.&exportSysEnv + ) + + if( args ) + result.add(args) + + return result.join(' ') + } + + /** + * @return A {@link Map} modeling driver pod specification + */ + protected Map makeLauncherSpec() { + assert runName + assert k8sClient + + // -- setup config file + String cmd = "source /etc/nextflow/init.sh; ${getLaunchCli()}" + + "; if [ -x /etc/nextflow/node-init-cleanup.sh ]; then /etc/nextflow/node-init-cleanup.sh || true; fi; " + + "exit \$status" + + // create the launcher pod + PodSpecBuilder builder = new PodSpecBuilder() + .withPodName(runName) + .withImageName(headImage ?: k8sConfig.getNextflowImageName()) + .withCommand(['/bin/bash', '-c', cmd]) + .withLabels([ app: 'nextflow', runName: runName ]) + .withNamespace(k8sClient.config.namespace) + .withServiceAccount(k8sClient.config.serviceAccount) + .withPodOptions(k8sConfig.getPodOptions()) + .withEnv( PodEnv.value('NXF_WORK', k8sConfig.getWorkDir()) ) + .withEnv( PodEnv.value('NXF_ASSETS', k8sConfig.getProjectDir()) ) + .withEnv( PodEnv.value('NXF_EXECUTOR', 'k8s')) + .withEnv( PodEnv.value('NXF_ANSI_LOG', 'false')) + .withMemory(headMemory?:"") + .withCpus(headCpus) + .withCpuLimits(k8sConfig.cpuLimitsEnabled()) + + if ( k8sConfig.useJobResource()) { + this.resourceType = ResourceType.Job + return builder.buildAsJob() + } + else { + return builder.build() + } + + // note: do *not* set the work directory because it may need to be created by the init script + } + + /** + * Creates and executes the nextflow driver pod + * @return A {@link recreationaltech.plugin.client.K8sResponseJson} response object + */ + protected createK8sLauncherPod() { + final spec = makeLauncherSpec() + if ( k8sConfig.useJobResource() ) { + k8sClient.jobCreate(spec, yamlDebugPath()) + } else { + k8sClient.podCreate(spec, yamlDebugPath()) + } + } + + protected Path yamlDebugPath() { + boolean debug = config.k8s.debug?.yaml?.toString() == 'true' + final result = debug ? Paths.get(".nextflow.${resourceType.lower()}.yaml") : null + if( result ) + log.info "Launcher ${resourceType.lower()} spec file: $result" + return result + } + + protected Path getScmFile() { + ProviderConfig.getScmConfigPath() + } + + String getPodImage() { + return podImage + } + + int getHeadCpus() { + return headCpus + } + + String getHeadMemory() { + return headMemory + } + + String getRunName() { + return runName + } + + CmdKubeRun getCmd() { + return cmd + } + + protected String getPipelineName() { + return pipelineName + } + + protected boolean getInteractive() { + return interactive + } + + protected ConfigObject getConfig() { + return config + } + + protected K8sConfig getK8sConfig() { + return k8sConfig + } + + protected K8sClient getK8sClient() { + return k8sClient + } + + /** + * Creates a K8s ConfigMap to share the nextflow configuration in the K8s cluster + */ + protected void createK8sConfigMap() { + Map configMap = [:] + + final launchDir = k8sConfig.getLaunchDir() + // init file + String initScript = '' + initScript += "mkdir -p '$launchDir'; if [ -d '$launchDir' ]; then cd '$launchDir'; else echo 'Cannot create directory: $launchDir'; exit 1; fi; " + initScript += '[ -f /etc/nextflow/scm ] && ln -s /etc/nextflow/scm $NXF_HOME/scm; ' + initScript += '[ -f /etc/nextflow/nextflow.config ] && cp /etc/nextflow/nextflow.config $PWD/nextflow.config; ' + if( headPreScript ) + initScript += "[ -f '$headPreScript' ] && '$headPreScript'; " + configMap['init.sh'] = initScript + + // nextflow config file + if( this.config ) { + configMap['nextflow.config'] = ConfigHelper.toCanonicalString( this.config ) + } + + // scm config file + final scmFile = getScmFile() + if( scmFile.exists() ) { + configMap['scm'] = scmFile.text + } + + // params file + if( cmd.paramsFile ) { + final file = FileHelper.asPath(cmd.paramsFile) + if( !file.exists() ) throw new NoSuchFileException("Params file does not exist: $file") + configMap[ file.getName() ] = file.text + paramsFile = "/etc/nextflow/$file.name" + } + + // pre-workflow pod cleanup + if ( background ) + configMap['node-init-cleanup.sh'] = initDeployer.buildCleanupScript() + + // create the config map + configMapName = makeConfigMapName(configMap) + tryCreateConfigMap(configMapName, configMap) + log.debug "Created K8s configMap with name: $configMapName" + k8sConfig.getPodOptions().getMountConfigMaps().add( new PodMountConfig(configMapName, '/etc/nextflow') ) + } + + protected void tryCreateConfigMap(String name, Map data) { + try { + k8sClient.configCreate(name, data) + } + catch( K8sResponseException e ) { + if( e.response.reason != 'AlreadyExists' ) + throw e + } + } + + protected String makeConfigMapName( Map configMap ) { + "nf-config-${hash(configMap.values())}" + } + + protected String hash(Collection text) { + def hasher = Hashing .murmur3_32() .newHasher() + def itr = text.iterator() + while( itr.hasNext() ) { + hasher.putUnencodedChars(itr.next()) + } + + return hasher.hash().toString() + } + + protected void launchLogin() { + def launchDir = k8sConfig.getLaunchDir() + def cmd = "kubectl -n ${k8sClient.config.namespace} exec -it $runName -- /bin/bash -c 'cd $launchDir; exec bash --login -i'" + def proc = new ProcessBuilder().command('bash','-c',cmd).inheritIO().start() + def result = proc.waitFor() + if( result == 0 ) { + if ( k8sConfig.useJobResource() ) + k8sClient.jobDelete(runName) + else + k8sClient.podDelete(runName) + } + } +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sExecutor.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sExecutor.groovy new file mode 100644 index 0000000..27cccc4 --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sExecutor.groovy @@ -0,0 +1,192 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin + +import groovy.transform.CompileDynamic +import recreationaltech.plugin.strategies.K8sDVFSSchedulingStrategy +import recreationaltech.plugin.strategies.K8sHashSchedulingStrategy + +import java.util.concurrent.TimeUnit + +import com.google.common.cache.Cache +import com.google.common.cache.CacheBuilder +import groovy.transform.CompileStatic +import groovy.transform.Memoized +import groovy.util.logging.Slf4j +import nextflow.executor.Executor +import nextflow.fusion.FusionHelper +import recreationaltech.plugin.client.K8sClient +import nextflow.processor.TaskHandler +import nextflow.processor.TaskMonitor +import nextflow.processor.TaskPollingMonitor +import nextflow.processor.TaskRun +import nextflow.util.Duration +import nextflow.util.ServiceName +import org.pf4j.ExtensionPoint + +/** + * Implement the Kubernetes executor + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +@ServiceName('k8s-dvfs') +class K8sExecutor extends Executor implements ExtensionPoint { + + /** + * Cache for the Kubernetes HTTP client. The client is refreshed periodically + * so that the service account token is re-read when it expires. + */ + private Cache clientCache + + private K8sTaskScheduler taskScheduler + private Thread schedulerThread + + K8sRuntimeRecorder runtimeRecorder + K8sRuntimeEstimator runtimeEstimator + + /** + * @return The Kubernetes HTTP client. Delegates to a Guava cache that refreshes + * the client (including the service account token) when the configured interval expires. + */ + K8sClient getClient() { + clientCache.get('client', () -> new K8sClient(k8sConfig.getClient())) + } + + protected K8sTaskScheduler getTaskScheduler() { + assert taskScheduler != null + return taskScheduler + } + + /** + * @return The `k8s` configuration scope in the nextflow configuration object + */ + @Memoized + protected K8sConfig getK8sConfig() { + new K8sConfig( (Map)session.config.k8s ) + } + + /** + * Initialise the executor setting-up the kubernetes client configuration + */ + @Override + protected void register() { + super.register() + final k8sConfig = getK8sConfig() + final refreshInterval = k8sConfig.clientRefreshInterval + this.clientCache = CacheBuilder.newBuilder() + .expireAfterWrite(refreshInterval.toMillis(), TimeUnit.MILLISECONDS) + .build() + final client = getClient() + + log.debug "[K8s] config=$k8sConfig; API client config=$client.config" + + this.runtimeRecorder = new K8sRuntimeRecorder(k8sConfig.recordTaskRuntimes, k8sConfig.runtimeRecordPath) + + if (k8sConfig.runtimeEstimator == "LinearFit") { + this.runtimeEstimator = new K8sLinearFitRuntimeEstimator(k8sConfig.runtimeRecordPath) + } else if (k8sConfig.runtimeEstimator == "Noise") { + this.runtimeEstimator = new K8sNoiseRuntimeEstimator(k8sConfig.runtimeEstimator, k8sConfig.noiseRuntimeEstimatorNoiseMagnitude) + } else { + log.error "[K8s] invalid runtime estimator ${k8sConfig.runtimeEstimator} - falling back on \"LinearFit\"" + this.runtimeEstimator = new K8sLinearFitRuntimeEstimator(k8sConfig.runtimeRecordPath) + } + + String[] nodes = getNodeList() + + K8sSchedulingStrategy strategy = null + if (k8sConfig.schedulingStrategy == "Hash") { + strategy = new K8sHashSchedulingStrategy() + } else if (k8sConfig.schedulingStrategy == "DVFS" || k8sConfig.schedulingStrategy == "DVFS-SPEED") { + String[] ips = new String[nodes.length] + for (int i = 0; i < nodes.length; i++) { + ips[i] = client.getPodIpAddress(K8sNodeInitDeployer.buildPodName(nodes[i])) + log.info "[K8s] node ${nodes[i]} -> ${ips[i]}" + } + strategy = new K8sDVFSSchedulingStrategy(this.runtimeEstimator, + new K8sDVFSClient(nodes, ips), + () -> getClient(), + k8sConfig.runtimeComparisonEpsilon, + k8sConfig.dvfsSchedulingNumTopRuntimes) + strategy.fullSpeedMode = k8sConfig.schedulingStrategy == "DVFS-SPEED" + } else { + log.error "[K8s] invalid scheduling strategy $k8sConfig.schedulingStrategy, falling back on \"Hash\"" + strategy = new K8sHashSchedulingStrategy() + } + + this.taskScheduler = new K8sTaskScheduler(nodes, strategy, k8sConfig.schedulerInterval) + this.schedulerThread = new Thread(this.taskScheduler) + this.schedulerThread.start() + } + + @CompileDynamic + private String[] getNodeList() { + final resp = getClient().nodeList() + ArrayList nodes = new ArrayList() + for ( Map item : resp.items ) { + nodes.add(item.metadata.name as String) + } + return nodes.toArray() + } + + @Override + void shutdown() { + this.runtimeRecorder.write() + this.taskScheduler.stop() + this.schedulerThread.join() + } + + /** + * @return {@code true} since containerised execution is managed by Kubernetes + */ + boolean isContainerNative() { + return true + } + + @Override + String containerConfigEngine() { + return 'docker' + } + + /** + * @return A {@link TaskMonitor} associated to this executor type + */ + @Override + protected TaskMonitor createTaskMonitor() { + TaskPollingMonitor.create(session, config, name, 100, Duration.of('5 sec')) + } + + /** + * Creates a {@link TaskHandler} for the given {@link TaskRun} instance + * + * @param task A {@link TaskRun} instance representing a process task to be executed + * @return A {@link K8sTaskHandler} instance modeling the execution in the K8s cluster + */ + @Override + TaskHandler createTaskHandler(TaskRun task) { + assert task + assert task.workDir + log.trace "[K8s] launching process > ${task.name} -- work folder: ${task.workDirStr}" + new K8sTaskHandler(task,this) + } + + @Override + boolean isFusionEnabled() { + return FusionHelper.isFusionEnabled(session) + } +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sLinearFitRuntimeEstimator.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sLinearFitRuntimeEstimator.groovy new file mode 100644 index 0000000..f0f09de --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sLinearFitRuntimeEstimator.groovy @@ -0,0 +1,99 @@ +package recreationaltech.plugin + +import groovy.util.logging.Slf4j + +import java.nio.file.Files +import java.nio.file.Path + +/** + * Provides runtime estimates for tasks based on task-name and input file size + */ +@Slf4j +class K8sLinearFitRuntimeEstimator extends K8sRuntimeEstimator { + + class Function { + private double m + private double n + + Function(double m, double n) { + this.m = m + this.n = n + } + + double estimate(long x) { + return m * (double)x + n + } + } + + HashMap estimators; + + /** + * Initialize the estimator with data recorded by K8sRuntimeRecorder + * @param dataFilePath + */ + K8sLinearFitRuntimeEstimator(String dataFilePath) { + def data = parseDataFile(dataFilePath) + fit(data) + } + + /** + * Initializes the runtime estimator with statically known data. + * @param data Map from task name to list of recordings, where each recording is a tuple (input-size, runtime-in-ms) + */ + K8sLinearFitRuntimeEstimator(HashMap>> data) { + fit(data) + } + + /** + * Returns an estimation of the task runtime in milliseconds + * @param taskName the name of the task + * @param inputSize the total input size in bytes + * @return estimated task runtime in milliseconds OR infinity if the task is unknown. + */ + double estimate(String taskName, long inputSize) { + Function estimator = estimators.get(taskName) + if (estimator == null) { + //log.warn "[K8s] Unable to estimate take ${taskName}: unknown task" + return Double.POSITIVE_INFINITY + } + return estimator.estimate(inputSize) + } + + private void fit(HashMap>> data) { + estimators = new HashMap<>() + for (Map.Entry>> t : data) { + Function f = fit(t.value) + estimators.put(t.key, f) + } + } + + /** + * Uses linear regression to fit a linear function (y = m * x + n) to the provided observations + * @param observations list of tuples (input size, runtime in ms) + * @return linear function fitted to the input + */ + private Function fit(ArrayList> observations) throws IllegalArgumentException { + int n = observations.size() + if (n > 1) { + double sumX = 0, sumY = 0, sumXY = 0, sumX2 = 0 + for (Tuple2 o : observations) { + double x = (double) o.get(0) + double y = (double) o.get(1) + sumX += x + sumY += y + sumXY += x * y + sumX2 += x * x + } + double m = (n * sumXY - sumX * sumY) / (n * sumX2 - sumX * sumX) + return new Function( + m, + (sumY - m * sumX) / n + ) + } else if (n == 1) { + // Special case: We only have 1 measurement. We will just assume that the runtime is constant, + // because in our observed data, it is. + return new Function(0.0, (double) observations[0].get(1)) + } + throw new IllegalArgumentException("requires at least 1 observation") + } +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sNodeInitConfig.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sNodeInitConfig.groovy new file mode 100644 index 0000000..8ca6402 --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sNodeInitConfig.groovy @@ -0,0 +1,47 @@ +package recreationaltech.plugin + +import groovy.util.logging.Slf4j +import nextflow.config.scopes.Config +import nextflow.config.spec.ConfigScope +import nextflow.config.spec.ScopeName +import nextflow.config.spec.ConfigOption +import groovy.transform.CompileStatic +import nextflow.script.dsl.Description + +@CompileStatic +@Slf4j +@ScopeName("nodeInit") +@Description("The nodeInit scope contains options for the pre-workflow execution initialization of nodes") +class K8sNodeInitConfig implements ConfigScope { + @ConfigOption + @Description("enables the pre-workflow execution deployment of pods") + final boolean enabled; + + @ConfigOption + @Description("the used image") + final String image; + + @ConfigOption + @Description("the start-command") + final List command; + + @ConfigOption + @Description("the pod state to wait on") + final String wait; + + @ConfigOption + @Description("enables cleanup of pre-workflow nodes.") + final boolean cleanup; + + K8sNodeInitConfig() { + this(Collections.emptyMap()) + } + + K8sNodeInitConfig(Map opts) { + enabled = opts.enabled as boolean + image = opts.image as String + command = opts.command as List + wait = opts.wait as String + cleanup = opts.cleanup as boolean + } +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sNodeInitDeployer.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sNodeInitDeployer.groovy new file mode 100644 index 0000000..8d874e5 --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sNodeInitDeployer.groovy @@ -0,0 +1,177 @@ +package recreationaltech.plugin + +import groovy.util.logging.Slf4j +import recreationaltech.plugin.client.K8sClient +import recreationaltech.plugin.model.PodHostMount +import recreationaltech.plugin.model.PodSpecBuilder + +@Slf4j +class K8sNodeInitDeployer { + private K8sClient client + private K8sConfig config + + K8sNodeInitDeployer(K8sClient client, K8sConfig config) { + this.client = client + this.config = config + } + + void deploy() { + final init = config.nodeInit + if ( !init?.enabled ) + return + + log.info("deploying init pods") + + final nodes = getNodes() + for ( String nodeName : nodes ) { + log.info(" ... deploying to " + nodeName) + final spec = makePodSpec(init, nodeName) + client.podCreate(spec) + } + + log.info("waiting for init pods") + waitForPods(nodes) + } + + void cleanup() { + final init = config.nodeInit + if ( !init?.enabled || !init?.cleanup ) + return + + final nodes = getNodes() + for ( String nodeName : nodes ) { + final podName = buildPodName(nodeName) + client.podDelete(podName) + } + } + + String buildCleanupScript() { + if ( !config.nodeInit?.enabled || !config.nodeInit?.cleanup ) + return "#!/usr/bin/env/bash\nexit 0" + + final nodes = getNodes() + final podNames = nodes.collect {buildPodName(it)} + + String script = ''' +#!/usr/bin/env bash +set -u + +PODS=(''' + for ( String podName : podNames ) { + script += "\"${podName}\"\n" + } + script += ''' +) + +SERVICE_ACCOUNT_DIR="/var/run/secrets/kubernetes.io/serviceaccount" +TOKEN="$(cat "${SERVICE_ACCOUNT_DIR}/token")" +NAMESPACE="$(cat "${SERVICE_ACCOUNT_DIR}/namespace")" +CA_CERT="${SERVICE_ACCOUNT_DIR}/ca.crt" + +KUBE_API="https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS:-443}" + +for POD in "${PODS[@]}"; do + echo "Deleting pod: ${POD}" + + HTTP_CODE="$( + curl \\ + --silent \\ + --show-error \\ + --output /tmp/delete-pod-response.json \\ + --write-out "%{http_code}" \\ + --request DELETE \\ + --cacert "${CA_CERT}" \\ + --header "Authorization: Bearer ${TOKEN}" \\ + --header "Accept: application/json" \\ + "${KUBE_API}/api/v1/namespaces/${NAMESPACE}/pods/${POD}" + )" + + case "${HTTP_CODE}" in + 200|202) + echo "Deleted pod: ${POD}" + ;; + 404) + echo "Pod already absent: ${POD}" + ;; + *) + echo "Failed to delete pod: ${POD}; HTTP ${HTTP_CODE}" >&2 + cat /tmp/delete-pod-response.json >&2 || true + ;; + esac +done + +exit 0 + ''' + return script + } + + private List getNodes() { + final resp = client.nodeList() + ArrayList nodes = new ArrayList() + for ( Map item: resp.items ) { + nodes.add(item.metadata.name as String) + } + return nodes + } + + public static String buildPodName(String nodeName) { + // TODO: Remove anything that is not lowercase alpha-numeric or dash + String sanitizedNodeName = nodeName.toLowerCase() + String name = "nf-init-${sanitizedNodeName}" + if ( name.length() > 63 ) + name = name.substring(0, 63) + return name + } + + private Map makePodSpec(K8sNodeInitConfig config, String nodeName) { + ArrayList mounts = new ArrayList() + mounts.add(new PodHostMount("/sys", "/sys")) + mounts.add(new PodHostMount("/dev", "/dev")) + mounts.add(new PodHostMount("/lib/modules", "/lib/modules")) + + PodSpecBuilder builder = new PodSpecBuilder() + return builder.withNodeName(nodeName) + .withImageName(config.image) + .withCommand(config.command) + .withPrivileged(true) + .withHostMounts(mounts) + .withPodName(buildPodName(nodeName)) + .withPort(8080) + .build() + } + + private void waitForPods(List nodes) { + if ( config.nodeInit.wait == 'Running' ) { + for (String nodeName : nodes) { + String podName = buildPodName(nodeName) + while (true) { + sleep 1000 + final state = client.podState(podName) + if (state && !state.containsKey('waiting')) { + break + } + } + } + } else if ( config.nodeInit.wait == 'Succeeded' ) { + for ( String nodeName : nodes ) { + final String podName = buildPodName(nodeName) + final currentState = client.podState(podName) + if ( currentState && currentState?.running instanceof Map ) { + try { + while (true) { + sleep 10000 + final state = client.podState(podName) + if (state && !(state?.running instanceof Map)) { + println "$podName has changed from running state $state" + break + } + } + } + catch (Exception e) { + println "Caught exception while waiting for ${podName} to stop running" + } + } + } + } + } +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sNoiseRuntimeEstimator.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sNoiseRuntimeEstimator.groovy new file mode 100644 index 0000000..8087cc8 --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sNoiseRuntimeEstimator.groovy @@ -0,0 +1,50 @@ +package recreationaltech.plugin + +import nextflow.util.Duration + +class K8sNoiseRuntimeEstimator extends K8sRuntimeEstimator { + // Magnitude of the noise we add + private long noiseMag + + private HashMap>> data; + private synchronized Random rnd; + + /** + * Initialize the estimator with data recorded by K8sRuntimeRecorder + * @param dataFilePath + * @param noiseMagnitude magnitude of the noise added to recordings + */ + K8sNoiseRuntimeEstimator(String dataFilePath, Duration noiseMagnitude) { + this.data = parseDataFile(dataFilePath) + this.noiseMag = noiseMagnitude.toMillis() + this.rnd = new Random() + } + + /** + * Initializes the runtime estimator with statically known data. + * @param data Map from task name to list of recordings, where each recording is a tuple (input-size, runtime-in-ms) + */ + K8sNoiseRuntimeEstimator(HashMap>> data, Duration noiseMagnitude) { + this.data = data; + this.noiseMag = noiseMagnitude.toMillis() + } + + /** + * Returns an estimation of the task runtime in milliseconds + * @param taskName the name of the task + * @param inputSize the total input size in bytes + * @return estimated task runtime in milliseconds OR infinity if the task is unknown. + */ + @Override + double estimate(String taskName, long inputSize) { + ArrayList recordings = data.get(taskName) + for (Tuple2 recording : recordings) { + if (recording.get(0).longValue() == inputSize) { + double noise = rnd.nextDouble(-1.0, 1.0) * (double)noiseMag + long ms = recording.get(1).longValue() + return (double)ms + noise + } + } + return 0 + } +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sDvfsPlugin.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sPlugin.groovy similarity index 78% rename from k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sDvfsPlugin.groovy rename to k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sPlugin.groovy index 0fb4fbb..ed6f1fa 100644 --- a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sDvfsPlugin.groovy +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sPlugin.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2025, Seqera Labs + * Copyright 2013-2026, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,12 +21,14 @@ import nextflow.plugin.BasePlugin import org.pf4j.PluginWrapper /** - * The plugin entry point + * Kubernetes plugin entry point + * + * @author Paolo Di Tommaso */ @CompileStatic -class K8sDvfsPlugin extends BasePlugin { +class K8sPlugin extends BasePlugin { - K8sDvfsPlugin(PluginWrapper wrapper) { + K8sPlugin(PluginWrapper wrapper) { super(wrapper) } } diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sRuntimeEstimator.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sRuntimeEstimator.groovy new file mode 100644 index 0000000..0a4aee0 --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sRuntimeEstimator.groovy @@ -0,0 +1,84 @@ +package recreationaltech.plugin + +import groovy.util.logging.Slf4j + +import java.nio.file.Files +import java.nio.file.Path + +/** + * Provides runtime estimates for tasks based on task-name and input file size + */ +@Slf4j +abstract class K8sRuntimeEstimator { + /** + * Returns an estimation of the task runtime in milliseconds + * @param handler the task handler + * @return estimated task runtime in milliseconds OR infinity if the task is unknown. + */ + double estimate(K8sTaskHandler handler) { + long x = getTaskHandlerInputSize(handler) + return estimate(handler.task.processor.name, x) + } + + /** + * Returns an estimation of the task runtime in milliseconds + * @param taskName the name of the task + * @param inputSize the total input size in bytes + * @return estimated task runtime in milliseconds OR infinity if the task is unknown. + */ + abstract double estimate(String taskName, long inputSize); + + protected long getTaskHandlerInputSize(K8sTaskHandler handler) { + long inputSizeSum = 0 + // File input + def inputFiles = handler.task.getInputFilesMap() + for (Map.Entry f : inputFiles) { + try { + inputSizeSum += Files.size(f.value) + } catch (IOException ex) { + log.error "[K8s] failed to get size of input file ${f.value} of task ${task.task.name}: ${ex.message}" + } + } + // Non file input + def inputs = handler.task.getInputs() + for (Map.Entry i : inputs) { + inputSizeSum += i.value.toString().length() + } + + return inputSizeSum + } + + /// @brief Parses the runtime recording data file + /// @return Map from task name to list of recordings, where each recording is a tuple (input-size, runtime-in-ms) + protected HashMap>> parseDataFile(String dataFilePath) { + HashMap>> data = new HashMap<>(); + try { + BufferedReader reader = new BufferedReader(new FileReader(dataFilePath)) + String line = reader.readLine() + while (line != null) { + // ,, + String[] elems = line.split(",") + if (elems.length != 3) { + log.warn "[K8s] ${dataFilePath}: Unexpected line ${line}" + continue + } + + ArrayList> taskData = data.get(elems[0]) + if (taskData == null) { + taskData = new ArrayList<>() + data.put(elems[0], taskData) + } + try { + taskData.add(new Tuple2(Long.parseLong(elems[1]), Long.parseLong(elems[2]))) + } catch (NumberFormatException ex) { + log.error "[K8s] ${dataFilePath} invalid data: ${ex.message}" + } + line = reader.readLine() + } + reader.close() + } catch (IOException ex) { + log.error "[K8s] Failed to load ${dataFilePath}: ${ex.message}" + } + return data + } +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sRuntimeRecord.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sRuntimeRecord.groovy new file mode 100644 index 0000000..4f268ee --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sRuntimeRecord.groovy @@ -0,0 +1,7 @@ +package recreationaltech.plugin + +record K8sRuntimeRecord( + String taskName, + long inputSize, + long runtimeMillis +) {} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sRuntimeRecorder.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sRuntimeRecorder.groovy new file mode 100644 index 0000000..f40bf2c --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sRuntimeRecorder.groovy @@ -0,0 +1,67 @@ +package recreationaltech.plugin + +import groovy.util.logging.Slf4j + +import java.nio.file.Files +import java.nio.file.Path + +@Slf4j +class K8sRuntimeRecorder { + private final boolean enabled + private final String recordPath + + private ArrayList records + + K8sRuntimeRecorder(boolean enabled, String recordPath) { + this.enabled = enabled + this.recordPath = recordPath + this.records = new ArrayList<>() + } + + /** + * Records the runtime of a task. + * This should be called for a finished task, which has its + * start and end timestamps set. + * @param task + */ + synchronized void record(K8sTaskHandler task) { + if (!enabled) + return; + long runtimeMilis = task.completeTimeMillis - task.startTimeMillis + long inputSizeSum = 0 + + // Input files + def inputFiles = task.task.getInputFilesMap() + for (Map.Entry f : inputFiles) { + try { + inputSizeSum += Files.size(f.value) + } catch (IOException ex) { + log.error "[K8s] failed to get size of input file ${f.value} of task ${task.task.name}: ${ex.message}" + } + } + + // Non file input + def inputs = task.task.getInputs() + for (Map.Entry i : inputs) { + inputSizeSum += i.value.toString().length() + } + + log.info "[K8s] task ${task.task.processor.name} - input ${inputSizeSum} bytes - ran ${runtimeMilis} ms (${task.startTimeMillis} -> ${task.completeTimeMillis})" + records.add(new K8sRuntimeRecord(task.task.processor.name, inputSizeSum, runtimeMilis)) + } + + void write() { + if (!enabled) + return; + try { + FileWriter out = new FileWriter(recordPath) + for (K8sRuntimeRecord record : records) { + out.write("${record.taskName},${record.inputSize},${record.runtimeMillis}\n") + } + out.close() + log.info "[K8s] written runtime recording to ${recordPath}" + } catch (IOException err) { + log.error "[K8s] failed to write runtime recording ${recordPath}: ${err.message}" + } + } +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sSchedulingDecision.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sSchedulingDecision.groovy new file mode 100644 index 0000000..aeb7125 --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sSchedulingDecision.groovy @@ -0,0 +1,14 @@ +package recreationaltech.plugin + +import groovy.transform.CompileStatic + +@CompileStatic +class K8sSchedulingDecision { + final String nodeName + final K8sSchedulingRequest request + + K8sSchedulingDecision(K8sSchedulingRequest request, String nodeName) { + this.request = request + this.nodeName = nodeName + } +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sSchedulingRequest.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sSchedulingRequest.groovy new file mode 100644 index 0000000..ab2c940 --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sSchedulingRequest.groovy @@ -0,0 +1,21 @@ +package recreationaltech.plugin + +import groovy.transform.CompileStatic +import nextflow.processor.TaskRun + +@CompileStatic +class K8sSchedulingRequest { + final K8sTaskHandler handler + final TaskRun task + final long submitTimeMillis + + K8sSchedulingRequest(K8sTaskHandler handler) { + this.handler = handler + this.task = handler.task + this.submitTimeMillis = System.currentTimeMillis() + } + + String getTaskName() { + return task.processor.name + } +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sSchedulingStrategy.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sSchedulingStrategy.groovy new file mode 100644 index 0000000..ca35ac1 --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sSchedulingStrategy.groovy @@ -0,0 +1,27 @@ +package recreationaltech.plugin + +interface K8sSchedulingStrategy { + /** + * Selects the next task to run from the given task queue. + * If the scheduler should wait, returns {@code null} instead + * + * @param scheduler the calling scheduler object + * @param queue Pending scheduling requests + * @return A launch decision, or {@code null} when no task should be launched now + */ + K8sSchedulingDecision schedule(K8sTaskScheduler scheduler, List queue) + + /** + * Decides if the scheduler should immediately invoke the @ref schedule method of the strategy + * @param scheduler the calling scheduler object + * @param queue pending scheduling requests + * @return {@code true} if @ref schedule should immediately be called + */ + boolean scheduleImmediately(K8sTaskScheduler scheduler, List queue) + + /** + * Called when a task has finished execution. + * @param task the task + */ + void taskFinished(K8sTaskHandler task) +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sTaskHandler.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sTaskHandler.groovy new file mode 100644 index 0000000..3b0343e --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sTaskHandler.groovy @@ -0,0 +1,586 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin + +import java.nio.file.FileAlreadyExistsException +import java.nio.file.Files +import java.nio.file.Path +import java.time.Instant +import java.time.format.DateTimeFormatter + +import groovy.transform.CompileDynamic +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.SysEnv +import nextflow.container.ContainerHelper +import nextflow.container.DockerBuilder +import nextflow.exception.NodeTerminationException +import recreationaltech.plugin.client.PodUnschedulableException +import nextflow.exception.ProcessSubmitException +import nextflow.executor.BashWrapperBuilder +import nextflow.fusion.FusionAwareTask +import recreationaltech.plugin.client.K8sClient +import recreationaltech.plugin.client.K8sResponseException +import recreationaltech.plugin.model.PodEnv +import recreationaltech.plugin.model.PodOptions +import recreationaltech.plugin.model.PodSpecBuilder +import recreationaltech.plugin.model.ResourceType +import nextflow.processor.TaskHandler +import nextflow.processor.TaskRun +import nextflow.processor.TaskStatus +import nextflow.trace.TraceRecord +import nextflow.util.Escape +import nextflow.util.PathTrie +import nextflow.util.TestOnly +/** + * Implements the {@link TaskHandler} interface for Kubernetes pods + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +class K8sTaskHandler extends TaskHandler implements FusionAwareTask { + + @Lazy + static private final String OWNER = { + if( System.getenv('NXF_OWNER') ) { + return System.getenv('NXF_OWNER') + } + else { + def p = ['bash','-c','echo -n $(id -u):$(id -g)'].execute(); + p.waitFor() + return p.text + } + + } () + + private ResourceType resourceType = ResourceType.Pod + + private K8sClient client + + private String podName + + private BashWrapperBuilder builder + + private Path outputFile + + private Path errorFile + + private Path exitFile + + private Map state + + private long timestamp + + private K8sExecutor executor + + private String runsOnNode = null + + K8sTaskHandler( TaskRun task, K8sExecutor executor ) { + super(task) + this.executor = executor + this.client = executor.getClient() + this.outputFile = task.workDir.resolve(TaskRun.CMD_OUTFILE) + this.errorFile = task.workDir.resolve(TaskRun.CMD_ERRFILE) + this.exitFile = task.workDir.resolve(TaskRun.CMD_EXIT) + this.resourceType = executor.k8sConfig.useJobResource() ? ResourceType.Job : ResourceType.Pod + } + + @TestOnly + protected K8sTaskHandler() {} + + /** + * @return The workflow execution unique run name + */ + protected String getRunName() { + executor.session.runName + } + + protected String getPodName() { + return podName + } + + protected K8sConfig getK8sConfig() { executor.getK8sConfig() } + + protected boolean useJobResource() { resourceType==ResourceType.Job } + + protected List getContainerMounts() { + + if( !k8sConfig.getAutoMountHostPaths() ) { + return Collections.emptyList() + } + + // get input files paths + final List paths = DockerBuilder.inputFilesToPaths(builder.getInputFiles()) + final List binDirs = builder.binDirs + final Path workDir = builder.workDir + // add standard paths + if( binDirs ) + paths.addAll(binDirs) + if( workDir ) + paths << workDir + + def trie = new PathTrie() + paths.each { trie.add(it) } + + // defines the mounts + trie.longest() + } + + protected BashWrapperBuilder createBashWrapper(TaskRun task) { + return fusionEnabled() + ? fusionLauncher() + : new K8sWrapperBuilder(task) + } + + protected List classicSubmitCli(TaskRun task) { + final workDir = Escape.path(task.workDir) + + final result = new ArrayList(BashWrapperBuilder.BASH) + result.add('-o') + result.add('pipefail') + result.add('-c') + result.add("bash ${workDir}/${TaskRun.CMD_RUN} 2>&1 | tee ${workDir}/${TaskRun.CMD_LOG}") + return result + } + + protected List getSubmitCommand(TaskRun task) { + return fusionEnabled() + ? fusionSubmitCli() + : classicSubmitCli(task) + } + + protected String getSyntheticPodName(TaskRun task) { + final suffix = System.currentTimeMillis().toString().md5()[-5..-1] + return "nf-${task.hash}-${suffix}" + } + + protected String getOwner() { OWNER } + + protected Boolean fixOwnership() { + ContainerHelper.fixOwnership(task.containerConfig) + } + + /** + * Creates a Pod specification that executed that specified task + * + * @param task A {@link TaskRun} instance representing the task to execute + * @return A {@link Map} object modeling a Pod specification + */ + protected Map newSubmitRequest(TaskRun task) { + return newSubmitRequest(task, null) + } + + /** + * Creates a Pod specification that executed that specified task + * + * @param task A {@link TaskRun} instance representing the task to execute + * @param nodeName The kubernetes node on which the task should run or {@code null}, if no specific node is requested + * @return A {@link Map} object modeling a Pod specification + */ + protected Map newSubmitRequest(TaskRun task, String nodeName) { + def imageName = task.container + if( !imageName ) + throw new ProcessSubmitException("Missing container image for process `$task.processor.name`") + + try { + newSubmitRequest0(task, imageName, nodeName) + } + catch( Throwable e ) { + throw new ProcessSubmitException("Failed to submit K8s ${resourceType.lower()} -- Cause: ${e.message ?: e}", e) + } + } + + protected boolean entrypointOverride() { + return executor.getK8sConfig().entrypointOverride() + } + + protected boolean cpuLimitsEnabled() { + return executor.getK8sConfig().cpuLimitsEnabled() + } + + protected Map newSubmitRequest0(TaskRun task, String imageName, String nodeName) { + + final launcher = getSubmitCommand(task) + final taskCfg = task.getConfig() + + final clientConfig = client.config + final builder = new PodSpecBuilder() + .withImageName(imageName) + .withPodName(getSyntheticPodName(task)) + .withNamespace(clientConfig.namespace) + .withServiceAccount(clientConfig.serviceAccount) + .withLabels(getLabels(task)) + .withAnnotations(getAnnotations()) + .withPodOptions(getPodOptions()) + .withCpuLimits(cpuLimitsEnabled()) + + // when `entrypointOverride` is false the launcher is run via `args` instead of `command` + // to not override the container entrypoint + if( !entrypointOverride() ) { + builder.withArgs(launcher) + } + else { + builder.withCommand(launcher) + } + + if( nodeName ) + builder.withNodeName(nodeName) + + // note: task environment is managed by the task bash wrapper + // do not add here -- see also #680 + if( fixOwnership() ) + builder.withEnv(PodEnv.value('NXF_OWNER', getOwner())) + + if( SysEnv.containsKey('NXF_DEBUG') ) + builder.withEnv(PodEnv.value('NXF_DEBUG', SysEnv.get('NXF_DEBUG'))) + + // add computing resources + final cpus = taskCfg.getCpus() + final mem = taskCfg.getMemory() + final disk = taskCfg.getDisk() + final acc = taskCfg.getAccelerator() + if( cpus ) + builder.withCpus(cpus) + if( mem ) + builder.withMemory(mem) + if( disk ) + builder.withDisk(disk) + if( acc ) + builder.withAccelerator(acc) + + final List hostMounts = getContainerMounts() + for( String mount : hostMounts ) { + builder.withHostMount(mount,mount) + } + + if ( taskCfg.time ) { + final duration = taskCfg.getTime() + builder.withActiveDeadline(duration.toSeconds() as int) + } + + if ( fusionEnabled() ) { + if( fusionConfig().privileged() ) + builder.withPrivileged(true) + else { + final device= k8sConfig.fuseDevicePlugin() + builder.withResourcesLimits(device) + } + + final env = fusionLauncher().fusionEnv() + for( Map.Entry it : env ) + builder.withEnv(PodEnv.value(it.key, it.value)) + } + + return useJobResource() + ? builder.buildAsJob() + : builder.build() + } + + protected PodOptions getPodOptions() { + // merge the pod options provided in the k8s config + // with the ones in process config + def opt1 = k8sConfig.getPodOptions() + def opt2 = taskPodOptions() + return opt1 + opt2 + } + + protected PodOptions taskPodOptions() { + new PodOptions((List)task.getConfig().get('pod')) + } + + protected Map getLabels(TaskRun task) { + final result = new LinkedHashMap(10) + final labels = k8sConfig.getLabels() + if( labels ) { + result.putAll(labels) + } + final resLabels = task.config.getResourceLabels() + if( resLabels ) + result.putAll(resLabels) + result.'nextflow.io/app' = 'nextflow' + result.'nextflow.io/runName' = getRunName() + result.'nextflow.io/taskName' = task.getName() + result.'nextflow.io/processName' = task.getProcessor().getName() + result.'nextflow.io/sessionId' = "uuid-${executor.getSession().uniqueId}" as String + if( task.config.queue ) + result.'nextflow.io/queue' = task.config.queue + return result + } + + protected Map getAnnotations() { + k8sConfig.getAnnotations() + } + + /** + * Prepares the task execution and enqueues it at the scheduler + */ + @Override + @CompileDynamic + void submit() { + builder = createBashWrapper(task) + builder.build() + log.info "[K8s] submitting task ${this.task.name}" + executor.taskScheduler.submit(this) + } + + /** + * Creates a new K8s pod executing the associated task + */ + @CompileDynamic + void submitNow(String nodeName) { + final req = newSubmitRequest(task, nodeName) + final resp = useJobResource() + ? client.jobCreate(req, yamlDebugPath()) + : client.podCreate(req, yamlDebugPath()) + + if( !resp.metadata?.name ) + throw new K8sResponseException("Missing created ${resourceType.lower()} name", resp) + this.podName = resp.metadata.name + this.status = TaskStatus.SUBMITTED + } + + @CompileDynamic + protected Path yamlDebugPath() { + boolean debug = k8sConfig.getDebug().getYaml() + return debug ? task.workDir.resolve('.command.yaml') : null + } + + /** + * @return Retrieve the submitted pod state + */ + protected Map getState() { + final now = System.currentTimeMillis() + try { + final delta = now - timestamp; + if( !state || delta >= 1_000) { + def newState = useJobResource() + ? client.jobState(podName) + : client.podState(podName) + if( newState ) { + log.trace "[K8s] Get ${resourceType.lower()}=$podName state=$newState" + state = newState + timestamp = now + } + } + return state + } + catch (NodeTerminationException | PodUnschedulableException e) { + // create a synthetic `state` object adding an extra `nodeTermination` + // attribute to return the error to the caller method + final instant = Instant.now() + final result = new HashMap(10) + result.terminated = [startedAt:instant.toString(), finishedAt:instant.toString()] + result.nodeTermination = e + timestamp = now + state = result + return state + } + } + + @Override + boolean checkIfRunning() { + if( !podName ) + return false + if(isSubmitted()) { + def state = getState() + // include `terminated` state to allow the handler status to progress + if (state && (state.running != null || state.terminated)) { + status = TaskStatus.RUNNING + determineNode() + return true + } + } + return false + } + + long getEpochMilli(String timeString) { + final time = DateTimeFormatter.ISO_INSTANT.parse(timeString) + return Instant.from(time).toEpochMilli() + } + + /** + * Update task start and end times based on pod timestamps. + * We update timestamps because it's possible for a task to run so quickly + * (less than 1 second) that it skips right over the RUNNING status. + * If this happens, the startTimeMillis never gets set and remains equal to 0. + * To make sure startTimeMillis is non-zero we update it with the pod start time. + * We update completeTimeMillis from the same pod info to be consistent. + */ + void updateTimestamps(Map terminated) { + try { + startTimeMillis = getEpochMilli(terminated.startedAt as String) + completeTimeMillis = getEpochMilli(terminated.finishedAt as String) + } catch( Exception e ) { + log.debug "Failed updating timestamps '${terminated.toString()}'", e + // Only update if startTimeMillis hasn't already been set. + // If startTimeMillis _has_ been set, then both startTimeMillis + // and completeTimeMillis will have been set with the normal + // TaskHandler mechanism, so there's no need to reset them here. + if (!startTimeMillis) { + startTimeMillis = System.currentTimeMillis() + completeTimeMillis = System.currentTimeMillis() + } + } + } + + @Override + boolean checkIfCompleted() { + if( !podName ) + return false + + final state = getState() + if( state && state.terminated ) { + if( state.nodeTermination instanceof NodeTerminationException || + state.nodeTermination instanceof PodUnschedulableException ) { + // keep track of the node termination error + task.error = (Throwable) state.nodeTermination + // mark the task as ABORTED since thr failure is caused by a node failure + task.aborted = true + } + else { + // finalize the task + // read the exit code from the K8s container terminated state, if missing + // take the exit code from the `.exitcode` file created by nextflow + // the rationale is that in case of error (e.g. OOMKilled, pod eviction), the exit code from + // the K8s API is more reliable because the container may terminate before the exit file is written + // See https://github.com/nextflow-io/nextflow/issues/6436 + // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#containerstateterminated-v1-core + log.trace("[k8s] Container Terminated state ${state.terminated}") + final k8sExitCode = (state.terminated as Map)?.exitCode as Integer + task.exitStatus = k8sExitCode != null ? k8sExitCode : readExitFile() + task.stdout = outputFile + task.stderr = errorFile + } + status = TaskStatus.COMPLETED + saveJobLogOnError(task) + deleteJobIfSuccessful(task) + updateTimestamps(state.terminated as Map) + determineNode() + + // Signal the scheduler that this task has finished running + if (executor != null) { + executor.taskScheduler.taskFinished(this) + executor.runtimeRecorder.record(this) + } + return true + } + + return false + } + + protected void saveJobLogOnError(TaskRun task) { + if( task.isSuccess() ) + return + + if( errorFile && !errorFile.empty() ) + return + + final session = executor.getSession() + if( session.isAborted() || session.isCancelled() || session.isTerminated() ) + return + + try { + final stream = useJobResource() + ? client.jobLog(podName) + : client.podLog(podName) + Files.copy(stream, task.workDir.resolve(TaskRun.CMD_LOG)) + } + catch( FileAlreadyExistsException e ) { + log.debug "Log file already exists for ${resourceType.lower()} $podName", e + } + catch( Exception e ) { + log.warn "Failed to copy log for ${resourceType.lower()} $podName", e + } + } + + protected int readExitFile() { + try { + exitFile.text as Integer + } + catch( Exception e ) { + log.debug "[K8s] Cannot read exitstatus for task: `$task.name` | ${e.message}" + return Integer.MAX_VALUE + } + } + + /** + * Terminates the current task execution + */ + @Override + protected void killTask() { + if( !podName ) + return + + if( cleanupDisabled() ) + return + + log.trace "[K8s] deleting ${resourceType.lower()} name=$podName" + delete0(podName) + } + + protected boolean cleanupDisabled() { + !k8sConfig.getCleanup() + } + + protected void deleteJobIfSuccessful(TaskRun task) { + if( !podName ) + return + + if( cleanupDisabled() ) + return + + // preserve failed pods for debugging purposes + if( !task.isSuccess() ) + return + + // k8s cluster will cleanup job on its own if TTL is set + if( useJobResource() && getPodOptions().getTtlSecondsAfterFinished() != null ) + return + + delete0(podName) + } + + private void delete0(String podName) { + try { + if ( useJobResource() ) + client.jobDelete(podName) + else + client.podDelete(podName) + } + catch( Exception e ) { + log.warn "Unable to delete ${resourceType.lower()}: $podName -- see the log file for details", e + } + } + + private void determineNode() { + try { + if ( k8sConfig.fetchNodeName() && !runsOnNode ) + runsOnNode = client.getNodeOfPod( podName ) + } catch ( Exception e ) { + log.warn ("Unable to get the node name of pod $podName -- see the log file for details", e) + } + } + + TraceRecord getTraceRecord() { + final result = super.getTraceRecord() + result.put('native_id', podName) + result.put( 'hostname', runsOnNode ) + return result + } + +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sTaskScheduler.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sTaskScheduler.groovy new file mode 100644 index 0000000..1623edc --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sTaskScheduler.groovy @@ -0,0 +1,96 @@ +package recreationaltech.plugin + + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.util.Duration +import java.util.concurrent.LinkedBlockingQueue + +@Slf4j +@CompileStatic +class K8sTaskScheduler implements Runnable { + private final Duration interval + private final K8sSchedulingStrategy strategy + private final LinkedBlockingQueue queue = new LinkedBlockingQueue<>() + private String[] nodes + + private synchronized boolean shouldStop + + K8sTaskScheduler(String[] nodes, K8sSchedulingStrategy strategy, Duration interval) { + this.interval = interval + this.strategy = strategy + this.nodes = nodes + } + + /** + * Adds a task to the queue of outstanding tasks + * @param handler + */ + void submit(K8sTaskHandler handler) { + log.info "[K8s] received queued task ${handler.task.name}" + queue.add(new K8sSchedulingRequest(handler)) + final pending = new ArrayList(queue) + if (strategy.scheduleImmediately(this, pending)) + schedule() + } + + /** + * Notify the scheduler that a task has finished execution + * @param handler + */ + void taskFinished(K8sTaskHandler handler) { + strategy.taskFinished(handler) + + /* If we have pending tasks, now would be a good time to schedule a new one. + * Because resources were freed right now */ + if (queue.size() > 0) + schedule() + } + + protected synchronized void schedule() { + final pending = new ArrayList(queue) + final decision = strategy.schedule(this, pending) + + if (!decision) + return + + if (!queue.remove(decision.request)) { + log.warn "[K8s] failed to remove selected task from queue ${decision.request.task.name}" + return + } + + log.info "[K8s] launching queued task ${decision.request.task.name} on node: ${decision.nodeName}" + decision.request.handler.submitNow(decision.nodeName) + } + + /* Scheduling Strategy Interface */ + List getNodes() { + return nodes.toList() + } + + /** + * Run is the scheduler threads main function + * */ + void run() { + this.shouldStop = false + final interval = this.interval + if (interval.toMillis() == 0L) { + log.info("[K8s] scheduler loop disabled (interval is 0)") + return + } + log.info("[K8s] launched scheduler loop (${interval.toString()} interval)") + while (!shouldStop) { + sleep(interval.toMillis()) + schedule() + } + log.info("[K8s] terminated scheduler loop") + } + + /** + * Stop terminates the scheduler thread + */ + void stop() { + log.info("[K8s] stopping scheduler loop") + this.shouldStop = true + } +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sDvfsFactory.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sWrapperBuilder.groovy similarity index 59% rename from k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sDvfsFactory.groovy rename to k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sWrapperBuilder.groovy index 4779973..6c9cf06 100644 --- a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sDvfsFactory.groovy +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sWrapperBuilder.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2025, Seqera Labs + * Copyright 2013-2026, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,21 +16,23 @@ package recreationaltech.plugin + import groovy.transform.CompileStatic -import nextflow.Session -import nextflow.trace.TraceObserver -import nextflow.trace.TraceObserverFactory +import nextflow.executor.BashWrapperBuilder +import nextflow.processor.TaskRun +import nextflow.util.Escape /** - * Implements a factory object required to create - * the {@link K8sDvfsObserver} instance. + * Implements a BASH wrapper for tasks executed by kubernetes cluster + * + * @author Paolo Di Tommaso */ @CompileStatic -class K8sDvfsFactory implements TraceObserverFactory { +class K8sWrapperBuilder extends BashWrapperBuilder { - @Override - Collection create(Session session) { - return List.of(new K8sDvfsObserver()) + K8sWrapperBuilder(TaskRun task) { + super(task) + this.headerScript = "NXF_CHDIR=${Escape.path(task.workDir)}" } } diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/cli/KubeCommandImpl.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/cli/KubeCommandImpl.groovy new file mode 100644 index 0000000..ed6fb91 --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/cli/KubeCommandImpl.groovy @@ -0,0 +1,49 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin.cli + +import groovy.transform.CompileStatic +import nextflow.cli.CmdKubeRun +import recreationaltech.plugin.K8sDriverLauncher + +/** + * Kuberun command implementation logic + * + * @author Paolo Di Tommaso + */ +@CompileStatic +class KubeCommandImpl implements CmdKubeRun.KubeCommand { + + @Override + int run(CmdKubeRun cmd, String pipeline, List args) { + // create + final driver = new K8sDriverLauncher( + cmd: cmd, + runName: cmd.runName, + headImage: cmd.headImage, + background: cmd.background(), + headCpus: cmd.headCpus, + headMemory: cmd.headMemory, + headPreScript: cmd.headPreScript, + plugins: cmd.plugins ) + // run it + driver.run(pipeline, args) + // return exit code + return driver.shutdown() + } + +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/client/ClientConfig.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/client/ClientConfig.groovy new file mode 100644 index 0000000..fda3d56 --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/client/ClientConfig.groovy @@ -0,0 +1,177 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin.client + +import groovy.util.logging.Slf4j +import nextflow.util.Duration + +import javax.net.ssl.KeyManager +import java.nio.file.Path +import java.nio.file.Paths + +import groovy.transform.CompileStatic +import groovy.transform.EqualsAndHashCode +/** + * Models the kubernetes cluster client configuration settings + * + * @author Paolo Di Tommaso + */ +@EqualsAndHashCode +@CompileStatic +@Slf4j +class ClientConfig { + + boolean verifySsl + + String server + + String namespace + + /** + * k8s service account name + * https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + */ + String serviceAccount + + String token + + byte[] sslCert + + byte[] clientCert + + byte[] clientKey + + KeyManager[] keyManagers + + K8sRetryConfig retryConfig + + /** + * Timeout when reading from Input stream when a connection is established to a resource. + * If the timeout expires before there is data available for read, a {@link java.net.SocketTimeoutException} is raised + */ + Duration httpReadTimeout + + /** + * Timeout when opening a communications link to the resource referenced by K8sClient request connection + * If the timeout expires before there is data available for read, a {@link java.net.SocketTimeoutException} is raised + */ + Duration httpConnectTimeout + + /** + * When true signal that the configuration was retrieved from within a K8s cluster + */ + boolean isFromCluster + + String getNamespace() { namespace ?: 'default' } + + ClientConfig() { + retryConfig = new K8sRetryConfig() + } + + String toString() { + "${this.class.getSimpleName()}[ server=$server, namespace=$namespace, serviceAccount=$serviceAccount, token=${cut(token)}, sslCert=${cut(sslCert)}, clientCert=${cut(clientCert)}, clientKey=${cut(clientKey)}, verifySsl=$verifySsl, fromFile=$isFromCluster, httpReadTimeout=$httpReadTimeout, httpConnectTimeout=$httpConnectTimeout, retryConfig=$retryConfig ]" + } + + private String cut(String str) { + if( !str ) return '-' + return str.size()<10 ? str : str[0..10].toString() + '..' + } + + private String cut(byte[] bytes) { + if( !bytes ) return '-' + cut(bytes.encodeBase64().toString()) + } + + static ClientConfig discover(String context, String namespace, String serviceAccount) { + new ConfigDiscovery().discover(context, namespace, serviceAccount) + } + + static ClientConfig fromNextflowConfig(Map opts, String namespace, String serviceAccount) { + final result = new ClientConfig() + + if( opts.server ) + result.server = opts.server + + if( opts.token ) + result.token = opts.token + else if( opts.tokenFile ) + result.token = Paths.get(opts.tokenFile.toString()).getText('UTF-8') + + result.namespace = namespace ?: opts.namespace ?: 'default' + + result.serviceAccount = serviceAccount ?: 'default' + + if( opts.verifySsl ) + result.verifySsl = opts.verifySsl as boolean + + if( opts.sslCert ) + result.sslCert = opts.sslCert.toString().decodeBase64() + else if( opts.sslCertFile ) + result.sslCert = Paths.get(opts.sslCertFile.toString()).bytes + + if( opts.clientCert ) + result.clientCert = opts.clientCert.toString().decodeBase64() + else if( opts.clientCertFile ) + result.clientCert = Paths.get(opts.clientCertFile.toString()).bytes + + if( opts.clientKey ) + result.clientKey = opts.clientKey.toString().decodeBase64() + else if( opts.clientKeyFile ) + result.clientKey = Paths.get(opts.clientKeyFile.toString()).bytes + + return result + } + + static ClientConfig fromUserAndCluster(Map user, Map cluster, Path location) { + final base = location.isDirectory() ? location : location.parent + final result = new ClientConfig() + if( user.token ) + result.token = user.token + + else if( user.tokenFile ) { + result.token = Paths.get(user.tokenFile.toString()).getText('UTF-8') + } + + if( user."client-certificate" ) + result.clientCert = base.resolve(user."client-certificate".toString()).bytes + + else if( user."client-certificate-data" ) + result.clientCert = user."client-certificate-data".toString().decodeBase64() + + if( user."client-key" ) + result.clientKey = base.resolve(user."client-key".toString()).bytes + + else if( user."client-key-data" ) + result.clientKey = user."client-key-data".toString().decodeBase64() + + // -- cluster settings + + if( cluster.server ) + result.server = cluster.server + + if( cluster."certificate-authority-data" ) + result.sslCert = cluster."certificate-authority-data".toString().decodeBase64() + + else if( cluster."certificate-authority" ) + result.sslCert = base.resolve(cluster."certificate-authority".toString()).bytes + + result.verifySsl = cluster."insecure-skip-tls-verify" != true + + return result + } + +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/client/ConfigDiscovery.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/client/ConfigDiscovery.groovy new file mode 100644 index 0000000..948ed4b --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/client/ConfigDiscovery.groovy @@ -0,0 +1,192 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin.client + +import javax.net.ssl.KeyManager +import javax.net.ssl.KeyManagerFactory +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.security.KeyStore +import static nextflow.util.StringUtils.formatHostName + +import groovy.util.logging.Slf4j +import org.yaml.snakeyaml.Yaml +/** + * Discover Kubernetes configuration from system environment + * + * @author Paolo Di Tommaso + */ +@Slf4j +class ConfigDiscovery { + + private Map env = System.getenv() + + ConfigDiscovery() { } + + /** + * Discover Kubernetes client configuration from current environment using + * either the .kube/config file or the pod service service account virtual + * file system when running in a pod. + * + * @param contextName The K8s config context name. + * @param namespace The K8s cluster namespace + * @param serviceAccount The K8s cluster service account + * @return The e + */ + ClientConfig discover(String contextName, String namespace, String serviceAccount) { + + // Note: System.getProperty('user.home') may not report the correct home path when + // running in a container. Use env HOME instead. + def home = System.getenv('HOME') + def kubeConfig = env.get('KUBECONFIG') ? env.get('KUBECONFIG') : "$home/.kube/config" + def configFile = Paths.get(kubeConfig) + + // determine the Kubernetes client configuration via the `.kube/config` file + if( configFile.exists() ) { + return fromKubeConfig(configFile, contextName, namespace, serviceAccount) + } + else { + log.debug "K8s config file does not exist: $configFile" + } + + // determine the Kubernetes client configuration via the pod environment + if( env.get('KUBERNETES_SERVICE_HOST') ) { + return fromCluster(env, namespace, serviceAccount) + } + else { + log.debug "K8s env variable KUBERNETES_SERVICE_HOST is not defined" + } + + throw new IllegalStateException("Unable to lookup Kubernetes cluster configuration") + } + + protected ClientConfig fromCluster(Map env, String cfgNamespace, String serviceAccount) { + + // See https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/#accessing-the-api-from-a-pod + + final host = env.get('KUBERNETES_SERVICE_HOST') + final port = env.get('KUBERNETES_SERVICE_PORT') + final server = formatHostName(host, port) + + final cert = path('/var/run/secrets/kubernetes.io/serviceaccount/ca.crt').bytes + final token = path('/var/run/secrets/kubernetes.io/serviceaccount/token').text + final namespace = path('/var/run/secrets/kubernetes.io/serviceaccount/namespace').text + + return new ClientConfig( + server: server, + token: token, + namespace: cfgNamespace ?: namespace, + serviceAccount: serviceAccount, + sslCert: cert, + isFromCluster: true ) + } + + protected Path path(String path) { + Paths.get(path) + } + + protected ClientConfig fromKubeConfig(Path path, String contextName, String namespace, String serviceAccount) { + def yaml = (Map)new Yaml().load(Files.newInputStream(path)) + + contextName ?= yaml."current-context" as String + + final allContext = yaml.contexts as List + final allClusters = yaml.clusters as List + final allUsers = yaml.users as List + final context = allContext.find { Map it -> it.name == contextName } ?.context + if( !context ) + throw new IllegalArgumentException("Unknown Kubernetes context: $contextName -- check config file: $path") + final userName = context?.user + final clusterName = context?.cluster + final user = allUsers.find{ Map it -> it.name == userName } ?.user ?: [:] + final cluster = allClusters.find{ Map it -> it.name == clusterName } ?.cluster ?: [:] + + final config = ClientConfig.fromUserAndCluster(user, cluster, path) + + // the namespace provided should have priority over the context current namespace + config.namespace = namespace ?: context?.namespace ?: 'default' + + config.serviceAccount = serviceAccount ?: 'default' + + if( config.clientCert && config.clientKey ) { + config.keyManagers = createKeyManagers(config.clientCert, config.clientKey) + } + else if( !config.token ) { + config.token = discoverAuthToken(contextName, config.namespace, config.serviceAccount) + } + + return config + } + + protected KeyStore createKeyStore0(byte[] clientCert, byte[] clientKey, char[] passphrase, String alg) { + def cert = new ByteArrayInputStream(clientCert) + def key = new ByteArrayInputStream(clientKey) + return SSLUtils.createKeyStore(cert, key, alg, passphrase, null, null) + } + + protected KeyStore createKeyStore(byte[] clientCert, byte[] clientKey, char[] passphrase) { + try { + // try first RSA algorithm + return createKeyStore0(clientCert, clientKey, passphrase, "RSA") + } + catch (Exception e1) { + // fallback to EC algorithm + try { + return createKeyStore0(clientCert, clientKey, passphrase, "EC") + } + catch (Exception e2) { + // if still fails, throws the first exception + throw e1 + } + } + } + + protected KeyManager[] createKeyManagers(byte[] clientCert, byte[] clientKey) { + final passphrase = "".toCharArray() + final keyStore = createKeyStore(clientCert, clientKey, passphrase) + final kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + kmf.init(keyStore, passphrase); + return kmf.getKeyManagers(); + } + + String discoverAuthToken(String context, String namespace, String serviceAccount) { + context ?= 'default' + namespace ?= 'default' + serviceAccount ?= 'default' + + final cmd = "kubectl --context $context -n ${namespace} get secret -o=jsonpath='{.items[?(@.metadata.annotations.kubernetes\\.io/service-account\\.name==\"$serviceAccount\")].data.token}'" + final proc = new ProcessBuilder('bash','-o','pipefail','-c', cmd).start() + final status = proc.waitFor() + final text = proc.inputStream?.text + if( status==0 && text ) { + try { + return new String(text.trim().decodeBase64()) + } + catch( Exception e ) { + log.warn "Unable to decode K8s cluster auth token '$text' -- cause: ${e.message}" + } + } + else { + final cause = proc.errorStream?.text ?: text + final msg = cause ? "\n- cmd : $cmd\n- exit : $status\n- cause:\n${cause.indent(' ')}" : '' + log.warn "[K8s] unable to fetch auth token ${msg}" + } + return null + } + +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/client/K8sClient.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/client/K8sClient.groovy new file mode 100644 index 0000000..cf991d0 --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/client/K8sClient.groovy @@ -0,0 +1,1246 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin.client + +import dev.failsafe.Failsafe +import dev.failsafe.FailsafeException +import dev.failsafe.RetryPolicy +import dev.failsafe.event.EventListener +import dev.failsafe.event.ExecutionAttemptedEvent +import dev.failsafe.function.CheckedSupplier +import nextflow.exception.K8sOutOfCpuException +import nextflow.exception.K8sOutOfMemoryException + +import javax.net.ssl.HostnameVerifier +import javax.net.ssl.HttpsURLConnection +import javax.net.ssl.SSLContext +import javax.net.ssl.SSLSession +import javax.net.ssl.TrustManager +import javax.net.ssl.TrustManagerFactory +import javax.net.ssl.X509TrustManager +import java.nio.file.Path +import java.security.KeyStore +import java.security.SecureRandom +import java.security.cert.CertificateFactory +import java.security.cert.X509Certificate + +import groovy.json.JsonOutput +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.exception.NodeTerminationException +import nextflow.exception.ProcessFailedException +import org.yaml.snakeyaml.Yaml + +import java.time.temporal.ChronoUnit +import java.util.concurrent.TimeoutException +import java.util.function.Predicate + +/** + * Kubernetes API client + * + * Tip: use the following command to find out your kubernetes master node + * kubectl cluster-info + * + * See + * https://v1-8.docs.kubernetes.io/docs/api-reference/v1.8/#-strong-api-overview-strong- + * + * Useful cheatsheet + * https://kubernetes.io/docs/reference/kubectl/cheatsheet/ + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +class K8sClient { + + protected ClientConfig config + + private TrustManager[] trustManagers + + private HostnameVerifier hostnameVerifier + + K8sClient() { + this(new ClientConfig()) + } + + ClientConfig getConfig() { config } + + /** + * Creates a kubernetes client using the configuration setting provided by the specified + * {@link ConfigDiscovery} instance + * + * @param config + */ + K8sClient(ClientConfig config) { + this.config = config + setupSslCert() + } + + + protected setupSslCert() { + + if( !config.verifySsl ) { + // -- no SSL is required - use fake trust manager + final trustAll = new X509TrustManager() { + @Override X509Certificate[] getAcceptedIssuers() { return null } + @Override void checkClientTrusted(X509Certificate[] certs, String authType) { } + @Override void checkServerTrusted(X509Certificate[] certs, String authType) { } + } + + trustManagers = [trustAll] as TrustManager[] + hostnameVerifier = new HostnameVerifier() { + @Override boolean verify(String hostname, SSLSession session) { return true } + } + + } + else if ( config.sslCert != null) { + char[] password = null + final factory = CertificateFactory.getInstance("X.509"); + final authority = new ByteArrayInputStream(config.sslCert) + final certificates = factory.generateCertificates(authority) + if (certificates.isEmpty()) { + throw new IllegalArgumentException("Trusted certificates set cannot be empty"); + } + + final keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); + keyStore.load(null, password); + certificates.eachWithIndex{ cert, index -> + String alias = "ca$index" + keyStore.setCertificateEntry(alias, cert); + } + + final trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + trustManagerFactory.init(keyStore); + trustManagers = trustManagerFactory.getTrustManagers(); + } + } + + K8sResponseJson secretesList() { + final action = "/api/v1/namespaces/$config.namespace/secrets" + final resp = get(action) + trace('GET', action, resp.text) + new K8sResponseJson(resp.text) + } + + K8sResponseJson secretDescribe(String name) { + assert name + final action = "/api/v1/namespaces/$config.namespace/secrets/$name" + final resp = get(action) + trace('GET', action, resp.text) + new K8sResponseJson(resp.text) + } + + /** + * Create a pod + * + * See + * https://v1-8.docs.kubernetes.io/docs/api-reference/v1.8/#create-55 + * https://v1-8.docs.kubernetes.io/docs/api-reference/v1.8/#pod-v1-core + * + * @param spec + * @return + */ + K8sResponseJson podCreate(String req) { + assert req + final action = "/api/v1/namespaces/$config.namespace/pods" + final resp = post(action, req) + trace('POST', action, resp.text) + return new K8sResponseJson(resp.text) + } + + K8sResponseJson podCreate(Map req, Path saveYamlPath=null) { + + if( saveYamlPath ) try { + saveYamlPath.text = new Yaml().dump(req).toString() + } + catch( Exception e ) { + log.debug "WARN: unable to save request yaml -- cause: ${e.message ?: e}" + } + + podCreate(JsonOutput.toJson(req)) + } + + /** + * Delete a pod + * + * See + * https://v1-8.docs.kubernetes.io/docs/api-reference/v1.8/#delete-58 + * + * @param name + * @return + */ + K8sResponseJson podDelete(String name) { + assert name + final action = "/api/v1/namespaces/$config.namespace/pods/$name" + final resp = delete(action) + trace('DELETE', action, resp.text) + new K8sResponseJson(resp.text) + } + + /** + * Create a job + * + * See + * https://v1-8.docs.kubernetes.io/docs/api-reference/v1.8/#create-55 + * https://v1-8.docs.kubernetes.io/docs/api-reference/v1.8/#job-v1-batch + * + * @param spec + * @return + */ + K8sResponseJson jobCreate(String req) { + assert req + final action = "/apis/batch/v1/namespaces/$config.namespace/jobs" + final resp = post(action, req) + trace('POST', action, resp.text) + return new K8sResponseJson(resp.text) + } + + K8sResponseJson jobCreate(Map req, Path saveYamlPath=null) { + + if( saveYamlPath ) try { + saveYamlPath.text = new Yaml().dump(req).toString() + } + catch( Exception e ) { + log.debug "WARN: unable to save request yaml -- cause: ${e.message ?: e}" + } + + jobCreate(JsonOutput.toJson(req)) + } + + /** + * Delete a job + * + * See + * https://v1-8.docs.kubernetes.io/docs/api-reference/v1.8/#delete-58 + * + * @param name + * @return + */ + + K8sResponseJson jobDelete(String name) { + assert name + + // get podList of a job + final action = "/api/v1/namespaces/$config.namespace/pods?labelSelector=job-name=$name" + final resp = get(action) + trace('GET', action, resp.text) + final podList = new K8sResponseJson(resp.text) + + // delete all pods in a job + if (podList.kind == "PodList") { + for (item in podList.items) { + try { + podDelete(((item as Map).metadata as Map).name as String) + } + catch(K8sResponseException err) { + if( err.response.code == 404 ) + log.debug("Unable to delete Pod for job $name, pod already gone") + else + throw err + } + } + } + + // delete job + final action1 = "/apis/batch/v1/namespaces/$config.namespace/jobs/$name" + final resp1 = delete(action1) + trace('DELETE', action1, resp1.text) + new K8sResponseJson(resp1.text) + } + + /* + * https://v1-8.docs.kubernetes.io/docs/api-reference/v1.8/#list-62 + * https://v1-8.docs.kubernetes.io/docs/api-reference/v1.8/#list-all-namespaces-63 + */ + + K8sResponseJson jobList(boolean allNamespaces=false) { + final String action = allNamespaces ? "jobs" : "namespaces/$config.namespace/jobs" + final resp = get("/apis/batch/v1/$action") + trace('GET', action, resp.text) + new K8sResponseJson(resp.text) + } + + /* + * https://v1-8.docs.kubernetes.io/docs/api-reference/v1.8/#list-62 + * https://v1-8.docs.kubernetes.io/docs/api-reference/v1.8/#list-all-namespaces-63 + */ + K8sResponseJson podList(boolean allNamespaces=false) { + final String action = allNamespaces ? "pods" : "namespaces/$config.namespace/pods" + final resp = get("/api/v1/$action") + trace('GET', action, resp.text) + new K8sResponseJson(resp.text) + } + + /* + * https://v1-8.docs.kubernetes.io/docs/api-reference/v1.8/#read-status-69 + */ + + // converts the name of job to the name of the latest created pod + String findPodNameForJob(String name){ + final action = "/api/v1/namespaces/$config.namespace/pods?labelSelector=job-name=$name" + final resp = get(action) + trace('GET', action, resp.text) + final podList = new K8sResponseJson(resp.text) + + String podName + + // find latest created pod + if (podList.kind == "PodList") { + final pods = podList.items + String latestPod = "0000-00-00T00:00:00Z" + + for (item in pods) { + final podMetadata = (Map) (item as Map).metadata + final podTimestamp = podMetadata.creationTimestamp + + if (podTimestamp.toString() > latestPod) { + latestPod = podTimestamp + podName = podMetadata.name + } + } + } + return podName + } + + K8sResponseJson jobStatus(String name) { + assert name + final action = "/apis/batch/v1/namespaces/$config.namespace/jobs/$name/status" + final resp = get(action) + trace('GET', action, resp.text) + return new K8sResponseJson(resp.text) + } + + K8sResponseJson podStatus(String name) { + assert name + final action = "/api/v1/namespaces/$config.namespace/pods/$name/status" + final resp = get(action) + trace('GET', action, resp.text) + return new K8sResponseJson(resp.text) + } + + String getPodIpAddress(String podName) { + assert podName + final K8sResponseJson resp = podStatus0(podName) + (resp?.status as Map)?.podIP as String + } + + /* + * https://v1-8.docs.kubernetes.io/docs/api-reference/v1.8/#read-status-69 + */ + + protected K8sResponseJson podStatus0(String name) { + try { + return podStatus(name) + } + catch (K8sResponseException err) { + if( err.response.code == 404 && isKindPods(err.response) ) { + // this may happen when K8s node is shutdown and the pod is evicted + // therefore process exception is thrown so that the failure + // can be managed by the nextflow as re-triable execution + throw new NodeTerminationException("Unable to find pod $name - The pod may be evicted by a node shutdown event") + } + throw err + } + } + + protected boolean isKindPods(K8sResponseJson resp) { + if( resp.details instanceof Map ) { + final details = (Map) resp.details + return details.kind == 'pods' + } + return false + } + + String getNodeOfPod(String podName){ + assert podName + final K8sResponseJson resp = podStatus0(podName) + (resp?.spec as Map)?.nodeName as String + } + + /** + * Get pod current state object + * + * @param podName The pod name + * @return + * A {@link Map} representing the container state object as shown below + * + * { + * "terminated": { + * "exitCode": 127, + * "reason": "ContainerCannotRun", + * "message": "OCI runtime create failed: container_linux.go:296: starting container process caused \"exec: \\\"bash\\\": executable file not found in $PATH\": unknown", + * "startedAt": "2018-01-12T22:04:25Z", + * "finishedAt": "2018-01-12T22:04:25Z", + * "containerID": "docker://730ef2e05be72ffc354f2682b4e8300610812137b9037b726c21e5c4e41b6dda" + * } + * + * See the following link for details https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#containerstate-v1-core + * An empty map is return if the pod is a `Pending` status and the container state is not + * yet available + * + * + */ + Map jobState( String jobName ) { + assert jobName + final podName = findPodNameForJob(jobName) + if( podName ) { + try { + return podState(podName) + } + /* pod might be deleted by control plane just after findPodNameForJob() call + * so try fallback to jobState + */ + catch (NodeTerminationException err) { + log.warn1("Job $jobName's Pod not found, probably cleaned by controlplane") + return jobStateFallback0(jobName) + } + } + else { + return jobStateFallback0(jobName) + } + } + + protected Map jobStateFallback0(String jobName) { + final K8sResponseJson jobResp = jobStatus(jobName) + final jobStatus = jobResp.status as Map + if( jobStatus?.succeeded == 1 && jobStatus.conditions instanceof List ) { + final allConditions = jobStatus.conditions as List + final cond = allConditions.find { cond -> cond.type == 'Complete' } + + if( cond?.status == 'True' ) { + log.warn1("Job $jobName already completed and Pod is gone") + final dummyPodStatus = [ + terminated: [ + reason: "Completed", + startedAt: jobStatus.startTime, + finishedAt: jobStatus.completionTime, + ] + ] + return dummyPodStatus + } else { + throw new ProcessFailedException("K8s Job $jobName succeeded but does not have Complete status. $allConditions") + } + } + + if( jobStatus?.failed && (int)(jobStatus.failed) > 0 ) { + String message = 'unknown' + if( jobStatus.conditions instanceof List ) { + final allConditions = jobStatus.conditions as List + final cond = allConditions.find { cond -> cond.type == 'Failed' } + message = cond?.message + } + throw new ProcessFailedException("K8s Job $jobName execution failed: $message") + } + + log.debug1("K8s Job $jobName does not have pod - Not yet scheduled?") + return Collections.emptyMap() + } + + /** + * Get pod current state object + * + * @param podName The pod name + * @return + * A {@link Map} representing the container state object as shown below + * + * { + * "terminated": { + * "exitCode": 127, + * "reason": "ContainerCannotRun", + * "message": "OCI runtime create failed: container_linux.go:296: starting container process caused \"exec: \\\"bash\\\": executable file not found in $PATH\": unknown", + * "startedAt": "2018-01-12T22:04:25Z", + * "finishedAt": "2018-01-12T22:04:25Z", + * "containerID": "docker://730ef2e05be72ffc354f2682b4e8300610812137b9037b726c21e5c4e41b6dda" + * } + * + * See the following link for details https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#containerstate-v1-core + * An empty map is return if the pod is a `Pending` status and the container state is not + * yet available + * + * + */ + Map podState( String podName ) { + assert podName + + final K8sResponseJson resp = podStatus0(podName) + final status = resp.status as Map + final containerStatuses = status?.containerStatuses as List + + if( containerStatuses?.size()>0 ) { + final container = containerStatuses.get(0) + // note: when the pod is created by a Job submission + // the `podName` does not match the container name because it + // contains a suffix random generated by K8s pod scheduler + if( !container.name || !podName.startsWith(container.name.toString()) ) + throw new K8sResponseException("K8s invalid status for pod: ${podName} (unexpected container name: ${container.name})", resp) + + if( !container.state ) + throw new K8sResponseException("K8s invalid status for pod: ${podName} (missing state object)", resp) + + final state = container.state as Map + if( state.waiting instanceof Map ) { + def waiting = state.waiting as Map + checkInvalidWaitingState(waiting, resp) + } + return state + } + + if( status?.phase == 'Pending' ){ + if( status.conditions instanceof List ) { + final allConditions = status.conditions as List + final cond = allConditions.find { cond -> cond.type == 'PodScheduled' } + if( cond?.reason == 'Unschedulable' ) { + def message = "K8s pod cannot be scheduled" + if( cond.message ) message += " -- $cond.message" + //def cause = new K8sResponseException(resp) + log.warn1(message) + } + } + // undetermined status -- return an empty response + return Collections.emptyMap() + } + + if( status?.phase == 'Failed' ) { + def msg = "K8s pod '$podName' execution failed" + if( status.reason ) msg += " - reason: ${status.reason}" + if( status.message ) msg += " - message: ${status.message}" + switch ( status.reason ) { + case 'OutOfcpu': throw new K8sOutOfCpuException(msg) + case 'OutOfmemory': throw new K8sOutOfMemoryException(msg) + case 'Shutdown': throw new NodeTerminationException(msg) + default: throw new ProcessFailedException(msg) + } + } + + throw new K8sResponseException("K8s undetermined status conditions for pod $podName", resp) + } + + /** + * Get list of all nodes in the cluster + * @return Response object. + */ + K8sResponseJson nodeList() { + final action = "/api/v1/nodes" + final resp = get(action) + trace('GET', action, resp.text) + new K8sResponseJson(resp.text) + } + + /** + * Get a specific node by name + * @param name The node name + * @return Response object containing the node details. + */ + K8sResponseJson nodeDescribe(String name) { + assert name + final action = "/api/v1/nodes/$name" + final resp = get(action) + trace('GET', action, resp.text) + new K8sResponseJson(resp.text) + } + + /** + * Query the CPU capacity of a node + * @param nodeName The name of the node + * @return The CPU capacity in cores (as a String representing the quantity, e.g., "4", "2.5") + */ + String getNodeCpuCapacity(String nodeName) { + assert nodeName + final resp = nodeDescribe(nodeName) + final status = resp.status as Map + final capacity = status?.capacity as Map + capacity?.cpu as String + } + + /** + * Query the currently used CPU of a node + * Note: This requires the metrics server to be installed in the cluster. + * Falls back to the allocated CPU from the node's allocated resources if metrics are not available. + * @param nodeName The name of the node + * @return The used CPU in cores (as a String representing the quantity, e.g., "1", "0.5") + */ + String getNodeCpuUsed(String nodeName) { + assert nodeName + + // First, try to get metrics from the metrics server + try { + final action = "/apis/metrics.k8s.io/v1beta1/nodes/$nodeName" + final resp = get(action) + trace('GET', action, resp.text) + final metrics = new K8sResponseJson(resp.text) + final usage = metrics.usage as Map + return usage?.cpu as String + } + catch (Exception e) { + // Fall back to allocated CPU from node status + log.debug("Metrics server not available or error fetching metrics for node $nodeName, falling back to allocated resources: ${e.message}") + final nodeResp = nodeDescribe(nodeName) + final status = nodeResp.status as Map + final allocatable = status?.allocatable as Map + final allocated = status?.allocated as Map + + // Calculate used as allocatable minus available (if we had that info) + // For now, return the allocated CPU if available + if (allocated?.cpu) { + return allocated.cpu as String + } + + // If we can't get used metrics, return null + return null + } + } + + /** + * Query the memory capacity of a node + * @param nodeName The name of the node + * @return The memory capacity in bytes (as a String representing the quantity, e.g., "16Gi", "8192Mi") + */ + String getNodeMemoryCapacity(String nodeName) { + assert nodeName + final resp = nodeDescribe(nodeName) + final status = resp.status as Map + final capacity = status?.capacity as Map + capacity?.memory as String + } + + /** + * Query the currently used memory of a node + * Note: This requires the metrics server to be installed in the cluster. + * Falls back to the allocated memory from the node's allocated resources if metrics are not available. + * @param nodeName The name of the node + * @return The used memory in bytes (as a String representing the quantity, e.g., "4Gi", "2048Mi") + */ + String getNodeMemoryUsed(String nodeName) { + assert nodeName + + // First, try to get metrics from the metrics server + try { + final action = "/apis/metrics.k8s.io/v1beta1/nodes/$nodeName" + final resp = get(action) + trace('GET', action, resp.text) + final metrics = new K8sResponseJson(resp.text) + final usage = metrics.usage as Map + return usage?.memory as String + } + catch (Exception e) { + // Fall back to allocated memory from node status + log.debug("Metrics server not available or error fetching metrics for node $nodeName, falling back to allocated resources: ${e.message}") + final nodeResp = nodeDescribe(nodeName) + final status = nodeResp.status as Map + final allocated = status?.allocated as Map + + // Return the allocated memory if available + if (allocated?.memory) { + return allocated.memory as String + } + + // If we can't get used metrics, return null + return null + } + } + + /** + * Parse a Kubernetes quantity string (e.g., "16Gi", "8192Mi", "1024Ki", "1500m") into a long value. + * For memory: supports binary (Ki, Mi, Gi, Ti, Pi, Ei) and the value is returned in bytes. + * For CPU: supports millicores (e.g., "1500m" = 1.5 cores = 1500 millicores) and cores (e.g., "2" = 2000 millicores). + * + * @param quantity The Kubernetes quantity string (e.g., "16Gi", "1500m", "2") + * @param forCpu If true, parses as CPU (millicores); if false, parses as memory (bytes) + * @return The value as a Long, or null if unparseable + */ + static Long parseK8sQuantity(String quantity, boolean forCpu=false) { + if (!quantity) return null + + // Reject quantities with spaces - Kubernetes quantity strings don't have spaces + if (quantity.contains(' ')) { + return null + } + + // Extract numeric value and suffix separately + // Kubernetes quantities: https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/#quantity + def suffix = null + def numericStr = quantity + + // Try to extract suffix - check for the longest suffixes first + // Binary: KiB, MiB, GiB, TiB, PiB, EiB, Ki, Mi, Gi, Ti, Pi, Ei + // Decimal: KB, MB, GB, TB, PB, EB, K, M, G, T, P, E + // CPU: m (millicores), n (nanocores) + def suffixes = ['kib', 'mib', 'gib', 'tib', 'pib', 'eib', + 'ki', 'mi', 'gi', 'ti', 'pi', 'ei', + 'kb', 'mb', 'gb', 'tb', 'pb', 'eb', + 'k', 'm', 'g', 't', 'p', 'e'] + + // For CPU, we only care about 'm' (millicores), 'u' (microcores) and 'n' (nanocores) suffixes + if (forCpu) { + suffixes = ['m', 'u', 'n'] + } + + for (def s : suffixes) { + if (quantity.toLowerCase().endsWith(s)) { + suffix = s + numericStr = quantity.substring(0, quantity.length() - s.length()) + break + } + } + + // If we couldn't extract a suffix, try to parse the whole thing as a number + // Validate that numericStr only contains valid numeric characters + if (numericStr && !numericStr.matches(/^[+-]?[0-9]*(\.[0-9]*)?$/)) { + return null + } + + def numericValue + try { + numericValue = numericStr.toDouble() + } catch (Exception e) { + return null + } + + if (forCpu) { + // CPU: handle millicores (m), microcores (u) and nanocores (n) suffixes + if (suffix == 'm') { + // Already in millicores + return (long)(numericValue) + } else if (suffix == 'n') { + // Nanocores: convert to millicores (1 millicore = 1,000,000 nanocores) + return (long)(numericValue / 1000000.0) + } else if (suffix == 'u') { + // Microcores: convert to millicores (1 millicore = 1,000 microcores) + return (long)(numericValue / 1000) + } else { + // Default is cores, convert to millicores + return (long)(numericValue * 1000) + } + } else { + // Memory: convert to bytes + def multiplier = 1L + + if (suffix == null || suffix == '') { + // No suffix - assume bytes + return (long)numericValue + } + + // Binary suffixes (Ki, Mi, Gi, Ti, Pi, Ei) + switch (suffix) { + case 'ki': case 'kib': + multiplier = 1024L + break + case 'mi': case 'mib': + multiplier = 1024L * 1024L + break + case 'gi': case 'gib': + multiplier = 1024L * 1024L * 1024L + break + case 'ti': case 'tib': + multiplier = 1024L * 1024L * 1024L * 1024L + break + case 'pi': case 'pib': + multiplier = 1024L * 1024L * 1024L * 1024L * 1024L + break + case 'ei': case 'eib': + multiplier = 1024L * 1024L * 1024L * 1024L * 1024L * 1024L + break + // Decimal suffixes (K, M, G, T, P, E) - Kubernetes uses binary by default + // but we support decimal for completeness + case 'k': case 'kb': + multiplier = 1000L + break + case 'm': case 'mb': + multiplier = 1000L * 1000L + break + case 'g': case 'gb': + multiplier = 1000L * 1000L * 1000L + break + case 't': case 'tb': + multiplier = 1000L * 1000L * 1000L * 1000L + break + case 'p': case 'pb': + multiplier = 1000L * 1000L * 1000L * 1000L * 1000L + break + case 'e': case 'eb': + multiplier = 1000L * 1000L * 1000L * 1000L * 1000L * 1000L + break + } + + return (long)(numericValue * multiplier) + } + } + + /** + * Query the CPU capacity of a node in millicores (1 core = 1000 millicores) + * @param nodeName The name of the node + * @return The CPU capacity in millicores as a Long, or null if unparseable + */ + Long getNodeCpuCapacityMillis(String nodeName) { + assert nodeName + final capacity = getNodeCpuCapacity(nodeName) + if (!capacity) { + log.debug("Node $nodeName CPU capacity not available") + return null + } + final result = parseK8sQuantity(capacity, true) + if (result == null) { + log.warn("Unable to parse CPU capacity for node $nodeName: '$capacity'") + } + return result + } + + /** + * Query the currently used CPU of a node in millicores (1 core = 1000 millicores) + * @param nodeName The name of the node + * @return The used CPU in millicores as a Long, or null if unparseable + */ + Long getNodeCpuUsedMillis(String nodeName) { + assert nodeName + final used = getNodeCpuUsed(nodeName) + if (!used) { + log.debug("Node $nodeName CPU usage not available (metrics server not installed or node has no allocated resources)") + return null + } + final result = parseK8sQuantity(used, true) + if (result == null) { + log.warn("Unable to parse CPU usage for node $nodeName: '$used'") + } + return result + } + + /** + * Query the memory capacity of a node in bytes + * @param nodeName The name of the node + * @return The memory capacity in bytes as a Long, or null if unparseable + */ + Long getNodeMemoryCapacityBytes(String nodeName) { + assert nodeName + final capacity = getNodeMemoryCapacity(nodeName) + if (!capacity) { + log.debug("Node $nodeName memory capacity not available") + return null + } + final result = parseK8sQuantity(capacity, false) + if (result == null) { + log.warn("Unable to parse memory capacity for node $nodeName: '$capacity'") + } + return result + } + + /** + * Query the currently used memory of a node in bytes + * @param nodeName The name of the node + * @return The used memory in bytes as a Long, or null if unparseable + */ + Long getNodeMemoryUsedBytes(String nodeName) { + assert nodeName + final used = getNodeMemoryUsed(nodeName) + if (!used) { + log.debug("Node $nodeName memory usage not available (metrics server not installed or node has no allocated resources)") + return null + } + final result = parseK8sQuantity(used, false) + if (result == null) { + log.warn("Unable to parse memory usage for node $nodeName: '$used'") + } + return result + } + + /** + * Query the total CPU allocated to pods on a node (sum of pod requests) + * @param nodeName The name of the node + * @return The allocated CPU in millicores as a Long, or null if unparseable + */ + Long getNodeCpuAllocatedMillis(String nodeName) { + assert nodeName + try { + // List all pods on the node and sum their CPU requests + final action = "/api/v1/pods?fieldSelector=spec.nodeName=$nodeName" + final resp = get(action) + trace('GET', action, resp.text) + final podList = new K8sResponseJson(resp.text) + final items = podList.items as List + + long totalAllocated = 0L + if (items) { + for (Map pod : items) { + final spec = pod.spec as Map + if (spec) { + final containers = spec.containers as List + if (containers) { + for (Map container : containers) { + final resources = container.resources as Map + final requests = resources?.requests as Map + if (requests?.cpu) { + final cpuStr = requests.cpu as String + final cpuMillis = parseK8sQuantity(cpuStr, true) + if (cpuMillis != null) { + totalAllocated += cpuMillis + } + } + } + } + } + } + } + return totalAllocated + } + catch (Exception e) { + log.warn("Failed to get allocated CPU for node $nodeName: ${e.message}") + return null + } + } + + /** + * Query the available CPU on a node (allocatable - allocated) + * For scheduling purposes, we need allocated (requests), not usage (from metrics server) + * @param nodeName The name of the node + * @return The available CPU in millicores as a Long, or null if unparseable + */ + Long getNodeCpuAvailableMillis(String nodeName) { + assert nodeName + final capacity = getNodeCpuCapacityMillis(nodeName) + final allocated = getNodeCpuAllocatedMillis(nodeName) + + if (capacity == null || allocated == null) { + return null + } + return capacity - allocated + } + + /** + * Query the total memory allocated to pods on a node (sum of pod requests) + * @param nodeName The name of the node + * @return The allocated memory in bytes as a Long, or null if unparseable + */ + Long getNodeMemoryAllocatedBytes(String nodeName) { + assert nodeName + try { + // List all pods on the node and sum their memory requests + final action = "/api/v1/pods?fieldSelector=spec.nodeName=$nodeName" + final resp = get(action) + trace('GET', action, resp.text) + final podList = new K8sResponseJson(resp.text) + final items = podList.items as List + + long totalAllocated = 0L + if (items) { + for (Map pod : items) { + final spec = pod.spec as Map + if (spec) { + final containers = spec.containers as List + if (containers) { + for (Map container : containers) { + final resources = container.resources as Map + final requests = resources?.requests as Map + if (requests?.memory) { + final memStr = requests.memory as String + final memBytes = parseK8sQuantity(memStr, false) + if (memBytes != null) { + totalAllocated += memBytes + } + } + } + } + } + } + } + return totalAllocated + } + catch (Exception e) { + log.warn("Failed to get allocated memory for node $nodeName: ${e.message}") + return null + } + } + + /** + * Query the available memory on a node (allocatable - allocated) + * For scheduling purposes, we need allocated (requests), not usage (from metrics server) + * @param nodeName The name of the node + * @return The available memory in bytes as a Long, or null if unparseable + */ + Long getNodeMemoryAvailableBytes(String nodeName) { + assert nodeName + final capacity = getNodeMemoryCapacityBytes(nodeName) + final allocated = getNodeMemoryAllocatedBytes(nodeName) + + if (capacity == null || allocated == null) { + return null + } + return capacity - allocated + } + + protected void checkInvalidWaitingState( Map waiting, K8sResponseJson resp ) { + if( waiting.reason == 'ErrImagePull' || waiting.reason == 'ImagePullBackOff') { + def message = "K8s pod image cannot be pulled" + if( waiting.message ) message += " -- $waiting.message" + final cause = new K8sResponseException(resp) + throw new PodUnschedulableException(message, cause) + } + if( waiting.reason == 'CreateContainerConfigError' ) { + def message = "K8s pod configuration failed" + if( waiting.message ) message += " -- $waiting.message" + final cause = new K8sResponseException(resp) + throw new PodUnschedulableException(message, cause) + } + if( waiting.reason =~ /.+Error$/ ) { + def message = "K8s pod waiting on unknown error state" + if( waiting.message ) message += " -- $waiting.message" + final cause = new K8sResponseException(resp) + throw new PodUnschedulableException(message, cause) + } + final status = resp.status as Map + if( status?.phase == 'Failed' ) { + def message = "K8s pod in Failed state" + final cause = new K8sResponseException(resp) + throw new PodUnschedulableException(message, cause) + } + } + + /* + * https://v1-8.docs.kubernetes.io/docs/api-reference/v1.8/#read-log + */ + InputStream jobLog(String name) { + jobLog( Collections.emptyMap(), name ) + } + + InputStream jobLog(Map params, String name) { + assert name + // -- compose the request action uri + String podName = findPodNameForJob(name) + def action = "/api/v1/namespaces/$config.namespace/pods/$podName/log" + int count=0 + for( String key : (params.keySet()) ) { + action += "${count++==0 ? '?' : '&'}${key}=${params.get(key)}" + } + // -- submit request + def resp = get(action) + resp.stream + } + + /* + * https://v1-8.docs.kubernetes.io/docs/api-reference/v1.8/#read-log + */ + InputStream podLog(String name) { + podLog( Collections.emptyMap(), name ) + } + + InputStream podLog(Map params, String name) { + assert name + // -- compose the request action uri + def action = "/api/v1/namespaces/$config.namespace/pods/$name/log" + int count=0 + for( String key : (params.keySet()) ) { + action += "${count++==0 ? '?' : '&'}${key}=${params.get(key)}" + } + // -- submit request + def resp = get(action) + resp.stream + } + + protected K8sResponseApi post(String path, String spec) { + makeRequest('POST', path, spec) + } + + protected K8sResponseApi delete(String path, String body=null) { + makeRequest('DELETE', path, body) + } + + protected HttpURLConnection createConnection0(String url) { + new URL(url).openConnection() as HttpURLConnection + } + + protected void setupHttpsConn( HttpsURLConnection conn ) { + if (config.httpReadTimeout != null) { + conn.setReadTimeout(config.httpReadTimeout.toMillis() as int) + } + if (config.httpConnectTimeout != null) { + conn.setConnectTimeout(config.httpConnectTimeout.toMillis() as int) + } + if (config.keyManagers != null || trustManagers != null) { + SSLContext sslContext = SSLContext.getInstance("TLS"); + sslContext.init(config.keyManagers, trustManagers, new SecureRandom()); + conn.setSSLSocketFactory(sslContext.getSocketFactory()) + } + + if( hostnameVerifier ) + conn.setHostnameVerifier(hostnameVerifier) + } + + /** + * Makes a HTTP(S) request the kubernetes master + * + * @param method The HTTP verb to use eg. {@code GET}, {@code POST}, etc + * @param path The API action path + * @param body The request payload + * @return + * A two elements list in which the first entry is an integer representing the HTTP response code, + * the second element is the text (json) response + */ + protected K8sResponseApi makeRequest(String method, String path, String body=null) throws K8sResponseException { + return apply(() -> makeRequestCall( method, path, body ) ) + } + + + private K8sResponseApi makeRequestCall(String method, String path, String body=null) throws K8sResponseException { + assert config.server, 'Missing Kubernetes server name' + assert path.startsWith('/'), 'Kubernetes API request path must starts with a `/` character' + + final prefix = config.server.contains("://") ? config.server : "https://$config.server" + final conn = createConnection0(prefix + path) + conn.setRequestProperty("Content-Type", "application/json") + if( config.token ) { + conn.setRequestProperty("Authorization", "Bearer $config.token") + } + + if( conn instanceof HttpsURLConnection ) { + setupHttpsConn(conn) + } + + if( !method ) method = body ? 'POST' : 'GET' + conn.setRequestMethod(method) + log.trace "[K8s] API request $method $path ${body ? '\n'+prettyPrint(body).indent() : ''}" + + if( body ) { + conn.setDoOutput(true); + conn.setDoInput(true); + conn.getOutputStream() << body + conn.getOutputStream().flush() + } + + final code = conn.getResponseCode() + final isError = code >= 400 + final stream = isError ? conn.getErrorStream() : conn.getInputStream() + if( isError ) + throw new K8sResponseException("Request $method $path returned an error code=$code", stream) + return new K8sResponseApi(code, stream) + } + + static private void trace(String method, String path, String text) { + log.trace "[K8s] API response $method $path \n${prettyPrint(text).indent()}" + } + + protected K8sResponseApi get(String path) { + makeRequest('GET',path) + } + + static protected String prettyPrint(String json) { + try { + JsonOutput.prettyPrint(json) + } + catch( Exception e ) { + return json + } + } + + K8sResponseJson configCreate(String name, Map data) { + + final spec = [ + apiVersion: 'v1', + kind: 'ConfigMap', + metadata: [ name: name, namespace: config.namespace ], + data: data + ] + + configCreate0(spec) + } + + protected K8sResponseJson configCreate0(Map spec) { + final action = "/api/v1/namespaces/${config.namespace}/configmaps" + final body = JsonOutput.toJson(spec) + def resp = post(action, body) + trace('POST', action, resp.text) + return new K8sResponseJson(resp.text) + } + + + K8sResponseJson configDelete(String name) { + final action = "/api/v1/namespaces/${config.namespace}/configmaps/$name" + def resp = delete(action) + trace('DELETE', action, resp.text) + return new K8sResponseJson(resp.text) + } + + K8sResponseJson configDeleteAll() { + final action = "/api/v1/namespaces/${config.namespace}/configmaps" + def resp = delete(action) + trace('DELETE', action, resp.text) + return new K8sResponseJson(resp.text) + } + + + K8sResponseJson volumeClaimRead(String name) { + final action = "/api/v1/namespaces/${config.namespace}/persistentvolumeclaims/${name}" + def resp = get(action) + trace('GET', action, resp.text) + return new K8sResponseJson(resp.text) + } + + /** + * Creates a retry policy using the configuration specified by {@link recreationaltech.plugin.client.K8sRetryConfig} + * + * @param cond A predicate that determines when a retry should be triggered + * @return The {@link dev.failsafe.RetryPolicy} instance + */ + protected RetryPolicy retryPolicy(Predicate cond) { + final cfg = config.retryConfig + final listener = new EventListener>() { + @Override + void accept(ExecutionAttemptedEvent event) throws Throwable { + log.debug("K8s response error - attempt: ${event.attemptCount}; reason: ${event.lastFailure.message}") + } + } + return RetryPolicy.builder() + .handleIf(cond) + .withBackoff(cfg.delay.toMillis(), cfg.maxDelay.toMillis(), ChronoUnit.MILLIS) + .withMaxAttempts(cfg.maxAttempts) + .withJitter(cfg.jitter) + .onRetry(listener) + .build() + } + + final private static List RETRY_CODES = List.of(408, 429, 500, 502, 503, 504) + + /** + * Carry out the invocation of the specified action using a retry policy. + * + * @param action A {@link dev.failsafe.function.CheckedSupplier} instance modeling the action to be performed in a safe manner + * @return The result of the supplied action + */ + protected T apply(CheckedSupplier action) { + // define the retry condition + final cond = new Predicate() { + @Override + boolean test(Throwable t) { + if ( t instanceof K8sResponseException && t.response.code in RETRY_CODES ) + return true + if( t instanceof SocketException || t.cause instanceof SocketException ) + return true + if( t instanceof SocketTimeoutException || t.cause instanceof SocketTimeoutException ) + return true + return false + } + } + // create the retry policy object + final policy = retryPolicy(cond) + // apply the action with and throw the original cause + try { + return Failsafe.with(policy).get(action) + }catch(FailsafeException e){ + throw e.getCause() + } + } + + +} + diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/client/K8sResponseApi.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/client/K8sResponseApi.groovy new file mode 100644 index 0000000..c62c4f7 --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/client/K8sResponseApi.groovy @@ -0,0 +1,54 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin.client + +import groovy.transform.CompileStatic + +/** + * Model a Kubernetes API response + * + * @author Paolo Di Tommaso + */ +@CompileStatic +class K8sResponseApi { + + private int code + + private InputStream stream + + private String text + + K8sResponseApi(int code, InputStream stream) { + this.code = code + this.stream = stream + } + + String toString() { + "code=$code; stream=$stream" + } + + int getCode() { code } + + InputStream getStream() { stream } + + String getText() { + if( text == null ) { + text = stream?.text + } + return text + } +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/client/K8sResponseException.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/client/K8sResponseException.groovy new file mode 100644 index 0000000..059e60e --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/client/K8sResponseException.groovy @@ -0,0 +1,83 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin.client + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +/** + * Model a kubernetes invalid response + * + * @author Paolo Di Tommaso + */ + +@Slf4j +@CompileStatic +class K8sResponseException extends Exception { + + K8sResponseJson response + + K8sResponseException(K8sResponseJson response) { + super(msg0(response)) + this.response = response + } + + K8sResponseException(String message, K8sResponseJson response) { + super(msg1(message,response)) + this.response = response + } + + K8sResponseException(String message, InputStream response) { + this(message, new K8sResponseJson(fetch(response))) + } + + static private String msg1( String msg, K8sResponseJson resp ) { + if( !msg && resp==null ) + return null + + if( msg && resp != null ) { + def sep = resp.isRawText() ? ' -- ' : '\n' + return "${msg}${sep}${msg0(resp)}" + } + else if( msg ) { + return msg + } + else { + return msg0(resp) + } + } + + static private String msg0( K8sResponseJson response ) { + if( response == null ) + return null + + if( response.isRawText() ) + response.getRawText() + else + "\n${response.toString().indent(' ')}" + } + + static private String fetch(InputStream stream) { + try { + return stream?.text + } + catch( Exception e ) { + log.debug "Unable to fetch response text -- Cause: ${e.message ?: e}" + return null + } + } + +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/client/K8sResponseJson.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/client/K8sResponseJson.groovy new file mode 100644 index 0000000..6c09ddf --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/client/K8sResponseJson.groovy @@ -0,0 +1,74 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin.client + +import groovy.json.JsonOutput +import groovy.json.JsonSlurper +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j + +/** + * Model the response of a kubernetes api request + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +class K8sResponseJson implements Map { + + @Delegate + private Map response + + private String rawText + + K8sResponseJson(Map response) { + this.response = response + } + + K8sResponseJson(String response) { + this.response = toJson(response) + this.rawText = response + } + + boolean isRawText() { !response && rawText } + + String getRawText() { rawText } + + static private Map toJson(String raw) { + try { + return (Map)new JsonSlurper().parseText(raw) + } + catch( Exception e ) { + log.trace "[K8s] cannot parse response to json -- raw: ${raw? '\n'+raw.indent(' ') :'null'}" + return Collections.emptyMap() + } + } + + static private String prettyPrint(String json) { + try { + JsonOutput.prettyPrint(json) + } + catch( Exception e ) { + return json + } + } + + String toString() { + response ? prettyPrint(JsonOutput.toJson(response)) : rawText + } + +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/client/K8sRetryConfig.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/client/K8sRetryConfig.groovy new file mode 100644 index 0000000..f28e9ff --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/client/K8sRetryConfig.groovy @@ -0,0 +1,75 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin.client + +import groovy.transform.CompileStatic +import groovy.transform.EqualsAndHashCode +import groovy.transform.ToString +import nextflow.config.spec.ConfigOption +import nextflow.config.spec.ConfigScope +import nextflow.script.dsl.Description +import nextflow.util.Duration + +/** + * Model retry policy configuration + * + * @author Paolo Di Tommaso + */ +@ToString(includePackage = false, includeNames = true) +@EqualsAndHashCode +@CompileStatic +class K8sRetryConfig implements ConfigScope { + + @ConfigOption + @Description(""" + Delay when retrying failed API requests (default: `250ms`). + """) + Duration delay = Duration.of('250ms') + + @ConfigOption + @Description(""" + Max delay when retrying failed API requests (default: `90s`). + """) + Duration maxDelay = Duration.of('90s') + + @ConfigOption + @Description(""" + Max attempts when retrying failed API requests (default: `4`). + """) + int maxAttempts = 4 + + @ConfigOption + @Description(""" + Jitter value when retrying failed API requests (default: `0.25`). + """) + double jitter = 0.25 + + K8sRetryConfig() { + this(Collections.emptyMap()) + } + + K8sRetryConfig(Map config) { + if( config.delay ) + delay = config.delay as Duration + if( config.maxDelay ) + maxDelay = config.maxDelay as Duration + if( config.maxAttempts ) + maxAttempts = config.maxAttempts as int + if( config.jitter ) + jitter = config.jitter as double + } +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sDvfsExtension.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/client/PodUnschedulableException.groovy similarity index 52% rename from k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sDvfsExtension.groovy rename to k8s-dvfs/src/main/groovy/recreationaltech/plugin/client/PodUnschedulableException.groovy index e4c2a86..7210487 100644 --- a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sDvfsExtension.groovy +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/client/PodUnschedulableException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2025, Seqera Labs + * Copyright 2013-2026, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,32 +14,24 @@ * limitations under the License. */ -package recreationaltech.plugin +package recreationaltech.plugin.client import groovy.transform.CompileStatic -import nextflow.Session -import nextflow.plugin.extension.Function -import nextflow.plugin.extension.PluginExtensionPoint +import nextflow.exception.ProcessException +import nextflow.exception.ShowOnlyExceptionMessage /** - * Implements a custom function which can be imported by - * Nextflow scripts. + * Exception raised when a pod cannot be scheduled because + * e.g. the container image cannot be pulled, required resources + * cannot be fulfilled, etc. + * + * @author Paolo Di Tommaso */ @CompileStatic -class K8sDvfsExtension extends PluginExtensionPoint { +class PodUnschedulableException extends ProcessException implements ShowOnlyExceptionMessage { - @Override - protected void init(Session session) { - } - - /** - * Say hello to the given target. - * - * @param target - */ - @Function - void sayHello(String target) { - println "Hello, ${target}!" + PodUnschedulableException(String message, Throwable cause) { + super(message,cause) } } diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/client/SSLUtils.java b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/client/SSLUtils.java new file mode 100644 index 0000000..2a6a49b --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/client/SSLUtils.java @@ -0,0 +1,318 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin.client; + +/** + * This file is derived from + * https://github.com/kubernetes-client/java/blob/master/util/src/main/java/io/kubernetes/client/util/SSLUtils.java + * + * @author Paolo Di Tommaso + */ + +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.math.BigInteger; +import java.security.KeyFactory; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.PrivateKey; +import java.security.Security; +import java.security.UnrecoverableKeyException; +import java.security.cert.Certificate; +import java.security.cert.CertificateException; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; +import java.security.spec.InvalidKeySpecException; +import java.security.spec.PKCS8EncodedKeySpec; +import java.security.spec.RSAPrivateCrtKeySpec; +import javax.net.ssl.KeyManager; +import javax.net.ssl.KeyManagerFactory; + +import org.apache.commons.codec.binary.Base64; +import org.bouncycastle.asn1.pkcs.PrivateKeyInfo; +import org.bouncycastle.openssl.PEMKeyPair; +import org.bouncycastle.openssl.PEMParser; +import org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter; + + +public class SSLUtils { + + public static boolean isNotNullOrEmpty(String val) { + return val != null && val.length() > 0; + } + + public static KeyManager[] keyManagers(String certData, String certFile, String keyData, String keyFile, + String algo, String passphrase, String keyStoreFile, String keyStorePassphrase) + throws NoSuchAlgorithmException, UnrecoverableKeyException, KeyStoreException, CertificateException, + InvalidKeySpecException, IOException { + KeyManager[] keyManagers = null; + if ((isNotNullOrEmpty(certData) || isNotNullOrEmpty(certFile)) + && (isNotNullOrEmpty(keyData) || isNotNullOrEmpty(keyFile))) { + KeyStore keyStore = createKeyStore(certData, certFile, keyData, keyFile, algo, passphrase, keyStoreFile, + keyStorePassphrase); + KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + kmf.init(keyStore, passphrase.toCharArray()); + keyManagers = kmf.getKeyManagers(); + } + return keyManagers; + } + + + public static KeyStore createKeyStore(String clientCertData, String clientCertFile, String clientKeyData, + String clientKeyFile, String clientKeyAlgo, String clientKeyPassphrase, String keyStoreFile, + String keyStorePassphrase) throws IOException, CertificateException, NoSuchAlgorithmException, + InvalidKeySpecException, KeyStoreException { + try (InputStream certInputStream = getInputStreamFromDataOrFile(clientCertData, clientCertFile); + InputStream keyInputStream = getInputStreamFromDataOrFile(clientKeyData, clientKeyFile)) { + return createKeyStore(certInputStream, keyInputStream, clientKeyAlgo, + clientKeyPassphrase != null ? clientKeyPassphrase.toCharArray() : null, + keyStoreFile, getKeyStorePassphrase(keyStorePassphrase)); + } + } + + static private PrivateKey generateEcKey(InputStream keyInputStream) throws IOException { + PrivateKey privateKey=null; + Security.addProvider(new org.bouncycastle.jce.provider.BouncyCastleProvider()); + Object object = new PEMParser(new InputStreamReader(keyInputStream)).readObject(); + if (object instanceof PEMKeyPair) { + PEMKeyPair keys = (PEMKeyPair) object; + privateKey = new JcaPEMKeyConverter().getKeyPair(keys).getPrivate(); + } + if( object instanceof PrivateKeyInfo) { + PrivateKeyInfo privateKeyInfo = (PrivateKeyInfo)object; + privateKey = new JcaPEMKeyConverter().getPrivateKey(privateKeyInfo); + } + if( privateKey == null) { + throw new IOException("Unsupported EC algorithm"); + } + return privateKey; + } + + static private PrivateKey generateStdKey(InputStream keyInputStream, String clientKeyAlgo) throws IOException, NoSuchAlgorithmException, InvalidKeySpecException { + byte[] keyBytes = decodePem(keyInputStream); + + KeyFactory keyFactory = KeyFactory.getInstance(clientKeyAlgo); + try { + // First let's try PKCS8 + return keyFactory.generatePrivate(new PKCS8EncodedKeySpec(keyBytes)); + } + catch (InvalidKeySpecException e) { + // Otherwise try PKCS1 + RSAPrivateCrtKeySpec keySpec = decodePKCS1(keyBytes); + return keyFactory.generatePrivate(keySpec); + } + } + + public static KeyStore createKeyStore(InputStream certInputStream, InputStream keyInputStream, String clientKeyAlgo, + char[] clientKeyPassphrase, String keyStoreFile, char[] keyStorePassphrase) throws IOException, + CertificateException, NoSuchAlgorithmException, InvalidKeySpecException, KeyStoreException { + CertificateFactory certFactory = CertificateFactory.getInstance("X509"); + X509Certificate cert = (X509Certificate) certFactory.generateCertificate(certInputStream); + + PrivateKey privateKey = clientKeyAlgo.equals("EC") + ? generateEcKey(keyInputStream) + : generateStdKey(keyInputStream, clientKeyAlgo); + + KeyStore keyStore = KeyStore.getInstance("JKS"); + if (keyStoreFile != null && keyStoreFile.length() > 0) { + keyStore.load(new FileInputStream(keyStoreFile), keyStorePassphrase); + } else { + loadDefaultKeyStoreFile(keyStore, keyStorePassphrase); + } + + String alias = cert.getSubjectX500Principal().getName(); + keyStore.setKeyEntry(alias, privateKey, clientKeyPassphrase, new Certificate[] { cert }); + + return keyStore; + } + + // This method is inspired and partly taken over from + // http://oauth.googlecode.com/svn/code/java/ + // All credits to belong to them. + private static byte[] decodePem(InputStream keyInputStream) throws IOException { + BufferedReader reader = new BufferedReader(new InputStreamReader(keyInputStream)); + try { + String line; + while ((line = reader.readLine()) != null) { + if (line.contains("-----BEGIN ")) { + return readBytes(reader, line.trim().replace("BEGIN", "END")); + } + } + throw new IOException("PEM is invalid: no begin marker"); + } finally { + reader.close(); + } + } + + private static byte[] readBytes(BufferedReader reader, String endMarker) throws IOException { + String line; + StringBuffer buf = new StringBuffer(); + + while ((line = reader.readLine()) != null) { + if (line.indexOf(endMarker) != -1) { + return Base64.decodeBase64(buf.toString()); + } + buf.append(line.trim()); + } + throw new IOException("PEM is invalid : No end marker"); + } + + public static RSAPrivateCrtKeySpec decodePKCS1(byte[] keyBytes) throws IOException { + DerParser parser = new DerParser(keyBytes); + Asn1Object sequence = parser.read(); + sequence.validateSequence(); + parser = new DerParser(sequence.getValue()); + parser.read(); + + return new RSAPrivateCrtKeySpec(next(parser), next(parser), next(parser), next(parser), next(parser), + next(parser), next(parser), next(parser)); + } + + private static BigInteger next(DerParser parser) throws IOException { + return parser.read().getInteger(); + } + + static class DerParser { + + private InputStream in; + + DerParser(byte[] bytes) throws IOException { + this.in = new ByteArrayInputStream(bytes); + } + + Asn1Object read() throws IOException { + int tag = in.read(); + + if (tag == -1) { + throw new IOException("Invalid DER: stream too short, missing tag"); + } + + int length = getLength(); + byte[] value = new byte[length]; + if (in.read(value) < length) { + throw new IOException("Invalid DER: stream too short, missing value"); + } + + return new Asn1Object(tag, value); + } + + private int getLength() throws IOException { + int i = in.read(); + if (i == -1) { + throw new IOException("Invalid DER: length missing"); + } + + if ((i & ~0x7F) == 0) { + return i; + } + + int num = i & 0x7F; + if (i >= 0xFF || num > 4) { + throw new IOException("Invalid DER: length field too big (" + i + ")"); + } + + byte[] bytes = new byte[num]; + if (in.read(bytes) < num) { + throw new IOException("Invalid DER: length too short"); + } + + return new BigInteger(1, bytes).intValue(); + } + } + + static class Asn1Object { + + private final int type; + private final byte[] value; + private final int tag; + + public Asn1Object(int tag, byte[] value) { + this.tag = tag; + this.type = tag & 0x1F; + this.value = value; + } + + public byte[] getValue() { + return value; + } + + BigInteger getInteger() throws IOException { + if (type != 0x02) { + throw new IOException("Invalid DER: object is not integer"); //$NON-NLS-1$ + } + return new BigInteger(value); + } + + void validateSequence() throws IOException { + if (type != 0x10) { + throw new IOException("Invalid DER: not a sequence"); + } + if ((tag & 0x20) != 0x20) { + throw new IOException("Invalid DER: can't parse primitive entity"); + } + } + } + + private static void loadDefaultKeyStoreFile(KeyStore keyStore, char[] keyStorePassphrase) + throws CertificateException, NoSuchAlgorithmException, IOException { + + String keyStorePath = System.getProperty("javax.net.ssl.keyStore"); + if (keyStorePath != null && keyStorePath.length() > 0) { + File keyStoreFile = new File(keyStorePath); + if (loadDefaultStoreFile(keyStore, keyStoreFile, keyStorePassphrase)) { + return; + } + } + + keyStore.load(null); + } + + private static boolean loadDefaultStoreFile(KeyStore keyStore, File fileToLoad, char[] passphrase) + throws CertificateException, NoSuchAlgorithmException, IOException { + if (fileToLoad.exists() && fileToLoad.isFile() && fileToLoad.length() > 0) { + keyStore.load(new FileInputStream(fileToLoad), passphrase); + return true; + } + return false; + } + + public static InputStream getInputStreamFromDataOrFile(String data, String file) throws FileNotFoundException { + if (data != null) { + byte[] bytes = Base64.decodeBase64(data); + // TODO handle non-base64 here? + return new ByteArrayInputStream(bytes); + } + if (file != null) { + return new FileInputStream(file); + } + return null; + } + + private static char[] getKeyStorePassphrase(String keyStorePassphrase) { + if (keyStorePassphrase == null || keyStorePassphrase.length() == 0) { + return System.getProperty("javax.net.ssl.keyStorePassword", "changeit").toCharArray(); + } + return keyStorePassphrase.toCharArray(); + } +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodEnv.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodEnv.groovy new file mode 100644 index 0000000..3e60058 --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodEnv.groovy @@ -0,0 +1,81 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin.model + +import groovy.transform.CompileStatic +import groovy.transform.EqualsAndHashCode +import groovy.transform.ToString + +/** + * Model a K8s pod environment variable definition + * + * @author Paolo Di Tommaso + */ +@CompileStatic +@ToString(includeNames = true) +@EqualsAndHashCode(includeFields = true) +class PodEnv { + + private Map spec + + private PodEnv(Map spec) { + this.spec = spec + } + + static PodEnv value(String env, String value) { + new PodEnv([name:env, value:value]) + } + + static PodEnv fieldPath(String env, String fieldPath) { + new PodEnv([ name: env, valueFrom: [fieldRef:[fieldPath: fieldPath]]]) + } + + static PodEnv config(String env, String config) { + final tokens = config.tokenize('/') + if( tokens.size() > 2 ) + throw new IllegalArgumentException("K8s invalid pod env file: $config -- Secret must be specified as /") + + final name = tokens[0] + final key = tokens[1] + + assert env, 'Missing pod env variable name' + assert name, 'Missing pod env config name' + + final ref = [ name: name, key: (key ?: env) ] + new PodEnv([ name: env, valueFrom: [configMapKeyRef: ref]]) + } + + static PodEnv secret(String env, String secret) { + + final tokens = secret.tokenize('/') + if( tokens.size() > 2 ) + throw new IllegalArgumentException("K8s invalid pod env secret: $secret -- Secret must be specified as /") + + final name = tokens[0] + final key = tokens[1] + + final ref = [ name: name, key: (key ?: env) ] + new PodEnv([ name: env, valueFrom: [secretKeyRef: ref]]) + } + + + Map toSpec() { spec } + + String toString() { + "PodEnv[ ${spec?.toString()} ]" + } +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sDvfsObserver.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodHostMount.groovy similarity index 56% rename from k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sDvfsObserver.groovy rename to k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodHostMount.groovy index 748116b..1c07167 100644 --- a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sDvfsObserver.groovy +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodHostMount.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2025, Seqera Labs + * Copyright 2013-2026, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,28 +14,28 @@ * limitations under the License. */ -package recreationaltech.plugin +package recreationaltech.plugin.model import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import nextflow.Session -import nextflow.trace.TraceObserver +import groovy.transform.EqualsAndHashCode +import groovy.transform.ToString /** - * Implements an observer that allows implementing custom - * logic on nextflow execution events. + * Model a K8s pod host mount definition + * + * @author Paolo Di Tommaso */ -@Slf4j +@EqualsAndHashCode +@ToString(includeNames = true) @CompileStatic -class K8sDvfsObserver implements TraceObserver { +class PodHostMount { - @Override - void onFlowCreate(Session session) { - println "Pipeline is starting! 🚀" - } + String hostPath - @Override - void onFlowComplete() { - println "Pipeline complete! 👋" + String mountPath + + PodHostMount(String host, String container) { + this.hostPath = host + this.mountPath = container } } diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodMountConfig.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodMountConfig.groovy new file mode 100644 index 0000000..d0dbb27 --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodMountConfig.groovy @@ -0,0 +1,66 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin.model + +import java.nio.file.Paths + +import groovy.transform.CompileStatic +import groovy.transform.EqualsAndHashCode +import groovy.transform.ToString + +/** + * Model a K8s pod ConfigMap mount + * + * See also https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/ + * + * @author Paolo Di Tommaso + */ +@CompileStatic +@ToString(includeNames = true) +@EqualsAndHashCode +class PodMountConfig { + + String mountPath + + String fileName + + String configName + + String configKey + + PodMountConfig( String config, String mount ) { + assert config + assert mount + + final path = Paths.get(mount) + final tokens = config.tokenize('/') + configName = tokens[0].trim() + configKey = tokens.size()>1 ? tokens[1].trim() : null + if( configKey ) { + mountPath = path.parent.toString() + fileName = path.fileName.toString() + } + else { + mountPath = path.toString() + } + } + + PodMountConfig( Map entry ) { + this(entry.config as String, entry.mountPath as String) + } + +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodMountCsiEphemeral.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodMountCsiEphemeral.groovy new file mode 100644 index 0000000..954636b --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodMountCsiEphemeral.groovy @@ -0,0 +1,53 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin.model + +import java.nio.file.Paths + +import groovy.transform.CompileStatic +import groovy.transform.EqualsAndHashCode +import groovy.transform.ToString + +/** + * Model a K8s pod CSI ephemeral volume mount + * + * See also https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/#csi-ephemeral-volumes + * + * @author Ben Sherman + */ +@CompileStatic +@ToString(includeNames = true) +@EqualsAndHashCode +class PodMountCsiEphemeral { + + String mountPath + + Map csi + + PodMountCsiEphemeral( Map csi, String mountPath ) { + assert csi + assert mountPath + + this.csi = csi + this.mountPath = mountPath + } + + PodMountCsiEphemeral( Map entry ) { + this(entry.csi as Map, entry.mountPath as String) + } + +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodMountEmptyDir.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodMountEmptyDir.groovy new file mode 100644 index 0000000..78072bc --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodMountEmptyDir.groovy @@ -0,0 +1,50 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin.model + +import groovy.transform.CompileStatic +import groovy.transform.EqualsAndHashCode +import groovy.transform.ToString + +/** + * Model a K8s pod emptyDir mount + * + * See also https://kubernetes.io/docs/concepts/storage/volumes/#emptydir + * + * @author Ben Sherman + */ +@CompileStatic +@ToString(includeNames = true) +@EqualsAndHashCode +class PodMountEmptyDir { + + String mountPath + + Map emptyDir + + PodMountEmptyDir( Map emptyDir, String mountPath ) { + assert mountPath + + this.emptyDir = emptyDir + this.mountPath = mountPath + } + + PodMountEmptyDir( Map entry ) { + this(entry.emptyDir as Map, entry.mountPath as String) + } + +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodMountSecret.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodMountSecret.groovy new file mode 100644 index 0000000..a7db5b9 --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodMountSecret.groovy @@ -0,0 +1,66 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin.model + +import java.nio.file.Paths + +import groovy.transform.CompileStatic +import groovy.transform.EqualsAndHashCode +import groovy.transform.ToString + +/** + * Model a K8s Secret file mount + * + * https://kubernetes.io/docs/concepts/configuration/secret/ + * + * @author Paolo Di Tommaso + */ +@CompileStatic +@ToString(includeNames = true) +@EqualsAndHashCode +class PodMountSecret { + + String mountPath + + String fileName + + String secretName + + String secretKey + + PodMountSecret(String secret, String mount) { + assert secret + assert mount + + final path = Paths.get(mount) + final tokens = secret.tokenize('/') + secretName = tokens[0].trim() + secretKey = tokens.size()>1 ? tokens[1].trim() : null + if( secretKey ) { + mountPath = path.parent.toString() + fileName = path.fileName.toString() + } + else { + mountPath = path.toString() + } + } + + PodMountSecret(Map entry) { + this(entry.secret as String, entry.mountPath as String) + } + +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodNodeSelector.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodNodeSelector.groovy new file mode 100644 index 0000000..1aa8e59 --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodNodeSelector.groovy @@ -0,0 +1,80 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin.model + +import groovy.transform.CompileStatic +import groovy.transform.EqualsAndHashCode +import groovy.transform.ToString + +/** + * Model a Pod nodeSelector spec + * + * https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector + * + * @author Paolo Di Tommaso + */ +@CompileStatic +@ToString(includeNames = true) +@EqualsAndHashCode(includeFields = true) +class PodNodeSelector { + + private Map spec = [:] + + PodNodeSelector(selector) { + if( selector instanceof CharSequence ) + createWithString(selector.toString()) + + else if( selector instanceof Map ) + createWithMap(selector) + + else if( selector != null ) + throw new IllegalArgumentException("K8s invalid pod nodeSelector value: $selector [${selector.getClass().getName()}]") + } + + private createWithMap(Map selection ) { + if(selection) { + for( Map.Entry entry : selection.entrySet() ) { + spec.put(entry.key.toString(), entry.value?.toString()) + } + } + } + + /** + * @param selector + * A string representing a comma separated list of pairs + * e.g. foo=1,bar=2 + * + */ + private createWithString( String selector ) { + if(!selector) return + def entries = selector.tokenize(',') + for( String item : entries ) { + def pair = item.tokenize('=') + spec.put( trim(pair[0]), trim(pair[1]) ?: 'true' ) + } + } + + private String trim(String v) { + v?.trim() + } + + Map toSpec() { spec } + + String toString() { + "PodNodeSelector[ ${spec?.toString()} ]" + } +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodOptions.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodOptions.groovy new file mode 100644 index 0000000..f33a31d --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodOptions.groovy @@ -0,0 +1,323 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin.model + +import groovy.transform.CompileStatic +import groovy.transform.EqualsAndHashCode +import groovy.transform.PackageScope +import groovy.transform.ToString + +/** + * Model K8s pod options such as environment variables, + * secret and config-maps + * + * @author Paolo Di Tommaso + */ +@CompileStatic +@ToString(includeNames = true) +@EqualsAndHashCode(includeFields = true) +class PodOptions { + + private String imagePullPolicy + + private String imagePullSecret + + private Collection envVars + + private Collection mountConfigMaps + + private Collection mountCsiEphemerals + + private Collection mountEmptyDirs + + private Collection mountSecrets + + private Collection mountClaims + + private Collection mountHostPaths + + private Map labels = [:] + + private Map annotations = [:] + + private PodNodeSelector nodeSelector + + private Map affinity + + private PodSecurityContext securityContext + + private boolean automountServiceAccountToken + + private String priorityClassName + + private List tolerations + + private Boolean privileged + + private String schedulerName + + private Integer ttlSecondsAfterFinished + + private String runtimeClassName + + PodOptions( List options=null ) { + int size = options ? options.size() : 0 + envVars = new HashSet<>(size) + mountConfigMaps = new HashSet<>(size) + mountCsiEphemerals = new HashSet<>(size) + mountEmptyDirs = new HashSet<>(size) + mountSecrets = new HashSet<>(size) + mountClaims = new HashSet<>(size) + mountHostPaths = new HashSet<>(10) + automountServiceAccountToken = true + tolerations = new ArrayList(size) + init(options) + } + + @PackageScope void init(List options) { + if( !options ) return + for( Map entry : options ) { + create(entry) + } + } + + @PackageScope void create(Map entry) { + if( entry.env && entry.value ) { + envVars << PodEnv.value(entry.env, entry.value) + } + else if( entry.env && entry.fieldPath ) { + envVars << PodEnv.fieldPath(entry.env, entry.fieldPath) + } + else if( entry.env && entry.secret ) { + envVars << PodEnv.secret(entry.env, entry.secret) + } + else if( entry.env && entry.config ) { + envVars << PodEnv.config(entry.env, entry.config) + } + else if( entry.mountPath && entry.secret ) { + mountSecrets << new PodMountSecret(entry) + } + else if( entry.mountPath && entry.config ) { + mountConfigMaps << new PodMountConfig(entry) + } + else if( entry.mountPath && entry.csi ) { + mountCsiEphemerals << new PodMountCsiEphemeral(entry) + } + else if( entry.mountPath && entry.emptyDir != null ) { + mountEmptyDirs << new PodMountEmptyDir(entry) + } + else if( entry.mountPath && entry.volumeClaim ) { + mountClaims << new PodVolumeClaim(entry) + } + else if( entry.mountPath && entry.hostPath instanceof CharSequence ) { + mountHostPaths << new PodHostMount(entry.hostPath, entry.mountPath) + } + else if( entry.pullPolicy || entry.imagePullPolicy ) { + this.imagePullPolicy = entry.pullPolicy ?: entry.imagePullPolicy as String + } + else if( entry.imagePullSecret || entry.imagePullSecrets ) { + this.imagePullSecret = entry.imagePullSecret ?: entry.imagePullSecrets + } + else if( entry.label && entry.value ) { + this.labels.put(entry.label as String, entry.value as String) + } + else if( entry.runAsUser != null ) { + this.securityContext = new PodSecurityContext(entry.runAsUser) + } + else if( entry.securityContext instanceof Map ) { + this.securityContext = new PodSecurityContext(entry.securityContext as Map) + } + else if( entry.nodeSelector ) { + this.nodeSelector = new PodNodeSelector(entry.nodeSelector) + } + else if( entry.affinity instanceof Map ) { + this.affinity = entry.affinity as Map + } + else if( entry.annotation && entry.value ) { + this.annotations.put(entry.annotation as String, entry.value as String) + } + else if( entry.automountServiceAccountToken instanceof Boolean ) { + this.automountServiceAccountToken = entry.automountServiceAccountToken as Boolean + } + else if( entry.priorityClassName ) { + this.priorityClassName = entry.priorityClassName + } + else if( entry.toleration instanceof Map ) { + tolerations << (entry.toleration as Map) + } + else if( entry.privileged instanceof Boolean ) { + this.privileged = entry.privileged as Boolean + } + else if( entry.schedulerName ) { + this.schedulerName = entry.schedulerName + } + else if( entry.ttlSecondsAfterFinished instanceof Integer ) { + this.ttlSecondsAfterFinished = entry.ttlSecondsAfterFinished as Integer + } + else if( entry.runtimeClassName ) { + this.runtimeClassName = entry.runtimeClassName + } + else + throw new IllegalArgumentException("Unknown pod options: $entry") + } + + + Collection getEnvVars() { envVars } + + Collection getMountConfigMaps() { mountConfigMaps } + + Collection getMountCsiEphemerals() { mountCsiEphemerals } + + Collection getMountEmptyDirs() { mountEmptyDirs } + + Collection getMountSecrets() { mountSecrets } + + Collection getMountHostPaths() { mountHostPaths } + + Collection getVolumeClaims() { mountClaims } + + Map getLabels() { labels } + + Map getAnnotations() { annotations } + + PodNodeSelector getNodeSelector() { nodeSelector } + + PodOptions setNodeSelector( PodNodeSelector sel ) { + nodeSelector = sel + return this + } + + Map getAffinity() { affinity } + + PodSecurityContext getSecurityContext() { securityContext } + + PodOptions setSecurityContext( PodSecurityContext ctx ) { + this.securityContext = ctx + return this + } + + String getImagePullSecret() { imagePullSecret } + + PodOptions setImagePullSecret( String secret ) { + this.imagePullSecret = secret + return this + } + + String getImagePullPolicy() { imagePullPolicy } + + PodOptions setImagePullPolicy( String policy ) { + this.imagePullPolicy = policy + return this + } + + boolean getAutomountServiceAccountToken() { automountServiceAccountToken } + + PodOptions setAutomountServiceAccountToken( boolean mount ) { + this.automountServiceAccountToken = mount + return this + } + + String getPriorityClassName() { priorityClassName } + + String getSchedulerName() { schedulerName } + + List getTolerations() { tolerations } + + Boolean getPrivileged() { privileged } + + Integer getTtlSecondsAfterFinished() { ttlSecondsAfterFinished } + + String getRuntimeClassName() { runtimeClassName } + + PodOptions plus( PodOptions other ) { + def result = new PodOptions() + + // env vars + result.envVars.addAll(envVars) + result.envVars.addAll( other.envVars ) + + // config maps + result.mountConfigMaps.addAll( mountConfigMaps ) + result.mountConfigMaps.addAll( other.mountConfigMaps ) + + // csi ephemeral volumes + result.mountCsiEphemerals.addAll( mountCsiEphemerals ) + result.mountCsiEphemerals.addAll( other.mountCsiEphemerals ) + + // empty dirs + result.mountEmptyDirs.addAll( mountEmptyDirs ) + result.mountEmptyDirs.addAll( other.mountEmptyDirs ) + + // host paths + result.mountHostPaths.addAll( mountHostPaths ) + result.mountHostPaths.addAll( other.mountHostPaths ) + + // secrets + result.mountSecrets.addAll( mountSecrets ) + result.mountSecrets.addAll( other.mountSecrets ) + + // volume claims + result.volumeClaims.addAll( volumeClaims ) + result.volumeClaims.addAll( other.volumeClaims ) + + // sec context + result.securityContext = other.securityContext ?: this.securityContext + + // node selector + result.nodeSelector = other.nodeSelector ?: this.nodeSelector + + // affinity + result.affinity = other.affinity ?: this.affinity + + // pull policy + result.imagePullPolicy = other.imagePullPolicy ?: this.imagePullPolicy + + // image secret + result.imagePullSecret = other.imagePullSecret ?: this.imagePullSecret + + // labels + result.labels.putAll(labels) + result.labels.putAll(other.labels) + + // annotations + result.annotations.putAll(annotations) + result.annotations.putAll(other.annotations) + + // automount service account token + result.automountServiceAccountToken = other.automountServiceAccountToken & this.automountServiceAccountToken + + // priority class name + result.priorityClassName = other.priorityClassName ?: this.priorityClassName + + // tolerations + result.tolerations = other.tolerations ?: this.tolerations + + // privileged execution + result.privileged = other.privileged!=null ? other.privileged : this.privileged + + // scheduler name + result.schedulerName = other.schedulerName ?: this.schedulerName + + // ttl seconds after finished (job) + result.ttlSecondsAfterFinished = other.ttlSecondsAfterFinished!=null ? other.ttlSecondsAfterFinished : this.ttlSecondsAfterFinished + + // runtime class name + result.runtimeClassName = other.runtimeClassName ?: this.runtimeClassName + + return result + } +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodSecurityContext.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodSecurityContext.groovy new file mode 100644 index 0000000..bb2227e --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodSecurityContext.groovy @@ -0,0 +1,52 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin.model + +import groovy.transform.CompileStatic +import groovy.transform.EqualsAndHashCode +import groovy.transform.ToString + +/** + * Models K8s pod security context + * + * See + * https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + * + * @author Paolo Di Tommaso + */ +@CompileStatic +@ToString(includeNames = true) +@EqualsAndHashCode(includeFields = true) +class PodSecurityContext { + + private Map spec + + PodSecurityContext(def user) { + spec = [runAsUser: user] + } + + PodSecurityContext(Map ctx) { + assert ctx + spec = ctx + } + + Map toSpec() { spec } + + String toString() { + "PodSecurityContext[ ${spec?.toString()} ]" + } +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodSpecBuilder.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodSpecBuilder.groovy new file mode 100644 index 0000000..9f1a21d --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodSpecBuilder.groovy @@ -0,0 +1,814 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin.model + +import java.nio.file.Path +import java.util.concurrent.atomic.AtomicInteger + +import groovy.transform.CompileDynamic +import groovy.transform.CompileStatic +import groovy.transform.PackageScope +import nextflow.executor.res.AcceleratorResource +import nextflow.util.MemoryUnit +import groovy.util.logging.Slf4j + +/** + * Object build for a K8s pod specification + * + * @author Paolo Di Tommaso + */ +@CompileStatic +@Slf4j +class PodSpecBuilder { + + static enum MetaType { LABEL, ANNOTATION } + + static enum SegmentType { + PREFIX (253), + NAME (63), + VALUE (63) + + private final int maxSize; + SegmentType(int maxSize) { + this.maxSize = maxSize; + } + } + + static @PackageScope AtomicInteger VOLUMES = new AtomicInteger() + + String podName + + String imageName + + String imagePullPolicy + + String imagePullSecret + + List command = [] + + List args = new ArrayList<>() + + Map labels = [:] + + Map annotations = [:] + + String namespace + + String restart + + List envVars = [] + + String workDir + + Integer cpus + + boolean cpuLimits + + String memory + + String disk + + String serviceAccount + + boolean automountServiceAccountToken = true + + AcceleratorResource accelerator + + Collection configMaps = [] + + Collection csiEphemerals = [] + + Collection emptyDirs = [] + + Collection secrets = [] + + Collection hostMounts = [] + + Collection volumeClaims = [] + + PodSecurityContext securityContext + + PodNodeSelector nodeSelector + + Map affinity + + String priorityClassName + + List tolerations = [] + + boolean privileged + + int activeDeadlineSeconds + + Map> capabilities + + List devices + + Map resourcesLimits + + String schedulerName + + Integer ttlSecondsAfterFinished + + String runtimeClassName + + String nodeName + + Integer port = null + + /** + * @return A sequential volume unique identifier + */ + static protected String nextVolName() { + "vol-${VOLUMES.incrementAndGet()}".toString() + } + + PodSpecBuilder withPodName(String name) { + this.podName = name + return this + } + + PodSpecBuilder withImageName(String name) { + this.imageName = name + return this + } + + PodSpecBuilder withImagePullPolicy(String policy) { + this.imagePullPolicy = policy + return this + } + + PodSpecBuilder withWorkDir( String path ) { + this.workDir = path + return this + } + + PodSpecBuilder withWorkDir(Path path ) { + this.workDir = path.toString() + return this + } + + PodSpecBuilder withNamespace(String name) { + this.namespace = name + return this + } + + PodSpecBuilder withServiceAccount(String name) { + this.serviceAccount = name + return this + } + + PodSpecBuilder withCommand( cmd ) { + if( cmd==null ) return this + assert cmd instanceof List || cmd instanceof CharSequence, "Missing or invalid K8s command parameter: $cmd" + this.command = cmd instanceof List ? cmd as List : ['/bin/bash','-c', cmd.toString()] + return this + } + + PodSpecBuilder withArgs( args ) { + if( args==null ) return this + assert args instanceof List || args instanceof CharSequence, "Missing or invalid K8s args parameter: $args" + this.args = args instanceof List ? args as List : ['/bin/bash','-c', args.toString()] + return this + } + + PodSpecBuilder withCpus( Integer cpus ) { + this.cpus = cpus + return this + } + + PodSpecBuilder withCpuLimits(boolean cpuLimits) { + this.cpuLimits = cpuLimits + return this + } + + PodSpecBuilder withMemory(String mem) { + this.memory = mem + return this + } + + PodSpecBuilder withMemory(MemoryUnit mem) { + this.memory = "${mem.mega}Mi".toString() + return this + } + + PodSpecBuilder withDisk(String disk) { + this.disk = disk + return this + } + + PodSpecBuilder withDisk(MemoryUnit disk) { + this.disk = "${disk.mega}Mi".toString() + return this + } + + PodSpecBuilder withAccelerator(AcceleratorResource acc) { + this.accelerator = acc + return this + } + + PodSpecBuilder withLabel( String name, String value ) { + this.labels.put(name, value) + return this + } + + PodSpecBuilder withLabels(Map labels) { + this.labels.putAll(labels) + return this + } + + PodSpecBuilder withAnnotation( String name, String value ) { + this.annotations.put(name, value) + return this + } + + PodSpecBuilder withAnnotations(Map annotations) { + this.annotations.putAll(annotations) + return this + } + + + PodSpecBuilder withEnv( PodEnv env ) { + envVars.add(env) + return this + } + + PodSpecBuilder withEnv( Collection envs ) { + envVars.addAll(envs) + return this + } + + PodSpecBuilder withVolumeClaim( PodVolumeClaim claim ) { + volumeClaims.add(claim) + return this + } + + PodSpecBuilder withVolumeClaims( Collection claims ) { + volumeClaims.addAll(claims) + return this + } + + PodSpecBuilder withConfigMaps( Collection configMaps ) { + this.configMaps.addAll(configMaps) + return this + } + + PodSpecBuilder withConfigMap( PodMountConfig configMap ) { + this.configMaps.add(configMap) + return this + } + + PodSpecBuilder withCsiEphemerals( Collection csiEphemerals ) { + this.csiEphemerals.addAll(csiEphemerals) + return this + } + + PodSpecBuilder withCsiEphemeral( PodMountCsiEphemeral csiEphemeral ) { + this.csiEphemerals.add(csiEphemeral) + return this + } + + PodSpecBuilder withEmptyDirs( Collection emptyDirs ) { + this.emptyDirs.addAll(emptyDirs) + return this + } + + PodSpecBuilder withEmptyDir( PodMountEmptyDir emptyDir ) { + this.emptyDirs.add(emptyDir) + return this + } + + PodSpecBuilder withSecrets( Collection secrets ) { + this.secrets.addAll(secrets) + return this + } + + PodSpecBuilder withSecret( PodMountSecret secret ) { + this.secrets.add(secret) + return this + } + + PodSpecBuilder withHostMounts( Collection mounts ) { + this.hostMounts.addAll(mounts) + return this + } + + PodSpecBuilder withHostMount( String host, String mount ) { + this.hostMounts.add( new PodHostMount(host, mount)) + return this + } + + PodSpecBuilder withPrivileged(boolean value) { + this.privileged = value + return this + } + + PodSpecBuilder withCapabilities(Map> cap) { + this.capabilities = cap + for( String it : cap.keySet() ) { + if( it !in ['add','drop']) throw new IllegalArgumentException("K8s capability action can be either 'add' or 'drop' - offending value '$it'") + } + return this + } + + PodSpecBuilder withActiveDeadline(int seconds) { + this.activeDeadlineSeconds = seconds + return this + } + + PodSpecBuilder withResourcesLimits(Map limits) { + this.resourcesLimits = limits + return this + } + + PodSpecBuilder withPodOptions(PodOptions opts) { + // -- pull policy + if( opts.imagePullPolicy ) + imagePullPolicy = opts.imagePullPolicy + if( opts.imagePullSecret ) + imagePullSecret = opts.imagePullSecret + // -- env vars + if( opts.getEnvVars() ) + envVars.addAll( opts.getEnvVars() ) + // -- configMaps + if( opts.getMountConfigMaps() ) + configMaps.addAll( opts.getMountConfigMaps() ) + // -- csi ephemeral volumes + if( opts.getMountCsiEphemerals() ) + csiEphemerals.addAll( opts.getMountCsiEphemerals() ) + // -- emptyDirs + if( opts.getMountEmptyDirs() ) + emptyDirs.addAll( opts.getMountEmptyDirs() ) + // -- host paths + if( opts.getMountHostPaths() ) + hostMounts.addAll( opts.getMountHostPaths() ) + // -- secrets + if( opts.getMountSecrets() ) + secrets.addAll( opts.getMountSecrets() ) + // -- volume claims + if( opts.getVolumeClaims() ) + volumeClaims.addAll( opts.getVolumeClaims() ) + // -- labels + if( opts.labels ) { + def keys = opts.labels.keySet() + if( 'app' in keys ) throw new IllegalArgumentException("Invalid pod label -- `app` is a reserved label") + if( 'runName' in keys ) throw new IllegalArgumentException("Invalid pod label -- `runName` is a reserved label") + labels.putAll( opts.labels ) + } + // - annotations + if( opts.annotations ) { + annotations.putAll( opts.annotations ) + } + // -- security context + if( opts.securityContext ) + securityContext = opts.securityContext + // -- node selector + if( opts.nodeSelector ) + nodeSelector = opts.nodeSelector + // -- affinity + if( opts.affinity ) + affinity = opts.affinity + // -- automount service account token + automountServiceAccountToken = opts.automountServiceAccountToken + // -- priority class name + priorityClassName = opts.priorityClassName + // -- tolerations + if( opts.tolerations ) + tolerations.addAll(opts.tolerations) + // -- privileged + privileged = opts.privileged + // -- scheduler name + schedulerName = opts.schedulerName + // -- ttl seconds after finished (job) + if( opts.ttlSecondsAfterFinished != null ) + ttlSecondsAfterFinished = opts.ttlSecondsAfterFinished + // runtime class name + if( opts.runtimeClassName != null ) + runtimeClassName = opts.runtimeClassName + + return this + } + + PodSpecBuilder withNodeName(String value) { + this.nodeName = value + return this + } + + PodSpecBuilder withPort(int value) { + this.port = new Integer(value) + return this + } + + @PackageScope List createPullSecret() { + def result = new ArrayList(1) + def entry = new LinkedHashMap(1) + entry.name = imagePullSecret + result.add(entry) + return result + } + + Map build() { + assert this.podName, 'Missing K8s podName parameter' + assert this.imageName, 'Missing K8s imageName parameter' + assert this.command || this.args, 'Missing K8s command parameter' + + final restart = this.restart ?: 'Never' + + final metadata = new LinkedHashMap() + metadata.name = podName + metadata.namespace = namespace ?: 'default' + + final labels = this.labels ?: [:] + final env = [] + for( PodEnv entry : this.envVars ) { + env.add(entry.toSpec()) + } + + final container = [ name: this.podName, image: this.imageName ] + if( this.command ) + container.command = this.command + if( this.args ) + container.args = args + + if( this.workDir ) + container.put('workingDir', workDir) + + if( imagePullPolicy ) + container.imagePullPolicy = imagePullPolicy + + if( this.port ) { + List ports = [] + ports << [containerPort: this.port.intValue()] + container.ports = ports + } + + final secContext = new LinkedHashMap(10) + if( privileged ) { + // note: privileged flag needs to be defined in the *container* securityContext + // not the 'spec' securityContext (see below) + secContext.privileged =true + } + if( capabilities ) { + secContext.capabilities = capabilities + } + if( secContext ) { + container.securityContext = secContext + } + + final spec = [ + restartPolicy: restart, + containers: [ container ], + ] + + if( nodeSelector ) + spec.nodeSelector = nodeSelector.toSpec() + + if( schedulerName ) + spec.schedulerName = schedulerName + + if( affinity ) + spec.affinity = affinity + + if( this.serviceAccount ) + spec.serviceAccountName = this.serviceAccount + + if( ! this.automountServiceAccountToken ) + spec.automountServiceAccountToken = false + + if( securityContext ) + spec.securityContext = securityContext.toSpec() + + if( imagePullSecret ) + spec.imagePullSecrets = createPullSecret() + + if( priorityClassName ) + spec.priorityClassName = priorityClassName + + // tolerations + if( this.tolerations ) + spec.tolerations = this.tolerations + + // add labels + if( labels ) + metadata.labels = sanitize(labels, MetaType.LABEL) + + if( annotations ) + metadata.annotations = sanitize(annotations, MetaType.ANNOTATION) + + // time directive + if ( activeDeadlineSeconds > 0) + spec.activeDeadlineSeconds = activeDeadlineSeconds + + if ( runtimeClassName ) + spec.runtimeClassName = runtimeClassName + + if ( nodeName ) + spec.nodeName = nodeName + + final pod = [ + apiVersion: 'v1', + kind: 'Pod', + metadata: metadata, + spec: spec + ] + + // add environment + if( env ) + container.env = env + + // add resources + if( this.cpus ) { + container.resources = addCpuResources(this.cpus, container.resources as Map) + } + + if( this.memory ) { + container.resources = addMemoryResources(this.memory, container.resources as Map) + } + + if( this.accelerator ) { + container.resources = addAcceleratorResources(this.accelerator, container.resources as Map) + } + + if( this.disk ) { + container.resources = addDiskResources(this.disk, container.resources as Map) + } + + if( this.resourcesLimits ) { + container.resources = addResourcesLimits(this.resourcesLimits, container.resources as Map) + } + + // add storage definitions ie. volumes and mounts + final List mounts = [] + final List volumes = [] + final namesMap = [:] + + // creates a volume name for each unique claim name + for( String claimName : volumeClaims.collect { it.claimName }.unique() ) { + final volName = nextVolName() + namesMap[claimName] = volName + volumes << [name: volName, persistentVolumeClaim: [claimName: claimName]] + } + + // -- persistent volume claims + for( PodVolumeClaim entry : volumeClaims ) { + //check if we already have a volume for the pvc + final name = namesMap.get(entry.claimName) + final claim = [name: name, mountPath: entry.mountPath ] + if( entry.subPath ) + claim.subPath = entry.subPath + if( entry.readOnly ) + claim.readOnly = entry.readOnly + mounts << claim + } + + // -- configMap volumes + for( PodMountConfig entry : configMaps ) { + final name = nextVolName() + configMapToSpec(name, entry, mounts, volumes) + } + + // -- csi ephemeral volumes + for( PodMountCsiEphemeral entry : csiEphemerals ) { + final name = nextVolName() + mounts << [name: name, mountPath: entry.mountPath, readOnly: entry.csi.readOnly ?: false] + volumes << [name: name, csi: entry.csi] + } + + // -- emptyDir volumes + for( PodMountEmptyDir entry : emptyDirs ) { + final name = nextVolName() + mounts << [name: name, mountPath: entry.mountPath] + volumes << [name: name, emptyDir: entry.emptyDir] + } + + // -- secret volumes + for( PodMountSecret entry : secrets ) { + final name = nextVolName() + secretToSpec(name, entry, mounts, volumes) + } + + // -- host path volumes + for( PodHostMount entry : hostMounts ) { + final name = nextVolName() + mounts << [name: name, mountPath: entry.mountPath] + volumes << [name: name, hostPath: [path: entry.hostPath]] + } + + + if( volumes ) + spec.volumes = volumes + if( mounts ) + container.volumeMounts = mounts + + return pod + } + + Map buildAsJob() { + final pod = build() + final spec = [ + backoffLimit: 0, + template: [ + metadata: pod.metadata, + spec: pod.spec + ] + ] + + if( ttlSecondsAfterFinished != null ) + spec.ttlSecondsAfterFinished = ttlSecondsAfterFinished + + return [ + apiVersion: 'batch/v1', + kind: 'Job', + metadata: pod.metadata, + spec: spec + ] + } + + @PackageScope + Map addResourcesLimits(Map limits, Map result) { + if( result == null ) + result = new LinkedHashMap(2) + + final limits0 = result.limits as Map ?: new LinkedHashMap(10) + limits0.putAll( limits ) + result.limits = limits0 + + return result + } + + @PackageScope + Map addCpuResources(Integer cpus, Map res) { + if( res == null ) + res = new LinkedHashMap(2) + + final requests0 = res.requests as Map ?: new LinkedHashMap<>(10) + requests0.cpu = cpus + res.requests = requests0 + + if( cpuLimits ) { + final limits0 = res.limits as Map ?: new LinkedHashMap(10) + limits0.cpu = cpus + res.limits = limits0 + } + + return res + } + + @PackageScope + Map addMemoryResources(String memory, Map res) { + if( res == null ) + res = new LinkedHashMap(2) + + final req = res.requests as Map ?: new LinkedHashMap(10) + req.memory = memory + res.requests = req + + final lim = res.limits as Map ?: new LinkedHashMap(10) + lim.memory = memory + res.limits = lim + + return res + } + + @PackageScope + Map addDiskResources(String diskRequest, Map res) { + if( res == null ) + res = new LinkedHashMap(2) + + final req = res.requests as Map ?: new LinkedHashMap(10) + req.'ephemeral-storage' = diskRequest + res.requests = req + + final lim = res.limits as Map ?: new LinkedHashMap(10) + lim.'ephemeral-storage' = diskRequest + res.limits = lim + + return res + } + + @PackageScope + String getAcceleratorType(AcceleratorResource accelerator) { + + def type = accelerator.type ?: 'nvidia.com' + + if ( type.contains('/') ) + // Assume the user has fully specified the resource type. + return type + + // Assume we're using GPU and update as necessary. + if( !type.contains('.') ) type += '.com' + type += '/gpu' + + return type + } + + + @PackageScope + Map addAcceleratorResources(AcceleratorResource accelerator, Map res) { + + if( res == null ) + res = new LinkedHashMap(2) + + def type = getAcceleratorType(accelerator) + + if( accelerator.request ) { + final req = res.requests as Map ?: new LinkedHashMap<>(2) + req.put(type, accelerator.request) + res.requests = req + } + if( accelerator.limit ) { + final lim = res.limits as Map ?: new LinkedHashMap<>(2) + lim.put(type, accelerator.limit) + res.limits = lim + } + + return res + } + + @PackageScope + @CompileDynamic + static void secretToSpec(String volName, PodMountSecret entry, List mounts, List volumes ) { + assert entry + + final secret = [secretName: entry.secretName] + if( entry.secretKey ) { + secret.items = [ [key: entry.secretKey, path: entry.fileName ] ] + } + + mounts << [name: volName, mountPath: entry.mountPath] + volumes << [name: volName, secret: secret ] + } + + @PackageScope + @CompileDynamic + static void configMapToSpec(String volName, PodMountConfig entry, List mounts, List volumes ) { + assert entry + + final config = [name: entry.configName] + if( entry.configKey ) { + config.items = [ [key: entry.configKey, path: entry.fileName ] ] + } + + mounts << [name: volName, mountPath: entry.mountPath] + volumes << [name: volName, configMap: config ] + } + + protected Map sanitize(Map map, MetaType kind) { + final result = new HashMap(map.size()) + for( Map.Entry entry : map ) { + final key = sanitizeKey(entry.key as String, kind) + final value = (kind == MetaType.LABEL) + ? sanitizeValue(entry.value, kind, SegmentType.VALUE) + : entry.value + + result.put(key, value) + } + return result + } + + protected String sanitizeKey(String value, MetaType kind) { + final parts = value.tokenize('/') + + if (parts.size() == 2) { + return "${sanitizeValue(parts[0], kind, SegmentType.PREFIX)}/${sanitizeValue(parts[1], kind, SegmentType.NAME)}" + } + if( parts.size() == 1 ) { + return sanitizeValue(parts[0], kind, SegmentType.NAME) + } + else { + throw new IllegalArgumentException("Invalid key in pod ${kind.toString().toLowerCase()} -- Key can only contain exactly one '/' character") + } + } + + + /** + * Sanitize a string value to contain only alphanumeric characters, '-', '_' or '.', + * and to start and end with an alphanumeric character. + */ + protected String sanitizeValue(value, MetaType kind, SegmentType segment) { + def str = String.valueOf(value) + if( str.length() > segment.maxSize ) { + log.debug "K8s $kind $segment exceeds allowed size: $segment.maxSize -- offending str=$str" + str = str.substring(0,segment.maxSize) + } + str = str.replaceAll(/[^a-zA-Z0-9\.\_\-]+/, '_') + str = str.replaceAll(/^[^a-zA-Z0-9]+/, '') + str = str.replaceAll(/[^a-zA-Z0-9]+$/, '') + return str + } + +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodVolumeClaim.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodVolumeClaim.groovy new file mode 100644 index 0000000..06d6930 --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodVolumeClaim.groovy @@ -0,0 +1,76 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin.model + +import groovy.transform.CompileStatic +import groovy.transform.EqualsAndHashCode +import groovy.transform.ToString + +/** + * Model a K8s pod persistent volume claim mount + * + * See https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim + * + * @author Paolo Di Tommaso + */ +@CompileStatic +@ToString(includeNames = true) +@EqualsAndHashCode +class PodVolumeClaim { + + String claimName + + String mountPath + + String subPath + + boolean readOnly + + PodVolumeClaim(String name, String mount, String subPath=null, boolean readOnly=false) { + assert name + assert mount + this.claimName = name + this.mountPath = sanitize(mount) + this.subPath = subPath + this.readOnly = readOnly + validate(mountPath) + } + + PodVolumeClaim(Map entry) { + assert entry.volumeClaim + assert entry.mountPath + this.claimName = entry.volumeClaim + this.mountPath = sanitize(entry.mountPath) + this.subPath = entry.subPath + this.readOnly = entry.readOnly ?: false + validate(mountPath) + } + + private static validate(String path) { + if( !path.startsWith('/') ) + throw new IllegalArgumentException("K8s volume claim path must be an absolute path: $path") + } + + static private String sanitize(path) { + if( !path ) return null + def result = path.toString().trim() + while( result.endsWith('/') && result.size()>1 ) + result = result.substring(0,result.size()-1) + return result + } + +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/ResourceType.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/ResourceType.groovy new file mode 100644 index 0000000..6eb4f53 --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/ResourceType.groovy @@ -0,0 +1,30 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin.model + +/** + * Model the resource type to be used to run nextflow tasks + * + * @author Paolo Di Tommaso + */ +enum ResourceType { + Pod, Job; + + String lower() { + return this.name().toLowerCase() + } +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/strategies/K8sDVFSSchedulingStrategy.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/strategies/K8sDVFSSchedulingStrategy.groovy new file mode 100644 index 0000000..2743f1b --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/strategies/K8sDVFSSchedulingStrategy.groovy @@ -0,0 +1,384 @@ +package recreationaltech.plugin.strategies + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import recreationaltech.plugin.K8sDVFSClient +import recreationaltech.plugin.K8sRuntimeEstimator +import recreationaltech.plugin.K8sSchedulingDecision +import recreationaltech.plugin.K8sSchedulingRequest +import recreationaltech.plugin.K8sSchedulingStrategy +import recreationaltech.plugin.K8sTaskHandler +import recreationaltech.plugin.K8sTaskScheduler +import recreationaltech.plugin.client.K8sClient +import nextflow.processor.TaskRun +import nextflow.util.Duration + +/** + * Implements a scheduling strategy utilizing dvfs to reduce the energy consumption + * of workflow execution, while attempting to maintain the same makespan. + */ +@Slf4j +@CompileStatic +class K8sDVFSSchedulingStrategy implements K8sSchedulingStrategy { + + /** Used for passing K8sExecutor.getClient. + * We cannot pass the client directly, becaue it can be refreshed + * during workflow execution. + */ + public interface K8sClientGetter { + K8sClient getClient() + } + + private static long getTaskMemoryRequirment(TaskRun task) { + return task.config.getMemory() ? task.config.getMemory().bytes : 64 * 1024 * 1024 + } + + private static int getTaskCPURequirement(TaskRun task) { + return task.config.hasCpus() ? task.config.getCpus() * 1000 : 1000 + } + + @Slf4j + private class WorkerNode { + private class AssignedTask { + TaskRun task + long frequency + + AssignedTask(TaskRun t, long f) { + this.task = t + this.frequency = f + } + } + + String name + + long maxFrequency + long minFrequency + long currentFrequency + + ArrayList tasks + + WorkerNode(String name, long maxF, long minF, long curF, K8sClientGetter clientGetter) { + this.name = name + this.maxFrequency = maxF + this.minFrequency = minF + this.currentFrequency = curF + this.tasks = new ArrayList<>() + } + + /* Return the number of available (unoccupied) bytes */ + long getAvailableMemory() { + def k8sClient = clientGetter.getClient() + Long available = k8sClient.getNodeMemoryAvailableBytes(this.name) + if (available == null) { + log.warn "[K8s] failed to retrieve available memory for node ${name}" + // Fallback: try capacity - allocated + Long capacity = k8sClient.getNodeMemoryCapacityBytes(this.name) + Long allocated = k8sClient.getNodeMemoryAllocatedBytes(this.name) + if (capacity != null && allocated != null) { + return capacity - allocated + } + return 0 + } + return available + } + + /* Return the number of available (unoccupied) cpu cores */ + long getAvailableCPUs() { + def k8sClient = clientGetter.getClient() + Long available = k8sClient.getNodeCpuAvailableMillis(this.name) + if (available == null) { + log.warn "[K8s] failed to retrieve available CPU for node ${name}" + // Fallback: try capacity - allocated + Long capacity = k8sClient.getNodeCpuCapacityMillis(this.name) + Long allocated = k8sClient.getNodeCpuAllocatedMillis(this.name) + if (capacity != null && allocated != null) { + return capacity - allocated + } + return 0 + } + return available + } + + /* Return the total amount of installed memory */ + long getMemoryAmount() { + def k8sClient = clientGetter.getClient() + Long capacity = k8sClient.getNodeMemoryCapacityBytes(this.name) + return capacity != null ? capacity.longValue() : 0 + } + + /* Return the total number of installed cpu cores */ + long getCPUCount() { + def k8sClient = clientGetter.getClient() + Long capacity = k8sClient.getNodeCpuCapacityMillis(this.name) + return capacity != null ? capacity.longValue(): 0 + } + + // Sets the frequency to the max. requested frequency of all currently running tasks. + private void updateFrequency(K8sDVFSClient dvfsClient) { + if (tasks.size() == 0) + return + long max = Long.MIN_VALUE + for (AssignedTask t : tasks) { + max = Long.max(t.frequency, max) + } + log.info "[K8s] node ${name} running at ${max} Hz / ${maxFrequency} Hz ${((double)max/(double)maxFrequency) * 100.0}%" + dvfsClient.setNodeFrequency(name, (int)max) + this.currentFrequency = max + } + + void assignTask(TaskRun task, long frequency, K8sDVFSClient dvfsClient) { + log.info "[K8s] node ${name}: task ${task.name} assigned with ${frequency}/${maxFrequency}" + this.tasks.add(new AssignedTask(task, frequency)) + updateFrequency(dvfsClient) + } + + void taskFinished(TaskRun task, K8sDVFSClient dvfsClient) { + log.info "[K8s] node ${name}: task ${task.name} finished" + this.tasks.removeIf {it.task == task} + updateFrequency(dvfsClient) + } + } + + class SchedulingRequestComparator implements Comparator { + K8sRuntimeEstimator runtimeEstimator + long currentTime + double epsilon + + @Override + int compare(K8sSchedulingRequest o1, K8sSchedulingRequest o2) { + // First, check if one of the tasks is (estimated to be) on the critical path + double t1 = runtimeEstimator.estimate(o1.handler) + double t2 = runtimeEstimator.estimate(o2.handler) + + if (t1 > t2 + epsilon) + return -1 + else if (t2 > t1 + epsilon) + return 1 + + // Both are not on the critical path. Sort based on the time they spent in the queue + long w1 = currentTime - o1.submitTimeMillis + long w2 = currentTime - o2.submitTimeMillis + if (w1 > w2) + return -1 + else if (w2 > w1) + return 1 + return 0 + } + } + + private K8sRuntimeEstimator runtimeEstimator + private K8sDVFSClient dvfsClient + + private ArrayList nodes + private HashMap taskToNode + private long globalMaxFrequency + private long globalMinFrequency + + private K8sClientGetter clientGetter + + private double comparisonEpsilonMillis + + private double[] topRuntimes + private double averageRuntime + private double finishedTaskCount + + boolean fullSpeedMode + + K8sDVFSSchedulingStrategy(K8sRuntimeEstimator runtimeEstimator, + K8sDVFSClient dvfsClient, + K8sClientGetter clientGetter, + Duration runtimeComparisonEpsilon, + int topRuntimeCount) { + this.runtimeEstimator = runtimeEstimator + this.dvfsClient = dvfsClient + this.nodes = new ArrayList<>() + this.taskToNode = new HashMap<>(); + this.clientGetter = clientGetter + this.comparisonEpsilonMillis = (double)runtimeComparisonEpsilon.toMillis() + this.topRuntimes = new double[topRuntimeCount] + for (int i = 0; i < topRuntimeCount; i++) { + this.topRuntimes[i] = 0.0 + } + this.averageRuntime = 0.0 + this.finishedTaskCount = 0.0 + } + + private boolean isInTopRuntimes(double rt) { + for (int i = 0; i < topRuntimes.size(); i++) { + if (rt >= topRuntimes[i]) + return true + } + return false + } + + private void updateTopRuntimes(double rt) { + for (int i = 0; i < topRuntimes.size(); i++) { + if (rt > topRuntimes[i]) { + /* Move all one down */ + for (int j = topRuntimes.size() - 1; j > i; j--) { + topRuntimes[j] = topRuntimes[j - 1]; + } + topRuntimes[i] = rt + break + } + } + } + + @Override + K8sSchedulingDecision schedule(K8sTaskScheduler scheduler, List queue) { + if (nodes.isEmpty()) { + if (!initNodes(scheduler)) + return null + } + + /* Step 1: Sort by task priority. We will attempt to schedule tasks "in order", so that the + * highest priority tasks are assigned to nodes as soon as possible. + * + * Priority is based on a) the tasks estimated runtime and b) the wait time of the task. + */ + SchedulingRequestComparator comparator = new SchedulingRequestComparator() + comparator.runtimeEstimator = runtimeEstimator + comparator.epsilon = comparisonEpsilonMillis + comparator.currentTime = System.currentTimeMillis() + queue.sort(comparator) + + /* Step 2: For each task attempt to schedule it onto a node */ + for (K8sSchedulingRequest req : queue) { + /* Step 2.1: Determine if the task is on the critical path. + * If yes, we just schedule it at max frequency on the node with the highest available + * frequency. If not, we determine a frequency (see below). + */ + final double taskEstimation = runtimeEstimator.estimate(req.handler) + final boolean isCriticalPath = isInTopRuntimes(taskEstimation) + long frequency = globalMaxFrequency + if (!isCriticalPath && !fullSpeedMode) { + /* Set frequency so that we expect the runtime to be close to the mean runtime. */ + frequency = (long)Math.floor((taskEstimation * globalMaxFrequency) / averageRuntime) + frequency = Math.max(frequency, globalMinFrequency) + frequency = Math.min(frequency, globalMaxFrequency) + } + + /* Step 2.2: Filter nodes based on task requirements */ + ArrayList suitableNodes = filterNodes(req.task) + if (suitableNodes.size() == 0) { + if (!anyNode(req.task)) { + log.error "[K8s] unable to schedule task ${req.task} - no node satisfies resource requirements ${getTaskMemoryRequirment(req.task)} bytes ${getTaskCPURequirement(req.task)} cpus" + return null + } + /* No node can currently execute this task, but it should be possible in the future */ + /* log.info "[K8s] ${req.task} can not be scheduled: ${getTaskMemoryRequirment(req.task)} bytes ${getTaskCPURequirement(req.task)} CPUs" + for (WorkerNode n : this.nodes) { + log.info "[K8s] node ${n.name} - ${n.availableMemory}, ${n.availableCPUs}" + } */ + continue + } + + /* Step 2.3: Assign to node based on "best fit" - current node frequency is closest to determined frequency */ + long minDist = Math.abs(suitableNodes[0].currentFrequency - frequency) + WorkerNode closest = suitableNodes[0] + for (WorkerNode node : suitableNodes) { + long dist = Math.abs(node.currentFrequency - frequency) + if (dist < minDist) { + closest = node + minDist = dist + } + } + + closest.assignTask(req.task, frequency, dvfsClient) + taskToNode.put(req.task.hash.toString(), closest) + log.info "[K8s] DVFS: Assigned task ${req.task} to node ${closest.name} - ${taskToNode.size()} assigned tasks" + return new K8sSchedulingDecision(req, closest.name) + } + + log.info "[K8s] unable to schedule any task. The queue contains ${queue.size()} tasks." + return null + } + + @Override + boolean scheduleImmediately(K8sTaskScheduler scheduler, List queue) { + if (nodes.isEmpty()) { + if (!initNodes(scheduler)) + return false + } + /* We want to schedule immediately if there are unoccupied nodes */ + boolean doIt = queue != null && queue.size() > 0 && taskToNode.size() < nodes.size() + log.info "[K8s] scheduleImmediately: ${queue.size()} tasks in queue, ${taskToNode.size()} tasks running on ${nodes.size()} nodes: ${doIt}" + return doIt + } + + @Override + synchronized void taskFinished(K8sTaskHandler task) { + /* TODO: This just uses elapsed wall-clock time, regardless of the frequency used to execute the task. + * This will skew the average towards longer runtimes, which is undesirable, because it will lead to more + * tasks classified as "critical path". + * A simple (rough) solution could be to keep track of the tasks "relative" frequency and just scale the + * elapsed time based on that. + */ + double runtime = (double)(task.getCompleteTimeMillis() - task.getStartTimeMillis()) + averageRuntime = (runtime + finishedTaskCount * averageRuntime) / (finishedTaskCount + 1.0) + finishedTaskCount += 1.0 + updateTopRuntimes(runtime) + + /* Free resources allocated by this task */ + WorkerNode node = taskToNode.get(task.task.hash.toString()) + if (node != null) { + node.taskFinished(task.task, dvfsClient) + taskToNode.remove(task.task.hash.toString()) + } else { + log.warn "[K8s] no node recorded for task ${task.toString()}" + } + + log.info "[K8s] task ${task.toString()} finished - ${taskToNode.size()} tasks running" + if (node != null) { + log.info "[K8s] task ran on node ${node.name} - ${node.availableMemory} bytes ${node.availableCPUs}" + } + } + + private synchronized boolean initNodes(K8sTaskScheduler scheduler) { + this.globalMaxFrequency = Long.MAX_VALUE + final nodes = scheduler.getNodes() + for (String node : nodes) { + final cur = dvfsClient.getNodeCurrentFrequency(node) + final min = dvfsClient.getNodeMinFrequency(node) + final max = dvfsClient.getNodeMaxFrequency(node) + final cpus = dvfsClient.getCPUCount(node) + final mem = dvfsClient.getMemoryAmount(node) + + if (cur.empty || min.empty || max.empty || cpus.empty || mem.empty) { + log.error "[K8s] failed to query node $node information" + continue + } + globalMaxFrequency = Long.min(globalMaxFrequency, max.asLong) + globalMinFrequency = Long.max(globalMinFrequency, min.asLong) + + log.info "[K8s] node ${node}: ${cpus.asLong} CPUs, ${mem.asLong} bytes RAM ${min.asLong} Hz - ${max.asLong} Hz current ${cur.asLong}" + this.nodes.add(new WorkerNode(node, max.asLong, min.asLong, cur.asLong, clientGetter)) + } + return !this.nodes.isEmpty() + } + + /* Returns a list of nodes that fulfill the tasks resource requirements + */ + private ArrayList filterNodes(TaskRun task) { + final long reqBytes = getTaskMemoryRequirment(task) + final int reqCPUs = getTaskCPURequirement(task) + ArrayList suitableNodes = new ArrayList<>() + for (WorkerNode n : nodes) { + if (n.availableMemory >= reqBytes && n.availableCPUs >= reqCPUs) { + log.info "[K8s] task ${task.name}: ${reqBytes} bytes ${reqCPUs} cpus: node ${n.name} has ${n.availableMemory} bytes, ${n.availableCPUs} cpus" + suitableNodes.add(n) + } + } + return suitableNodes + } + + private boolean anyNode(TaskRun task) { + final long reqBytes = getTaskMemoryRequirment(task) + final int reqCPUs = getTaskCPURequirement(task) + for (WorkerNode n : nodes) { + if (n.memoryAmount >= reqBytes && n.CPUCount >= reqCPUs) + return true + } + return false + } +} diff --git a/k8s-dvfs/src/main/groovy/recreationaltech/plugin/strategies/K8sHashSchedulingStrategy.groovy b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/strategies/K8sHashSchedulingStrategy.groovy new file mode 100644 index 0000000..de7e44a --- /dev/null +++ b/k8s-dvfs/src/main/groovy/recreationaltech/plugin/strategies/K8sHashSchedulingStrategy.groovy @@ -0,0 +1,34 @@ +package recreationaltech.plugin.strategies + +import groovy.transform.CompileStatic +import recreationaltech.plugin.K8sSchedulingDecision +import recreationaltech.plugin.K8sSchedulingRequest +import recreationaltech.plugin.K8sSchedulingStrategy +import recreationaltech.plugin.K8sTaskHandler +import recreationaltech.plugin.K8sTaskScheduler + +@CompileStatic +class K8sHashSchedulingStrategy implements K8sSchedulingStrategy { + + @Override + K8sSchedulingDecision schedule(K8sTaskScheduler scheduler, List queue) { + if (!queue) + return null + + final freeNodes = scheduler.nodes + if (freeNodes.size() == 0) + return null + + final request = queue[0] + final index = Math.floorMod(request.task.hash.asInt(), freeNodes.size()) + return new K8sSchedulingDecision(request, freeNodes[index]) + } + + @Override + boolean scheduleImmediately(K8sTaskScheduler scheduler, List queue) { + return false + } + + @Override + void taskFinished(K8sTaskHandler task) { /* nop */ } +} diff --git a/k8s-dvfs/src/test/groovy/recreationaltech/plugin/K8sConfigTest.groovy b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/K8sConfigTest.groovy new file mode 100644 index 0000000..84edd81 --- /dev/null +++ b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/K8sConfigTest.groovy @@ -0,0 +1,497 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin + +import nextflow.BuildInfo +import nextflow.SysEnv +import recreationaltech.plugin.client.ClientConfig +import recreationaltech.plugin.model.PodEnv +import recreationaltech.plugin.model.PodSecurityContext +import recreationaltech.plugin.model.PodVolumeClaim +import nextflow.util.Duration +import spock.lang.Specification +import spock.lang.Unroll +/** + * + * @author Paolo Di Tommaso + */ +class K8sConfigTest extends Specification { + + def 'should create config object' () { + + when: + def cfg = new K8sConfig() + then: + cfg.getNamespace() == null + cfg.getServiceAccount() == null + !cfg.getDebug().getYaml() + + when: + cfg = new K8sConfig( namespace:'foo', serviceAccount: 'bar', debug: [yaml: true] ) + then: + cfg.getNamespace() == 'foo' + cfg.getServiceAccount() == 'bar' + cfg.getDebug().getYaml() + cfg.debug.yaml + + } + + def 'should set cleanup' () { + given: + K8sConfig cfg + + when: + cfg = new K8sConfig() + then: 'it should return true when missing value' + cfg.getCleanup() + + when: + cfg = new K8sConfig() + then: 'it should return false specified as default' + !cfg.getCleanup(false) + + when: + cfg = new K8sConfig(cleanup:false) + then: 'it should return false' + !cfg.getCleanup() + + when: + cfg = new K8sConfig(cleanup:true) + then: 'it should return true' + cfg.getCleanup() + + when: + cfg = new K8sConfig(cleanup:true) + then: 'the default value should be ignored' + cfg.getCleanup(false) + } + + def 'should create config with storage claims' () { + + when: + def cfg = new K8sConfig(storageClaimName: 'pvc-1') + then: + cfg.getStorageClaimName() == 'pvc-1' + cfg.getStorageMountPath() == '/workspace' + cfg.getPodOptions().getVolumeClaims() == [ new PodVolumeClaim('pvc-1', '/workspace') ] as Set + + when: + cfg = new K8sConfig([ + storageClaimName: 'pvc-2', + storageMountPath: '/data', + pod: [ [volumeClaim:'foo', mountPath: '/here'], + [volumeClaim: 'bar', mountPath: '/there']] ]) + then: + cfg.getStorageClaimName() == 'pvc-2' + cfg.getStorageMountPath() == '/data' + cfg.getPodOptions().getVolumeClaims() == [ + new PodVolumeClaim('pvc-2', '/data'), + new PodVolumeClaim('foo', '/here'), + new PodVolumeClaim('bar', '/there') + ] as Set + + + when: + cfg = new K8sConfig(storageClaimName: 'pvc-3', storageMountPath: '/some/path', storageSubPath: '/bar') + then: + cfg.getStorageClaimName() == 'pvc-3' + cfg.getStorageMountPath() == '/some/path' + cfg.getStorageSubPath() == '/bar' + cfg.getPodOptions().getVolumeClaims() == [ new PodVolumeClaim('pvc-3', '/some/path', '/bar') ] as Set + + } + + def 'should set device plugin' () { + when: + def cfg = new K8sConfig([:]) + then: + cfg.fuseDevicePlugin() == ['nextflow.io/fuse':1] + + when: + cfg = new K8sConfig([fuseDevicePlugin:['foo/fuse':10]]) + then: + cfg.fuseDevicePlugin() == ['foo/fuse':10] + } + + def 'should create client config' () { + + given: + def CONFIG = [namespace: 'this', serviceAccount: 'that', client: [server: 'http://foo']] + + when: + def config = new K8sConfig(CONFIG) + def client = config.getClient() + then: + client.server == 'http://foo' + client.namespace == 'this' + client.serviceAccount == 'that' + client.httpConnectTimeout == null // testing default null + client.httpReadTimeout == null // testing default null + client.retryConfig.maxAttempts == 4 + + } + + def 'should create client config with http request timeouts' () { + + given: + def CONFIG = [ + namespace: 'this', + serviceAccount: 'that', + client: [server: 'http://foo'], + httpReadTimeout: '20s', + httpConnectTimeout: '25s' ] + + when: + def config = new K8sConfig(CONFIG) + def client = config.getClient() + then: + client.server == 'http://foo' + client.namespace == 'this' + client.serviceAccount == 'that' + client.httpConnectTimeout == Duration.of('25s') + client.httpReadTimeout == Duration.of('20s') + + } + + @Unroll + def 'should create client config with discovery' () { + + given: + def CONFIG = [context: CONTEXT, namespace: NAMESPACE, serviceAccount: SERVICE_ACCOUNT] + K8sConfig config = Spy(K8sConfig, constructorArgs: [ CONFIG ]) + + when: + def client = config.getClient() + then: + 1 * config.clientDiscovery(CONTEXT, NAMESPACE, SERVICE_ACCOUNT) >> new ClientConfig(namespace: NAMESPACE, server: SERVER) + and: + client.server == SERVER + client.namespace == NAMESPACE ?: 'default' + client.serviceAccount == SERVICE_ACCOUNT ?: 'default' + + where: + CONTEXT | SERVER | NAMESPACE | SERVICE_ACCOUNT + 'foo' | 'host.com'| null | null + 'bar' | 'this.com'| 'ns1' | 'sa2' + + } + + def 'should get nextflow image name' () { + + when: + def cfg = new K8sConfig() + then: + cfg.getNextflowImageName() == "nextflow/nextflow:${BuildInfo.version}" + + } + + def 'should get autoMountHostPaths' () { + + when: + def cfg = new K8sConfig() + then: + !cfg.getAutoMountHostPaths() + + when: + cfg = new K8sConfig(autoMountHostPaths: true) + then: + cfg.getAutoMountHostPaths() + + when: + cfg = new K8sConfig(autoMountHostPaths: false) + then: + !cfg.getAutoMountHostPaths() + } + + + def 'should get podOptions' () { + + when: + def cfg = new K8sConfig() + def opts = cfg.getPodOptions() + then: + opts.envVars == [] as Set + opts.mountSecrets == [] as Set + opts.mountConfigMaps == [] as Set + opts.volumeClaims == [] as Set + + + when: + opts = new K8sConfig(pod: [ [pullPolicy: 'Always'], [env: 'HELLO', value: 'WORLD'] ]).getPodOptions() + then: + opts.getImagePullPolicy() == 'Always' + opts.getEnvVars() == [ PodEnv.value('HELLO','WORLD') ] as Set + } + + def 'should return user name' () { + + when: + def cfg = new K8sConfig() + then: + cfg.getUserName() == System.properties.get('user.name') + + when: + cfg = new K8sConfig(userName: 'foo') + then: + cfg.getUserName() == 'foo' + } + + def 'should return user dir' () { + when: + def cfg = new K8sConfig() + then: + cfg.getLaunchDir() == '/workspace/' + System.properties.get('user.name') + + when: + cfg = new K8sConfig(storageMountPath: '/this/path', userName: 'foo') + then: + cfg.getLaunchDir() == '/this/path/foo' + + when: + cfg = new K8sConfig(storageMountPath: '/this/path', userName: 'foo', launchDir: '/my/path') + then: + cfg.getLaunchDir() == '/my/path' + + } + + def 'should return work dir' () { + when: + def cfg = new K8sConfig() + then: + cfg.getWorkDir() == "/workspace/${System.properties.get('user.name')}/work" + + when: + cfg = new K8sConfig(launchDir: '/my/dir') + then: + cfg.getWorkDir() == "/my/dir/work" + + when: + cfg = new K8sConfig(launchDir: '/my/dir', workDir: '/the/wor/dir') + then: + cfg.getWorkDir() == "/the/wor/dir" + } + + def 'should return project dir' () { + when: + def cfg = new K8sConfig() + then: + cfg.getProjectDir() == '/workspace/projects' + + when: + cfg = new K8sConfig(storageMountPath: '/foo') + then: + cfg.getProjectDir() == '/foo/projects' + + when: + cfg = new K8sConfig(storageMountPath: '/foo', projectDir: '/my/project/dir') + then: + cfg.getProjectDir() == '/my/project/dir' + } + + def 'should return storage dir' () { + + when: + def cfg = new K8sConfig() + then: + cfg.getStorageMountPath() == '/workspace' + + when: + cfg = new K8sConfig(storageMountPath: '/mnt/there') + then: + cfg.getStorageMountPath() == '/mnt/there' + + } + + def 'should return compute resource type' () { + + when: + def cfg = new K8sConfig() + then: + !cfg.useJobResource() + + when: + cfg = new K8sConfig(computeResourceType: 'Job') + then: + cfg.useJobResource() + + } + + def 'should return storage claim name' () { + when: + def cfg = new K8sConfig() + then: + cfg.getStorageClaimName() == null + + when: + cfg = new K8sConfig(storageClaimName: 'xxx') + then: + cfg.getStorageClaimName() == 'xxx' + } + + def 'should create k8s config with one volume claim' () { + + when: + def cfg = new K8sConfig( pod: [runAsUser: 1000] ) + then: + cfg.getPodOptions().getSecurityContext() == new PodSecurityContext(1000) + cfg.getPodOptions().getVolumeClaims().size() == 0 + + when: + cfg = new K8sConfig( pod: [volumeClaim: 'nf-0001', mountPath: '/workspace'] ) + then: + cfg.getPodOptions().getSecurityContext() == null + cfg.getPodOptions().getVolumeClaims() == [new PodVolumeClaim('nf-0001', '/workspace')] as Set + + + when: + cfg = new K8sConfig( pod: [ + [runAsUser: 1000], + [volumeClaim: 'nf-0001', mountPath: '/workspace'], + [volumeClaim: 'nf-0002', mountPath: '/data', subPath: '/home'] + ]) + then: + cfg.getPodOptions().getSecurityContext() == new PodSecurityContext(1000) + cfg.getPodOptions().getVolumeClaims() == [ + new PodVolumeClaim('nf-0001', '/workspace'), + new PodVolumeClaim('nf-0002', '/data', '/home') + ] as Set + + } + + + def 'should set the sec context'( ) { + + given: + def ctx = [runAsUser: 500, fsGroup: 200, allowPrivilegeEscalation: true, seLinuxOptions: [level: "s0:c123,c456"]] + + when: + def cfg = new K8sConfig( runAsUser: 500 ) + then: + cfg.getPodOptions().getSecurityContext() == new PodSecurityContext(500) + + when: + cfg = new K8sConfig( securityContext: ctx ) + then: + cfg.getPodOptions().getSecurityContext() == new PodSecurityContext(ctx) + + } + + def 'should set env and sec context' () { + given: + def ctx = [ + [env: 'FUSION_BUCKETS', value: 's3://nextflow-ci'], + [securityContext: [privileged: true]]] + + when: + def cfg = new K8sConfig( pod: ctx ) + then: + cfg.getPodOptions().getEnvVars().first() == PodEnv.value('FUSION_BUCKETS', 's3://nextflow-ci') + cfg.getPodOptions().getSecurityContext().toSpec() == [privileged:true] + + } + + def 'should set the image pull policy' () { + when: + def cfg = new K8sConfig( pullPolicy: 'always' ) + then: + cfg.getPodOptions().getImagePullPolicy() == 'always' + } + + def 'should set preserve entrypoint setting'( ) { + + when: + def cfg = new K8sConfig([:]) + then: + !cfg.entrypointOverride() + + when: + SysEnv.push(NXF_CONTAINER_ENTRYPOINT_OVERRIDE: 'true') + cfg = new K8sConfig() + def result = cfg.entrypointOverride() + SysEnv.pop() + then: + result + + } + + def 'should set debug.yaml' () { + when: + def cfg = new K8sConfig( debug: [yaml: true] ) + then: + cfg.getDebug().getYaml() + + when: + cfg = new K8sConfig( debug: [yaml: false] ) + then: + !cfg.getDebug().getYaml() + + when: + cfg = new K8sConfig( debug: null ) + then: + !cfg.getDebug().getYaml() + + when: + cfg = new K8sConfig( debug: [:] ) + then: + !cfg.getDebug().getYaml() + } + + def 'should set fetchNodeName' () { + when: + def cfg = new K8sConfig( fetchNodeName: true ) + then: + cfg.fetchNodeName() == true + + when: + cfg = new K8sConfig( fetchNodeName: false ) + then: + cfg.fetchNodeName() == false + + when: + cfg = new K8sConfig() + then: + cfg.fetchNodeName() == false + } + + def 'should set clientRefreshInterval' () { + when: + def cfg = new K8sConfig() + then: + cfg.clientRefreshInterval == Duration.of('50m') + + when: + cfg = new K8sConfig(clientRefreshInterval: '30m') + then: + cfg.clientRefreshInterval == Duration.of('30m') + + when: + cfg = new K8sConfig(clientRefreshInterval: '1h') + then: + cfg.clientRefreshInterval == Duration.of('1h') + } + + def 'should have nodeInit image' () { + when: + def cfg = new K8sConfig( + nodeInit: [ + image: 'some-image:0' + ] + ) + then: + cfg.nodeInit.image == 'some-image:0' + } +} diff --git a/k8s-dvfs/src/test/groovy/recreationaltech/plugin/K8sDriverLauncherTest.groovy b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/K8sDriverLauncherTest.groovy new file mode 100644 index 0000000..6c5b775 --- /dev/null +++ b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/K8sDriverLauncherTest.groovy @@ -0,0 +1,674 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin + +import java.nio.file.Files + +import nextflow.cli.CliOptions +import nextflow.cli.CmdKubeRun +import nextflow.cli.Launcher +import recreationaltech.plugin.client.ClientConfig +import recreationaltech.plugin.client.K8sClient +import recreationaltech.plugin.model.PodMountConfig +import recreationaltech.plugin.model.PodOptions +import recreationaltech.plugin.model.PodSpecBuilder +import recreationaltech.plugin.model.PodVolumeClaim +import spock.lang.Specification +import spock.lang.Unroll +/** + * + * @author Paolo Di Tommaso + */ +class K8sDriverLauncherTest extends Specification { + + def setup() { + PodSpecBuilder.VOLUMES.set(0) + } + + def 'should execute run' () { + given: + def NAME = 'nxf-foo' + def NF_CONFIG = [process:[executor:'k8s']] + def K8S_CONFIG = Mock(K8sConfig) + def K8S_CLIENT = Mock(K8sClient) + def driver = Spy(K8sDriverLauncher) + + when: + driver.run(NAME, ['a','b','c']) + then: + 1 * driver.makeConfig(NAME) >> NF_CONFIG + 1 * driver.makeK8sConfig(NF_CONFIG) >> K8S_CONFIG + 1 * driver.makeK8sClient(K8S_CONFIG) >> K8S_CLIENT + 1 * K8S_CONFIG.checkStorageAndPaths(K8S_CLIENT) + 1 * driver.createK8sConfigMap() >> null + 1 * driver.createK8sLauncherPod() >> null + 1 * driver.waitPodStart() >> null + 1 * driver.printK8sPodOutput() >> null + + driver.pipelineName == NAME + driver.interactive == false + driver.config == NF_CONFIG + driver.k8sConfig == K8S_CONFIG + driver.k8sClient == K8S_CLIENT + } + + def 'should make k8s config' () { + + given: + K8sConfig k8sConfig + K8sDriverLauncher driver = Spy(K8sDriverLauncher) + + when: + k8sConfig = driver.makeK8sConfig([:]) + then: + k8sConfig != null + + when: + k8sConfig = driver.makeK8sConfig(k8s: [storageClaimName: 'foo', storageMountPath: '/mnt']) + then: + k8sConfig.getStorageClaimName() == 'foo' + k8sConfig.getStorageMountPath() == '/mnt' + + } + + + @Unroll + def 'should get cmd cli' () { + + given: + def l = new K8sDriverLauncher(cmd: cmd, pipelineName: 'foo') + + when: + cmd.launcher = new Launcher(options: new CliOptions()) + then: + l.getLaunchCli() == expected + + where: + cmd | expected + new CmdKubeRun() | 'nextflow run foo' + new CmdKubeRun(cacheable: false) | 'nextflow run foo -cache false' + new CmdKubeRun(resume: true) | 'nextflow run foo -resume true' + new CmdKubeRun(poolSize: 10) | 'nextflow run foo -ps 10' + new CmdKubeRun(pollInterval: 5) | 'nextflow run foo -pi 5' + new CmdKubeRun(queueSize: 9) | 'nextflow run foo -qs 9' + new CmdKubeRun(revision: 'xyz') | 'nextflow run foo -r xyz' + new CmdKubeRun(latest: true) | 'nextflow run foo -latest true' + new CmdKubeRun(withTrace: true) | 'nextflow run foo -with-trace true' + new CmdKubeRun(withTimeline: true) | 'nextflow run foo -with-timeline true' + new CmdKubeRun(withDag: true) | 'nextflow run foo -with-dag true' + new CmdKubeRun(dumpHashes: true) | 'nextflow run foo -dump-hashes true' + new CmdKubeRun(dumpChannels: 'lala') | 'nextflow run foo -dump-channels lala' + new CmdKubeRun(env: [XX:'hello', YY: 'world']) | 'nextflow run foo -e.XX hello -e.YY world' + new CmdKubeRun(process: [mem: '100',cpus:'2']) | 'nextflow run foo -process.mem 100 -process.cpus 2' + new CmdKubeRun(params: [alpha:'x', beta:'y']) | 'nextflow run foo --alpha x --beta y' + new CmdKubeRun(params: [alpha: '/path/*.txt']) | 'nextflow run foo --alpha /path/\\*.txt' + new CmdKubeRun(entryName: 'lala') | 'nextflow run foo -entry lala' + } + + def 'should set the run name' () { + given: + def cmd = new CmdKubeRun() + cmd.launcher = new Launcher(options: new CliOptions()) + + when: + def l = new K8sDriverLauncher(cmd: cmd, pipelineName: 'foo', runName: 'bar') + then: + l.getLaunchCli() == 'nextflow run foo -name bar' + } + + + + def 'should create launcher spec pod' () { + + given: + def pod = Mock(PodOptions) + pod.getVolumeClaims() >> [ new PodVolumeClaim('pvc-1', '/mnt/path/data') ] + pod.getMountConfigMaps() >> [ new PodMountConfig('cfg-2', '/mnt/path/cfg') ] + pod.automountServiceAccountToken = false + + def k8s = Mock(K8sConfig) + k8s.getNextflowImageName() >> 'the-image' + k8s.getLaunchDir() >> '/the/user/dir' + k8s.getWorkDir() >> '/the/work/dir' + k8s.getProjectDir() >> '/the/project/dir' + k8s.getPodOptions() >> pod + + and: + def driver = Spy(K8sDriverLauncher) + driver.@runName = 'foo-boo' + driver.@k8sClient = new K8sClient(new ClientConfig(namespace: 'foo', serviceAccount: 'bar')) + driver.@k8sConfig = k8s + + when: + def spec = driver.makeLauncherSpec() + then: + driver.getLaunchCli() >> 'nextflow run foo' + + spec == [ + apiVersion: 'v1', + kind: 'Pod', + metadata: [name:'foo-boo', namespace:'foo', labels:[app:'nextflow', runName:'foo-boo']], + spec: [ + restartPolicy: 'Never', + containers: [ + [ + name: 'foo-boo', + image: 'the-image', + command: ['/bin/bash', '-c', "source /etc/nextflow/init.sh; nextflow run foo; if [ -x /etc/nextflow/node-init-cleanup.sh ]; then /etc/nextflow/node-init-cleanup.sh || true; fi; exit \$status"], + env: [ + [name:'NXF_WORK', value:'/the/work/dir'], + [name:'NXF_ASSETS', value:'/the/project/dir'], + [name:'NXF_EXECUTOR', value:'k8s'], + [name:'NXF_ANSI_LOG', value: 'false'] + ], + volumeMounts: [ + [name:'vol-1', mountPath:'/mnt/path/data'], + [name:'vol-2', mountPath:'/mnt/path/cfg'] + ] + ] + ], + serviceAccountName: 'bar', + automountServiceAccountToken: false, + volumes: [ + [name:'vol-1', persistentVolumeClaim:[claimName:'pvc-1']], + [name:'vol-2', configMap:[name:'cfg-2']] + ] + ] + ] + + } + + def 'should create launcher spec job' () { + + given: + def pod = Mock(PodOptions) + pod.getVolumeClaims() >> [ new PodVolumeClaim('pvc-1', '/mnt/path/data') ] + pod.getMountConfigMaps() >> [ new PodMountConfig('cfg-2', '/mnt/path/cfg') ] + + def k8s = Mock(K8sConfig) + k8s.getNextflowImageName() >> 'the-image' + k8s.getLaunchDir() >> '/the/user/dir' + k8s.getWorkDir() >> '/the/work/dir' + k8s.getProjectDir() >> '/the/project/dir' + k8s.getPodOptions() >> pod + k8s.useJobResource() >> true + + and: + def driver = Spy(K8sDriverLauncher) + driver.@runName = 'foo-boo' + driver.@k8sClient = new K8sClient(new ClientConfig(namespace: 'foo', serviceAccount: 'bar')) + driver.@k8sConfig = k8s + + and: + def metadata = [name: 'foo-boo', namespace: 'foo', labels: [app: 'nextflow', runName: 'foo-boo']] + + when: + def spec = driver.makeLauncherSpec() + then: + driver.getLaunchCli() >> 'nextflow run foo' + + spec == [ + apiVersion: 'batch/v1', + kind: 'Job', + metadata: metadata, + spec: [ + backoffLimit: 0, + template: [ + metadata: metadata, + spec: [ + restartPolicy: 'Never', + containers: [ + [ + name: 'foo-boo', + image: 'the-image', + command: ['/bin/bash', '-c', "source /etc/nextflow/init.sh; nextflow run foo; if [ -x /etc/nextflow/node-init-cleanup.sh ]; then /etc/nextflow/node-init-cleanup.sh || true; fi; exit \$status"], + env: [ + [name:'NXF_WORK', value:'/the/work/dir'], + [name:'NXF_ASSETS', value:'/the/project/dir'], + [name:'NXF_EXECUTOR', value:'k8s'], + [name:'NXF_ANSI_LOG', value: 'false'] + ], + volumeMounts: [ + [name:'vol-1', mountPath:'/mnt/path/data'], + [name:'vol-2', mountPath:'/mnt/path/cfg'] + ] + ] + ], + serviceAccountName: 'bar', + automountServiceAccountToken: false, + volumes: [ + [name:'vol-1', persistentVolumeClaim:[claimName:'pvc-1']], + [name:'vol-2', configMap:[name:'cfg-2']] + ] + ] + ] + ] + ] + } + + def 'should use user provided pod image' () { + + given: + def pod = Mock(PodOptions) + pod.getVolumeClaims() >> [ new PodVolumeClaim('pvc-1', '/mnt/path/data') ] + pod.getMountConfigMaps() >> [ new PodMountConfig('cfg-2', '/mnt/path/cfg') ] + + def k8s = Mock(K8sConfig) + k8s.getLaunchDir() >> '/the/user/dir' + k8s.getWorkDir() >> '/the/work/dir' + k8s.getProjectDir() >> '/the/project/dir' + k8s.getPodOptions() >> pod + + and: + def driver = Spy(K8sDriverLauncher) + driver.@runName = 'foo-boo' + driver.@k8sClient = new K8sClient(new ClientConfig(namespace: 'foo', serviceAccount: 'bar')) + driver.@k8sConfig = k8s + driver.@headImage = 'foo/bar' + + when: + def result = driver.makeLauncherSpec() + then: + driver.getLaunchCli() >> 'nextflow run foo' + and: + result.spec.containers[0].image == 'foo/bar' + + } + + def 'should use user provided head-cpu and head-memory request' () { + given: + def pod = Mock(PodOptions) + pod.getVolumeClaims() >> [ new PodVolumeClaim('pvc-1', '/mnt/path/data') ] + pod.getMountConfigMaps() >> [ new PodMountConfig('cfg-2', '/mnt/path/cfg') ] + + def k8s = Mock(K8sConfig) + k8s.getNextflowImageName() >> 'the-image' + k8s.getLaunchDir() >> '/the/user/dir' + k8s.getWorkDir() >> '/the/work/dir' + k8s.getProjectDir() >> '/the/project/dir' + k8s.getPodOptions() >> pod + + and: + def driver = Spy(K8sDriverLauncher) + driver.@runName = 'foo-boo' + driver.@k8sClient = new K8sClient(new ClientConfig(namespace: 'foo', serviceAccount: 'bar')) + driver.@k8sConfig = k8s + driver.@headCpus = 2 + driver.@headMemory = '200Mi' + + when: + def result = driver.makeLauncherSpec() + then: + driver.getLaunchCli() >> 'nextflow run foo' + and: + result.spec.containers[0].resources == [ + requests: [cpu: 2, memory: '200Mi'], + limits: [memory: '200Mi'] + ] + } + + def 'should use user provided head-cpu and head-memory limits' () { + given: + def pod = Mock(PodOptions) + pod.getVolumeClaims() >> [ new PodVolumeClaim('pvc-1', '/mnt/path/data') ] + pod.getMountConfigMaps() >> [ new PodMountConfig('cfg-2', '/mnt/path/cfg') ] + + def k8s = Mock(K8sConfig) + k8s.getNextflowImageName() >> 'the-image' + k8s.getLaunchDir() >> '/the/user/dir' + k8s.getWorkDir() >> '/the/work/dir' + k8s.getProjectDir() >> '/the/project/dir' + k8s.getPodOptions() >> pod + k8s.cpuLimitsEnabled() >> true + + and: + def driver = Spy(K8sDriverLauncher) + driver.@runName = 'foo-boo' + driver.@k8sClient = new K8sClient(new ClientConfig(namespace: 'foo', serviceAccount: 'bar')) + driver.@k8sConfig = k8s + driver.@headCpus = 2 + driver.@headMemory = '200Mi' + + when: + def result = driver.makeLauncherSpec() + then: + driver.getLaunchCli() >> 'nextflow run foo' + and: + result.spec.containers[0].resources == [ + requests: [cpu: 2, memory: '200Mi'], + limits: [cpu: 2, memory: '200Mi'] + ] + } + + def 'should create config map' () { + + given: + def folder = Files.createTempDirectory('foo') + + def params = folder.resolve('params.json') + params.text = 'bla-bla' + def driver = Spy(K8sDriverLauncher) + def NXF_CONFIG = [foo: 'bar'].toConfigObject() + + def SCM_FILE = folder.resolve('scm') + SCM_FILE.text = "hello = 'world'\n" + + + def EXPECTED = [:] + EXPECTED['init.sh'] == '' + + def POD_OPTIONS = new PodOptions() + + def K8S_CONFIG = Mock(K8sConfig) + K8S_CONFIG.getLaunchDir() >> '/launch/dir' + K8S_CONFIG.getPodOptions() >> POD_OPTIONS + + when: + driver.@config = NXF_CONFIG + driver.@k8sConfig = K8S_CONFIG + driver.@cmd = new CmdKubeRun(paramsFile: params.toString()) + + driver.createK8sConfigMap() + then: + 1 * driver.getScmFile() >> SCM_FILE + 1 * driver.makeConfigMapName(_ as Map) >> 'nf-config-123' + 1 * driver.tryCreateConfigMap('nf-config-123', _ as Map) >> { name, cfg -> + assert cfg.'init.sh' == "mkdir -p '/launch/dir'; if [ -d '/launch/dir' ]; then cd '/launch/dir'; else echo 'Cannot create directory: /launch/dir'; exit 1; fi; [ -f /etc/nextflow/scm ] && ln -s /etc/nextflow/scm \$NXF_HOME/scm; [ -f /etc/nextflow/nextflow.config ] && cp /etc/nextflow/nextflow.config \$PWD/nextflow.config; " + assert cfg.'nextflow.config' == "foo = 'bar'\n" + assert cfg.'scm' == "hello = 'world'\n" + assert cfg.'params.json' == 'bla-bla' + return null + } + + POD_OPTIONS.getMountConfigMaps() == [ new PodMountConfig('nf-config-123', '/etc/nextflow') ] as Set + + cleanup: + folder?.deleteDir() + } + + def 'should create config map with pre-script' () { + + given: + def folder = Files.createTempDirectory('foo') + + def params = folder.resolve('params.json') + params.text = 'bla-bla' + def driver = Spy(K8sDriverLauncher) + driver.@headPreScript = '/bin/foo.sh' + def NXF_CONFIG = [foo: 'bar'].toConfigObject() + + def SCM_FILE = folder.resolve('scm') + SCM_FILE.text = "hello = 'world'\n" + + + def EXPECTED = [:] + EXPECTED['init.sh'] == '' + + def POD_OPTIONS = new PodOptions() + + def K8S_CONFIG = Mock(K8sConfig) + K8S_CONFIG.getLaunchDir() >> '/launch/dir' + K8S_CONFIG.getPodOptions() >> POD_OPTIONS + + when: + driver.@config = NXF_CONFIG + driver.@k8sConfig = K8S_CONFIG + driver.@cmd = new CmdKubeRun(paramsFile: params.toString()) + + driver.createK8sConfigMap() + then: + 1 * driver.getScmFile() >> SCM_FILE + 1 * driver.makeConfigMapName(_ as Map) >> 'nf-config-123' + 1 * driver.tryCreateConfigMap('nf-config-123', _ as Map) >> { name, cfg -> + assert cfg.'init.sh' == "mkdir -p '/launch/dir'; if [ -d '/launch/dir' ]; then cd '/launch/dir'; else echo 'Cannot create directory: /launch/dir'; exit 1; fi; [ -f /etc/nextflow/scm ] && ln -s /etc/nextflow/scm \$NXF_HOME/scm; [ -f /etc/nextflow/nextflow.config ] && cp /etc/nextflow/nextflow.config \$PWD/nextflow.config; [ -f '/bin/foo.sh' ] && '/bin/foo.sh'; " + assert cfg.'nextflow.config' == "foo = 'bar'\n" + assert cfg.'scm' == "hello = 'world'\n" + assert cfg.'params.json' == 'bla-bla' + return null + } + + POD_OPTIONS.getMountConfigMaps() == [ new PodMountConfig('nf-config-123', '/etc/nextflow') ] as Set + + cleanup: + folder?.deleteDir() + } + + + def 'should make config' () { + given: + Map config + def driver = Spy(K8sDriverLauncher) + def NAME = 'somePipelineName' + def CFG_EMPTY = new ConfigObject() + def CFG_WITH_MOUNTS = new ConfigObject() + CFG_WITH_MOUNTS.k8s.storageClaimName = 'pvc' + CFG_WITH_MOUNTS.k8s.storageMountPath = '/foo' + + when: + driver.@cmd = new CmdKubeRun() + config = driver.makeConfig(NAME).toMap() + then: + 1 * driver.loadConfig(NAME) >> CFG_EMPTY + config.process.executor == 'k8s' + config.k8s.pod == null + config.k8s.storageMountPath == null + config.k8s.storageClaimName == null + + when: + driver.@cmd = new CmdKubeRun() + config = driver.makeConfig(NAME).toMap() + then: + 1 * driver.loadConfig(NAME) >> CFG_WITH_MOUNTS + config.process.executor == 'k8s' + config.k8s.storageClaimName == 'pvc' + config.k8s.storageMountPath == '/foo' + and: + new K8sConfig(config.k8s).getStorageClaimName() == 'pvc' + new K8sConfig(config.k8s).getStorageMountPath() == '/foo' + new K8sConfig(config.k8s).getPodOptions() == new PodOptions([ [volumeClaim:'pvc', mountPath: '/foo'] ]) + + when: + driver.@cmd = new CmdKubeRun(volMounts: ['pvc-1:/this','pvc-2:/that'] ) + config = driver.makeConfig(NAME).toMap() + then: + 1 * driver.loadConfig(NAME) >> CFG_EMPTY + config.process.executor == 'k8s' + config.k8s.storageClaimName == 'pvc-1' + config.k8s.storageMountPath == '/this' + config.k8s.pod == [ [volumeClaim: 'pvc-2', mountPath: '/that'] ] + and: + new K8sConfig(config.k8s).getStorageClaimName() == 'pvc-1' + new K8sConfig(config.k8s).getStorageMountPath() == '/this' + new K8sConfig(config.k8s).getPodOptions() == new PodOptions([ + [volumeClaim:'pvc-1', mountPath: '/this'], + [volumeClaim:'pvc-2', mountPath: '/that'] + ]) + + + when: + driver.@cmd = new CmdKubeRun(volMounts: ['xyz:/this'] ) + config = driver.makeConfig(NAME).toMap() + then: + 1 * driver.loadConfig(NAME) >> CFG_WITH_MOUNTS + config.process.executor == 'k8s' + config.k8s.storageClaimName == 'xyz' + config.k8s.storageMountPath == '/this' + config.k8s.pod == null + and: + new K8sConfig(config.k8s).getStorageClaimName() == 'xyz' + new K8sConfig(config.k8s).getStorageMountPath() == '/this' + new K8sConfig(config.k8s).getPodOptions() == new PodOptions([ + [volumeClaim:'xyz', mountPath: '/this'] + ]) + + + when: + driver.@cmd = new CmdKubeRun(volMounts: ['xyz', 'bar:/mnt/bar'] ) + config = driver.makeConfig(NAME).toMap() + then: + 1 * driver.loadConfig(NAME) >> CFG_WITH_MOUNTS + config.process.executor == 'k8s' + config.k8s.storageClaimName == 'xyz' + config.k8s.storageMountPath == null + config.k8s.pod == [ [volumeClaim: 'bar', mountPath: '/mnt/bar'] ] + and: + new K8sConfig(config.k8s).getStorageClaimName() == 'xyz' + new K8sConfig(config.k8s).getStorageMountPath() == '/workspace' + new K8sConfig(config.k8s).getPodOptions() == new PodOptions([ + [volumeClaim:'xyz', mountPath: '/workspace'], + [volumeClaim:'bar', mountPath: '/mnt/bar'] + ]) + + } + + def 'should add the plugin into the config' () { + given: + def cmd = new CmdKubeRun() + cmd.launcher = new Launcher(options: new CliOptions()) + + when: + def l = new K8sDriverLauncher(cmd: cmd, plugins: 'nf-cws@1.0.0', runName: 'bar') + then: + l.makeConfig( "/bar").get('plugins') == [ 'nf-cws@1.0.0' ] + } + + def 'should make config - deprecated' () { + + given: + Map config + def driver = Spy(K8sDriverLauncher) + def NAME = 'somePipelineName' + def CFG_EMPTY = new ConfigObject() + def CFG_WITH_MOUNTS = new ConfigObject() + CFG_WITH_MOUNTS.k8s.volumeClaims = [ pvc: [mountPath:'/foo'] ] + + when: + driver.@cmd = new CmdKubeRun() + config = driver.makeConfig(NAME).toMap() + then: + 1 * driver.loadConfig(NAME) >> CFG_EMPTY + config.process.executor == 'k8s' + + when: + driver.@cmd = new CmdKubeRun() + config = driver.makeConfig(NAME).toMap() + then: + 1 * driver.loadConfig(NAME) >> CFG_WITH_MOUNTS + config.process.executor == 'k8s' + config.k8s.storageClaimName == 'pvc' + config.k8s.storageMountPath == '/foo' + + when: + driver.@cmd = new CmdKubeRun(volMounts: ['pvc-1:/this','pvc-2:/that'] ) + config = driver.makeConfig(NAME).toMap() + then: + 1 * driver.loadConfig(NAME) >> CFG_EMPTY + config.process.executor == 'k8s' + config.k8s.storageClaimName == 'pvc-1' + config.k8s.storageMountPath == '/this' + config.k8s.pod == [ [volumeClaim: 'pvc-2', mountPath: '/that'] ] + + + when: + driver.@cmd = new CmdKubeRun(volMounts: ['xyz:/this'] ) + config = driver.makeConfig(NAME).toMap() + then: + 1 * driver.loadConfig(NAME) >> CFG_WITH_MOUNTS + config.process.executor == 'k8s' + config.k8s.storageClaimName == 'xyz' + config.k8s.storageMountPath == '/this' + config.k8s.pod == null + and: + new K8sConfig(config.k8s).getStorageClaimName() == 'xyz' + new K8sConfig(config.k8s).getStorageMountPath() == '/this' + new K8sConfig(config.k8s).getPodOptions() == new PodOptions([ + [volumeClaim:'xyz', mountPath: '/this'] + ]) + + } + + def 'should return pod exit status' () { + given: + def POD_NAME = 'pod-x' + def client = Mock(K8sClient) + def driver = Spy(K8sDriverLauncher) + driver.@k8sClient = client + driver.@runName = POD_NAME + driver.@k8sConfig = Mock(K8sConfig) + + when: + def status = driver.waitPodTermination() + then: + 1 * client.podState(POD_NAME) >> [terminated: [exitCode: 99]] + 1 * driver.k8sConfig.useJobResource() >> [:] + then: + status == 99 + + when: + status = driver.waitPodTermination() + then: + 1 * client.podState(POD_NAME) >> [:] + then: + 1 * client.podState(POD_NAME) >> [terminated: [exitCode: 99]] + then: + status == 99 + + when: + status = driver.waitPodTermination() + then: + 1 * client.podState(POD_NAME) >> [:] + 1 * driver.isWaitTimedOut(_) >> true + then: + status == 127 + } + + + def 'should delete configMap' () { + given: + def POD_NAME = 'pod-x' + def config = Mock(K8sConfig) + def driver = Spy(K8sDriverLauncher) + driver.@k8sConfig = config + driver.@runName = POD_NAME + driver.@initDeployer = new K8sNodeInitDeployer(driver.k8sClient, config) + + when: + driver.shutdown() + then: + 1 * driver.waitPodTermination() >> 0 + then: + 1 * config.getCleanup(true) >> true + 1 * driver.deleteConfigMap() >> null + + when: + driver.shutdown() + then: + 1 * driver.waitPodTermination() >> 1 + then: + 1 * config.getCleanup(false) >> true + 1 * driver.deleteConfigMap() >> null + + when: + driver.shutdown() + then: + 1 * driver.waitPodTermination() >> 1 + then: + 1 * config.getCleanup(false) >> false + 0 * driver.deleteConfigMap() >> null + + } +} diff --git a/k8s-dvfs/src/test/groovy/recreationaltech/plugin/K8sDvfsObserverTest.groovy b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/K8sDvfsObserverTest.groovy deleted file mode 100644 index b76b4f8..0000000 --- a/k8s-dvfs/src/test/groovy/recreationaltech/plugin/K8sDvfsObserverTest.groovy +++ /dev/null @@ -1,22 +0,0 @@ -package recreationaltech.plugin - -import nextflow.Session -import spock.lang.Specification - -/** - * Implements a basic factory test - * - */ -class K8sDvfsObserverTest extends Specification { - - def 'should create the observer instance' () { - given: - def factory = new K8sDvfsFactory() - when: - def result = factory.create(Mock(Session)) - then: - result.size() == 1 - result.first() instanceof K8sDvfsObserver - } - -} diff --git a/k8s-dvfs/src/test/groovy/recreationaltech/plugin/K8sExecutorTest.groovy b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/K8sExecutorTest.groovy new file mode 100644 index 0000000..4214f03 --- /dev/null +++ b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/K8sExecutorTest.groovy @@ -0,0 +1,66 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin + +import java.util.concurrent.TimeUnit + +import com.google.common.cache.CacheBuilder +import recreationaltech.plugin.client.ClientConfig +import recreationaltech.plugin.client.K8sClient +import spock.lang.Specification + +/** + * @author Paolo Di Tommaso + */ +class K8sExecutorTest extends Specification { + + def 'should cache k8s client and refresh after expiration' () { + given: + def CONFIG = new K8sConfig( + client: [server: 'http://k8s-server'], + namespace: 'test-ns', + serviceAccount: 'test-sa', + clientRefreshInterval: '100ms' + ) + and: + def executor = Spy(K8sExecutor) + executor.getK8sConfig() >> CONFIG + // use a short-lived cache for the test + executor.@clientCache = CacheBuilder.newBuilder() + .expireAfterWrite(100, TimeUnit.MILLISECONDS) + .build() + + when: 'first call to getClient' + def client1 = executor.getClient() + then: 'a new K8sClient is created' + client1 instanceof K8sClient + client1.config.server == 'http://k8s-server' + + when: 'second call within cache interval' + def client2 = executor.getClient() + then: 'returns the same cached instance' + client2.is(client1) + + when: 'call after cache expiration' + sleep(150) + def client3 = executor.getClient() + then: 'a new K8sClient instance is created' + client3 instanceof K8sClient + !client3.is(client1) + } + +} diff --git a/k8s-dvfs/src/test/groovy/recreationaltech/plugin/K8sNodeInitDeployerTest.groovy b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/K8sNodeInitDeployerTest.groovy new file mode 100644 index 0000000..6eb6688 --- /dev/null +++ b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/K8sNodeInitDeployerTest.groovy @@ -0,0 +1,164 @@ +package recreationaltech.plugin + +import recreationaltech.plugin.client.K8sClient +import recreationaltech.plugin.model.PodSpecBuilder +import spock.lang.Specification + +class K8sNodeInitDeployerTest extends Specification { + + def setup() { + PodSpecBuilder.VOLUMES.set(0) + } + + def 'should not deploy pods when node init is disabled' () { + given: + def client = Mock(K8sClient) + def config = new K8sConfig(nodeInit: [enabled: false]) + def deployer = new K8sNodeInitDeployer(client, config) + + when: + deployer.deploy() + + then: + 0 * client.nodeList() + 0 * client.podCreate(_) + } + + def 'should deploy one init pod for each node' () { + given: + def client = Mock(K8sClient) + def config = new K8sConfig(nodeInit: [ + enabled: true, + image: 'ubuntu:latest', + command: ['/bin/bash', '-c', 'echo init'] + ]) + def deployer = new K8sNodeInitDeployer(client, config) + + when: + deployer.deploy() + + then: + 1 * client.nodeList() >> [ + items: [ + [metadata: [name: 'node-a']], + [metadata: [name: 'node-b']] + ] + ] + + then: + 1 * client.podCreate({ Map spec -> + spec.kind == 'Pod' + spec.metadata.name == 'nf-init-node-a' + spec.metadata.namespace == 'default' + spec.spec.nodeName == 'node-a' + spec.spec.restartPolicy == 'Never' + + def container = spec.spec.containers[0] + container.name == 'nf-init-node-a' + container.image == 'ubuntu:latest' + container.command == ['/bin/bash', '-c', 'echo init'] + container.securityContext.privileged == true + + spec.spec.volumes*.hostPath*.path as Set == ['/sys', '/dev', '/lib/modules'] as Set + container.volumeMounts*.mountPath as Set == ['/sys', '/dev', '/lib/modules'] as Set + }) + + then: + 1 * client.podCreate({ Map spec -> + spec.kind == 'Pod' + spec.metadata.name == 'nf-init-node-b' + spec.spec.nodeName == 'node-b' + + def container = spec.spec.containers[0] + container.name == 'nf-init-node-b' + container.image == 'ubuntu:latest' + container.command == ['/bin/bash', '-c', 'echo init'] + container.securityContext.privileged == true + }) + + 0 * client._ + } + + def 'should lowercase and truncate generated pod names' () { + given: + def client = Mock(K8sClient) + def config = new K8sConfig(nodeInit: [ + enabled: true, + image: 'ubuntu:latest', + command: ['true'] + ]) + def deployer = new K8sNodeInitDeployer(client, config) + + when: + deployer.deploy() + + then: + 1 * client.nodeList() >> [ + items: [ + [metadata: [name: 'NODE-WITH-A-VERY-LONG-NAME-ABCDEFGHIJKLMNOPQRSTUVWXYZ-0123']] + ] + ] + + then: + 1 * client.podCreate({ Map spec -> + spec.metadata.name == 'nf-init-node-with-a-very-long-name-abcdefghijklmnopqrstuvwxyz-0' + spec.metadata.name.size() == 63 + spec.metadata.name == spec.metadata.name.toLowerCase() + spec.spec.nodeName == 'NODE-WITH-A-VERY-LONG-NAME-ABCDEFGHIJKLMNOPQRSTUVWXYZ-0123' + }) + + then: + 0 * client._ + } + + def 'should not cleanup pods when node init is disabled' () { + given: + def client = Mock(K8sClient) + def config = new K8sConfig(nodeInit: [enabled: false, cleanup: true]) + def deployer = new K8sNodeInitDeployer(client, config) + + when: + deployer.cleanup() + + then: + 0 * client.nodeList() + 0 * client.podDelete(_) + } + + def 'should not cleanup pods when cleanup is disabled' () { + given: + def client = Mock(K8sClient) + def config = new K8sConfig(nodeInit: [enabled: true, cleanup: false]) + def deployer = new K8sNodeInitDeployer(client, config) + + when: + deployer.cleanup() + + then: + 0 * client.nodeList() + 0 * client.podDelete(_) + } + + def 'should cleanup one init pod for each node' () { + given: + def client = Mock(K8sClient) + def config = new K8sConfig(nodeInit: [enabled: true, cleanup: true]) + def deployer = new K8sNodeInitDeployer(client, config) + + when: + deployer.cleanup() + + then: + 1 * client.nodeList() >> [ + items: [ + [metadata: [name: 'node-a']], + [metadata: [name: 'NODE-B']] + ] + ] + + then: + 1 * client.podDelete('nf-init-node-a') + 1 * client.podDelete('nf-init-node-b') + 0 * client._ + } +} diff --git a/k8s-dvfs/src/test/groovy/recreationaltech/plugin/K8sRuntimeEstimatorTest.groovy b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/K8sRuntimeEstimatorTest.groovy new file mode 100644 index 0000000..ab24ec2 --- /dev/null +++ b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/K8sRuntimeEstimatorTest.groovy @@ -0,0 +1,55 @@ +package recreationaltech.plugin + +import spock.lang.Specification + +class K8sRuntimeEstimatorTest extends Specification { + + def 'does extract a function' () { + given: + def observations = [a: [new Tuple2(0, 0), new Tuple2(100, 100)]] + def estimator = new K8sRuntimeEstimator(observations) + + when: + def estimation = estimator.estimate("a", 1234) + + then: + estimation != Double.POSITIVE_INFINITY + estimation == 1234.0 + } + + def 'can extract a function from single measurement' () { + given: + def observations = [a: [new Tuple2(100, 100)]] + def estimator = new K8sRuntimeEstimator(observations) + + when: + def estimation = estimator.estimate("a", 1234) + + then: + estimation == 100.0 /* Single measurement implies constant runtime */ + } + + def 'unknown function returns positive infinity' () { + given: + def observations = [] as HashMap + def estimator = new K8sRuntimeEstimator(observations) + + when: + def estimation = estimator.estimate("b", 1234) + + then: + estimation == Double.POSITIVE_INFINITY + } + + def 'extracts function from multiple measurements' () { + given: + def observations = [a: [new Tuple2(10, 15), new Tuple2(20, 34), new Tuple2(30, 46)]] + def estimator = new K8sRuntimeEstimator(observations) + + when: + def estimation = estimator.estimate("a", 40) + + then: + estimation != Double.POSITIVE_INFINITY + } +} diff --git a/k8s-dvfs/src/test/groovy/recreationaltech/plugin/K8sSchedulingRequestTest.groovy b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/K8sSchedulingRequestTest.groovy new file mode 100644 index 0000000..0cad035 --- /dev/null +++ b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/K8sSchedulingRequestTest.groovy @@ -0,0 +1,23 @@ +package recreationaltech.plugin + +import nextflow.processor.TaskRun +import spock.lang.Specification + +class K8sSchedulingRequestTest extends Specification { + + def 'should create scheduling request from task handler' () { + given: + def task = Mock(TaskRun) + def handler = Spy(K8sTaskHandler) + handler.task = task + + when: + def request = new K8sSchedulingRequest(handler) + + then: + request.handler == handler + request.task == task + request.submitTimeMillis > 0 + request.submitTimeMillis <= System.currentTimeMillis() + } +} diff --git a/k8s-dvfs/src/test/groovy/recreationaltech/plugin/K8sTaskHandlerTest.groovy b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/K8sTaskHandlerTest.groovy new file mode 100644 index 0000000..87d23ea --- /dev/null +++ b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/K8sTaskHandlerTest.groovy @@ -0,0 +1,1198 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin + +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths + +import nextflow.Session +import nextflow.SysEnv +import nextflow.exception.NodeTerminationException +import nextflow.file.http.XPath +import nextflow.fusion.FusionConfig +import nextflow.fusion.FusionScriptLauncher +import recreationaltech.plugin.client.ClientConfig +import recreationaltech.plugin.client.K8sClient +import recreationaltech.plugin.client.K8sResponseException +import recreationaltech.plugin.client.K8sResponseJson +import recreationaltech.plugin.client.PodUnschedulableException +import recreationaltech.plugin.model.PodEnv +import recreationaltech.plugin.model.PodHostMount +import recreationaltech.plugin.model.PodMountConfig +import recreationaltech.plugin.model.PodMountSecret +import recreationaltech.plugin.model.PodOptions +import recreationaltech.plugin.model.PodSpecBuilder +import recreationaltech.plugin.model.PodVolumeClaim +import nextflow.processor.TaskBean +import nextflow.processor.TaskConfig +import nextflow.processor.TaskProcessor +import nextflow.processor.TaskRun +import nextflow.processor.TaskStatus +import nextflow.util.MemoryUnit +import spock.lang.Specification +/** + * + * @author Paolo Di Tommaso + */ +class K8sTaskHandlerTest extends Specification { + + def setup() { + PodSpecBuilder.VOLUMES.set(0) + } + + def 'should return a new pod request' () { + given: + def WORK_DIR = Paths.get('/some/work/dir') + def config = Mock(TaskConfig) + def task = Mock(TaskRun) + def client = Mock(K8sClient) + def builder = Mock(K8sWrapperBuilder) + def handler = Spy(new K8sTaskHandler(builder:builder, client: client)) + Map result + + when: + result = handler.newSubmitRequest(task) + then: + _ * handler.fusionEnabled() >> false + 1 * handler.fixOwnership() >> false + 1 * handler.entrypointOverride() >> false + 1 * handler.cpuLimitsEnabled() >> false + 1 * handler.getPodOptions() >> new PodOptions() + 1 * handler.getSyntheticPodName(task) >> 'nf-123' + 1 * handler.getLabels(task) >> [:] + 1 * handler.getAnnotations() >> [:] + 1 * handler.getContainerMounts() >> [] + 1 * task.getContainer() >> 'debian:latest' + 1 * task.getWorkDir() >> WORK_DIR + 1 * task.getConfig() >> config + 1 * config.getCpus() >> 0 + 1 * config.getMemory() >> null + 1 * client.getConfig() >> new ClientConfig() + and: + result == [ + apiVersion: 'v1', + kind: 'Pod', + metadata: [ + name:'nf-123', + namespace:'default' + ], + spec: [ + restartPolicy:'Never', + containers: [[ + name:'nf-123', + image:'debian:latest', + args:['/bin/bash', '-ue', '-o', 'pipefail', '-c', 'bash /some/work/dir/.command.run 2>&1 | tee /some/work/dir/.command.log'] + ]] + ] + ] + + when: + result = handler.newSubmitRequest(task) + then: + _ * handler.fusionEnabled() >> false + 1 * handler.entrypointOverride() >> true + 1 * handler.cpuLimitsEnabled() >> false + 1 * handler.getSyntheticPodName(task) >> 'nf-foo' + 1 * handler.getLabels(task) >> [sessionId:'xxx'] + 1 * handler.getAnnotations() >> [evict: 'false'] + 1 * handler.getPodOptions() >> new PodOptions() + 1 * handler.getContainerMounts() >> [] + 1 * handler.fixOwnership() >> true + 1 * handler.getOwner() >> '501:502' + 1 * task.getContainer() >> 'debian:latest' + 1 * task.getWorkDir() >> WORK_DIR + 1 * task.getConfig() >> config + 1 * config.getCpus() >> 1 + 1 * config.getMemory() >> null + 1 * client.getConfig() >> new ClientConfig() + and: + result.metadata.labels == [sessionId: 'xxx'] + result.metadata.annotations == [evict: 'false'] + result.spec.containers[0].command == ['/bin/bash', '-ue', '-o', 'pipefail', '-c', 'bash /some/work/dir/.command.run 2>&1 | tee /some/work/dir/.command.log'] + result.spec.containers[0].resources == [ requests: [cpu:1] ] + result.spec.containers[0].env == [ [name:'NXF_OWNER', value:'501:502'] ] + + when: + result = handler.newSubmitRequest(task) + then: + _ * handler.fusionEnabled() >> false + 1 * handler.fixOwnership() >> false + 1 * handler.entrypointOverride() >> true + 1 * handler.cpuLimitsEnabled() >> false + 1 * handler.getSyntheticPodName(task) >> 'nf-abc' + 1 * handler.getLabels(task) >> [:] + 1 * handler.getAnnotations() >> [:] + 1 * handler.getPodOptions() >> new PodOptions() + 1 * handler.getContainerMounts() >> [] + 1 * task.getContainer() >> 'user/alpine:1.0' + 1 * task.getWorkDir() >> WORK_DIR + 1 * task.getConfig() >> config + 1 * config.getCpus() >> 4 + 1 * config.getMemory() >> MemoryUnit.of('16GB') + 1 * client.getConfig() >> new ClientConfig(namespace: 'namespace-x') + and: + result.metadata.namespace == 'namespace-x' + result.spec.containers[0].image == 'user/alpine:1.0' + result.spec.containers[0].command == ['/bin/bash', '-ue', '-o', 'pipefail', '-c', 'bash /some/work/dir/.command.run 2>&1 | tee /some/work/dir/.command.log'] + result.spec.containers[0].resources == [ requests: [cpu:4, memory:'16384Mi'], limits: [memory:'16384Mi'] ] + + } + + def 'should create a pod with debug options' () { + given: + SysEnv.push([NXF_DEBUG:'true']) + and: + def WORK_DIR = Paths.get('/some/work/dir') + def config = Mock(TaskConfig) + def task = Mock(TaskRun) + def client = Mock(K8sClient) + def builder = Mock(K8sWrapperBuilder) + def handler = Spy(new K8sTaskHandler(builder: builder, client:client)) + Map result + + when: + result = handler.newSubmitRequest(task) + then: + _ * handler.fusionEnabled() >> false + 1 * handler.fixOwnership() >> false + 1 * handler.entrypointOverride() >> false + 1 * handler.cpuLimitsEnabled() >> false + 1 * handler.getPodOptions() >> new PodOptions() + 1 * handler.getSyntheticPodName(task) >> 'nf-123' + 1 * handler.getLabels(task) >> [:] + 1 * handler.getAnnotations() >> [:] + 1 * handler.getContainerMounts() >> [] + 1 * task.getContainer() >> 'debian:latest' + 1 * task.getWorkDir() >> WORK_DIR + 1 * task.getConfig() >> config + 1 * config.getCpus() >> 0 + 1 * config.getMemory() >> null + 1 * client.getConfig() >> new ClientConfig() + and: + result.spec.containers[0].env == [[name:'NXF_DEBUG', value:'true']] + + cleanup: + SysEnv.pop() + } + + def 'should create a pod with specified client configs' () { + + given: + def WORK_DIR = Paths.get('/some/work/dir') + def task = Mock(TaskRun) + def client = Mock(K8sClient) + def builder = Mock(K8sWrapperBuilder) + def config = Mock(ClientConfig) + def handler = Spy(new K8sTaskHandler(builder: builder, client: client)) + Map result + + when: + result = handler.newSubmitRequest(task) + then: + _ * handler.fusionEnabled() >> false + 1 * handler.fixOwnership() >> false + 1 * handler.entrypointOverride() >> true + 1 * handler.cpuLimitsEnabled() >> false + 1 * handler.getSyntheticPodName(task) >> 'nf-123' + 1 * handler.getPodOptions() >> new PodOptions() + 1 * handler.getLabels(task) >> [:] + 1 * handler.getAnnotations() >> [:] + 1 * handler.getContainerMounts() >> [] + 1 * task.getContainer() >> 'debian:latest' + 1 * task.getWorkDir() >> WORK_DIR + 1 * task.getConfig() >> new TaskConfig() + 1 * client.getConfig() >> config + 1 * config.getNamespace() >> 'just-a-namespace' + 1 * config.getServiceAccount() >> 'pedantic-kallisto' + and: + result.metadata.namespace == 'just-a-namespace' + result.spec.serviceAccountName == 'pedantic-kallisto' + + } + + def 'should create a pod with specified pod options' () { + + given: + def WORK_DIR = Paths.get('/some/work/dir') + def task = Mock(TaskRun) + def client = Mock(K8sClient) + def builder = Mock(K8sWrapperBuilder) + def config = Mock(TaskConfig) + def handler = Spy(new K8sTaskHandler(builder:builder, client:client)) + def podOptions = Mock(PodOptions) + and: + Map result + + when: + result = handler.newSubmitRequest(task) + then: + 1 * client.getConfig() >> new ClientConfig() + _ * handler.fusionEnabled() >> false + 1 * handler.fixOwnership() >> false + 1 * handler.entrypointOverride() >> true + 1 * handler.cpuLimitsEnabled() >> false + 1 * handler.getSyntheticPodName(task) >> 'nf-123' + 1 * handler.getLabels(task) >> [:] + 1 * handler.getAnnotations() >> [:] + 1 * handler.getPodOptions() >> podOptions + 1 * handler.getContainerMounts() >> [] + 1 * task.getContainer() >> 'debian:latest' + 1 * task.getWorkDir() >> WORK_DIR + 1 * task.getConfig() >> config + 2 * podOptions.getEnvVars() >> [ PodEnv.value('FOO','bar') ] + 2 * podOptions.getMountSecrets() >> [ new PodMountSecret('my-secret/key-z', '/data/secret.txt') ] + 2 * podOptions.getMountConfigMaps() >> [ new PodMountConfig('my-data/key-x', '/etc/file.txt') ] + 2 * podOptions.getMountHostPaths() >> [ new PodHostMount('/host/x', '/mnt/x') ] + and: + result.spec.containers[0].env == [[name:'FOO', value:'bar']] + result.spec.containers[0].volumeMounts == [ + [name:'vol-1', mountPath:'/etc'], + [name:'vol-2', mountPath:'/data'], + [name:'vol-3', mountPath:'/mnt/x'] + ] + result.spec.volumes == [ + [name:'vol-1', configMap:[name:'my-data', items:[[key:'key-x', path:'file.txt']]]], + [name:'vol-2', secret:[secretName:'my-secret', items:[[key:'key-z', path:'secret.txt']]]], + [name:'vol-3', 'hostPath':[path:'/host/x']] + ] + + } + + def 'should create a request with vols and mounts' () { + + given: + def WORK_DIR = Paths.get('/some/work/dir') + def config = Mock(TaskConfig) + def task = Mock(TaskRun) + def client = Mock(K8sClient) + def builder = Mock(K8sWrapperBuilder) + def handler = Spy(new K8sTaskHandler(builder:builder, client:client)) + def podOptions = Mock(PodOptions) + and: + Map result + + when: + result = handler.newSubmitRequest(task) + then: + _ * handler.fusionEnabled() >> false + 1 * handler.fixOwnership() >> false + 1 * handler.entrypointOverride() >> true + 1 * handler.cpuLimitsEnabled() >> false + 1 * handler.getSyntheticPodName(task) >> 'nf-123' + 1 * handler.getContainerMounts() >> [] + 1 * handler.getLabels(task) >> [:] + 1 * handler.getAnnotations() >> [:] + 1 * handler.getPodOptions() >> podOptions + 1 * task.getContainer() >> 'debian:latest' + 1 * task.getWorkDir() >> WORK_DIR + 1 * task.getConfig() >> config + 1 * config.getCpus() >> 0 + 1 * config.getMemory() >> null + 1 * client.getConfig() >> new ClientConfig() + 2 * podOptions.getVolumeClaims() >> [ + new PodVolumeClaim('first','/work'), + new PodVolumeClaim('second','/data') + ] + and: + result.spec.containers[0].volumeMounts == [ + [name:'vol-1', mountPath:'/work'], + [name:'vol-2', mountPath:'/data'] + ] + result.spec.volumes == [ + [name:'vol-1', persistentVolumeClaim:[claimName: 'first']], + [name:'vol-2', persistentVolumeClaim:[claimName: 'second']] + ] + + when: + result = handler.newSubmitRequest(task) + then: + _ * handler.fusionEnabled() >> false + 1 * handler.fixOwnership() >> false + 1 * handler.entrypointOverride() >> true + 1 * handler.cpuLimitsEnabled() >> false + 1 * handler.getSyntheticPodName(task) >> 'nf-123' + 1 * handler.getContainerMounts() >> ['/tmp', '/data'] + 1 * handler.getLabels(task) >> [:] + 1 * handler.getAnnotations() >> [:] + 1 * handler.getPodOptions() >> new PodOptions() + 1 * task.getContainer() >> 'debian:latest' + 1 * task.getWorkDir() >> WORK_DIR + 1 * task.getConfig() >> config + 1 * config.getCpus() >> 0 + 1 * config.getMemory() >> null + 1 * client.getConfig() >> new ClientConfig() + and: + result.spec.containers[0].volumeMounts == [ + [name:'vol-3', mountPath:'/tmp'], + [name:'vol-4', mountPath: '/data'] + ] + result.spec.volumes == [ + [name:'vol-3', hostPath:[path:'/tmp']], + [name:'vol-4', hostPath:[path:'/data']] + ] + + } + + def 'should submit a job' () { + given: + def WORK_DIR = Paths.get('/some/work/dir') + def task = Mock(TaskRun) + def client = Mock(K8sClient) + def builder = Mock(K8sWrapperBuilder) + def config = Mock(TaskConfig) + def executor = Mock(K8sExecutor) + def handler = Spy(new K8sTaskHandler(builder: builder, client: client, executor: executor)) + def podOptions = Mock(PodOptions) + and: + Map result + + when: + result = handler.newSubmitRequest(task) + then: + 1 * client.getConfig() >> new ClientConfig() + _ * handler.fusionEnabled() >> false + 1 * handler.fixOwnership() >> false + 1 * handler.useJobResource() >> true + 1 * handler.entrypointOverride() >> true + 1 * handler.cpuLimitsEnabled() >> false + 1 * handler.getSyntheticPodName(task) >> 'nf-123' + 1 * handler.getLabels(task) >> [:] + 1 * handler.getAnnotations() >> [:] + 1 * handler.getContainerMounts() >> [] + 1 * handler.getPodOptions() >> podOptions + 1 * task.getContainer() >> 'debian:latest' + 1 * task.getWorkDir() >> WORK_DIR + 1 * task.getConfig() >> config + + result == [ + apiVersion: 'batch/v1', + kind: 'Job', + metadata: [name: 'nf-123', namespace: 'default'], + spec: [ + backoffLimit: 0, + template: [ + metadata: [name: 'nf-123', namespace: 'default'], + spec: [ + automountServiceAccountToken: false, + restartPolicy: 'Never', + containers: [[ + name: 'nf-123', + image: 'debian:latest', + command: ['/bin/bash', '-ue', '-o', 'pipefail', '-c', 'bash /some/work/dir/.command.run 2>&1 | tee /some/work/dir/.command.log'] + ]] + ] + ] + ] + ] + } + + def 'should check if running' () { + given: + def POD_NAME = 'pod-xyz' + def client = Mock(K8sClient) + def handler = Spy(new K8sTaskHandler(client: client, podName: POD_NAME, status: TaskStatus.SUBMITTED)) + + when: + def result = handler.checkIfRunning() + then: + 1 * handler.getState() >> [:] + result == false + + when: + result = handler.checkIfRunning() + then: + 1 * handler.getState() >> null + result == false + + when: + result = handler.checkIfRunning() + then: + 1 * handler.getState() >> [running:["startedAt": "2018-01-13T10:19:16Z"]] + result == true + } + + def 'should check if completed' () { + given: + def ERR_FILE = Paths.get('err.file') + def OUT_FILE = Paths.get('out.filex') + def POD_NAME = 'pod-xyz' + def EXIT_STATUS = 111 + def task = new TaskRun() + def client = Mock(K8sClient) + def termState = [ reason: "Completed", + startedAt: "2018-01-13T10:09:36Z", + finishedAt: "2018-01-13T10:19:36Z", + exitCode: 0 ] + def fullState = [terminated: termState] + def noExitCodeTermState = [ reason: "Completed", + startedAt: "2018-01-13T10:09:36Z", + finishedAt: "2018-01-13T10:19:36Z" ] + def noExitCodeState = [terminated: noExitCodeTermState] + and: + def handler = Spy(new K8sTaskHandler(task: task, client:client, podName: POD_NAME, outputFile: OUT_FILE, errorFile: ERR_FILE)) + + when: + def result = handler.checkIfCompleted() + then: + 1 * handler.getState() >> [:] + handler.status != TaskStatus.COMPLETED + result == false + + when: + result = handler.checkIfCompleted() + then: + 1 * handler.getState() >> null + handler.status != TaskStatus.COMPLETED + result == false + + when: + result = handler.checkIfCompleted() + then: + 1 * handler.getState() >> fullState + 1 * handler.updateTimestamps(termState) + 1 * handler.deleteJobIfSuccessful(task) >> null + 1 * handler.saveJobLogOnError(task) >> null + handler.task.exitStatus == 0 + handler.task.@stdout == OUT_FILE + handler.task.@stderr == ERR_FILE + handler.status == TaskStatus.COMPLETED + handler.startTimeMillis == 1515838176000 + handler.completeTimeMillis == 1515838776000 + result == true + + when: + result = handler.checkIfCompleted() + then: + 1 * handler.getState() >> noExitCodeState + 1 * handler.updateTimestamps(noExitCodeTermState) + 1 * handler.readExitFile() >> EXIT_STATUS + 1 * handler.deleteJobIfSuccessful(task) >> null + 1 * handler.saveJobLogOnError(task) >> null + handler.task.exitStatus == EXIT_STATUS + handler.task.@stdout == OUT_FILE + handler.task.@stderr == ERR_FILE + handler.status == TaskStatus.COMPLETED + handler.startTimeMillis == 1515838176000 + handler.completeTimeMillis == 1515838776000 + result == true + + } + + def 'should use K8s exit code when available' () { + given: + def ERR_FILE = Paths.get('err.file') + def OUT_FILE = Paths.get('out.file') + def POD_NAME = 'pod-xyz' + def client = Mock(K8sClient) + def termState = [ reason: "Error", + startedAt: "2018-01-13T10:09:36Z", + finishedAt: "2018-01-13T10:19:36Z", + exitCode: 137 ] + def task = new TaskRun() + def handler = Spy(new K8sTaskHandler(task: task, client:client, podName: POD_NAME, outputFile: OUT_FILE, errorFile: ERR_FILE)) + + when: + def result = handler.checkIfCompleted() + then: + 1 * handler.getState() >> [terminated: termState] + 1 * handler.updateTimestamps(termState) + 0 * handler.readExitFile() + 1 * handler.deleteJobIfSuccessful(task) >> null + 1 * handler.saveJobLogOnError(task) >> null + handler.task.exitStatus == 137 + handler.status == TaskStatus.COMPLETED + result == true + } + + def 'should kill a pod' () { + given: + def POD_NAME = 'pod-xyz' + def client = Mock(K8sClient) + def handler = Spy(new K8sTaskHandler(client:client, podName: POD_NAME)) + + when: + handler.killTask() + then: + 1 * handler.cleanupDisabled() >> false + 1 * client.podDelete(POD_NAME) >> null + + when: + handler.killTask() + then: + 1 * handler.cleanupDisabled() >> true + 0 * client.podDelete(POD_NAME) >> null + } + + def 'should check task cached state' () { + given: + def POD_NAME = 'pod-xyz' + def client = Mock(K8sClient) + def handler = Spy(new K8sTaskHandler(client:client, podName: POD_NAME)) + and: + Map STATE1 = [status:'pending'] + Map STATE2 = [status:'running'] + Map STATE3 = [status:'complete'] + Map state + + // first time `client.podState` is invoked + when: + state = handler.getState() + then: + 1 * client.podState(POD_NAME) >> STATE1 + state == STATE1 + + // second time `client.podState` NOT invoked + // the cached status is returned + when: + state = handler.getState() + then: + 0 * client.podState(POD_NAME) + state == STATE1 + + // after more than a second `client.podState` is invoked + // an empty value is returned, therefore the previous status is returned + when: + sleep 1_500 + state = handler.getState() + then: + 1 * client.podState(POD_NAME) >> [:] + state == STATE1 + + // still an empty status + // the previous status is returned + when: + state = handler.getState() + then: + 1 * client.podState(POD_NAME) >> [:] + state == STATE1 + + // now, the a valid status is returned + // the status is cached + when: + state = handler.getState() + then: + 1 * client.podState(POD_NAME) >> STATE2 + state == STATE2 + + // following invocation is cached + // the previous status is returned + when: + state = handler.getState() + then: + 0 * client.podState(POD_NAME) + state == STATE2 + + // after a second, the a new invocation is executed + // the new status is returned + when: + sleep 1_500 + state = handler.getState() + then: + 1 * client.podState(POD_NAME) >> STATE3 + state == STATE3 + } + + def 'should return nodeTermination state' () { + given: + def POD_NAME = 'pod-xyz' + def client = Mock(K8sClient) + def handler = Spy(new K8sTaskHandler(client:client, podName: POD_NAME)) + + when: + def state = handler.getState() + then: + 1 * client.podState(POD_NAME) >> { throw new NodeTerminationException("Node shutdown happened") } + then: + state.terminated.startedAt + state.terminated.finishedAt + and: + state.nodeTermination instanceof NodeTerminationException + state.nodeTermination.message == "Node shutdown happened" + } + + def 'should return other nodeTermination state' () { + given: + def POD_NAME = 'pod-xyz' + def client = Mock(K8sClient) + def handler = Spy(new K8sTaskHandler(client:client, podName: POD_NAME)) + + when: + def state = handler.getState() + then: + 1 * client.podState(POD_NAME) >> { throw new PodUnschedulableException("Pod failed for unknown reason", new Exception("cause")) } + then: + state.terminated.startedAt + state.terminated.finishedAt + and: + state.nodeTermination instanceof PodUnschedulableException + state.nodeTermination.message == "Pod failed for unknown reason" + } + + def 'should return container mounts' () { + + given: + def wrapper = Mock(K8sWrapperBuilder) + def k8sConfig = Mock(K8sConfig) + and: + def handler = Spy(new K8sTaskHandler(builder: wrapper)) + handler.getK8sConfig() >> k8sConfig + + when: + def mounts = handler.getContainerMounts() + then: + 1 * k8sConfig.getAutoMountHostPaths() >> false + mounts == [] + + when: + mounts = handler.getContainerMounts() + then: + 1 * k8sConfig.getAutoMountHostPaths() >> true + 1 * wrapper.getInputFiles() >> ['foo': Paths.get('/base_path/foo.txt'), 'bar': Paths.get('/base_path/bar.txt')] + 1 * wrapper.getBinDirs() >> [ Paths.get('/user/bin') ] + 1 * wrapper.getWorkDir() >> Paths.get('/work/dir') + mounts == ['/base_path', '/user/bin', '/work/dir'] + + } + + def 'should return labels map' () { + given: + def uuid = UUID.randomUUID() + def task = Mock(TaskRun) + def exec = Mock(K8sExecutor) + def proc = Mock(TaskProcessor) + def sess = Mock(Session) + def handler = Spy(new K8sTaskHandler(executor: exec)) + + when: + def labels = handler.getLabels(task) + then: + handler.getRunName() >> 'pedantic-joe' + task.getName() >> 'hello-world-1' + task.getProcessor() >> proc + task.getConfig() >> Mock(TaskConfig) { + getResourceLabels() >> [mylabel: 'myvalue'] + } + proc.getName() >> 'hello-proc' + exec.getSession() >> sess + sess.getUniqueId() >> uuid + exec.getK8sConfig() >> [pod: [ + [label: 'foo', value: 'bar'], + [label: 'app', value: 'nextflow'], + [label: 'x', value: 'hello_world'] + ]] + and: + labels.mylabel == 'myvalue' + and: + labels.app == 'nextflow' + labels.foo == 'bar' + labels.x == 'hello_world' + and: + labels.'nextflow.io/app' == 'nextflow' + labels.'nextflow.io/processName' == 'hello-proc' + labels.'nextflow.io/taskName' == 'hello-world-1' + labels.'nextflow.io/sessionId' instanceof String + labels.'nextflow.io/sessionId' == "uuid-${uuid.toString()}".toString() + and: + !labels.containsKey('nextflow.io/queue') + } + + def 'should return process queue as a label'() { + given: + def uuid = UUID.randomUUID() + def task = Mock(TaskRun) + def exec = Mock(K8sExecutor) + def proc = Mock(TaskProcessor) + def sess = Mock(Session) + def handler = Spy(new K8sTaskHandler(executor: exec)) + + when: + def labels = handler.getLabels(task) + then: + handler.getRunName() >> 'pedantic-joe' + task.getName() >> 'hello-world-1' + task.getProcessor() >> proc + task.getConfig() >> new TaskConfig(queue: 'him-mem-queue') + proc.getName() >> 'hello-proc' + exec.getSession() >> sess + sess.getUniqueId() >> uuid + exec.getK8sConfig() >> [:] + and: + labels.'nextflow.io/queue' == 'him-mem-queue' + and: + labels.'nextflow.io/app' == 'nextflow' + labels.'nextflow.io/processName' == 'hello-proc' + labels.'nextflow.io/taskName' == 'hello-world-1' + labels.'nextflow.io/sessionId' instanceof String + labels.'nextflow.io/sessionId' == "uuid-${uuid.toString()}".toString() + } + + def 'should delete pod if complete' () { + + given: + def POD_NAME = 'the-pod-name' + def executor = Mock(K8sExecutor) + def client = Mock(K8sClient) + def handler = Spy(new K8sTaskHandler(podName: POD_NAME, executor:executor, client:client)) + handler.useJobResource() >> false + and: + def TASK_OK = Mock(TaskRun); TASK_OK.isSuccess() >> true + def TASK_FAIL = Mock(TaskRun); TASK_FAIL.isSuccess() >> false + + when: + handler.deleteJobIfSuccessful(TASK_OK) + then: + 1 * executor.getK8sConfig() >> new K8sConfig() + 1 * client.podDelete(POD_NAME) >> null + + when: + handler.deleteJobIfSuccessful(TASK_OK) + then: + 1 * executor.getK8sConfig() >> new K8sConfig(cleanup: true) + 1 * client.podDelete(POD_NAME) >> null + + when: + handler.deleteJobIfSuccessful(TASK_FAIL) + then: + 1 * executor.getK8sConfig() >> new K8sConfig(cleanup: false) + 0 * client.podDelete(POD_NAME) >> null + + } + + def 'should not delete job if ttlSecondsAfterFinished is set' () { + + given: + def POD_NAME = 'the-job-name' + def executor = Mock(K8sExecutor) + def client = Mock(K8sClient) + def handler = Spy(new K8sTaskHandler(podName: POD_NAME, executor:executor, client:client)) + handler.useJobResource() >> true + and: + def TASK_OK = Mock(TaskRun); TASK_OK.isSuccess() >> true + + when: 'job with ttlSecondsAfterFinished should not be deleted' + handler.deleteJobIfSuccessful(TASK_OK) + then: + 1 * executor.getK8sConfig() >> new K8sConfig() + 1 * handler.getPodOptions() >> new PodOptions([[ttlSecondsAfterFinished: 100]]) + 0 * client.jobDelete(POD_NAME) + + when: 'job without ttlSecondsAfterFinished should be deleted' + handler.deleteJobIfSuccessful(TASK_OK) + then: + 1 * executor.getK8sConfig() >> new K8sConfig() + 1 * handler.getPodOptions() >> new PodOptions() + 1 * client.jobDelete(POD_NAME) >> null + } + + def 'should save pod log' () { + + given: + def folder = Files.createTempDirectory('test') + def POD_NAME = 'the-pod-name' + def POD_MESSAGE = 'Hello world!' + def POD_LOG = new ByteArrayInputStream(new String(POD_MESSAGE).bytes) + def session = Mock(Session) + def task = Mock(TaskRun) + def executor = Mock(K8sExecutor) + def client = Mock(K8sClient) + and: + def handler = Spy(new K8sTaskHandler(executor: executor, client: client, podName: POD_NAME)) + + when: + handler.saveJobLogOnError(task) + then: + task.isSuccess() >> true + 0 * client.podLog(_) + + when: + handler.saveJobLogOnError(task) + then: + task.isSuccess() >> false + task.getWorkDir() >> folder + executor.getSession() >> session + session.isTerminated() >> false + session.isCancelled() >> false + session.isAborted() >> false + 1 * client.podLog(POD_NAME) >> POD_LOG + + folder.resolve( TaskRun.CMD_LOG ).text == POD_MESSAGE + cleanup: + folder?.deleteDir() + + } + + def 'should merge pod options' () { + given: + PodOptions opts + + def taskConfig = Mock(TaskConfig) + def task = Mock(TaskRun) + task.getConfig() >> taskConfig + + def k8sConfig = Mock(K8sConfig) + def handler = Spy(K8sTaskHandler) + handler.getK8sConfig() >> k8sConfig + handler.task = task + + when: + opts = handler.getPodOptions() + then: + 1 * handler.taskPodOptions() >> new PodOptions() + 1 * k8sConfig.getPodOptions() >> new PodOptions() + opts == new PodOptions() + + when: + opts = handler.getPodOptions() + then: + 1 * handler.taskPodOptions() >> new PodOptions([[env:'HELLO', value:'WORLD']]) + 1 * k8sConfig.getPodOptions() >> new PodOptions() + opts == new PodOptions([[env:'HELLO', value:'WORLD']]) + + + when: + opts = handler.getPodOptions() + then: + 1 * handler.taskPodOptions() >> new PodOptions([[env:'HELLO', value:'WORLD']]) + 1 * k8sConfig.getPodOptions() >> new PodOptions([ [env:'BRAVO', value:'HOTEL'] ]) + opts == new PodOptions([[env:'HELLO', value:'WORLD'], [env:'BRAVO', value:'HOTEL']]) + + when: + opts = handler.getPodOptions() + then: + 1 * k8sConfig.getPodOptions() >> new PodOptions([[env: 'FUSION_BUCKETS', value: 's3://nextflow-ci'], [privileged: true]]) + and: + 1 * handler.taskPodOptions() >> new PodOptions([:]) + and: + opts == new PodOptions([[env: 'FUSION_BUCKETS', value: 's3://nextflow-ci'], [privileged: true]]) + } + + def 'should update startTimeMillis and completeTimeMillis with terminated state' () { + + given: + def handler = Spy(K8sTaskHandler) + def termState = [ startedAt: "2018-01-13T10:09:36Z", + finishedAt: "2018-01-13T10:19:36Z" ] + + when: + handler.updateTimestamps(termState) + then: + handler.startTimeMillis == 1515838176000 + handler.completeTimeMillis == 1515838776000 + } + + def 'should update timestamps with current time with missing or malformed time' () { + + given: + def handler = Spy(K8sTaskHandler) + def malformedTime = [ startedAt: "2018-01-13 10:09:36", + finishedAt: "2018-01-13T10:19:36Z" ] + + def garbage = [ what: "nope" ] + + when: + handler.updateTimestamps(malformedTime) + then: + handler.startTimeMillis > 0 // confirms that timestamps have been updated + handler.startTimeMillis <= handler.completeTimeMillis // confirms that order is sane + + when: + handler.updateTimestamps(garbage) + then: + handler.startTimeMillis > 0 + handler.startTimeMillis <= handler.completeTimeMillis + } + + def 'should not update timestamps with malformed time and when startTimeMillis already set' () { + + given: + def handler = Spy(K8sTaskHandler) + handler.startTimeMillis = 10 + handler.completeTimeMillis = 20 + def malformedTime = [ startedAt: "2018-01-13 10:09:36", + finishedAt: "2018-01-13T10:19:36Z" ] + + when: + handler.updateTimestamps(malformedTime) + then: + handler.startTimeMillis == 10 + handler.completeTimeMillis == 20 + } + + def 'should create a fusion privileged pod' () { + given: + def WORK_DIR = XPath.get('http://some/work/dir') + def config = Mock(TaskConfig) + def task = Mock(TaskRun) + def client = Mock(K8sClient) + def builder = Mock(K8sWrapperBuilder) + def launcher = Mock(FusionScriptLauncher) + def handler = Spy(new K8sTaskHandler(builder:builder, client: client)) + Map result + + when: + result = handler.newSubmitRequest(task) + then: + launcher.fusionEnv() >> [FUSION_BUCKETS: 'this,that'] + launcher.toContainerMount(WORK_DIR.resolve('.command.run')) >> Path.of('/fusion/http/work/dir/.command.run') + launcher.fusionSubmitCli(task) >> ['/usr/bin/fusion', 'bash', '/fusion/http/work/dir/.command.run'] + and: + handler.getTask() >> task + handler.fusionEnabled() >> true + handler.fusionLauncher() >> launcher + and: + task.getContainer() >> 'debian:latest' + task.getWorkDir() >> WORK_DIR + task.getConfig() >> config + and: + 1 * handler.fixOwnership() >> false + 1 * handler.entrypointOverride() >> false + 1 * handler.cpuLimitsEnabled() >> false + 1 * handler.getPodOptions() >> new PodOptions() + 1 * handler.getSyntheticPodName(task) >> 'nf-123' + 1 * handler.getLabels(task) >> [:] + 1 * handler.getAnnotations() >> [:] + 1 * handler.getContainerMounts() >> [] + and: + 1 * config.getCpus() >> 0 + 1 * config.getMemory() >> null + 1 * client.getConfig() >> new ClientConfig() + and: + result.spec.containers[0].args == ['/usr/bin/fusion', 'bash', '/fusion/http/work/dir/.command.run'] + result.spec.containers[0].securityContext == [privileged:true] + result.spec.containers[0].env == [[name:'FUSION_BUCKETS', value:'this,that']] + } + + def 'should create a fusion unprivileged pod' () { + given: + def WORK_DIR = XPath.get('http://some/work/dir') + def config = Mock(TaskConfig) + def task = Mock(TaskRun) + def client = Mock(K8sClient) + def builder = Mock(K8sWrapperBuilder) + def launcher = Mock(FusionScriptLauncher) + def k8sConfig = Spy(K8sConfig) + def exec = Mock(K8sExecutor) { getK8sConfig()>>k8sConfig } + def handler = Spy(new K8sTaskHandler(builder:builder, client: client, executor: exec)) + Map result + + when: + result = handler.newSubmitRequest(task) + then: + launcher.fusionEnv() >> [FUSION_BUCKETS: 'this,that'] + launcher.toContainerMount(WORK_DIR.resolve('.command.run')) >> Path.of('/fusion/http/work/dir/.command.run') + launcher.fusionSubmitCli(task) >> ['/usr/bin/fusion', 'bash', '/fusion/http/work/dir/.command.run'] + and: + handler.getTask() >> task + handler.fusionEnabled() >> true + handler.fusionLauncher() >> launcher + handler.fusionConfig() >> new FusionConfig(privileged: false) + and: + task.getContainer() >> 'debian:latest' + task.getWorkDir() >> WORK_DIR + task.getConfig() >> config + and: + 1 * handler.fixOwnership() >> false + 1 * handler.entrypointOverride() >> false + 1 * handler.getPodOptions() >> new PodOptions() + 1 * handler.getSyntheticPodName(task) >> 'nf-123' + 1 * handler.getLabels(task) >> [:] + 1 * handler.getAnnotations() >> [:] + 1 * handler.getContainerMounts() >> [] + and: + 1 * config.getCpus() >> 0 + 1 * config.getMemory() >> null + 1 * client.getConfig() >> new ClientConfig() + and: + result.spec.containers[0].args == ['/usr/bin/fusion', 'bash', '/fusion/http/work/dir/.command.run'] + result.spec.containers[0].env == [[name:'FUSION_BUCKETS', value:'this,that']] + result.spec.containers[0].resources == [limits:['nextflow.io/fuse':1]] + !result.spec.containers[0].securityContext + + + /* + * use custom fuse device + */ + when: + result = handler.newSubmitRequest(task) + then: + launcher.fusionEnv() >> [FUSION_BUCKETS: 'this,that'] + launcher.toContainerMount(WORK_DIR.resolve('.command.run')) >> Path.of('/fusion/http/work/dir/.command.run') + launcher.fusionSubmitCli(task) >> ['/usr/bin/fusion', 'bash', '/fusion/http/work/dir/.command.run'] + and: + k8sConfig.fuseDevicePlugin() >> ['custom/device/fuse': 1] + and: + handler.getTask() >> task + handler.fusionEnabled() >> true + handler.fusionLauncher() >> launcher + handler.fusionConfig() >> new FusionConfig(privileged: false) + and: + task.getContainer() >> 'debian:latest' + task.getWorkDir() >> WORK_DIR + task.getConfig() >> config + and: + 1 * handler.fixOwnership() >> false + 1 * handler.entrypointOverride() >> false + 1 * handler.getPodOptions() >> new PodOptions() + 1 * handler.getSyntheticPodName(task) >> 'nf-123' + 1 * handler.getLabels(task) >> [:] + 1 * handler.getAnnotations() >> [:] + 1 * handler.getContainerMounts() >> [] + and: + 1 * config.getCpus() >> 0 + 1 * config.getMemory() >> null + 1 * client.getConfig() >> new ClientConfig() + and: + result.spec.containers[0].args == ['/usr/bin/fusion', 'bash', '/fusion/http/work/dir/.command.run'] + result.spec.containers[0].env == [[name:'FUSION_BUCKETS', value:'this,that']] + result.spec.containers[0].resources == [limits:['custom/device/fuse':1]] + !result.spec.containers[0].securityContext + } + + def 'get fusion submit command' () { + given: + def remoteWorkDir = XPath.get('http://foo/work/dir') + def handler = Spy(K8sTaskHandler) { + fusionEnabled() >> true + fusionLauncher() >> new FusionScriptLauncher(Mock(TaskBean), 'http', remoteWorkDir) + getTask() >> Mock(TaskRun) { + getWorkDir() >> remoteWorkDir + } + } + + when: + def result = handler.getSubmitCommand(Mock(TaskRun)) + then: + result.join(' ') == '/usr/bin/fusion bash /fusion/http/foo/work/dir/.command.run' + } + + def 'should get task pod options' () { + given: + def taskConfig = Mock(TaskConfig) + def task = Mock(TaskRun) + task.getConfig() >> taskConfig + + def handler = Spy(K8sTaskHandler) + handler.task = task + + when: + def podOptions = handler.taskPodOptions() + then: + taskConfig.get('pod') >> null + and: + podOptions == new PodOptions(null) + + when: + podOptions = handler.taskPodOptions() + then: + taskConfig.get('pod') >> [ [env: 'FOO', value: 'BAR'] ] + and: + podOptions == new PodOptions([ [env: 'FOO', value: 'BAR'] ]) + + } + + def 'should enqueue task instead of submitting pod immediately' () { + given: + def task = Mock(TaskRun) + def wrapper = Mock(K8sWrapperBuilder) + def scheduler = Mock(K8sTaskScheduler) + def executor = Mock(K8sExecutor) + def handler = Spy(new K8sTaskHandler(task: task, executor: executor)) + + when: + handler.submit() + + then: + 1 * handler.createBashWrapper(task) >> wrapper + 1 * wrapper.build() + 1 * executor.getTaskScheduler() >> scheduler + 1 * scheduler.submit(handler) + + and: + 0 * handler.newSubmitRequest(_, _) + } + + def 'should return false for queued task that has no pod name yet' () { + given: + def handler = Spy(K8sTaskHandler) + + expect: + !handler.checkIfRunning() + !handler.checkIfCompleted() + } + + def 'should create a pod request assigned to selected node' () { + given: + def WORK_DIR = Paths.get('/some/work/dir') + def config = Mock(TaskConfig) + def task = Mock(TaskRun) + def client = Mock(K8sClient) + def builder = Mock(K8sWrapperBuilder) + def handler = Spy(new K8sTaskHandler(builder: builder, client: client)) + Map result + + when: + result = handler.newSubmitRequest(task, 'node-a') + + then: + _ * handler.fusionEnabled() >> false + 1 * handler.fixOwnership() >> false + 1 * handler.entrypointOverride() >> false + 1 * handler.cpuLimitsEnabled() >> false + 1 * handler.getPodOptions() >> new PodOptions() + 1 * handler.getSyntheticPodName(task) >> 'nf-123' + 1 * handler.getLabels(task) >> [:] + 1 * handler.getAnnotations() >> [:] + 1 * handler.getContainerMounts() >> [] + 1 * task.getContainer() >> 'debian:latest' + 1 * task.getWorkDir() >> WORK_DIR + 1 * task.getConfig() >> config + 1 * config.getCpus() >> 0 + 1 * config.getMemory() >> null + 1 * client.getConfig() >> new ClientConfig() + + and: + result.spec.nodeName == 'node-a' + result.spec.containers[0].image == 'debian:latest' + } + + def 'should submit now to selected node and set pod name' () { + given: + def task = Mock(TaskRun) + def client = Mock(K8sClient) + def handler = Spy(new K8sTaskHandler(task: task, client: client)) + def request = [ + apiVersion: 'v1', + kind: 'Pod', + metadata: [name: 'nf-123'], + spec: [:] + ] + + when: + handler.submitNow('node-a') + + then: + 1 * handler.newSubmitRequest(task, 'node-a') >> request + 1 * handler.useJobResource() >> false + 1 * handler.yamlDebugPath() >> null + 1 * client.podCreate(request, null) >> [metadata: [name: 'nf-123']] + + and: + handler.getPodName() == 'nf-123' + handler.status == TaskStatus.SUBMITTED + } +} diff --git a/k8s-dvfs/src/test/groovy/recreationaltech/plugin/K8sWrapperBuilderTest.groovy b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/K8sWrapperBuilderTest.groovy new file mode 100644 index 0000000..0d08d3a --- /dev/null +++ b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/K8sWrapperBuilderTest.groovy @@ -0,0 +1,62 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin + +import java.nio.file.Files + +import nextflow.Session +import nextflow.executor.Executor +import nextflow.processor.TaskConfig +import nextflow.processor.TaskProcessor +import nextflow.processor.TaskRun +import spock.lang.Specification +/** + * + * @author Paolo Di Tommaso + */ +class K8sWrapperBuilderTest extends Specification { + + def 'should render launcher script' () { + given: + def folder = Files.createTempDirectory('test') + and: + def sess = Mock(Session) + def exec = Mock(Executor) + def proc = Mock(TaskProcessor) { getSession() >> sess; getExecutor() >> exec } + def config = new TaskConfig() + def task = Mock(TaskRun) { + getName() >> 'foo' + getConfig() >> config + getProcessor() >> proc + getWorkDir() >> folder + getInputFilesMap() >> [:] + getOutputFilesNames() >> [] + } + + and: + def builder = Spy(new K8sWrapperBuilder(task)) { getSecretsEnv() >> null; fixOwnership() >> false } + + when: + def binding = builder.makeBinding() + + then: + binding.header_script == "NXF_CHDIR=${folder}" + + cleanup: + folder?.deleteDir() + } +} diff --git a/k8s-dvfs/src/test/groovy/recreationaltech/plugin/client/ClientConfigTest.groovy b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/client/ClientConfigTest.groovy new file mode 100644 index 0000000..6160302 --- /dev/null +++ b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/client/ClientConfigTest.groovy @@ -0,0 +1,120 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin.client + +import java.nio.file.Files + +import spock.lang.Specification +/** + * + * @author Paolo Di Tommaso + */ +class ClientConfigTest extends Specification { + + def 'should stringify a config' () { + + when: + final CERT = 'LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNTRENDQVRDZ0F3SUJBZ0lJRlFsM1l2Y2k1TWN3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB4T0RBeE1UTXlNREEyTlRaYUZ3MHhPVEF4TVRNeU1EQTJOVFphTURNeApGREFTQmdOVkJBb1RDMFJ2WTJ0bGNpQkpibU11TVJzd0dRWURWUVFERXhKa2IyTnJaWEl0Wm05eUxXUmxjMnQwCmIzQXdnWjh3RFFZSktvWklodmNOQVFFQkJRQURnWTBBTUlHSkFvR0JBUEtYT0ZsV2t2THIzb29ETGNFOElyME0KTzNBMHZqQlVvUzZ0bUdBbFRYYTd0QWQwM3BTMXNJNit0WVRwVlU2YXR6ZU9vU0VrOWhmaWxBdVNYdG1hSHZCUAp1czFEcG1LZEZRMWI3OFRkSnQ4OGV3c3BRajFxYUwvQldHeitMUzUrRHUrNUJuUGtmZlhDS1UxQTdUc2tZamJyClhxeDhlN2FWZURWTmFjZXc0Z0RqQWdNQkFBR2pBakFBTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFBMXVtVlAKR29EZTVCRXJrb21qWXdITXhiTTd4UStibTYrUDE1T0pINUo0UGNQeU11d25ocC9ORVp1NnpsTTZSUUo3SUNKQgpHWTRBMnFKVmJsWUkwQkJzRkF1TXMreTAyazdVVVVoK0NRYVd0SXhBcFNmbkQ4dUVXQ0g5VE1ZNGdLbTZjTDhVCk1OVVl1RnpUQ2hmTS96RjdUMXVaZWxJYXNrYXFaWSt3a3hxa3YyRUQxQ2F5MDUxSXRWRXZVbDIvSVZyVHdrT20KZ25nL3Q4L2RkeDhpOUkzTFJrMTlTaERKdXlQZ1NrTTZRSWlSd09mRHk4V0ZFaURpd0hBS0ErSEZhTGhOOFJTMwpieDUvdEhEN01id0FpdnorNTU4YUFEQjNEd1ZpekthM2d5Wm4yUzRjUGFqZnNwODFqRkNIQS9QekdQdTU2MzJwCkxRN0gyRW1aYmJuUHFYTFgKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo='.decodeBase64() + final KEY = 'LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlDWGdJQkFBS0JnUUR5bHpoWlZwTHk2OTZLQXkzQlBDSzlERHR3Tkw0d1ZLRXVyWmhnSlUxMnU3UUhkTjZVCnRiQ092cldFNlZWT21yYzNqcUVoSlBZWDRwUUxrbDdabWg3d1Q3ck5RNlppblJVTlcrL0UzU2JmUEhzTEtVSTkKYW1pL3dWaHMvaTB1Zmc3dnVRWno1SDMxd2lsTlFPMDdKR0kyNjE2c2ZIdTJsWGcxVFduSHNPSUE0d0lEQVFBQgpBb0dBYWRUOCtVU2lvU1d6bFVRanZ1eHNQMHRKMXY2N2hqdzFnVGFzaGkxZjZRK2tUNmgxdml5eGxPU3dMZ2JaCmQ0eFpwL3dxWVZwTm5rZnp6RVNUNnB5cEo5WTEwdHY1cFpSWG9HbG1NT2tIZSswUW45N0c5ZDRzL2JCV3lmYXYKRzhRTC9tZFN6Vy85YUdrSkpiNWU0VDlsSURvRDNFVDgwYUFWbzl2V0NPVUxsdWtDUVFEK0hINU5ucVBuSTdnTApWOUJKZzlRRVBwUTVYa2traW8rejZ2YkRHQU5rR1VPV1dmRURKUHE2Q2JBb1dqeWh1Qy9KS1dYRWs4Rkt0M1Y2CkhVNllYeVpGQWtFQTlHVE9XOFM4KzVNNHE5R3lNeURxN1ZkVHA3M2daeSsvNjVQam5hNlpDUnhTZklxL2xKUVoKY2F6MkhGYVRzRFdLbkdhWGNxTmdBVXNEODNyWTlzM3hCd0pCQUt5Vjc1YUtPMm0rRWI3cWVsV2p5bmpEZytwZQp4akNpUnkxOFZQSjJPYjlmaFU3MWNVS2dlQVdvbE5NalRuREw1dkNxUkNzNTZ4cnk5VC9sN2I2QlNUMENRUURnCjRoV2xDZTdnQzhOZEQzTkxhdUhpRGJZenB4dmp0Mk9Ca2E4ai9ISmptTVVxUnI0dEtPNFUxUlFPVlhoRzc2MmgKWnlHNjRpeklZOCs1N3ZQUWZ3Wm5Ba0VBdW9RWW1lUi90UWhIakhRNFlhZGRHbkNBQ2hZZ29ObEFzSGhGTElxVQo1ZTZaMXN2Q3VKU285TDVVRCtrclFUYWlGU01pRHZwZlJyVE1ZKzZ5Q0tTajd3PT0KLS0tLS1FTkQgUlNBIFBSSVZBVEUgS0VZLS0tLS0K'.decodeBase64() + + def config = new ClientConfig() + config.clientCert = CERT + config.clientKey = KEY + config.sslCert = CERT + println config.toString() + + then: + noExceptionThrown() + + } + + def 'should create a client config from a map' () { + + given: + def MAP = [ + server:'foo.com', + token: 'blah-blah', + namespace: 'my-namespace', + verifySsl: true, + sslCert: 'fizzbuzz'.bytes.encodeBase64().toString(), + clientCert: 'hello'.bytes.encodeBase64().toString(), + clientKey: 'world'.bytes.encodeBase64().toString() ] + + when: + def result = ClientConfig.fromNextflowConfig(MAP, null, null) + + then: + result.server == 'foo.com' + result.token == 'blah-blah' + result.namespace == 'my-namespace' + result.serviceAccount == 'default' + result.verifySsl + result.clientCert == 'hello'.bytes + result.clientKey == 'world'.bytes + result.sslCert == 'fizzbuzz'.bytes + + when: + result = ClientConfig.fromNextflowConfig(MAP, 'ns1', 'sa2') + then: + result.server == 'foo.com' + result.token == 'blah-blah' + result.namespace == 'ns1' + result.serviceAccount == 'sa2' + result.verifySsl + result.clientCert == 'hello'.bytes + result.clientKey == 'world'.bytes + result.sslCert == 'fizzbuzz'.bytes + } + + def 'should create a client config from a map with files' () { + + given: + def folder = Files.createTempDirectory('test') + def file1 = folder.resolve('file1') + def file2 = folder.resolve('file2') + def file3 = folder.resolve('file3') + file1.text = 'fizzbuzz'.bytes.encodeBase64().toString() + file2.text = 'hello'.bytes.encodeBase64().toString() + file3.text = 'world'.bytes.encodeBase64().toString() + + def MAP = [ + server:'foo.com', + token: 'blah-blah', + namespace: 'my-namespace', + verifySsl: false, + sslCertFile: file1, + clientCertFile: file2, + clientKeyFile: file3 ] + + when: + def result = ClientConfig.fromNextflowConfig(MAP, null, null) + + then: + result.server == 'foo.com' + result.token == 'blah-blah' + result.namespace == 'my-namespace' + result.serviceAccount == 'default' + !result.verifySsl + result.sslCert == file1.text.bytes + result.clientCert == file2.text.bytes + result.clientKey == file3.text.bytes + + cleanup: + folder?.deleteDir() + } + +} diff --git a/k8s-dvfs/src/test/groovy/recreationaltech/plugin/client/ConfigDiscoveryTest.groovy b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/client/ConfigDiscoveryTest.groovy new file mode 100644 index 0000000..d9ef35c --- /dev/null +++ b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/client/ConfigDiscoveryTest.groovy @@ -0,0 +1,437 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin.client +import javax.net.ssl.KeyManager +import java.nio.file.Files + +import spock.lang.Specification +import test.TestHelper +/** + * + * @author Paolo Di Tommaso + */ +class ConfigDiscoveryTest extends Specification { + + def 'should read config from file' () { + + given: + final CERT_DATA = "d29ybGQgaGVsbG8=" + final CLIENT_CERT = "aGVsbG8gd29ybGQ=" + final CLIENT_KEY = "Y2lhbyBtaWFv" + + def CONFIG = TestHelper.createInMemTempFile('config') + CONFIG.text = """ + apiVersion: v1 + clusters: + - cluster: + insecure-skip-tls-verify: true + server: https://localhost:6443 + certificate-authority-data: $CERT_DATA + name: docker-for-desktop-cluster + contexts: + - context: + cluster: docker-for-desktop-cluster + user: docker-for-desktop + name: docker-for-desktop + current-context: docker-for-desktop + kind: Config + preferences: {} + users: + - name: docker-for-desktop + user: + client-certificate-data: $CLIENT_CERT + client-key-data: $CLIENT_KEY + """ + .stripIndent() + + def discovery = Spy(ConfigDiscovery) + def KEY_MANAGERS = [] as KeyManager[] + + when: + def config = discovery.fromKubeConfig(CONFIG, null, null, null) + then: + 0 * discovery.discoverAuthToken(_, 'default',null) >> 'secret-token' + 1 * discovery.createKeyManagers(CLIENT_CERT.decodeBase64(), CLIENT_KEY.decodeBase64()) >> KEY_MANAGERS + config.server == 'https://localhost:6443' + config.token == null + config.namespace == 'default' + config.serviceAccount == 'default' + config.clientCert == CLIENT_CERT.decodeBase64() + config.clientKey == CLIENT_KEY.decodeBase64() + config.sslCert == CERT_DATA.decodeBase64() + config.keyManagers.is( KEY_MANAGERS ) + !config.verifySsl + !config.isFromCluster + + } + + def 'should read config from file with provided namespace' () { + + given: + final CERT_DATA = "d29ybGQgaGVsbG8=" + final CLIENT_CERT = "aGVsbG8gd29ybGQ=" + final CLIENT_KEY = "Y2lhbyBtaWFv" + + def CONFIG = TestHelper.createInMemTempFile('config') + CONFIG.text = """ + apiVersion: v1 + clusters: + - cluster: + insecure-skip-tls-verify: true + server: https://localhost:6443 + certificate-authority-data: $CERT_DATA + name: docker-for-desktop-cluster + contexts: + - context: + cluster: docker-for-desktop-cluster + user: docker-for-desktop + name: docker-for-desktop + current-context: docker-for-desktop + kind: Config + preferences: {} + users: + - name: docker-for-desktop + user: + client-certificate-data: $CLIENT_CERT + client-key-data: $CLIENT_KEY + """ + .stripIndent() + + def discovery = Spy(ConfigDiscovery) + def KEY_MANAGERS = [] as KeyManager[] + + when: + def config = discovery.fromKubeConfig(CONFIG, 'docker-for-desktop', 'ns1', 'sa2') + then: + 0 * discovery.discoverAuthToken('docker-for-desktop','ns1','sa2') >> 'secret-token' + 1 * discovery.createKeyManagers(CLIENT_CERT.decodeBase64(), CLIENT_KEY.decodeBase64()) >> KEY_MANAGERS + config.server == 'https://localhost:6443' + config.token == null + config.namespace == 'ns1' + config.serviceAccount == 'sa2' + config.clientCert == CLIENT_CERT.decodeBase64() + config.clientKey == CLIENT_KEY.decodeBase64() + config.sslCert == CERT_DATA.decodeBase64() + config.keyManagers.is( KEY_MANAGERS ) + !config.verifySsl + !config.isFromCluster + + } + + def 'should read config from file with cert files' () { + + given: + def folder = Files.createTempDirectory(null) + def CA_FILE = folder.resolve('ca'); CA_FILE.text = 'ca-content' + def CLIENT_CERT_FILE = folder.resolve('client-cert'); CLIENT_CERT_FILE.text = 'client-cert-content' + def CLIENT_KEY_FILE = folder.resolve('client-key'); CLIENT_KEY_FILE.text = 'client-key-content' + def CONFIG = folder.resolve('config') + def KEY_MANAGERS = [] as KeyManager[] + + CONFIG.text = """ + apiVersion: v1 + clusters: + - cluster: + insecure-skip-tls-verify: true + server: https://localhost:6443 + certificate-authority: $CA_FILE + name: docker-for-desktop-cluster + contexts: + - context: + cluster: docker-for-desktop-cluster + user: docker-for-desktop + name: docker-for-desktop + current-context: docker-for-desktop + kind: Config + preferences: {} + users: + - name: docker-for-desktop + user: + client-certificate: $CLIENT_CERT_FILE + client-key: $CLIENT_KEY_FILE + """ + .stripIndent() + + def discovery = Spy(ConfigDiscovery) + + when: + def config = discovery.fromKubeConfig(CONFIG, null, null, null) + then: + 1 * discovery.createKeyManagers( CLIENT_CERT_FILE.bytes, CLIENT_KEY_FILE.bytes ) >> KEY_MANAGERS + config.server == 'https://localhost:6443' + config.token == null + config.namespace == 'default' + config.serviceAccount == 'default' + config.clientCert == CLIENT_CERT_FILE.bytes + config.clientKey == CLIENT_KEY_FILE.bytes + config.sslCert == CA_FILE.bytes + config.keyManagers.is( KEY_MANAGERS ) + !config.verifySsl + !config.isFromCluster + + cleanup: + folder?.deleteDir() + } + + def 'should read config and use token' () { + + given: + def folder = Files.createTempDirectory(null) + def CONFIG = folder.resolve('config') + + CONFIG.text = """ + apiVersion: v1 + clusters: + - cluster: + insecure-skip-tls-verify: true + server: https://localhost:6443 + name: docker-for-desktop-cluster + contexts: + - context: + cluster: docker-for-desktop-cluster + user: docker-for-desktop + name: docker-for-desktop + current-context: docker-for-desktop + kind: Config + preferences: {} + users: + - name: docker-for-desktop + user: + token: 90s090s98s7f8s + """ + .stripIndent() + + def discovery = Spy(ConfigDiscovery) + + when: + def config = discovery.fromKubeConfig(CONFIG, null, null, null) + then: + 0 * discovery.discoverAuthToken(_,_,_) >> 'secret-token' + 0 * discovery.createKeyManagers( _, _ ) >> null + config.server == 'https://localhost:6443' + config.token == '90s090s98s7f8s' + config.namespace == 'default' + config.serviceAccount == 'default' + !config.verifySsl + !config.isFromCluster + + cleanup: + folder?.deleteDir() + } + + def 'should read config and discover token' () { + + given: + def folder = Files.createTempDirectory(null) + def CONFIG = folder.resolve('config') + + CONFIG.text = """ + apiVersion: v1 + clusters: + - cluster: + insecure-skip-tls-verify: true + server: https://localhost:6443 + name: docker-for-desktop-cluster + contexts: + - context: + cluster: docker-for-desktop-cluster + user: docker-for-desktop + name: docker-for-desktop + current-context: docker-for-desktop + kind: Config + preferences: {} + users: + - name: docker-for-desktop + user: + foo: bar + """ + .stripIndent() + + def discovery = Spy(ConfigDiscovery) + + when: + def config = discovery.fromKubeConfig(CONFIG, null, null, null) + then: + 1 * discovery.discoverAuthToken(_, _, _) >> 'secret-token' + 0 * discovery.createKeyManagers( _, _ ) >> null + config.server == 'https://localhost:6443' + config.token == 'secret-token' + config.namespace == 'default' + config.serviceAccount == 'default' + !config.verifySsl + !config.isFromCluster + + cleanup: + folder?.deleteDir() + } + + def 'should read config from given context' () { + + given: + def folder = Files.createTempDirectory(null) + folder.resolve('fake-cert-file').text = 'fake-cert-content' + folder.resolve('fake-key-file').text = 'fake-key-content' + folder.resolve('fake-ca-file').text = 'fake-ca-content' + + def CONFIG = folder.resolve('config') + CONFIG.text = ''' + apiVersion: v1 + clusters: + - cluster: + certificate-authority: fake-ca-file + server: https://1.2.3.4 + name: development + - cluster: + insecure-skip-tls-verify: true + server: https://5.6.7.8 + name: scratch + contexts: + - context: + cluster: development + namespace: frontend + user: developer + name: dev-frontend + - context: + cluster: development + namespace: storage + user: developer + name: dev-storage + - context: + cluster: scratch + namespace: default + user: experimenter + name: exp-scratch + current-context: "" + kind: Config + preferences: {} + users: + - name: developer + user: + client-certificate: fake-cert-file + - name: experimenter + user: + password: some-password + username: exp + '''.stripIndent() + + when: + def cfg1 = new ConfigDiscovery().fromKubeConfig(CONFIG, 'dev-frontend', null, null) + then: + cfg1.server == 'https://1.2.3.4' + cfg1.sslCert == 'fake-ca-content'.bytes + cfg1.isVerifySsl() + cfg1.namespace == 'frontend' + cfg1.serviceAccount == 'default' + cfg1.clientCert == 'fake-cert-content'.bytes + + when: + def cfg2 = new ConfigDiscovery().fromKubeConfig(CONFIG, 'dev-storage', null, null) + then: + cfg2.server == 'https://1.2.3.4' + cfg2.sslCert == 'fake-ca-content'.bytes + cfg2.isVerifySsl() + cfg2.namespace == 'storage' + cfg2.serviceAccount == 'default' + cfg2.clientCert == 'fake-cert-content'.bytes + + when: + def cfg3 = new ConfigDiscovery().fromKubeConfig(CONFIG, 'exp-scratch', null, null) + then: + cfg3.server == 'https://5.6.7.8' + cfg3.sslCert == null + !cfg3.isVerifySsl() + cfg3.namespace == 'default' + cfg3.serviceAccount == 'default' + + when: + new ConfigDiscovery().fromKubeConfig(CONFIG, 'foo', null, null) + then: + thrown(IllegalArgumentException) + + true + cleanup: + folder.deleteDir() + } + + def 'should load from cluster env' () { + given: + def CERT_FILE = TestHelper.createInMemTempFile('ca'); CERT_FILE.text = 'ca-content' + def TOKEN_FILE = TestHelper.createInMemTempFile('token'); TOKEN_FILE.text = 'my-token' + def NAMESPACE_FILE = TestHelper.createInMemTempFile('namespace'); NAMESPACE_FILE.text = 'foo-namespace' + + def discovery = Spy(ConfigDiscovery) + + when: + def env = [ KUBERNETES_SERVICE_HOST: 'foo.com', KUBERNETES_SERVICE_PORT: '4343' ] + def config = discovery.fromCluster(env, null, null) + then: + 1 * discovery.path('/var/run/secrets/kubernetes.io/serviceaccount/ca.crt') >> CERT_FILE + 1 * discovery.path('/var/run/secrets/kubernetes.io/serviceaccount/token') >> TOKEN_FILE + 1 * discovery.path('/var/run/secrets/kubernetes.io/serviceaccount/namespace') >> NAMESPACE_FILE + 0 * discovery.createKeyManagers(_,_) >> null + and: + config.server == 'foo.com:4343' + config.namespace == 'foo-namespace' + config.token == 'my-token' + config.sslCert == CERT_FILE.text.bytes + config.isFromCluster + + when: + env = [ KUBERNETES_SERVICE_HOST: 'https://host.com' ] + config = discovery.fromCluster(env, 'my-namespace', null) + then: + 1 * discovery.path('/var/run/secrets/kubernetes.io/serviceaccount/ca.crt') >> CERT_FILE + 1 * discovery.path('/var/run/secrets/kubernetes.io/serviceaccount/token') >> TOKEN_FILE + 1 * discovery.path('/var/run/secrets/kubernetes.io/serviceaccount/namespace') >> NAMESPACE_FILE + and: + config.server == 'https://host.com' + config.namespace == 'my-namespace' + } + + def 'should create key managers' () { + given: + final CERT = 'LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNTRENDQVRDZ0F3SUJBZ0lJRlFsM1l2Y2k1TWN3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB4T0RBeE1UTXlNREEyTlRaYUZ3MHhPVEF4TVRNeU1EQTJOVFphTURNeApGREFTQmdOVkJBb1RDMFJ2WTJ0bGNpQkpibU11TVJzd0dRWURWUVFERXhKa2IyTnJaWEl0Wm05eUxXUmxjMnQwCmIzQXdnWjh3RFFZSktvWklodmNOQVFFQkJRQURnWTBBTUlHSkFvR0JBUEtYT0ZsV2t2THIzb29ETGNFOElyME0KTzNBMHZqQlVvUzZ0bUdBbFRYYTd0QWQwM3BTMXNJNit0WVRwVlU2YXR6ZU9vU0VrOWhmaWxBdVNYdG1hSHZCUAp1czFEcG1LZEZRMWI3OFRkSnQ4OGV3c3BRajFxYUwvQldHeitMUzUrRHUrNUJuUGtmZlhDS1UxQTdUc2tZamJyClhxeDhlN2FWZURWTmFjZXc0Z0RqQWdNQkFBR2pBakFBTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFBMXVtVlAKR29EZTVCRXJrb21qWXdITXhiTTd4UStibTYrUDE1T0pINUo0UGNQeU11d25ocC9ORVp1NnpsTTZSUUo3SUNKQgpHWTRBMnFKVmJsWUkwQkJzRkF1TXMreTAyazdVVVVoK0NRYVd0SXhBcFNmbkQ4dUVXQ0g5VE1ZNGdLbTZjTDhVCk1OVVl1RnpUQ2hmTS96RjdUMXVaZWxJYXNrYXFaWSt3a3hxa3YyRUQxQ2F5MDUxSXRWRXZVbDIvSVZyVHdrT20KZ25nL3Q4L2RkeDhpOUkzTFJrMTlTaERKdXlQZ1NrTTZRSWlSd09mRHk4V0ZFaURpd0hBS0ErSEZhTGhOOFJTMwpieDUvdEhEN01id0FpdnorNTU4YUFEQjNEd1ZpekthM2d5Wm4yUzRjUGFqZnNwODFqRkNIQS9QekdQdTU2MzJwCkxRN0gyRW1aYmJuUHFYTFgKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo='.decodeBase64() + final KEY = 'LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlDWGdJQkFBS0JnUUR5bHpoWlZwTHk2OTZLQXkzQlBDSzlERHR3Tkw0d1ZLRXVyWmhnSlUxMnU3UUhkTjZVCnRiQ092cldFNlZWT21yYzNqcUVoSlBZWDRwUUxrbDdabWg3d1Q3ck5RNlppblJVTlcrL0UzU2JmUEhzTEtVSTkKYW1pL3dWaHMvaTB1Zmc3dnVRWno1SDMxd2lsTlFPMDdKR0kyNjE2c2ZIdTJsWGcxVFduSHNPSUE0d0lEQVFBQgpBb0dBYWRUOCtVU2lvU1d6bFVRanZ1eHNQMHRKMXY2N2hqdzFnVGFzaGkxZjZRK2tUNmgxdml5eGxPU3dMZ2JaCmQ0eFpwL3dxWVZwTm5rZnp6RVNUNnB5cEo5WTEwdHY1cFpSWG9HbG1NT2tIZSswUW45N0c5ZDRzL2JCV3lmYXYKRzhRTC9tZFN6Vy85YUdrSkpiNWU0VDlsSURvRDNFVDgwYUFWbzl2V0NPVUxsdWtDUVFEK0hINU5ucVBuSTdnTApWOUJKZzlRRVBwUTVYa2traW8rejZ2YkRHQU5rR1VPV1dmRURKUHE2Q2JBb1dqeWh1Qy9KS1dYRWs4Rkt0M1Y2CkhVNllYeVpGQWtFQTlHVE9XOFM4KzVNNHE5R3lNeURxN1ZkVHA3M2daeSsvNjVQam5hNlpDUnhTZklxL2xKUVoKY2F6MkhGYVRzRFdLbkdhWGNxTmdBVXNEODNyWTlzM3hCd0pCQUt5Vjc1YUtPMm0rRWI3cWVsV2p5bmpEZytwZQp4akNpUnkxOFZQSjJPYjlmaFU3MWNVS2dlQVdvbE5NalRuREw1dkNxUkNzNTZ4cnk5VC9sN2I2QlNUMENRUURnCjRoV2xDZTdnQzhOZEQzTkxhdUhpRGJZenB4dmp0Mk9Ca2E4ai9ISmptTVVxUnI0dEtPNFUxUlFPVlhoRzc2MmgKWnlHNjRpeklZOCs1N3ZQUWZ3Wm5Ba0VBdW9RWW1lUi90UWhIakhRNFlhZGRHbkNBQ2hZZ29ObEFzSGhGTElxVQo1ZTZaMXN2Q3VKU285TDVVRCtrclFUYWlGU01pRHZwZlJyVE1ZKzZ5Q0tTajd3PT0KLS0tLS1FTkQgUlNBIFBSSVZBVEUgS0VZLS0tLS0K'.decodeBase64() + final discovery = new ConfigDiscovery() + when: + def managers = discovery.createKeyManagers(CERT, KEY) + then: + managers.size()==1 + } + + def 'should create key managers from an EC client key' () { + given: + final CERT = 'LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNTRENDQVRDZ0F3SUJBZ0lJRlFsM1l2Y2k1TWN3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB4T0RBeE1UTXlNREEyTlRaYUZ3MHhPVEF4TVRNeU1EQTJOVFphTURNeApGREFTQmdOVkJBb1RDMFJ2WTJ0bGNpQkpibU11TVJzd0dRWURWUVFERXhKa2IyTnJaWEl0Wm05eUxXUmxjMnQwCmIzQXdnWjh3RFFZSktvWklodmNOQVFFQkJRQURnWTBBTUlHSkFvR0JBUEtYT0ZsV2t2THIzb29ETGNFOElyME0KTzNBMHZqQlVvUzZ0bUdBbFRYYTd0QWQwM3BTMXNJNit0WVRwVlU2YXR6ZU9vU0VrOWhmaWxBdVNYdG1hSHZCUAp1czFEcG1LZEZRMWI3OFRkSnQ4OGV3c3BRajFxYUwvQldHeitMUzUrRHUrNUJuUGtmZlhDS1UxQTdUc2tZamJyClhxeDhlN2FWZURWTmFjZXc0Z0RqQWdNQkFBR2pBakFBTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFBMXVtVlAKR29EZTVCRXJrb21qWXdITXhiTTd4UStibTYrUDE1T0pINUo0UGNQeU11d25ocC9ORVp1NnpsTTZSUUo3SUNKQgpHWTRBMnFKVmJsWUkwQkJzRkF1TXMreTAyazdVVVVoK0NRYVd0SXhBcFNmbkQ4dUVXQ0g5VE1ZNGdLbTZjTDhVCk1OVVl1RnpUQ2hmTS96RjdUMXVaZWxJYXNrYXFaWSt3a3hxa3YyRUQxQ2F5MDUxSXRWRXZVbDIvSVZyVHdrT20KZ25nL3Q4L2RkeDhpOUkzTFJrMTlTaERKdXlQZ1NrTTZRSWlSd09mRHk4V0ZFaURpd0hBS0ErSEZhTGhOOFJTMwpieDUvdEhEN01id0FpdnorNTU4YUFEQjNEd1ZpekthM2d5Wm4yUzRjUGFqZnNwODFqRkNIQS9QekdQdTU2MzJwCkxRN0gyRW1aYmJuUHFYTFgKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo='.decodeBase64() + final KEY = 'LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JR0hBZ0VBTUJNR0J5cUdTTTQ5QWdFR0NDcUdTTTQ5QXdFSEJHMHdhd0lCQVFRZ21aZFZ3NmJRU0w1T1l5RjQKbzJ4V0hUQ05BSW1hRTkycGd2dGMzK2Z2UDVxaFJBTkNBQVJSd0RpUVptTUNqcWxvbFBzRTdiZjgwWjhrZkRXTworS2U4NUdVSll2MlBubWVxbDhkYjdwcmFlMHFPQUJaaXR2Mmh2SmJFeFdsUFR0MS9CYTNMK1B5NAotLS0tLUVORCBQUklWQVRFIEtFWS0tLS0tCg=='.decodeBase64() + final discovery = new ConfigDiscovery() + when: + def managers = discovery.createKeyManagers(CERT, KEY) + then: + managers.size()==1 + } + + def 'should create key managers from an EC-encrypted client key' () { + given: + final CERT = 'LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrVENDQVRlZ0F3SUJBZ0lJSGw1Zmx0UmRTdDB3Q2dZSUtvWkl6ajBFQXdJd0l6RWhNQjhHQTFVRUF3d1kKYXpOekxXTnNhV1Z1ZEMxallVQXhOekUwTnpRM09UVTNNQjRYRFRJME1EVXdNekUwTlRJek4xb1hEVEkxTURVdwpNekUwTlRJek4xb3dNREVYTUJVR0ExVUVDaE1PYzNsemRHVnRPbTFoYzNSbGNuTXhGVEFUQmdOVkJBTVRESE41CmMzUmxiVHBoWkcxcGJqQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJQN1Q5RHVvUlllLzBlUkwKUmNHV2RoYnl2Q3BucXlsSVIyaUwxdGkwc1hVdEpZZjUrVXhIOWFBMjdzY2FSYW1qbjdnTTFrKzZNaVk5cm15OApyRmdoWm1xalNEQkdNQTRHQTFVZER3RUIvd1FFQXdJRm9EQVRCZ05WSFNVRUREQUtCZ2dyQmdFRkJRY0RBakFmCkJnTlZIU01FR0RBV2dCU0NhdXFoQVEvWEdoaFRtaFBoY21vRVdOeWluakFLQmdncWhrak9QUVFEQWdOSUFEQkYKQWlCZzRaNmlWeFV3Mk5uMHBQTG02VlovUGttQnVuTDEwZG50dEg3UVdIcklCd0loQU0vTDhVMGxQN0IyeFEyZwpsZjlhNHNhbzJ1bE5ONnQvQ0dibzlxTlo1QzZHCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0KLS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJkekNDQVIyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWpNU0V3SHdZRFZRUUREQmhyTTNNdFkyeHAKWlc1MExXTmhRREUzTVRRM05EYzVOVGN3SGhjTk1qUXdOVEF6TVRRMU1qTTNXaGNOTXpRd05UQXhNVFExTWpNMwpXakFqTVNFd0h3WURWUVFEREJock0zTXRZMnhwWlc1MExXTmhRREUzTVRRM05EYzVOVGN3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFRZHFYVHdIQS9mVjRKZGdYa2FubXB1OVE0QStwUGRGaXZGdytiUmVhdEYKUXVOUTBKWndIbzlaa2ltb2lEUU5qb2h0TWdHckdtTVlsTTZuaXM4ZVFvM3RvMEl3UURBT0JnTlZIUThCQWY4RQpCQU1DQXFRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVWdtcnFvUUVQMXhvWVU1b1Q0WEpxCkJGamNvcDR3Q2dZSUtvWkl6ajBFQXdJRFNBQXdSUUloQUlvb2ZmNzdvb1VYS2hmNVo3aVRzdExhOTVwU2VaRmUKRHZjMXdFQXVEa3NTQWlBNzJQajJxNnpBclhpYkpUa0s2RTBHTEtVODdhTHhHc3BmS29uVVJnalI2Zz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K'.decodeBase64() + final KEY = 'LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUNvQTNvRHkzN3NXdmszM3JGRGtRdlZ1Wkh1cCt1Uk40V3RqbUlPR1c4cHBvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFL3RQME82aEZoNy9SNUV0RndaWjJGdks4S21lcktVaEhhSXZXMkxTeGRTMGxoL241VEVmMQpvRGJ1eHhwRnFhT2Z1QXpXVDdveUpqMnViTHlzV0NGbWFnPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo='.decodeBase64() + final discovery = new ConfigDiscovery() + when: + def managers = discovery.createKeyManagers(CERT, KEY) + then: + managers.size()==1 + } +} diff --git a/k8s-dvfs/src/test/groovy/recreationaltech/plugin/client/K8sClientTest.groovy b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/client/K8sClientTest.groovy new file mode 100644 index 0000000..b4c66c9 --- /dev/null +++ b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/client/K8sClientTest.groovy @@ -0,0 +1,1630 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin.client + +import nextflow.exception.K8sOutOfCpuException +import nextflow.exception.K8sOutOfMemoryException + +import javax.net.ssl.HttpsURLConnection + +import nextflow.exception.NodeTerminationException +import spock.lang.Specification +/** + * + * @author Paolo Di Tommaso + */ +class K8sClientTest extends Specification { + + def 'should create a request' () { + + given: + final TOKEN = '8d09d0ds' + final client = Spy(K8sClient) + + def HTTPS_CONN = Mock(HttpsURLConnection) + def HTTP_CONN = Mock(HttpURLConnection) + + when: + client.config.server = 'host.com:443' + client.config.token = TOKEN + def resp = client.makeRequest('GET', '/foo/bar') + then: + 1 * client.createConnection0("https://host.com:443/foo/bar") >> HTTPS_CONN + 1 * client.setupHttpsConn(HTTPS_CONN) >> null + 1 * HTTPS_CONN.setRequestMethod('GET') >> null + 1 * HTTPS_CONN.setRequestProperty("Authorization", "Bearer $TOKEN") + 1 * HTTPS_CONN.setRequestProperty("Content-Type", "application/json") + 1 * HTTPS_CONN.getResponseCode() >> 200 + 1 * HTTPS_CONN.getInputStream() >> { new ByteArrayInputStream('{"field_x":"hello"}'.bytes) } + resp instanceof K8sResponseApi + resp.text == '{"field_x":"hello"}' + + when: + client.config.server = 'http://my-server.com' + client.config.token = TOKEN + client.makeRequest('POST', '/foo/bar') + then: + 1 * client.createConnection0("http://my-server.com/foo/bar") >> HTTP_CONN + 0 * client.setupHttpsConn(_) >> null + 1 * HTTP_CONN.setRequestMethod('POST') >> null + 1 * HTTP_CONN.getResponseCode() >> 401 + 1 * HTTP_CONN.getErrorStream() >> { new ByteArrayInputStream('{"field_x":"oops.."}'.bytes) } + def e = thrown(K8sResponseException) + e.response.field_x == 'oops..' + + } + + def 'should make a get request' () { + + given: + def client = Spy(K8sClient) + when: + client.get('/foo/bar') + then: + 1 * client.makeRequest('GET', '/foo/bar') >> null + } + + def 'should make a post request' () { + + given: + def client = Spy(K8sClient) + when: + client.post('/foo/bar', '{ the: body }') + then: + 1 * client.makeRequest('POST', '/foo/bar', '{ the: body }') >> null + } + + def 'should make a delete request' () { + + given: + def client = Spy(K8sClient) + when: + client.delete('/foo/bar', '{ the: body }') + then: + 1 * client.makeRequest('DELETE', '/foo/bar', '{ the: body }') >> null + } + + def 'should delete a pod' () { + + given: + K8sResponseJson result + def client = Spy(K8sClient) + def RESP = Mock(K8sResponseApi) + RESP.getText() >> '{"field":"OK"}' + + when: + result = client.podDelete('foo') + then: + 1 * client.delete("/api/v1/namespaces/default/pods/foo",null) >> RESP + result.field == "OK" + + when: + client.config.namespace = 'bar' + result = client.podDelete('foo') + then: + 1 * client.delete("/api/v1/namespaces/bar/pods/foo",null) >> RESP + result.field == "OK" + + } + + def 'should list pods' () { + given: + K8sResponseJson result + def client = Spy(K8sClient) + def RESP = Mock(K8sResponseApi) + RESP.getText() >> '{"response":"hello"}' + + when: + result = client.podList(true) + then: + 1 * client.get('/api/v1/pods') >> RESP + result.response == 'hello' + + when: + result = client.podList() + then: + 1 * client.get('/api/v1/namespaces/default/pods') >> RESP + result.response == 'hello' + + when: + client.config.namespace = 'foo' + result = client.podList() + then: + 1 * client.get('/api/v1/namespaces/foo/pods') >> RESP + result.response == 'hello' + } + + def 'should list secrets' () { + given: + K8sResponseJson result + def client = Spy(K8sClient) + def RESP = Mock(K8sResponseApi) + RESP.getText() >> '{"response":"hello"}' + + when: + result = client.secretesList() + then: + 1 * client.get("/api/v1/namespaces/default/secrets") >> RESP + result.response == 'hello' + + when: + client.config.namespace = 'pippo' + result = client.podList() + then: + 1 * client.get('/api/v1/namespaces/pippo/pods') >> RESP + result.response == 'hello' + + } + + def 'should describe secret' () { + given: + K8sResponseJson result + def client = Spy(K8sClient) + def RESP = Mock(K8sResponseApi) + RESP.getText() >> '{"response":"hello"}' + + when: + result = client.secretDescribe('my-secret') + then: + 1 * client.get("/api/v1/namespaces/default/secrets/my-secret") >> RESP + result.response == 'hello' + + when: + client.config.namespace = 'paperino' + result = client.secretDescribe('pluto') + then: + 1 * client.get('/api/v1/namespaces/paperino/secrets/pluto') >> RESP + result.response == 'hello' + } + + def 'should log pod' () { + + given: + InputStream result + def client = Spy(K8sClient) + def STREAM = Mock(InputStream) + def RESP = Mock(K8sResponseApi) + RESP.getStream() >> STREAM + + when: + result = client.podLog('pod-123') + then: + 1 * client.get('/api/v1/namespaces/default/pods/pod-123/log') >> RESP + result == STREAM + + when: + result = client.podLog('pod-123', follow: true) + then: + 1 * client.get('/api/v1/namespaces/default/pods/pod-123/log?follow=true') >> RESP + result == STREAM + + when: + result = client.podLog('pod-123', follow: true, foo:1, bar: 'x') + then: + 1 * client.get('/api/v1/namespaces/default/pods/pod-123/log?follow=true&foo=1&bar=x') >> RESP + result == STREAM + + } + + def 'should create config' () { + + given: + K8sResponseJson result + def client = Spy(K8sClient) + def RESP = Mock(K8sResponseApi) + RESP.getText() >> '{"response":"done"}' + def CONFIG = [foo: 'hello', bar:'world'] + def JSON1 = '{"apiVersion":"v1","kind":"ConfigMap","metadata":{"name":"foo","namespace":"default"},"data":{"foo":"hello","bar":"world"}}' + def JSON2 = '{"apiVersion":"v1","kind":"ConfigMap","metadata":{"name":"foo","namespace":"bar"},"data":{"foo":"hello","bar":"world"}}' + + when: + result = client.configCreate('foo' , CONFIG) + then: + 1 * client.post("/api/v1/namespaces/default/configmaps", JSON1) >> RESP + result.response == 'done' + + when: + client.config.namespace = 'bar' + result = client.configCreate('foo' , CONFIG) + then: + 1 * client.post("/api/v1/namespaces/bar/configmaps", JSON2) >> RESP + result.response == 'done' + + } + + def 'should delete config' () { + + given: + K8sResponseJson result + def client = Spy(K8sClient) + def RESP = Mock(K8sResponseApi) + RESP.getText() >> '{"response":"done"}' + + when: + result = client.configDelete('foo') + then: + 1 * client.delete("/api/v1/namespaces/default/configmaps/foo") >> RESP + result.response == 'done' + + when: + client.config.namespace = 'ns-1' + result = client.configDelete('foo') + then: + 1 * client.delete("/api/v1/namespaces/ns-1/configmaps/foo") >> RESP + result.response == 'done' + + } + + def 'should delete all configs' () { + + given: + K8sResponseJson result + def client = Spy(K8sClient) + def RESP = Mock(K8sResponseApi) + RESP.getText() >> '{"response":"done"}' + + when: + result = client.configDeleteAll() + then: + 1 * client.delete("/api/v1/namespaces/default/configmaps") >> RESP + result.response == 'done' + + when: + client.config.namespace = 'ns-1' + result = client.configDeleteAll() + then: + 1 * client.delete("/api/v1/namespaces/ns-1/configmaps") >> RESP + result.response == 'done' + + } + + def 'should get a pod state' () { + + given: + def JSON = ''' + { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "name": "pod-xyz", + "namespace": "default", + "selfLink": "/api/v1/namespaces/default/pods/pod-xyz/status", + "uid": "33390e2c-f84b-11e7-a89d-025000000001", + "resourceVersion": "119932", + "creationTimestamp": "2018-01-13T10:19:12Z", + "labels": { + "app": "nextflow" + } + }, + + + "status": { + "phase": "Succeeded", + "conditions": [ + { + "type": "Initialized", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2018-01-13T10:19:12Z", + "reason": "PodCompleted" + }, + { + "type": "Ready", + "status": "False", + "lastProbeTime": null, + "lastTransitionTime": "2018-01-13T10:19:37Z", + "reason": "PodCompleted" + }, + { + "type": "PodScheduled", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2018-01-13T10:19:12Z" + } + ], + "hostIP": "192.168.65.3", + "podIP": "10.1.0.25", + "startTime": "2018-01-13T10:19:12Z", + "containerStatuses": [ + { + "name": "pod-xyz", + "state": { + "terminated": { + "exitCode": 0, + "reason": "Completed", + "startedAt": "2018-01-13T10:19:16Z", + "finishedAt": "2018-01-13T10:19:36Z", + "containerID": "docker://90d447d4a2518642b12c8979474aa2bbb5fe8e96ed9e5caf3c979bb3b751519a" + } + }, + "lastState": { + + }, + "ready": false, + "restartCount": 0, + "image": "debian:latest", + "imageID": "docker-pullable://debian@sha256:0a5fcee6f52d5170f557ee2447d7a10a5bdcf715dd7f0250be0b678c556a501b", + "containerID": "docker://90d447d4a2518642b12c8979474aa2bbb5fe8e96ed9e5caf3c979bb3b751519a" + } + ], + "qosClass": "BestEffort" + } + } + ''' + + def client = Spy(K8sClient) + final POD_NAME = 'pod-xyz' + + when: + def result = client.podState(POD_NAME) + then: + 1 * client.podStatus(POD_NAME) >> new K8sResponseJson(JSON) + + result == [terminated: [exitCode:0, + reason: 'Completed', + startedAt: "2018-01-13T10:19:16Z", + finishedAt: "2018-01-13T10:19:36Z", + containerID: "docker://90d447d4a2518642b12c8979474aa2bbb5fe8e96ed9e5caf3c979bb3b751519a"]] + + + } + + def 'should get a pod its node' () { + + given: + def JSON = ''' + { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "name": "pod-xyz", + "namespace": "default", + "selfLink": "/api/v1/namespaces/default/pods/pod-xyz/status", + "uid": "33390e2c-f84b-11e7-a89d-025000000001", + "resourceVersion": "119932", + "creationTimestamp": "2018-01-13T10:19:12Z", + "labels": { + "app": "nextflow" + } + }, + "spec": { + "restartPolicy": "Never", + "terminationGracePeriodSeconds": 30, + "dnsPolicy": "ClusterFirst", + "serviceAccountName": "default", + "serviceAccount": "default", + "nodeName": "gke-bioinformatics-s-pipeline-pool-sm-bbac2e1c-k1tw", + "priority": 0, + "enableServiceLinks": true, + "preemptionPolicy": "PreemptLowerPriority" + } + } + ''' + + def client = Spy(K8sClient) + final POD_NAME = 'pod-xyz' + + when: + def result = client.getNodeOfPod(POD_NAME) + then: + 1 * client.podStatus(POD_NAME) >> new K8sResponseJson(JSON) + + result == "gke-bioinformatics-s-pipeline-pool-sm-bbac2e1c-k1tw" + + } + + def 'should return undetermined status' () { + given: + def JSON = ''' + { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "name": "nf-eb853c8010b8e173b23d8d15489d1a31", + "namespace": "default", + "selfLink": "/api/v1/namespaces/default/pods/nf-eb853c8010b8e173b23d8d15489d1a31/status", + "uid": "5ccdeb23-4f69-11e8-89b1-fa163e31bb09", + "resourceVersion": "2847182", + "creationTimestamp": "2018-05-04T07:04:18Z", + "labels": { + "app": "nextflow", + "processName": "markDuplicates", + "runName": "grave-jones", + "sessionId": "uuid-f51cd941-c21c-447b-86ca-eaebafa5ad9b", + "taskName": "markDuplicates_22028_2_118_1AlignedByCoord.out" + } + }, + + "status": { + "phase": "Pending", + "conditions": [ + { + "type": "PodScheduled", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2018-05-04T07:04:18Z" + } + ], + "qosClass": "Guaranteed" + } + } + ''' + + def client = Spy(K8sClient) + final POD_NAME = 'nf-eb853c8010b8e173b23d8d15489d1a31' + + when: + def result = client.podState(POD_NAME) + then: + 1 * client.podStatus(POD_NAME) >> new K8sResponseJson(JSON) + + result == [:] + } + + def 'should return undetermined status when status conditions are missing' () { + given: + def JSON = ''' + { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "name": "nf-89b34d7c2d1dc11daad72b1fcf7e0540", + "namespace": "default", + "selfLink": "/api/v1/namespaces/default/pods/nf-89b34d7c2d1dc11daad72b1fcf7e0540/status", + "uid": "02c3a34d-7720-11e9-8e1a-0a8b849038f8", + "resourceVersion": "21753960", + "creationTimestamp": "2019-05-15T14:44:58Z", + "labels": { + "app": "nextflow", + "processName": "split_input_file", + "runName": "hopeful_lalande", + "sessionId": "uuid-6b10c771-1a70-4af0-92be-7c0280e0e17f", + "taskName": "split_input_file_variants.gord_chr11_72000001-78000000", + } + }, + + "status": { + "phase": "Pending", + "qosClass": "Guaranteed" + } + } + ''' + + def client = Spy(K8sClient) + final POD_NAME = 'nf-89b34d7c2d1dc11daad72b1fcf7e0540' + + when: + def result = client.podState(POD_NAME) + then: + 1 * client.podStatus(POD_NAME) >> new K8sResponseJson(JSON) + + result == [:] + } + + def 'should return a process execution on pod not found' () { + given: + def JSON = ''' + { + "kind": "Status", + "apiVersion": "v1", + "metadata": { + + }, + "status": "Failure", + "message": "pods \\"nf-7cee928c1dd05b39cd50ab79a3e742f9\\" not found", + "reason": "NotFound", + "details": { + "name": "nf-7cee928c1dd05b39cd50ab79a3e742f9", + "kind": "pods" + }, + "code": 404 + } + ''' + + def client = Spy(K8sClient) + final POD_NAME = 'pod-xyz' + + when: + client.podState(POD_NAME) + then: + 1 * client.podStatus(POD_NAME) >> { throw new K8sResponseException("Request GET /api/v1/namespaces/xyz/pods/nf-xyz/status returned an error code=404", new ByteArrayInputStream(JSON.bytes)) } + + and: + thrown(NodeTerminationException) + } + + def 'should fail to get pod state' () { + + given: + def client = Spy(K8sClient) + final POD_NAME = 'pod-xyz' + final STATE = [foo:1, bar:2] + + when: + def e + client.podState(POD_NAME) + then: + 1 * client.podStatus(POD_NAME) >> new K8sResponseJson([:]) + e = thrown(K8sResponseException) + e.message.startsWith('K8s undetermined status conditions for pod') + + when: + client.podState(POD_NAME) + then: + 1 * client.podStatus(POD_NAME) >> new K8sResponseJson([status:[containerStatuses: []]]) + e = thrown(K8sResponseException) + e.message.startsWith('K8s undetermined status conditions for pod') + + when: + client.podState(POD_NAME) + then: + 1 * client.podStatus(POD_NAME) >> new K8sResponseJson([status:[containerStatuses: [ [name: 'foo'] ]]]) + e = thrown(K8sResponseException) + e.message.startsWith('K8s invalid status for pod: pod-xyz (unexpected container name: foo)') + + when: + client.podState(POD_NAME) + then: + 1 * client.podStatus(POD_NAME) >> new K8sResponseJson([status:[containerStatuses: [ [name: POD_NAME] ]]]) + e = thrown(K8sResponseException) + e.message.startsWith('K8s invalid status for pod: pod-xyz (missing state object)') + + when: + def result = client.podState(POD_NAME) + then: + 1 * client.podStatus(POD_NAME) >> new K8sResponseJson([status:[containerStatuses: [ [name: POD_NAME, state: STATE] ]]]) + result == STATE + } + + def 'client should throw an exception when container status returns ErrImagePull' () { + given: + def JSON = ''' + { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "name": "nf-c6e49ee9ebc79486f774a47924a743d7", + "namespace": "default", + "selfLink": "/api/v1/namespaces/default/pods/nf-c6e49ee9ebc79486f774a47924a743d7/status", + "uid": "18954b48-5811-11e8-8e71-025000000001", + "resourceVersion": "3615842", + "creationTimestamp": "2018-05-15T07:25:08Z", + "labels": { + "app": "nextflow", + "processName": "sayHello", + "runName": "lethal-jones", + "sessionId": "uuid-b6243a3d-5c07-44f4-97eb-9de499747800", + "taskName": "sayHello_2" + } + }, + "spec": { + + }, + "status": { + "phase": "Pending", + "conditions": [ + { + "type": "Initialized", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2018-05-15T07:25:08Z" + }, + { + "type": "Ready", + "status": "False", + "lastProbeTime": null, + "lastTransitionTime": "2018-05-15T07:25:08Z", + "reason": "ContainersNotReady", + "message": "containers with unready status: [nf-c6e49ee9ebc79486f774a47924a743d7]" + }, + { + "type": "PodScheduled", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2018-05-15T07:25:08Z" + } + ], + "hostIP": "192.168.65.3", + "podIP": "10.1.3.173", + "startTime": "2018-05-15T07:25:08Z", + "containerStatuses": [ + { + "name": "nf-c6e49ee9ebc79486f774a47924a743d7", + "state": { + "waiting": { + "reason": "ErrImagePull", + "message": "rpc error: code = Unknown desc = Error response from daemon: pull access denied for nextflow/foo, repository does not exist or may require 'docker login'" + } + }, + "lastState": { + + }, + "ready": false, + "restartCount": 0, + "image": "nextflow/foo", + "imageID": "" + } + ], + "qosClass": "BestEffort" + } + } +''' + + def client = Spy(K8sClient) + final POD_NAME = 'nf-c6e49ee9ebc79486f774a47924a743d7' + + when: + client.podState(POD_NAME) + then: + 1 * client.podStatus(POD_NAME) >> new K8sResponseJson(JSON) + def e = thrown(PodUnschedulableException) + e.message == "K8s pod image cannot be pulled -- rpc error: code = Unknown desc = Error response from daemon: pull access denied for nextflow/foo, repository does not exist or may require 'docker login'" + } + + def 'client should throw an exception when k8s is out of cpu' () { + given: + def JSON = ''' + { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "name": "nf-3b344812fe0aeb9554424bcf6caa7ffb", + "namespace": "default", + "uid": "0dd3c071-82a3-4b20-bdd4-0d34ff3d90bd", + "resourceVersion": "55320", + "creationTimestamp": "2022-09-23T13:43:48Z", + "labels": { + "app": "nextflow", + "processName": "combineFiles", + "runName": "insane-kare", + "sessionId": "uuid-85951b91-b5bc-4566-8938-fef4256c21c8", + "taskName": "combineFiles_1" + }, + }, + "spec": { + }, + "status": { + "phase": "Failed", + "message": "Pod Node didn't have enough resource: cpu, requested: 4000, used: 2100, capacity: 6000", + "reason": "OutOfcpu", + "startTime": "2022-09-23T13:43:48Z" + } + } + ''' + + def client = Spy(K8sClient) + final POD_NAME = 'nf-3b344812fe0aeb9554424bcf6caa7ffb' + + when: + client.podState(POD_NAME) + then: + 1 * client.podStatus(POD_NAME) >> new K8sResponseJson(JSON) + def e = thrown(K8sOutOfCpuException) + e.message == "K8s pod 'nf-3b344812fe0aeb9554424bcf6caa7ffb' execution failed - reason: OutOfcpu - message: Pod Node didn't have enough resource: cpu, requested: 4000, used: 2100, capacity: 6000" + } + + def 'client should throw an exception when k8s is out of memory' () { + given: + def JSON = ''' + { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "name": "nf-3b344812fe0aeb9554424bcf6caa7ffb", + "namespace": "default", + "uid": "0dd3c071-82a3-4b20-bdd4-0d34ff3d90bd", + "resourceVersion": "55320", + "creationTimestamp": "2022-09-23T13:43:48Z", + "labels": { + "app": "nextflow", + "processName": "combineFiles", + "runName": "insane-kare", + "sessionId": "uuid-85951b91-b5bc-4566-8938-fef4256c21c8", + "taskName": "combineFiles_2" + }, + }, + "spec": { + }, + "status": { + "phase": "Failed", + "message": "Pod Node didn't have enough resource: memory, requested: 16106127360, used: 16158556160, capacity: 16778358784", + "reason": "OutOfmemory", + "startTime": "2022-09-23T13:42:59Z" + } + } + ''' + + def client = Spy(K8sClient) + final POD_NAME = 'nf-3b344812fe0aeb9554424bcf6caa7ffb' + + when: + client.podState(POD_NAME) + then: + 1 * client.podStatus(POD_NAME) >> new K8sResponseJson(JSON) + def e = thrown(K8sOutOfMemoryException) + e.message == "K8s pod 'nf-3b344812fe0aeb9554424bcf6caa7ffb' execution failed - reason: OutOfmemory - message: Pod Node didn't have enough resource: memory, requested: 16106127360, used: 16158556160, capacity: 16778358784" + } + + def 'client should throw an exception when container status returns ImagePullBackOff' () { + def JSON = ''' + { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "name": "nf-c6e49ee9ebc79486f774a47924a743d7", + "namespace": "default", + "selfLink": "/api/v1/namespaces/default/pods/nf-c6e49ee9ebc79486f774a47924a743d7/status", + "uid": "18954b48-5811-11e8-8e71-025000000001", + "resourceVersion": "3615866", + "creationTimestamp": "2018-05-15T07:25:08Z", + "labels": { + "app": "nextflow", + "processName": "sayHello", + "runName": "lethal-jones", + "sessionId": "uuid-b6243a3d-5c07-44f4-97eb-9de499747800", + "taskName": "sayHello_2" + } + }, + "spec": { }, + "status": { + "phase": "Pending", + "conditions": [ + { + "type": "Initialized", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2018-05-15T07:25:08Z" + }, + { + "type": "Ready", + "status": "False", + "lastProbeTime": null, + "lastTransitionTime": "2018-05-15T07:25:08Z", + "reason": "ContainersNotReady", + "message": "containers with unready status: [nf-c6e49ee9ebc79486f774a47924a743d7]" + }, + { + "type": "PodScheduled", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2018-05-15T07:25:08Z" + } + ], + "hostIP": "192.168.65.3", + "podIP": "10.1.3.173", + "startTime": "2018-05-15T07:25:08Z", + "containerStatuses": [ + { + "name": "nf-c6e49ee9ebc79486f774a47924a743d7", + "state": { + "waiting": { + "reason": "ImagePullBackOff", + "message": "Back-off pulling image \\"nextflow/foo\\"" + } + }, + "lastState": { + + }, + "ready": false, + "restartCount": 0, + "image": "nextflow/foo", + "imageID": "" + } + ], + "qosClass": "BestEffort" + } + } +''' + def client = Spy(K8sClient) + final POD_NAME = 'nf-c6e49ee9ebc79486f774a47924a743d7' + + when: + client.podState(POD_NAME) + then: + 1 * client.podStatus(POD_NAME) >> new K8sResponseJson(JSON) + def e = thrown(PodUnschedulableException) + e.message == "K8s pod image cannot be pulled -- Back-off pulling image \"nextflow/foo\"" + + } + + def 'client should throw process exception on failed state' () { + def JSON = ''' + { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "name": "nf-f34e124e1471736f27c8ef1aa52d02be", + "namespace": "tower-nf", + "uid": "8ddb536f-da4c-4a99-a5c5-3b5b53df6411", + "resourceVersion": "31269576", + "creationTimestamp": "2021-08-09T17:58:42Z", + "labels": { + "app": "nextflow", + "processName": "bulk_rnaseq_trim_galore", + "runName": "insane_agnesi", + "sessionId": "uuid-35a8fe1a-2622-4582-a1b2-1cc21bd85b57", + "taskName": "bulk_rnaseq_trim_galore_d0_tdTOM-0701" + } + }, + "spec": { + "restartPolicy": "Never", + "terminationGracePeriodSeconds": 30, + "dnsPolicy": "ClusterFirst", + "serviceAccountName": "default", + "serviceAccount": "default", + "nodeName": "gke-bioinformatics-s-pipeline-pool-sm-bbac2e1c-k1tw", + "priority": 0, + "enableServiceLinks": true, + "preemptionPolicy": "PreemptLowerPriority" + }, + "status": { + "phase": "Failed", + "message": "Node is shutting, evicting pods", + "reason": "Shutdown", + "startTime": "2021-08-09T17:59:31Z" + } + } +''' + def client = Spy(K8sClient) + final POD_NAME = 'nf-xyz' + + when: + client.podState(POD_NAME) + then: + 1 * client.podStatus(POD_NAME) >> new K8sResponseJson(JSON) + and: + def e = thrown(NodeTerminationException) + e.message == "K8s pod 'nf-xyz' execution failed - reason: Shutdown - message: Node is shutting, evicting pods" + + } + + def 'client should fail when config fail' () { + given: + def JSON = ''' + { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "name": "angry-blackwell", + "namespace": "default", + "selfLink": "/api/v1/namespaces/default/pods/angry-blackwell/status", + "uid": "83a35c1e-73b6-11e8-8259-025000000001", + "resourceVersion": "465382", + "creationTimestamp": "2018-06-19T11:47:16Z", + "labels": { + "app": "nextflow", + "runName": "angry-blackwell" + } + }, + "spec": { + + }, + "status": { + "phase": "Pending", + "conditions": [ + { + "type": "Initialized", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2018-06-19T11:47:16Z" + }, + { + "type": "Ready", + "status": "False", + "lastProbeTime": null, + "lastTransitionTime": "2018-06-19T11:47:16Z", + "reason": "ContainersNotReady", + "message": "containers with unready status: [angry-blackwell]" + }, + { + "type": "PodScheduled", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2018-06-19T11:47:16Z" + } + ], + "hostIP": "192.168.65.3", + "podIP": "10.1.4.20", + "startTime": "2018-06-19T11:47:16Z", + "containerStatuses": [ + { + "name": "angry-blackwell", + "state": { + "waiting": { + "reason": "CreateContainerConfigError", + "message": "secrets \\"my-env\\" not found" + } + }, + "lastState": { + + }, + "ready": false, + "restartCount": 0, + "image": "nextflow/nextflow:0.30.2", + "imageID": "" + } + ], + "qosClass": "BestEffort" + } + } + ''' + + def client = Spy(K8sClient) + final POD_NAME = 'angry-blackwell' + + when: + client.podState(POD_NAME) + then: + 1 * client.podStatus(POD_NAME) >> new K8sResponseJson(JSON) + def e = thrown(PodUnschedulableException) + e.message == 'K8s pod configuration failed -- secrets "my-env" not found' + + } + + def 'client should throw process exception when initContainer fails' () { + // Addresses bug 2428, which is when an initContainer exists and fails + // leaving the containerStatuses in "PodInitializing", but the pod + // phase is "Failed". + def JSON = ''' + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2021-11-03T21:37:45Z", + "labels": { + "app": "nextflow", + "magic-version": "v7.5.3", + "processName": "some_special_magic", + "runName": "lonely_tuckerman", + "sessionId": "uuid-41ef6f42-bab5-438f-83e4-85e91a6386ea", + "taskName": "some_special_magic_1" + }, + "name": "nf-45d62fd8462ba390bacfa20ad9065bfe", + "namespace": "default", + "resourceVersion": "90692", + "uid": "280bf29c-4c14-4451-a300-080760ff7a0e" + }, + "status": { + "containerStatuses": [ + { + "image": "docker/asdf/magictest:7.5.3", + "imageID": "", + "lastState": {}, + "name": "nf-45d62fd8462ba390bacfa20ad9065bfe", + "ready": false, + "restartCount": 0, + "started": false, + "state": { + "waiting": { + "reason": "PodInitializing" + } + } + } + ], + "hostIP": "10.12.205.49", + "initContainerStatuses": [ + { + "containerID": "containerd://8ac143e547ef8c417f3655da31a81018e9d14db08c218e56ee3d181c70e0b755", + "image": "docker/asdf/magic-special-handling:0.4.0", + "imageID": "sha256:587f96e98fd0a2b17772e9a505522e4c0e1cf5194a4f08924c339848df42100c", + "lastState": {}, + "name": "magic-init", + "ready": false, + "restartCount": 0, + "state": { + "terminated": { + "containerID": "containerd://8ac143e547ef8c417f3655da31a81018e9d14db08c218e56ee3d181c70e0b755", + "exitCode": 1, + "finishedAt": "2021-11-03T21:37:51Z", + "reason": "Error", + "startedAt": "2021-11-03T21:37:46Z" + } + } + } + ], + "phase": "Failed", + "podIP": "10.42.0.67", + "podIPs": [ + { + "ip": "10.42.0.67" + } + ], + "qosClass": "Burstable", + "startTime": "2021-11-03T21:37:45Z" + } + } +''' + def client = Spy(K8sClient) + final POD_NAME = 'nf-45d62fd8462ba390bacfa20ad9065bfe' + + when: + client.podState(POD_NAME) + then: + 1 * client.podStatus(POD_NAME) >> new K8sResponseJson(JSON) + and: + def e = thrown(PodUnschedulableException) + e.message == "K8s pod in Failed state" + } + + def 'should fallback to job status when pod is gone and not return hardcoded exit code' () { + given: + def JOB_STATUS_JSON = ''' + { + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "name": "test-job" + }, + "status": { + "succeeded": 1, + "startTime": "2025-01-15T10:00:00Z", + "completionTime": "2025-01-15T10:05:00Z", + "conditions": [ + { + "type": "Complete", + "status": "True", + "lastProbeTime": "2025-01-15T10:05:00Z", + "lastTransitionTime": "2025-01-15T10:05:00Z" + } + ] + } + } + ''' + def client = Spy(K8sClient) + final JOB_NAME = 'test-job' + + when: + def result = client.jobStateFallback0(JOB_NAME) + + then: + 1 * client.jobStatus(JOB_NAME) >> new K8sResponseJson(JOB_STATUS_JSON) + + and: + result.terminated != null + result.terminated.reason == 'Completed' + result.terminated.startedAt == '2025-01-15T10:00:00Z' + result.terminated.finishedAt == '2025-01-15T10:05:00Z' + // The key assertion: exitCode should not be present (null) so fallback to .exitcode file works + result.terminated.exitCode == null + result.terminated.exitcode == null + } + + def 'should describe a node' () { + given: + def JSON = ''' + { + "kind": "Node", + "apiVersion": "v1", + "metadata": { + "name": "test-node", + "namespace": "default" + }, + "status": { + "capacity": { + "cpu": "4", + "memory": "16Gi" + }, + "allocatable": { + "cpu": "3.5", + "memory": "14Gi" + } + } + } + ''' + + def client = Spy(K8sClient) + final NODE_NAME = 'test-node' + def RESP = Mock(K8sResponseApi) + RESP.getText() >> JSON + + when: + def result = client.nodeDescribe(NODE_NAME) + then: + 1 * client.get('/api/v1/nodes/test-node') >> RESP + result.kind == 'Node' + result.metadata.name == 'test-node' + } + + def 'should get node CPU capacity' () { + given: + def JSON = ''' + { + "kind": "Node", + "apiVersion": "v1", + "metadata": { + "name": "test-node" + }, + "status": { + "capacity": { + "cpu": "4", + "memory": "16Gi" + } + } + } + ''' + + def client = Spy(K8sClient) + final NODE_NAME = 'test-node' + def RESP = Mock(K8sResponseApi) + RESP.getText() >> JSON + + when: + def result = client.getNodeCpuCapacity(NODE_NAME) + then: + 1 * client.nodeDescribe(NODE_NAME) >> new K8sResponseJson(JSON) + result == '4' + } + + def 'should get node memory capacity' () { + given: + def JSON = ''' + { + "kind": "Node", + "apiVersion": "v1", + "metadata": { + "name": "test-node" + }, + "status": { + "capacity": { + "cpu": "4", + "memory": "16Gi" + } + } + } + ''' + + def client = Spy(K8sClient) + final NODE_NAME = 'test-node' + + when: + def result = client.getNodeMemoryCapacity(NODE_NAME) + then: + 1 * client.nodeDescribe(NODE_NAME) >> new K8sResponseJson(JSON) + result == '16Gi' + } + + def 'should get node CPU used from metrics server' () { + given: + def NODE_JSON = ''' + { + "kind": "Node", + "apiVersion": "v1", + "metadata": { + "name": "test-node" + }, + "status": { + "capacity": { + "cpu": "4", + "memory": "16Gi" + } + } + } + ''' + def METRICS_JSON = ''' + { + "kind": "NodeMetrics", + "apiVersion": "metrics.k8s.io/v1beta1", + "metadata": { + "name": "test-node" + }, + "usage": { + "cpu": "1500m", + "memory": "2Gi" + } + } + ''' + + def client = Spy(K8sClient) + final NODE_NAME = 'test-node' + def NODE_RESP = Mock(K8sResponseApi) + NODE_RESP.getText() >> NODE_JSON + def METRICS_RESP = Mock(K8sResponseApi) + METRICS_RESP.getText() >> METRICS_JSON + + when: + def result = client.getNodeCpuUsed(NODE_NAME) + then: + 1 * client.get('/apis/metrics.k8s.io/v1beta1/nodes/test-node') >> METRICS_RESP + result == '1500m' + } + + def 'should get node memory used from metrics server' () { + given: + def NODE_JSON = ''' + { + "kind": "Node", + "apiVersion": "v1", + "metadata": { + "name": "test-node" + }, + "status": { + "capacity": { + "cpu": "4", + "memory": "16Gi" + } + } + } + ''' + def METRICS_JSON = ''' + { + "kind": "NodeMetrics", + "apiVersion": "metrics.k8s.io/v1beta1", + "metadata": { + "name": "test-node" + }, + "usage": { + "cpu": "1500m", + "memory": "2Gi" + } + } + ''' + + def client = Spy(K8sClient) + final NODE_NAME = 'test-node' + def METRICS_RESP = Mock(K8sResponseApi) + METRICS_RESP.getText() >> METRICS_JSON + + when: + def result = client.getNodeMemoryUsed(NODE_NAME) + then: + 1 * client.get('/apis/metrics.k8s.io/v1beta1/nodes/test-node') >> METRICS_RESP + result == '2Gi' + } + + def 'should fallback when metrics server not available for CPU' () { + given: + def NODE_JSON = ''' + { + "kind": "Node", + "apiVersion": "v1", + "metadata": { + "name": "test-node" + }, + "status": { + "capacity": { + "cpu": "4", + "memory": "16Gi" + }, + "allocated": { + "cpu": "2", + "memory": "8Gi" + } + } + } + ''' + + def client = Spy(K8sClient) + final NODE_NAME = 'test-node' + def NODE_RESP = Mock(K8sResponseApi) + NODE_RESP.getText() >> NODE_JSON + + when: + def result = client.getNodeCpuUsed(NODE_NAME) + then: + // First attempt to metrics server fails + 1 * client.get('/apis/metrics.k8s.io/v1beta1/nodes/test-node') >> { throw new K8sResponseException("Metrics not available", new ByteArrayInputStream('{}'.bytes)) } + // Fallback to node describe - which calls get internally + 1 * client.get('/api/v1/nodes/test-node') >> NODE_RESP + result == '2' + } + + def 'should fallback when metrics server not available for memory' () { + given: + def NODE_JSON = ''' + { + "kind": "Node", + "apiVersion": "v1", + "metadata": { + "name": "test-node" + }, + "status": { + "capacity": { + "cpu": "4", + "memory": "16Gi" + }, + "allocated": { + "cpu": "2", + "memory": "8Gi" + } + } + } + ''' + + def client = Spy(K8sClient) + final NODE_NAME = 'test-node' + def NODE_RESP = Mock(K8sResponseApi) + NODE_RESP.getText() >> NODE_JSON + + when: + def result = client.getNodeMemoryUsed(NODE_NAME) + then: + // First attempt to metrics server fails + 1 * client.get('/apis/metrics.k8s.io/v1beta1/nodes/test-node') >> { throw new K8sResponseException("Metrics not available", new ByteArrayInputStream('{}'.bytes)) } + // Fallback to node describe - which calls get internally + 1 * client.get('/api/v1/nodes/test-node') >> NODE_RESP + result == '8Gi' + } + + def 'should return null when no allocated resources available' () { + given: + def NODE_JSON = ''' + { + "kind": "Node", + "apiVersion": "v1", + "metadata": { + "name": "test-node" + }, + "status": { + "capacity": { + "cpu": "4", + "memory": "16Gi" + } + } + } + ''' + + def client = Spy(K8sClient) + final NODE_NAME = 'test-node' + def NODE_RESP = Mock(K8sResponseApi) + NODE_RESP.getText() >> NODE_JSON + + when: + def resultCpu = client.getNodeCpuUsed(NODE_NAME) + def resultMem = client.getNodeMemoryUsed(NODE_NAME) + then: + // First attempt to metrics server fails for CPU + 1 * client.get('/apis/metrics.k8s.io/v1beta1/nodes/test-node') >> { throw new K8sResponseException("Metrics not available", new ByteArrayInputStream('{}'.bytes)) } + // Fallback to node describe for CPU + 1 * client.get('/api/v1/nodes/test-node') >> NODE_RESP + // First attempt to metrics server fails for Memory + 1 * client.get('/apis/metrics.k8s.io/v1beta1/nodes/test-node') >> { throw new K8sResponseException("Metrics not available", new ByteArrayInputStream('{}'.bytes)) } + // Fallback to node describe for Memory + 1 * client.get('/api/v1/nodes/test-node') >> NODE_RESP + resultCpu == null + resultMem == null + } + + def 'should parse memory quantity strings' () { + expect: + K8sClient.parseK8sQuantity(null) == null + K8sClient.parseK8sQuantity('') == null + K8sClient.parseK8sQuantity('1024') == 1024L + K8sClient.parseK8sQuantity('1Ki') == 1024L + K8sClient.parseK8sQuantity('1KiB') == 1024L + K8sClient.parseK8sQuantity('1Mi') == 1024L * 1024L + K8sClient.parseK8sQuantity('1MiB') == 1024L * 1024L + K8sClient.parseK8sQuantity('1Gi') == 1024L * 1024L * 1024L + K8sClient.parseK8sQuantity('1GiB') == 1024L * 1024L * 1024L + K8sClient.parseK8sQuantity('16Gi') == 16L * 1024L * 1024L * 1024L + K8sClient.parseK8sQuantity('8192Mi') == 8192L * 1024L * 1024L + K8sClient.parseK8sQuantity('0.5Gi') == (long)(0.5 * 1024L * 1024L * 1024L) + } + + def 'should parse CPU quantity strings' () { + expect: + K8sClient.parseK8sQuantity(null, true) == null + K8sClient.parseK8sQuantity('', true) == null + K8sClient.parseK8sQuantity('1', true) == 1000L // 1 core = 1000 millicores + K8sClient.parseK8sQuantity('2', true) == 2000L // 2 cores = 2000 millicores + K8sClient.parseK8sQuantity('0.5', true) == 500L // 0.5 cores = 500 millicores + K8sClient.parseK8sQuantity('1500m', true) == 1500L // 1500 millicores + K8sClient.parseK8sQuantity('100m', true) == 100L // 100 millicores + // Nanocores: 1 core = 1,000,000,000 nanocores, so we divide by 1,000,000 to get millicores + K8sClient.parseK8sQuantity('1000000n', true) == 1L // 1,000,000 nanocores = 1 millicore + K8sClient.parseK8sQuantity('1000000000n', true) == 1000L // 1,000,000,000 nanocores = 1000 millicores = 1 core + K8sClient.parseK8sQuantity('298332671n', true) == 298L // 298,332,671 nanocores ≈ 298.33 millicores + K8sClient.parseK8sQuantity('434126984n', true) == 434L // 434,126,984 nanocores ≈ 434.13 millicores + } + + def 'should get node CPU capacity in millicores' () { + given: + def NODE_JSON = ''' + { + "kind": "Node", + "apiVersion": "v1", + "metadata": { + "name": "test-node" + }, + "status": { + "capacity": { + "cpu": "4", + "memory": "16Gi" + } + } + } + ''' + + def client = Spy(K8sClient) + final NODE_NAME = 'test-node' + + when: + def result = client.getNodeCpuCapacityMillis(NODE_NAME) + then: + 1 * client.nodeDescribe(NODE_NAME) >> new K8sResponseJson(NODE_JSON) + result == 4000L // 4 cores = 4000 millicores + } + + def 'should get node CPU used in millicores' () { + given: + def METRICS_JSON = ''' + { + "kind": "NodeMetrics", + "apiVersion": "metrics.k8s.io/v1beta1", + "metadata": { + "name": "test-node" + }, + "usage": { + "cpu": "1500m", + "memory": "2Gi" + } + } + ''' + + def client = Spy(K8sClient) + final NODE_NAME = 'test-node' + def METRICS_RESP = Mock(K8sResponseApi) + METRICS_RESP.getText() >> METRICS_JSON + + when: + def result = client.getNodeCpuUsedMillis(NODE_NAME) + then: + 1 * client.get('/apis/metrics.k8s.io/v1beta1/nodes/test-node') >> METRICS_RESP + result == 1500L // 1500 millicores + } + + def 'should get node memory capacity in bytes' () { + given: + def NODE_JSON = ''' + { + "kind": "Node", + "apiVersion": "v1", + "metadata": { + "name": "test-node" + }, + "status": { + "capacity": { + "cpu": "4", + "memory": "16Gi" + } + } + } + ''' + + def client = Spy(K8sClient) + final NODE_NAME = 'test-node' + + when: + def result = client.getNodeMemoryCapacityBytes(NODE_NAME) + then: + 1 * client.nodeDescribe(NODE_NAME) >> new K8sResponseJson(NODE_JSON) + result == 16L * 1024L * 1024L * 1024L // 16 GiB in bytes + } + + def 'should get node memory used in bytes' () { + given: + def METRICS_JSON = ''' + { + "kind": "NodeMetrics", + "apiVersion": "metrics.k8s.io/v1beta1", + "metadata": { + "name": "test-node" + }, + "usage": { + "cpu": "1500m", + "memory": "2Gi" + } + } + ''' + + def client = Spy(K8sClient) + final NODE_NAME = 'test-node' + def METRICS_RESP = Mock(K8sResponseApi) + METRICS_RESP.getText() >> METRICS_JSON + + when: + def result = client.getNodeMemoryUsedBytes(NODE_NAME) + then: + 1 * client.get('/apis/metrics.k8s.io/v1beta1/nodes/test-node') >> METRICS_RESP + result == 2L * 1024L * 1024L * 1024L // 2 GiB in bytes + } + + def 'should handle fractional CPU values' () { + expect: + K8sClient.parseK8sQuantity('2.5', true) == 2500L // 2.5 cores = 2500 millicores + K8sClient.parseK8sQuantity('0.25', true) == 250L // 0.25 cores = 250 millicores + } + + def 'should handle fractional memory values' () { + expect: + K8sClient.parseK8sQuantity('1.5Gi') == (long)(1.5 * 1024L * 1024L * 1024L) + K8sClient.parseK8sQuantity('0.5Mi') == (long)(0.5 * 1024L * 1024L) + } + + def 'should handle decimal suffixes for memory' () { + expect: + K8sClient.parseK8sQuantity('1K') == 1000L + K8sClient.parseK8sQuantity('1M') == 1000L * 1000L + K8sClient.parseK8sQuantity('1G') == 1000L * 1000L * 1000L + } + + def 'should return null for invalid quantity strings' () { + expect: + K8sClient.parseK8sQuantity('invalid') == null + K8sClient.parseK8sQuantity('16 GiB') == null // space not allowed + K8sClient.parseK8sQuantity('Gi16') == null // suffix before number (doesn't match any suffix pattern) + K8sClient.parseK8sQuantity('16.5.5') == null // multiple decimal points + K8sClient.parseK8sQuantity('abc123') == null // non-numeric start + K8sClient.parseK8sQuantity('123abc') == null // non-numeric end (no valid suffix) + } + + def 'should handle large memory values like 8058776Ki' () { + expect: + K8sClient.parseK8sQuantity('8058776Ki') == 8058776L * 1024L + } + + def 'should handle large memory values like 8058776ki' () { + expect: + K8sClient.parseK8sQuantity('8058776ki') == 8058776L * 1024L + } + + def 'should handle memory values with B suffix' () { + expect: + K8sClient.parseK8sQuantity('1KiB') == 1024L + K8sClient.parseK8sQuantity('1MiB') == 1024L * 1024L + K8sClient.parseK8sQuantity('1GiB') == 1024L * 1024L * 1024L + } + + def 'should handle decimal memory suffixes' () { + expect: + K8sClient.parseK8sQuantity('1K') == 1000L + K8sClient.parseK8sQuantity('1KB') == 1000L + K8sClient.parseK8sQuantity('1M') == 1000L * 1000L + K8sClient.parseK8sQuantity('1MB') == 1000L * 1000L + K8sClient.parseK8sQuantity('1G') == 1000L * 1000L * 1000L + } + + def 'should handle real world memory value 8058776Ki' () { + expect: + K8sClient.parseK8sQuantity('8058776Ki') == 8058776L * 1024L + K8sClient.parseK8sQuantity('8058776ki') == 8058776L * 1024L + K8sClient.parseK8sQuantity('8058776KI') == 8058776L * 1024L + } + + def 'should handle CPU values without suffix' () { + expect: + K8sClient.parseK8sQuantity('1', true) == 1000L + K8sClient.parseK8sQuantity('2', true) == 2000L + K8sClient.parseK8sQuantity('0.5', true) == 500L + K8sClient.parseK8sQuantity('2.5', true) == 2500L + } + + def 'should handle fractional memory values with suffixes' () { + expect: + K8sClient.parseK8sQuantity('1.5Gi') == (long)(1.5 * 1024 * 1024 * 1024) + K8sClient.parseK8sQuantity('0.5Mi') == (long)(0.5 * 1024 * 1024) + K8sClient.parseK8sQuantity('2.25Ki') == (long)(2.25 * 1024) + } + + def 'should handle edge cases' () { + expect: + K8sClient.parseK8sQuantity('0') == 0L + K8sClient.parseK8sQuantity('0Ki') == 0L + K8sClient.parseK8sQuantity('0m', true) == 0L + K8sClient.parseK8sQuantity('+100') == 100L + K8sClient.parseK8sQuantity('-50') == -50L + } +} diff --git a/k8s-dvfs/src/test/groovy/recreationaltech/plugin/client/K8sResponseExceptionTest.groovy b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/client/K8sResponseExceptionTest.groovy new file mode 100644 index 0000000..6361afa --- /dev/null +++ b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/client/K8sResponseExceptionTest.groovy @@ -0,0 +1,66 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin.client + +import spock.lang.Specification + +/** + * + * @author Paolo Di Tommaso + */ +class K8sResponseExceptionTest extends Specification { + + def 'should create response from valid json' () { + + when: + def resp = new K8sResponseException( + 'Request /this/that failed', + new K8sResponseJson('{"foo":"one","bar":"two"}')) + then: + resp.getMessage() == ''' + Request /this/that failed + + { + "foo": "one", + "bar": "two" + } + '''.stripIndent().leftTrim() + } + + + def 'should create response from error message' () { + + when: + def resp = new K8sResponseException( + 'Request /this/that failed', + new K8sResponseJson('Oops.. it crashed badly')) + then: + resp.getMessage() == 'Request /this/that failed -- Oops.. it crashed badly' + } + + def 'should contain the response object passed to it' () { + + given: + def resp_json = new K8sResponseJson('{"error": "out of cheese error"}') + + when: + def resp = new K8sResponseException("Error occurred",resp_json) + + then: + resp.response == resp_json + } +} diff --git a/k8s-dvfs/src/test/groovy/recreationaltech/plugin/client/K8sResponseJsonTest.groovy b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/client/K8sResponseJsonTest.groovy new file mode 100644 index 0000000..3e849db --- /dev/null +++ b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/client/K8sResponseJsonTest.groovy @@ -0,0 +1,70 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin.client + +import spock.lang.Specification + +/** + * + * @author Paolo Di Tommaso + */ +class K8sResponseJsonTest extends Specification { + + def 'should create a response from a map' () { + + given: + def MAP = [foo: 'one', bar: 'two'] + + when: + def resp = new K8sResponseJson(MAP) + then: + resp.foo == 'one' + resp.bar == 'two' + resp.toString() == ''' + { + "foo": "one", + "bar": "two" + } + '''.stripIndent().trim() + + } + + def 'should create a response from a json string' () { + + when: + def resp = new K8sResponseJson('{"foo":"one","bar":"two"}') + then: + resp.foo == 'one' + resp.bar == 'two' + resp.toString() == ''' + { + "foo": "one", + "bar": "two" + } + '''.stripIndent().trim() + + } + + def 'should create a response from an error message' () { + when: + def resp = new K8sResponseJson('Ooops .. this crashed') + then: + resp.toString() == 'Ooops .. this crashed' + } + + +} diff --git a/k8s-dvfs/src/test/groovy/recreationaltech/plugin/model/PodEnvTest.groovy b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/model/PodEnvTest.groovy new file mode 100644 index 0000000..5980394 --- /dev/null +++ b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/model/PodEnvTest.groovy @@ -0,0 +1,62 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin.model + +import spock.lang.Specification + +/** + * + * @author Paolo Di Tommaso + */ + class PodEnvTest extends Specification { + + def 'should return env spec' () { + expect: + PodEnv.value('ALPHA', 'aaa').toSpec() == [name:'ALPHA', value:'aaa'] + } + + def 'should create env fieldPath spec' () { + expect: + PodEnv.fieldPath('ALPHA', 'aaa').toSpec() == [ + name:'ALPHA', + valueFrom: [fieldRef:[fieldPath: 'aaa']] + ] + } + + def 'should create env secret spec' () { + expect: + PodEnv.secret('ALPHA', 'data/key-1').toSpec() == [ + name: 'ALPHA', + valueFrom: [secretKeyRef:[name:'data', key:'key-1']] + ] + + PodEnv.secret('ALPHA', 'data').toSpec() == [ + name: 'ALPHA', + valueFrom: [secretKeyRef:[name:'data', key:'ALPHA']] + ] + + } + + def 'should create env config spec' () { + expect: + PodEnv.config('ALPHA', 'data/key-1').toSpec() == [ + name: 'ALPHA', + valueFrom: [configMapKeyRef:[name:'data', key:'key-1']] + ] + } + +} diff --git a/k8s-dvfs/src/test/groovy/recreationaltech/plugin/model/PodMountConfigTest.groovy b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/model/PodMountConfigTest.groovy new file mode 100644 index 0000000..1801c35 --- /dev/null +++ b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/model/PodMountConfigTest.groovy @@ -0,0 +1,47 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin.model + +import spock.lang.Specification + +/** + * + * @author Paolo Di Tommaso + */ +class PodMountConfigTest extends Specification { + + def 'should create mount for configmap' () { + + when: + def opt = new PodMountConfig(mountPath: '/etc/some/name', config: 'here' ) + then: + opt.mountPath == '/etc/some/name' + opt.fileName == null + opt.configName == 'here' + opt.configKey == null + + when: + opt = new PodMountConfig(mountPath: '/etc/some/name', config: 'here/there.txt' ) + then: + opt.mountPath == '/etc/some' + opt.fileName == 'name' + opt.configName == 'here' + opt.configKey == 'there.txt' + + } + +} diff --git a/k8s-dvfs/src/test/groovy/recreationaltech/plugin/model/PodMountSecretTest.groovy b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/model/PodMountSecretTest.groovy new file mode 100644 index 0000000..3f5b82a --- /dev/null +++ b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/model/PodMountSecretTest.groovy @@ -0,0 +1,46 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin.model + +import spock.lang.Specification + +/** + * + * @author Paolo Di Tommaso + */ +class PodMountSecretTest extends Specification { + + def 'should create mount for configmap' () { + + when: + def opt = new PodMountSecret(mountPath: '/etc/some/name', secret: 'here' ) + then: + opt.mountPath == '/etc/some/name' + opt.fileName == null + opt.secretName == 'here' + opt.secretKey == null + + when: + opt = new PodMountSecret(mountPath: '/etc/some/name', secret: 'here/there.txt' ) + then: + opt.mountPath == '/etc/some' + opt.fileName == 'name' + opt.secretName == 'here' + opt.secretKey == 'there.txt' + + } +} diff --git a/k8s-dvfs/src/test/groovy/recreationaltech/plugin/model/PodNodeSelectorTest.groovy b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/model/PodNodeSelectorTest.groovy new file mode 100644 index 0000000..73be3b0 --- /dev/null +++ b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/model/PodNodeSelectorTest.groovy @@ -0,0 +1,43 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin.model + +import spock.lang.Specification + +/** + * + * @author Paolo Di Tommaso + */ +class PodNodeSelectorTest extends Specification { + + def 'should create node selector' () { + + expect: + new PodNodeSelector(selector).toSpec() == spec + + where: + selector | spec + '' | [:] + 'foo=1' | [foo:'1'] + 'x=a,y=2,z=9' | [x:'a',y:'2',z:'9'] + 'x= a , y=2 , z =9' | [x:'a',y:'2',z:'9'] + 'gpu,intel' | [gpu:'true',intel: 'true'] + [foo:1, bar: 'two'] | [foo:'1', bar:'two'] + } + + +} diff --git a/k8s-dvfs/src/test/groovy/recreationaltech/plugin/model/PodOptionsTest.groovy b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/model/PodOptionsTest.groovy new file mode 100644 index 0000000..5d72199 --- /dev/null +++ b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/model/PodOptionsTest.groovy @@ -0,0 +1,568 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin.model + +import spock.lang.Specification + +/** + * + * @author Paolo Di Tommaso + */ +class PodOptionsTest extends Specification { + + + def 'should create empty options' () { + + when: + def options = new PodOptions(null) + then: + options.getEnvVars() == [] as Set + options.getMountConfigMaps() == [] as Set + options.getMountCsiEphemerals() == [] as Set + options.getMountEmptyDirs() == [] as Set + options.getMountSecrets() == [] as Set + options.getAutomountServiceAccountToken() == true + } + + def 'should set pullPolicy' () { + + when: + def options = new PodOptions() + then: + options.getImagePullPolicy() == null + + when: + options = new PodOptions([ [pullPolicy:'Always'] ]) + then: + options.getImagePullPolicy() == 'Always' + + when: + options = new PodOptions([ [imagePullPolicy:'latest'] ]) + then: + options.getImagePullPolicy() == 'latest' + } + + def 'should set imagePullSecret' () { + + when: + def options = new PodOptions() + then: + options.imagePullSecret == null + + when: + options = new PodOptions([ [imagePullSecret:'foo'] ]) + then: + options.imagePullSecret == 'foo' + + when: + options = new PodOptions([ [imagePullSecrets:'bar'] ]) + then: + options.imagePullSecret == 'bar' + } + + def 'should return config mounts' () { + + given: + def options = [ + [mountPath: '/this/path1.txt', config: 'name/key1'], + [mountPath: '/this/path2.txt', config: 'name/key2'], + [mountPath: '/this/path2.txt', config: 'name/key2'], // <-- identical entry are ignored + [mountPath: '/this/path2.txt', secret: 'name/secret'], + [env: 'FOO', config: '/name/foo'] + ] + + when: + def configs = new PodOptions(options).getMountConfigMaps() + then: + configs.size() == 2 + configs == [ + new PodMountConfig(mountPath: '/this/path1.txt', config: 'name/key1'), + new PodMountConfig(mountPath: '/this/path2.txt', config: 'name/key2') + ] as Set + } + + def 'should return csi ephemeral mounts' () { + + given: + def options = [ + [ + mountPath: '/data', + csi: [ + driver: 'inline.storage.kubernetes.io', + volumeAttributes: [foo: 'bar'] + ] + ] + ] + + when: + def csiEphemerals = new PodOptions(options).getMountCsiEphemerals() + then: + csiEphemerals == [ + new PodMountCsiEphemeral(mountPath: '/data', csi: options[0].csi) + ] as Set + } + + def 'should return emptyDir mounts' () { + + given: + def options = [ + [mountPath: '/scratch1', emptyDir: [medium: 'Memory']], + [mountPath: '/scratch2', emptyDir: [medium: 'Disk']] + ] + + when: + def emptyDirs = new PodOptions(options).getMountEmptyDirs() + then: + emptyDirs.size() == 2 + emptyDirs == [ + new PodMountEmptyDir(options[0]), + new PodMountEmptyDir(options[1]) ] as Set + } + + def 'should return secret mounts' () { + + given: + def options = [ + [mountPath: '/this/path1.txt', config: 'name/key1'], + [mountPath: '/this/alpha.txt', secret: 'name/secret1'], + [mountPath: '/this/beta.txt', secret: 'name/secret2'], + [mountPath: '/this/beta.txt', secret: 'name/secret2'], + [env: 'FOO', config: '/name/foo'] + ] + + when: + def secrets = new PodOptions(options).getMountSecrets() + then: + secrets.size() == 2 + secrets == [ + new PodMountSecret(mountPath: '/this/alpha.txt', secret: 'name/secret1'), + new PodMountSecret(mountPath: '/this/beta.txt', secret: 'name/secret2'), + ] as Set + } + + + def 'should return env definitions' () { + + given: + def options = [ + [mountPath: '/this/path1.txt', config: 'name/key1'], + [mountPath: '/this/alpha.txt', secret: 'name/secret1'], + [mountPath: '/this/beta.txt', secret: 'name/secret2'], + [env: 'FOO', config: '/name/foo'], + [env: 'FOO', config: '/name/foo'], + [env: 'BAR', config: '/name/BAR'], + [env: 'ALPHA', value: 'aaa'], + [env: 'ALPHA', value: 'aaa'], + [env: 'BETA', value: 'bbb'], + [env: 'PASSWORD', secret:'name/key'], + [env: 'PASSWORD', secret:'name/key'], + ] + + when: + def env = new PodOptions(options).getEnvVars() + then: + env.size() == 5 + env == [ + PodEnv.config('FOO', '/name/foo'), + PodEnv.config('BAR', '/name/BAR'), + PodEnv.value('ALPHA', 'aaa'), + PodEnv.value('BETA', 'bbb'), + PodEnv.secret('PASSWORD', 'name/key'), + ] as Set + + } + + def 'should create persistent volume claims' () { + given: + def options = [ + [volumeClaim:'pvc1', mountPath: '/this/path'], + [volumeClaim:'pvc2', mountPath: '/that/path'], + [volumeClaim:'pvc3', mountPath: '/some/data', subPath: '/foo'] + ] + + when: + def claims = new PodOptions(options).getVolumeClaims() + then: + claims.size() == 3 + claims == [ + new PodVolumeClaim('pvc1', '/this/path'), + new PodVolumeClaim('pvc2', '/that/path'), + new PodVolumeClaim('pvc3', '/some/data', '/foo') + ] as Set + + } + + def 'should create host path' () { + given: + def options = [ + [hostPath: '/host/one', mountPath: '/pod/1'], + [hostPath: '/host/two', mountPath: '/pod/2'] + ] + when: + def mounts = new PodOptions(options).getMountHostPaths() + + then: + mounts == [ + new PodHostMount('/host/one', '/pod/1'), + new PodHostMount('/host/two', '/pod/2') + ] as Set + + } + + def 'should not create env' () { + when: + new PodOptions([ [env:'FOO'] ]) + then: + thrown(IllegalArgumentException) + + when: + new PodOptions([ [secret:'FOO'] ]) + then: + thrown(IllegalArgumentException) + + when: + new PodOptions([ [config:'FOO'] ]) + then: + thrown(IllegalArgumentException) + + when: + new PodOptions([ [volumeClaim:'FOO'] ]) + then: + thrown(IllegalArgumentException) + } + + def 'should merge podOptions' () { + + given: + def list1 = [ + [env: 'HELLO', value: 'WORLD'], + [config: 'data/key', mountPath: '/data/file.txt'], + [secret: 'secret/key', mountPath: '/etc/secret'], + [volumeClaim: 'pvc', mountPath: '/mnt/claim'], + [runAsUser: 500] + ] + + def list2 = [ + [env: 'ALPHA', value: 'GAMMA'], + [config: 'bar/key', mountPath: '/b/bb'], + [secret: 'foo/key', mountPath: '/a/aa'], + [volumeClaim: 'cvp', mountPath: '/c/cc'], + + [env: 'DELTA', value: 'LAMBDA'], + [config: 'y', mountPath: '/y'], + [secret: 'x', mountPath: '/x'], + [volumeClaim: 'z', mountPath: '/z'], + ] + + def list3 = [ + [env: 'HELLO', value: 'WORLD'], + [config: 'data/key', mountPath: '/data/file.txt'], + [secret: 'secret/key', mountPath: '/etc/secret'], + [volumeClaim: 'pvc', mountPath: '/mnt/claim'], + + [env: 'DELTA', value: 'LAMBDA'], + [config: 'y', mountPath: '/y'], + [secret: 'x', mountPath: '/x'], + [volumeClaim: 'z', mountPath: '/z'], + + [csi: [driver: 'inline.storage.kubernetes.io'], mountPath: '/data'], + [emptyDir: [:], mountPath: '/scratch1'], + [securityContext: [runAsUser: 1000, fsGroup: 200, allowPrivilegeEscalation: true]], + [nodeSelector: 'foo=X, bar=Y'], + [automountServiceAccountToken: false], + [priorityClassName: 'high-priority'] + ] + + PodOptions opts + + when: + opts = new PodOptions() + new PodOptions() + then: + opts == new PodOptions() + + when: + opts = new PodOptions(list1) + new PodOptions() + then: + opts == new PodOptions(list1) + opts.securityContext.toSpec() == [runAsUser:500] + + when: + opts = new PodOptions() + new PodOptions(list1) + then: + opts == new PodOptions(list1) + opts.securityContext.toSpec() == [runAsUser:500] + + when: + opts = new PodOptions(list1) + new PodOptions(list1) + then: + opts == new PodOptions(list1) + opts.securityContext.toSpec() == [runAsUser:500] + + when: + opts = new PodOptions(list1) + new PodOptions(list2) + then: + opts == new PodOptions(list1 + list2) + opts.securityContext.toSpec() == [runAsUser:500] + + when: + opts = new PodOptions(list1) + new PodOptions(list3) + then: + opts.getEnvVars() == [ + PodEnv.value('HELLO','WORLD'), + PodEnv.value('DELTA','LAMBDA') + ] as Set + + opts.getMountConfigMaps() == [ + new PodMountConfig('data/key', '/data/file.txt'), + new PodMountConfig('y', '/y'), + ] as Set + + opts.getMountCsiEphemerals() == [ + new PodMountCsiEphemeral([driver: 'inline.storage.kubernetes.io'], '/data') + ] as Set + + opts.getMountEmptyDirs() == [ + new PodMountEmptyDir([:], '/scratch1'), + ] as Set + + opts.getMountSecrets() == [ + new PodMountSecret('secret/key', '/etc/secret'), + new PodMountSecret('x', '/x') + ] as Set + + opts.getVolumeClaims() == [ + new PodVolumeClaim('pvc','/mnt/claim'), + new PodVolumeClaim('z','/z'), + ] as Set + + opts.securityContext.toSpec() == [runAsUser: 1000, fsGroup: 200, allowPrivilegeEscalation: true] + + opts.nodeSelector.toSpec() == [foo: 'X', bar: "Y"] + + opts.getAutomountServiceAccountToken() == false + + opts.getPriorityClassName() == 'high-priority' + } + + def 'should copy image pull policy' (){ + given: + def data = [ + [imagePullPolicy : 'FOO'] + ] + + when: + def opts = new PodOptions() + new PodOptions(data) + then: + opts.imagePullPolicy == 'FOO' + + when: + opts = new PodOptions(data) + new PodOptions() + then: + opts.imagePullPolicy == 'FOO' + } + + def 'should copy image pull secret' (){ + given: + def data = [ + [imagePullSecret : 'BAR'] + ] + + when: + def opts = new PodOptions() + new PodOptions(data) + then: + opts.imagePullSecret == 'BAR' + + when: + opts = new PodOptions(data) + new PodOptions() + then: + opts.imagePullSecret == 'BAR' + } + + def 'should copy pod labels' (){ + given: + def data = [ + [label: "LABEL", value: 'VALUE'] + ] + + when: + def opts = new PodOptions() + new PodOptions(data) + then: + opts.labels == ["LABEL": "VALUE"] + + when: + opts = new PodOptions(data) + new PodOptions() + then: + opts.labels == ["LABEL": "VALUE"] + + when: + opts = new PodOptions([[label:"FOO", value:'one']]) + new PodOptions([[label:"BAR", value:'two']]) + then: + opts.labels == [FOO: 'one', BAR: 'two'] + } + + def 'should copy host paths' (){ + given: + def data = [ + [hostPath: "/foo", mountPath: '/one'] + ] + + when: + def opts = new PodOptions() + new PodOptions(data) + then: + opts.getMountHostPaths() == [new PodHostMount('/foo', '/one')] as Set + + when: + opts = new PodOptions(data) + new PodOptions() + then: + opts.getMountHostPaths() == [new PodHostMount('/foo', '/one')] as Set + + when: + opts = new PodOptions([[hostPath:"/foo", mountPath: '/one']]) + new PodOptions([[hostPath:"/bar", mountPath: '/two']]) + then: + opts.getMountHostPaths() == [ + new PodHostMount('/foo','/one'), + new PodHostMount('/bar','/two') + ] as Set + } + + def 'should create pod labels' () { + + given: + def options = [ + [label: 'ALPHA', value: 'aaa'], + [label: 'DELTA', value: 'bbb'], + [label: 'DELTA', value: 'ddd'] + ] + + when: + def opts = new PodOptions(options) + then: + opts.labels.size() == 2 + opts.labels == [ALPHA: 'aaa', DELTA: 'ddd'] + + } + + def 'should copy pod annotations' (){ + given: + def data = [ + [annotation: "ANNOTATION", value: 'VALUE'] + ] + + when: + def opts = new PodOptions() + new PodOptions(data) + then: + opts.annotations == ["ANNOTATION": "VALUE"] + + when: + opts = new PodOptions(data) + new PodOptions() + then: + opts.annotations == ["ANNOTATION": "VALUE"] + + when: + opts = new PodOptions([[annotation:"FOO", value:'one']]) + new PodOptions([[annotation:"BAR", value:'two']]) + then: + opts.annotations == [FOO: 'one', BAR: 'two'] + } + + def 'should create pod annotations' () { + + given: + def options = [ + [annotation: 'ALPHA', value: 'aaa'], + [annotation: 'DELTA', value: 'bbb'], + [annotation: 'DELTA', value: 'ddd'] + ] + + when: + def opts = new PodOptions(options) + then: + opts.annotations.size() == 2 + opts.annotations == [ALPHA: 'aaa', DELTA: 'ddd'] + + } + + def 'should create user security context' () { + when: + def opts = new PodOptions([ [runAsUser: 1000] ]) + then: + opts.getSecurityContext() == new PodSecurityContext(1000) + + when: + opts = new PodOptions([ [runAsUser: 'foo'] ]) + then: + opts.getSecurityContext() == new PodSecurityContext('foo') + + when: + opts = new PodOptions([ [runAsUser: 'foo'] ]) + then: + opts.getSecurityContext() != new PodSecurityContext('bar') + + when: + def ctx = [runAsUser: 500, fsGroup: 200, allowPrivilegeEscalation: true, seLinuxOptions: [level: "s0:c123,c456"]] + def expected = new PodSecurityContext(ctx) + opts = new PodOptions([ [securityContext: ctx] ]) + then: + opts.getSecurityContext() == expected + opts.getSecurityContext().toSpec() == ctx + } + + def 'should create pod node selector' () { + when: + def opts = new PodOptions([ [nodeSelector: 'foo=1, bar=true, baz=Z'] ]) + then: + opts.nodeSelector.toSpec() == [foo: '1', bar: 'true', baz: 'Z'] + + } + + def 'should set pod automount service token' () { + when: + def opts = new PodOptions([[automountServiceAccountToken: false]]) + then: + opts.getAutomountServiceAccountToken() == false + } + + def 'should set pod priority class name' () { + when: + def opts = new PodOptions([[priorityClassName: 'high-priority']]) + then: + opts.getPriorityClassName() == 'high-priority' + } + + def 'should set pod privileged' () { + when: + def opts = new PodOptions([:]) + then: + !opts.getPrivileged() + + when: + opts = new PodOptions([[privileged: true]]) + then: + opts.getPrivileged() + } + + def 'should set pod schedulerName' () { + when: + def opts = new PodOptions() + then: + opts.getSchedulerName() == null + + when: + opts = new PodOptions([ [schedulerName:'my-scheduler'] ]) + then: + opts.getSchedulerName() == 'my-scheduler' + } +} diff --git a/k8s-dvfs/src/test/groovy/recreationaltech/plugin/model/PodSpecBuilderTest.groovy b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/model/PodSpecBuilderTest.groovy new file mode 100644 index 0000000..37b680b --- /dev/null +++ b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/model/PodSpecBuilderTest.groovy @@ -0,0 +1,914 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin.model + +import nextflow.executor.res.AcceleratorResource +import nextflow.util.MemoryUnit +import spock.lang.Specification +import spock.lang.Unroll +/** + * + * @author Paolo Di Tommaso + */ +class PodSpecBuilderTest extends Specification { + + def setup() { + PodSpecBuilder.VOLUMES.set(0) + } + + + def 'should create pod spec' () { + + when: + def spec = new PodSpecBuilder() + .withPodName('foo') + .withImageName('busybox') + .withWorkDir('/some/work/dir') + .withCommand(['echo', 'hello']) + .build() + + then: + spec == [ + apiVersion: 'v1', + kind: 'Pod', + metadata: [name:'foo', namespace:'default'], + spec: [ + restartPolicy:'Never', + containers:[[ + name:'foo', + image:'busybox', + command:['echo', 'hello'], + workingDir:'/some/work/dir' + ]] + ] + ] + + } + + def 'should create pod spec with args' () { + + when: + def pod = new PodSpecBuilder() + .withPodName('foo') + .withImageName('busybox') + .withArgs(['echo', 'hello']) + .build() + + then: + pod.spec.containers[0].args == ['echo', 'hello'] + + } + + def 'should create pod spec with args string' () { + + when: + def pod = new PodSpecBuilder() + .withPodName('foo') + .withImageName('busybox') + .withArgs('echo foo') + .build() + + then: + pod.spec.containers[0].args == ['/bin/bash', '-c', 'echo foo'] + + } + + def 'should create pod spec with privileged' () { + + when: + def pod = new PodSpecBuilder() + .withPodName('foo') + .withImageName('busybox') + .withCommand('echo foo') + .withPrivileged(true) + .build() + + then: + pod.spec.containers[0].securityContext == [privileged: true] + + } + + def 'should create pod with resources limits' () { + when: + def pod1 = new PodSpecBuilder() + .withPodName('foo') + .withImageName('busybox') + .withCommand('echo foo') + .withResourcesLimits('nextflow.io/fuse': 1) + .build() + + then: + pod1.spec.containers[0].resources == [limits:['nextflow.io/fuse':1]] + + + when: + def pod2 = new PodSpecBuilder() + .withPodName('foo') + .withImageName('busybox') + .withCommand('echo foo') + .withCpus(8) + .withCpuLimits(true) + .withMemory(MemoryUnit.of('10GB')) + .withResourcesLimits('nextflow.io/fuse': 1) + .build() + + then: + pod2.spec.containers[0].resources == [ + requests: ['cpu':8, 'memory':'10240Mi'], + limits: ['cpu':8, 'memory':'10240Mi', 'nextflow.io/fuse':1] ] + } + + def 'should set namespace, labels and annotations' () { + + when: + def pod = new PodSpecBuilder() + .withPodName('foo') + .withImageName('busybox') + .withCommand(['sh', '-c', 'echo hello']) + .withNamespace('xyz') + .withLabel('app','myApp') + .withLabel('runName','something') + .withLabel('version','3.6.1') + .withAnnotation("anno1", "value1") + .withAnnotations([anno2: "value2", anno3: "value3"]) + .build() + + then: + pod.metadata.namespace == 'xyz' + pod.metadata.labels == [ + app: 'myApp', + runName: 'something', + version: '3.6.1' + ] + pod.metadata.annotations == [ + anno1: "value1", + anno2: "value2", + anno3: "value3" + ] + } + + def 'should truncate labels longer than 63 chars' () { + + when: + def pod = new PodSpecBuilder() + .withPodName('foo') + .withImageName('busybox') + .withCommand(['sh', '-c', 'echo hello']) + .withLabel('app','myApp') + .withLabel('runName','something') + .withLabel('tag','somethingreallylonggggggggggggggggggggggggggggggggggggggggggendEXTRABIT') + .withLabels([tag2: 'somethingreallylonggggggggggggggggggggggggggggggggggggggggggendEXTRABIT', tag3: 'somethingreallylonggggggggggggggggggggggggggggggggggggggggggendEXTRABIT']) + .build() + + then: + pod.metadata.labels == [ + app: 'myApp', + runName: 'something', + tag: 'somethingreallylonggggggggggggggggggggggggggggggggggggggggggend', + tag2: 'somethingreallylonggggggggggggggggggggggggggggggggggggggggggend', + tag3: 'somethingreallylonggggggggggggggggggggggggggggggggggggggggggend' + ] + } + + + def 'should set resources and env' () { + + when: + def pod = new PodSpecBuilder() + .withPodName('foo') + .withImageName('busybox') + .withCommand('echo hello') + .withEnv(PodEnv.value('ALPHA','hello')) + .withEnv(PodEnv.value('DELTA', 'world')) + .withCpus(8) + .withAccelerator( new AcceleratorResource(request: 5, limit:10, type: 'foo.org') ) + .withMemory('100Gi') + .withDisk('10Gi') + .build() + + then: + pod.spec.containers[0].env == [ + [name:'ALPHA', value:'hello'], + [name:'DELTA', value:'world'] + ] + pod.spec.containers[0].resources == [ + requests: ['foo.org/gpu':5, cpu:8, memory:'100Gi', 'ephemeral-storage':'10Gi'], + limits: ['foo.org/gpu':10, memory:'100Gi', 'ephemeral-storage':'10Gi'] + ] + } + + def 'should get storage spec for volume claims' () { + + when: + def pod = new PodSpecBuilder() + .withPodName('foo') + .withImageName('busybox') + .withCommand(['echo']) + .withVolumeClaim(new PodVolumeClaim('first','/work')) + .withVolumeClaim(new PodVolumeClaim('second', '/data', '/foo')) + .withVolumeClaim(new PodVolumeClaim('third', '/things', null, true)) + .build() + then: + pod.spec.containers[0].volumeMounts == [ + [name:'vol-1', mountPath:'/work'], + [name:'vol-2', mountPath:'/data', subPath: '/foo'], + [name:'vol-3', mountPath:'/things', readOnly: true] + ] + pod.spec.volumes == [ + [name:'vol-1', persistentVolumeClaim:[claimName:'first']], + [name:'vol-2', persistentVolumeClaim:[claimName:'second']], + [name:'vol-3', persistentVolumeClaim:[claimName:'third']] + ] + + } + + def 'should only define one volume per persistentVolumeClaim' () { + + when: + def pod = new PodSpecBuilder() + .withPodName('foo') + .withImageName('busybox') + .withCommand(['echo']) + .withVolumeClaim(new PodVolumeClaim('first','/work')) + .withVolumeClaim(new PodVolumeClaim('first','/work2', '/bar')) + .withVolumeClaim(new PodVolumeClaim('second', '/data', '/foo')) + .withVolumeClaim(new PodVolumeClaim('second', '/data2', '/fooz')) + .build() + then: + pod.spec.containers[0].volumeMounts == [ + [name:'vol-1', mountPath:'/work'], + [name:'vol-1', mountPath:'/work2', subPath: '/bar'], + [name:'vol-2', mountPath:'/data', subPath: '/foo'], + [name:'vol-2', mountPath:'/data2', subPath: '/fooz'] + ] + pod.spec.volumes == [ + [name:'vol-1', persistentVolumeClaim:[claimName:'first']], + [name:'vol-2', persistentVolumeClaim:[claimName:'second']] + ] + + } + + def 'should get config map mounts' () { + + when: + def pod = new PodSpecBuilder() + .withPodName('foo') + .withImageName('busybox') + .withCommand(['echo']) + .withConfigMap(new PodMountConfig(config: 'cfg1', mountPath: '/etc/config')) + .withConfigMap(new PodMountConfig(config: 'data2', mountPath: '/data/path')) + .build() + then: + pod.spec.containers[0].volumeMounts == [ + [name:'vol-1', mountPath:'/etc/config'], + [name:'vol-2', mountPath:'/data/path'] + ] + pod.spec.volumes == [ + [name:'vol-1', configMap:[name:'cfg1']], + [name:'vol-2', configMap:[name:'data2']] + ] + + } + + def 'should get csi ephemeral mounts' () { + + when: + def pod = new PodSpecBuilder() + .withPodName('foo') + .withImageName('busybox') + .withCommand(['echo']) + .withCsiEphemeral(new PodMountCsiEphemeral(csi: [driver: 'inline.storage.kubernetes.io', readOnly: true], mountPath: '/data')) + .build() + then: + pod.spec.containers[0].volumeMounts == [ + [name: 'vol-1', mountPath: '/data', readOnly: true] + ] + pod.spec.volumes == [ + [name: 'vol-1', csi: [driver: 'inline.storage.kubernetes.io', readOnly: true]] + ] + } + + def 'should get empty dir mounts' () { + + when: + def pod = new PodSpecBuilder() + .withPodName('foo') + .withImageName('busybox') + .withCommand(['echo']) + .withEmptyDir(new PodMountEmptyDir(mountPath: '/scratch1', emptyDir: [medium: 'Disk'])) + .withEmptyDir(new PodMountEmptyDir(mountPath: '/scratch2', emptyDir: [medium: 'Memory'])) + .build() + then: + pod.spec.containers[0].volumeMounts == [ + [name: 'vol-1', mountPath: '/scratch1'], + [name: 'vol-2', mountPath: '/scratch2'] + ] + pod.spec.volumes == [ + [name: 'vol-1', emptyDir: [medium: 'Disk']], + [name: 'vol-2', emptyDir: [medium: 'Memory']] + ] + } + + def 'should consume env secrets' () { + + when: + def pod = new PodSpecBuilder() + .withPodName('foo') + .withImageName('busybox') + .withCommand(['echo']) + .withEnv( PodEnv.value('FOO','abc')) + .withEnv( PodEnv.secret('VAR_X', 'delta/bar')) + .withEnv( PodEnv.secret('VAR_Y', 'gamma')) + .build() + + then: + pod.spec.containers[0].env == [ + [name: 'FOO', value: 'abc'], + [name: 'VAR_X', valueFrom: [secretKeyRef: [name:'delta', key:'bar']]], + [name: 'VAR_Y', valueFrom: [secretKeyRef: [name:'gamma', key:'VAR_Y']]] + ] + } + + def 'should consume env configMap' () { + + when: + def pod = new PodSpecBuilder() + .withPodName('foo') + .withImageName('busybox') + .withCommand(['echo']) + .withEnv( PodEnv.value('FOO','abc')) + .withEnv( PodEnv.config('VAR_X', 'data')) + .withEnv( PodEnv.config('VAR_Y', 'omega/bar-2')) + .build() + + then: + pod.spec.containers[0].env == [ + [name: 'FOO', value: 'abc'], + [name: 'VAR_X', valueFrom: [configMapKeyRef: [name:'data', key:'VAR_X']]], + [name: 'VAR_Y', valueFrom: [configMapKeyRef: [name:'omega', key:'bar-2']]] + ] + } + + def 'should consume file secrets' () { + + when: + def pod = new PodSpecBuilder() + .withPodName('foo') + .withImageName('busybox') + .withCommand(['echo']) + .withSecret(new PodMountSecret(secret: 'alpha', mountPath: '/this/and/that')) + .withSecret(new PodMountSecret(secret: 'delta/foo', mountPath: '/etc/mnt/bar.txt')) + .build() + + then: + pod.spec.containers[0].volumeMounts == [ + [name:'vol-1', mountPath:'/this/and/that'], + [name:'vol-2', mountPath:'/etc/mnt'] + ] + pod.spec.volumes == [ + [name:'vol-1', secret:[secretName: 'alpha']], + [name:'vol-2', secret:[ + secretName: 'delta', + items: [ + [ key: 'foo', path:'bar.txt' ] + ] + ]] + ] + } + + def 'should get host path mounts' () { + + when: + def pod = new PodSpecBuilder() + .withPodName('foo') + .withImageName('busybox') + .withCommand(['echo']) + .withHostMount('/tmp','/scratch') + .withHostMount('/host/data','/mnt/container') + .build() + + then: + pod.spec.containers[0].volumeMounts == [ + [name:'vol-1', mountPath:'/scratch'], + [name:'vol-2', mountPath:'/mnt/container'] + ] + pod.spec.volumes == [ + [name:'vol-1', hostPath: [path:'/tmp']], + [name:'vol-2', hostPath: [path:'/host/data']] + ] + + } + + + def 'should return secret file volume and mounts' () { + + given: + List mounts + List volumes + def builder = new PodSpecBuilder() + + when: + def secret1 = new PodMountSecret(secret:'foo', mountPath: '/etc/conf') + builder.secretToSpec( 'vol1', secret1, mounts=[], volumes=[] ) + + then: + mounts == [ + [ name: 'vol1', mountPath: '/etc/conf'] + ] + + volumes == [ + [ name: 'vol1', secret: [secretName: 'foo']] + ] + + + when: + def secret2 = new PodMountSecret(secret:'bar/hello.txt', mountPath: '/etc/conf/world.txt') + builder.secretToSpec( 'vol2', secret2, mounts=[], volumes=[] ) + + then: + mounts == [ + [ name: 'vol2', mountPath: '/etc/conf'] + ] + + volumes == [ + [ name: 'vol2', secret: [ + secretName: 'bar', + items: [ [key: 'hello.txt', path:'world.txt'] ] + ]] + ] + + } + + def 'should return configmap file volume and mounts' () { + + given: + List mounts + List volumes + def builder = new PodSpecBuilder() + + when: + def config1 = new PodMountConfig(config:'foo', mountPath: '/etc/conf') + builder.configMapToSpec( 'vol1', config1, mounts=[], volumes=[] ) + + then: + mounts == [ + [ name: 'vol1', mountPath: '/etc/conf'] + ] + + volumes == [ + [ name: 'vol1', configMap: [name: 'foo']] + ] + + + when: + def config2 = new PodMountConfig(config:'bar/hello.txt', mountPath: '/etc/conf/world.txt') + builder.configMapToSpec( 'vol2', config2, mounts=[], volumes=[] ) + + then: + mounts == [ + [ name: 'vol2', mountPath: '/etc/conf'] + ] + + volumes == [ + [ name: 'vol2', configMap: [ + name: 'bar', + items: [ [key: 'hello.txt', path:'world.txt'] ] + ]] + ] + + } + + + def 'should create pod spec with pod options' () { + + given: + def affinity = [ + nodeAffinity: [ + requiredDuringSchedulingIgnoredDuringExecution: [ + nodeSelectorTerms: [ + [key: 'foo', operator: 'In', values: ['bar', 'baz']] + ] + ] + ] + ] + def tolerations = [[ + key: 'example-key', + operator: 'Exists', + effect: 'NoSchedule' + ]] + def opts = Mock(PodOptions) + and: + def builder = new PodSpecBuilder() + .withPodName('foo') + .withImageName('busybox') + .withCommand(['echo']) + .withLabel('runName', 'crazy_john') + .withAnnotation('evict', 'false') + + when: + def pod = builder.withPodOptions(opts).build() + then: + _ * opts.getAffinity() >> affinity + _ * opts.getAnnotations() >> [OMEGA:'zzz', SIGMA:'www'] + _ * opts.getAutomountServiceAccountToken() >> false + 2 * opts.getEnvVars() >> [ PodEnv.value('HELLO','WORLD') ] + 2 * opts.getImagePullPolicy() >> 'always' + 2 * opts.getImagePullSecret() >> 'myPullSecret' + _ * opts.getLabels() >> [ALPHA: 'xxx', GAMMA: 'yyy'] + 2 * opts.getVolumeClaims() >> [ new PodVolumeClaim('pvc1', '/work') ] + 2 * opts.getMountConfigMaps() >> [ new PodMountConfig('data', '/home/user') ] + 2 * opts.getMountSecrets() >> [ new PodMountSecret('blah', '/etc/secret.txt') ] + _ * opts.getNodeSelector() >> new PodNodeSelector(gpu:true, queue: 'fast') + _ * opts.getPriorityClassName() >> 'high-priority' + _ * opts.getSecurityContext() >> new PodSecurityContext(1000) + _ * opts.getTolerations() >> tolerations + and: + pod.metadata == [ + name:'foo', + namespace:'default', + labels:[runName:'crazy_john', ALPHA:'xxx', GAMMA:'yyy'], + annotations: [evict: 'false', OMEGA:'zzz', SIGMA:'www'] + ] + and: + pod.spec.affinity == affinity + pod.spec.automountServiceAccountToken == false + pod.spec.imagePullSecrets == [[ name: 'myPullSecret' ]] + pod.spec.nodeSelector == [gpu: 'true', queue: 'fast'] + pod.spec.priorityClassName == 'high-priority' + pod.spec.securityContext == [ runAsUser: 1000 ] + pod.spec.tolerations == tolerations + pod.spec.containers[0].imagePullPolicy == 'always' + pod.spec.containers[0].env == [[name:'HELLO', value:'WORLD']] + pod.spec.containers[0].volumeMounts == [ + [name:'vol-1', mountPath:'/work'], + [name:'vol-2', mountPath:'/home/user'], + [name:'vol-3', mountPath:'/etc/secret.txt'] + ] + and: + pod.spec.volumes == [ + [name:'vol-1', persistentVolumeClaim:[claimName:'pvc1']], + [name:'vol-2', configMap:[name:'data']], + [name:'vol-3', secret:[secretName:'blah']] + ] + + } + + def 'should create pod spec with activeDeadlineSeconds' () { + + when: + def pod = new PodSpecBuilder() + .withPodName('foo') + .withImageName('busybox') + .withCommand(['echo', 'hello']) + .withActiveDeadline(100) + .build() + + then: + pod.spec.activeDeadlineSeconds == 100 + + } + + def 'should create pod spec with schedulerName' () { + + when: + def pod = new PodSpecBuilder() + .withPodName('foo') + .withImageName('busybox') + .withCommand(['echo', 'hello']) + .withPodOptions(new PodOptions(schedulerName: 'my-scheduler')) + .build() + + then: + pod.spec.schedulerName == 'my-scheduler' + + } + + def 'should create image pull request map' () { + given: + def builder = new PodSpecBuilder(imagePullSecret: 'MySecret') + when: + def result = builder.createPullSecret() + then: + result.size() == 1 + result.get(0).name == 'MySecret' + } + + + def 'should return the resources map' () { + + given: + def builder = new PodSpecBuilder() + + when: + def res = builder.addAcceleratorResources(new AcceleratorResource(request:2, limit: 5), null) + then: + res.requests == ['nvidia.com/gpu': 2] + res.limits == ['nvidia.com/gpu': 5] + + when: + res = builder.addAcceleratorResources(new AcceleratorResource(limit: 5, type:'foo'), null) + then: + res.requests == ['foo.com/gpu': 5] + res.limits == ['foo.com/gpu': 5] + + when: + res = builder.addAcceleratorResources(new AcceleratorResource(request: 5, type:'foo.org'), null) + then: + res.requests == ['foo.org/gpu': 5] + res.limits == null + + when: + res = builder.addAcceleratorResources(new AcceleratorResource(request: 5, type: 'foo.org'), [requests: [cpu: 2]]) + then: + res.requests == [cpu: 2, 'foo.org/gpu': 5] + res.limits == null + + when: + res = builder.addAcceleratorResources(new AcceleratorResource(request: 5, limit: 10, type: 'foo.org'), [requests: [cpu: 2]]) + then: + res.requests == [cpu: 2, 'foo.org/gpu': 5] + res.limits == ['foo.org/gpu': 10] + + when: + res = builder.addAcceleratorResources(new AcceleratorResource(request: 5, type:'example.com/fpga'), null) + then: + res.requests == ['example.com/fpga': 5] + res.limits == null + + when: + res = builder.addAcceleratorResources(new AcceleratorResource(request: 5, limit: 10, type: 'example.com/fpga'), [requests: [cpu: 2]]) + then: + res.requests == [cpu: 2, 'example.com/fpga': 5] + res.limits == ['example.com/fpga': 10] + } + + def 'should add resources limits' () { + given: + def builder = new PodSpecBuilder() + Map resources + + when: + resources = builder.addResourcesLimits(['foo':1], null) + then: + resources == [limits:[foo:1]] + + when: + resources = builder.addResourcesLimits(['foo':1], [requests: ['x':1], limits: ['y': 2]]) + then: + resources == [requests:[x:1], limits:[y:2, foo:1]] + } + + + @Unroll + def 'should sanitize k8s label value: #label' () { + given: + def builder = new PodSpecBuilder() + + expect: + builder.sanitizeValue(label, PodSpecBuilder.MetaType.LABEL, PodSpecBuilder.SegmentType.VALUE) == str + + where: + label | str + null | 'null' + 'hello' | 'hello' + 'hello world' | 'hello_world' + 'hello world' | 'hello_world' + 'hello.world' | 'hello.world' + 'hello-world' | 'hello-world' + 'hello_world' | 'hello_world' + 'hello_world-' | 'hello_world' + 'hello_world_' | 'hello_world' + 'hello_world.' | 'hello_world' + 'hello_123' | 'hello_123' + 'HELLO 123' | 'HELLO_123' + '123hello' | '123hello' + 'x2345678901234567890123456789012345678901234567890123456789012345' | 'x23456789012345678901234567890123456789012345678901234567890123' + } + + @Unroll + def 'should sanitize k8s label key: #label_key' () { + given: + def builder = new PodSpecBuilder() + + expect: + builder.sanitizeKey(label_key, PodSpecBuilder.MetaType.LABEL) == str + + where: + label_key | str + 'foo' | 'foo' + 'key 1' | 'key_1' + 'foo.bar/key 2' | 'foo.bar/key_2' + 'foo.bar/' | 'foo.bar' + '/foo.bar' | 'foo.bar' + 'x2345678901234567890123456789012345678901234567890123456789012345' | 'x23456789012345678901234567890123456789012345678901234567890123' + 'x23456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345/key 2' | 'x234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123/key_2' + 'foo.bar/x2345678901234567890123456789012345678901234567890123456789012345' | 'foo.bar/x23456789012345678901234567890123456789012345678901234567890123' + } + + @Unroll + def 'should report error if sanitizing k8s label with more than one slash character: #label_key' () { + given: + def builder = new PodSpecBuilder() + + when: + builder.sanitizeKey(label_key, PodSpecBuilder.MetaType.LABEL) + + then: + def error = thrown(expectedException) + error.message == expectedMessage + + where: + label_key | expectedException | expectedMessage + 'foo.bar/key 2/key 3' | IllegalArgumentException | "Invalid key in pod label -- Key can only contain exactly one '/' character" + 'foo.bar/foo/bar/bar' | IllegalArgumentException | "Invalid key in pod label -- Key can only contain exactly one '/' character" + } + + @Unroll + def 'should sanitize k8s label map' () { + given: + def builder = new PodSpecBuilder() + + expect: + builder.sanitize(KEY_VALUE, PodSpecBuilder.MetaType.LABEL) == EXPECTED + + where: + KEY_VALUE | EXPECTED + [foo:'bar'] | [foo:'bar'] + ['key 1':'value 2'] | [key_1:'value_2'] + ['foo.bar/key 2':'value 3'] | ['foo.bar/key_2':'value_3'] + } + + @Unroll + def 'should sanitize k8s annotation key' () { + given: + def builder = new PodSpecBuilder() + + expect: + builder.sanitize(KEY_VALUE, PodSpecBuilder.MetaType.ANNOTATION) == EXPECTED + + where: + KEY_VALUE | EXPECTED + [foo:'bar'] | [foo:'bar'] + ['key 1':'value 2'] | [key_1:'value 2'] + ['foo.bar/key 2':'value 3'] | ['foo.bar/key_2':'value 3'] + ['x2345678901234567890123456789012345678901234567890123456789012345':'value 5'] | ['x23456789012345678901234567890123456789012345678901234567890123':'value 5'] + ['x23456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345/key 4':'value 6'] | ['x234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123/key_4':'value 6'] + ['foo.bar/x2345678901234567890123456789012345678901234567890123456789012345':'value 7'] | ['foo.bar/x23456789012345678901234567890123456789012345678901234567890123':'value 7'] + } + + @Unroll + def 'should report error if sanitizing k8s annotation key with more than one slash character: #annotation_key' () { + given: + def builder = new PodSpecBuilder() + + when: + builder.sanitizeKey(annotation_key, PodSpecBuilder.MetaType.ANNOTATION) + + then: + def error = thrown(expectedException) + error.message == expectedMessage + + where: + annotation_key | expectedException | expectedMessage + 'foo.bar/key 2/key 3' | IllegalArgumentException | "Invalid key in pod annotation -- Key can only contain exactly one '/' character" + 'foo.bar/foo/bar/bar' | IllegalArgumentException | "Invalid key in pod annotation -- Key can only contain exactly one '/' character" + } + + @Unroll + def 'should not sanitize k8s annotation value' () { + given: + def builder = new PodSpecBuilder() + + expect: + builder.sanitize(ANNOTATION, PodSpecBuilder.MetaType.ANNOTATION) == EXPECTED + + where: + ANNOTATION | EXPECTED + ['foo':'value 1'] | ['foo':'value 1'] + ['foo':'foo.bar / *'] | ['foo':'foo.bar / *'] + ['foo':'value 2 \n value 3'] | ['foo':'value 2 \n value 3'] + ['foo':'value 3'] | ['foo':'value 3'] + ['foo':'x2345678901234567890123456789012345678901234567890123456789012345'] | ['foo':'x2345678901234567890123456789012345678901234567890123456789012345'] + } + + def 'should create job spec' () { + + when: + def spec = new PodSpecBuilder() + .withPodName('foo') + .withImageName('busybox') + .withCommand(['echo', 'hello']) + .buildAsJob() + + then: + spec == [ + apiVersion: 'batch/v1', + kind: 'Job', + metadata: [name: 'foo', namespace: 'default'], + spec: [ + backoffLimit: 0, + template: [ + metadata: [name: 'foo', namespace: 'default'], + spec: [ + restartPolicy: 'Never', + containers: [[ + name: 'foo', + image: 'busybox', + command: ['echo', 'hello'], + ]] + ] + ] + ] + ] + } + + def 'should create job spec with labels and annotations' () { + + when: + def job = new PodSpecBuilder() + .withPodName('foo') + .withImageName('busybox') + .withCommand(['echo', 'hello']) + .withLabel('app','someApp') + .withLabel('runName','someName') + .withLabel('version','3.8.1') + .withAnnotation('anno1', 'val1') + .withAnnotations([anno2: 'val2', anno3: 'val3']) + .buildAsJob() + + def metadata = [ + name: 'foo', + namespace: 'default', + labels: [ + app: 'someApp', + runName: 'someName', + version: '3.8.1' + ], + annotations: [ + anno1: 'val1', + anno2: 'val2', + anno3: 'val3' + ] + ] + + then: + job.metadata == metadata + job.spec.template.metadata == metadata + } + + def 'should create job spec with ttl seconds' () { + when: + def job = new PodSpecBuilder() + .withPodName('foo') + .withImageName('busybox') + .withCommand(['echo', 'hello']) + .buildAsJob() + then: + !job.spec.ttlSecondsAfterFinished + + when: + job = new PodSpecBuilder() + .withPodName('foo') + .withImageName('busybox') + .withCommand(['echo', 'hello']) + .withPodOptions( new PodOptions(ttlSecondsAfterFinished: 60) ) + .buildAsJob() + then: + job.spec.ttlSecondsAfterFinished == 60 + } + + def 'should create pod spec with runtimeClassName' () { + when: + def pod = new PodSpecBuilder() + .withPodName('foo') + .withImageName('busybox') + .withCommand(['echo', 'hello']) + .build() + then: + !pod.spec.runtimeClassName + + when: + pod = new PodSpecBuilder() + .withPodName('foo') + .withImageName('busybox') + .withCommand(['echo', 'hello']) + .withPodOptions(new PodOptions(runtimeClassName: 'val1')) + .build() + then: + pod.spec.runtimeClassName == 'val1' + } +} diff --git a/k8s-dvfs/src/test/groovy/recreationaltech/plugin/model/PodVolumeClaimTest.groovy b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/model/PodVolumeClaimTest.groovy new file mode 100644 index 0000000..bf7bfe7 --- /dev/null +++ b/k8s-dvfs/src/test/groovy/recreationaltech/plugin/model/PodVolumeClaimTest.groovy @@ -0,0 +1,74 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package recreationaltech.plugin.model + +import spock.lang.Specification +/** + * + * @author Paolo Di Tommaso + */ +class PodVolumeClaimTest extends Specification { + + + def 'should create a pod volume claim' () { + when: + def vol1 = new PodVolumeClaim('foo', '/bar') + + then: + vol1.claimName == 'foo' + vol1.mountPath == '/bar' + vol1.readOnly == false + + when: + def vol2 = new PodVolumeClaim(volumeClaim: 'alpha', mountPath: '/gamma') + then: + vol2.claimName == 'alpha' + vol2.mountPath == '/gamma' + vol2.readOnly == false + + when: + def vol3 = new PodVolumeClaim('aaa', '/bbb', null, true) + + then: + vol3.claimName == 'aaa' + vol3.mountPath == '/bbb' + vol3.readOnly == true + + when: + def vol4 = new PodVolumeClaim(volumeClaim: 'ccc', mountPath: '/ddd', readOnly: true) + + then: + vol4.claimName == 'ccc' + vol4.mountPath == '/ddd' + vol4.readOnly == true + + } + + def 'should sanitize paths' () { + + expect : + new PodVolumeClaim('foo','/data/work//').mountPath == '/data/work' + new PodVolumeClaim('foo','//').mountPath == '/' + new PodVolumeClaim('foo','/data').mountPath == '/data' + + when: + new PodVolumeClaim('foo','data') + then: + thrown(IllegalArgumentException) + } + +}