move to standalone plugin
This commit is contained in:
@@ -0,0 +1,527 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin
|
||||
|
||||
import nextflow.config.scopes.Config
|
||||
import recreationaltech.plugin.client.K8sRetryConfig
|
||||
|
||||
import javax.annotation.Nullable
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.PackageScope
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.BuildInfo
|
||||
import nextflow.config.spec.ConfigOption
|
||||
import nextflow.config.spec.ConfigScope
|
||||
import nextflow.config.spec.ScopeName
|
||||
import nextflow.container.ContainerHelper
|
||||
import nextflow.script.dsl.Description
|
||||
import nextflow.exception.AbortOperationException
|
||||
import recreationaltech.plugin.client.ClientConfig
|
||||
import recreationaltech.plugin.client.K8sClient
|
||||
import recreationaltech.plugin.client.K8sResponseException
|
||||
import recreationaltech.plugin.model.PodOptions
|
||||
import recreationaltech.plugin.model.PodSecurityContext
|
||||
import recreationaltech.plugin.model.PodVolumeClaim
|
||||
import recreationaltech.plugin.model.ResourceType
|
||||
import nextflow.util.Duration
|
||||
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
/**
|
||||
* Model Kubernetes specific settings defined in the nextflow
|
||||
* configuration file
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@ScopeName("k8s")
|
||||
@Description("""
|
||||
The `k8s` scope controls the deployment and execution of workflow applications in a Kubernetes cluster.
|
||||
""")
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class K8sConfig implements ConfigScope {
|
||||
|
||||
static final private Map<String,?> DEFAULT_FUSE_PLUGIN = Map.of('nextflow.io/fuse', 1)
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Automatically mount host paths into the task pods (default: `false`). Only intended for development purposes when using a single node.
|
||||
""")
|
||||
final boolean autoMountHostPaths
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Whether to use Kubernetes `Pod` or `Job` resource type to carry out Nextflow tasks (default: `Pod`).
|
||||
""")
|
||||
final String computeResourceType
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
When `true`, successful pods are automatically deleted (default: `true`).
|
||||
""")
|
||||
final private Boolean cleanup
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Map of options for the K8s client.
|
||||
|
||||
If this option is specified, it will be used instead of `.kube/config`.
|
||||
""")
|
||||
final Map client
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The interval after which the Kubernetes client configuration is refreshed (default: `50m`).
|
||||
""")
|
||||
final Duration clientRefreshInterval
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The Kubernetes [configuration context](https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/) to use.
|
||||
""")
|
||||
final String context
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
When `true`, set both the pod CPUs `request` and `limit` to the value specified by the `cpus` directive, otherwise set only the `request` (default: `false`).
|
||||
""")
|
||||
final boolean cpuLimits
|
||||
|
||||
final K8sDebug debug
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Include the hostname of each task in the execution trace (default: `false`).
|
||||
""")
|
||||
final boolean fetchNodeName
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The FUSE device plugin to be used when enabling Fusion in unprivileged mode (default: `['nextflow.io/fuse': 1]`).
|
||||
""")
|
||||
final Map fuseDevicePlugin
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The Kubernetes HTTP client request connection timeout e.g. `'60s'`.
|
||||
""")
|
||||
final Duration httpConnectTimeout
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The Kubernetes HTTP client request connection read timeout e.g. `'60s'`.
|
||||
""")
|
||||
final Duration httpReadTimeout
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The strategy for pulling container images. Can be `IfNotPresent`, `Always`, `Never`.
|
||||
|
||||
[Read more](https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy)
|
||||
""")
|
||||
final String imagePullPolicy
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The path where the workflow is launched and the user data is stored (default: `<volume-claim-mount-path>/<user-name>`). Must be a path in a shared K8s persistent volume.
|
||||
""")
|
||||
final String launchDir
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The Kubernetes namespace to use (default: `default`).
|
||||
""")
|
||||
final String namespace
|
||||
|
||||
@ConfigOption(types=[List, Map])
|
||||
@Description("""
|
||||
Additional pod configuration options such as environment variables, config maps, secrets, etc. Allows the same settings as the [pod](https://nextflow.io/docs/latest/process.html#pod) process directive.
|
||||
""")
|
||||
final PodOptions pod
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The path where Nextflow projects are downloaded (default: `<volume-claim-mount-path>/projects`). Must be a path in a shared K8s persistent volume.
|
||||
""")
|
||||
final String projectDir
|
||||
|
||||
@Deprecated
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
""")
|
||||
final String pullPolicy
|
||||
|
||||
final K8sRetryConfig retryPolicy
|
||||
|
||||
@ConfigOption(types=[Integer, String])
|
||||
@Description("""
|
||||
The user ID to be used to run the containers. Shortcut for the `securityContext` option.
|
||||
""")
|
||||
final Object runAsUser
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The [security context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) to use for all pods.
|
||||
""")
|
||||
final Map securityContext
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The Kubernetes [service account name](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/) to use.
|
||||
""")
|
||||
final String serviceAccount
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The name of the persistent volume claim where the shared work directory is stored.
|
||||
""")
|
||||
final String storageClaimName
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The mount path for the persistent volume claim (default: `/workspace`).
|
||||
""")
|
||||
final String storageMountPath
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The path in the persistent volume to be mounted (default: `/`).
|
||||
""")
|
||||
final String storageSubPath
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
""")
|
||||
final String userName
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The path of the shared work directory (default: `<user-dir>/work`). Must be a path in a shared K8s persistent volume.
|
||||
""")
|
||||
final String workDir
|
||||
|
||||
@Description("Node initialization config")
|
||||
final K8sNodeInitConfig nodeInit
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The image name of the nextflow launcher image
|
||||
""")
|
||||
final String nextflowImage
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The run interval of the kubernetes scheduler
|
||||
""")
|
||||
final Duration schedulerInterval
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Enables task runtime recording
|
||||
""")
|
||||
final boolean recordTaskRuntimes
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The runtime recording file
|
||||
""")
|
||||
final String runtimeRecordPath
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Selects a scheduling strategy
|
||||
""")
|
||||
final String schedulingStrategy
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Selects a runtime estimator
|
||||
""")
|
||||
final String runtimeEstimator
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Noise magnitude used for the noise runtime estimator
|
||||
""")
|
||||
final Duration noiseRuntimeEstimatorNoiseMagnitude
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Max. amount of time two runtime estimates can differ to be considered equal.
|
||||
""")
|
||||
final Duration runtimeComparisonEpsilon
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Number of saved top runtimes used to classify a task as critical.
|
||||
""")
|
||||
final int dvfsSchedulingNumTopRuntimes
|
||||
|
||||
/* required by extension point -- do not remove */
|
||||
K8sConfig() {
|
||||
this(Collections.emptyMap())
|
||||
}
|
||||
|
||||
K8sConfig(Map opts) {
|
||||
autoMountHostPaths = opts.autoMountHostPaths as boolean
|
||||
cleanup = opts.cleanup as Boolean
|
||||
client = opts.client as Map
|
||||
clientRefreshInterval = opts.clientRefreshInterval as Duration ?: Duration.of('50m')
|
||||
computeResourceType = opts.computeResourceType
|
||||
context = opts.context
|
||||
cpuLimits = opts.cpuLimits as boolean
|
||||
debug = new K8sDebug(opts.debug as Map ?: Collections.emptyMap())
|
||||
fetchNodeName = opts.fetchNodeName as boolean
|
||||
fuseDevicePlugin = parseFuseDevicePlugin(opts.fuseDevicePlugin)
|
||||
httpConnectTimeout = opts.httpConnectTimeout as Duration
|
||||
httpReadTimeout = opts.httpReadTimeout as Duration
|
||||
imagePullPolicy = opts.pullPolicy ?: opts.imagePullPolicy
|
||||
namespace = opts.namespace
|
||||
pod = createPodOptions(opts.pod)
|
||||
retryPolicy = new K8sRetryConfig(opts.retryPolicy as Map ?: Collections.emptyMap())
|
||||
runAsUser = opts.runAsUser
|
||||
securityContext = opts.securityContext as Map
|
||||
serviceAccount = opts.serviceAccount
|
||||
storageClaimName = opts.storageClaimName
|
||||
storageMountPath = opts.storageMountPath ?: '/workspace'
|
||||
storageSubPath = opts.storageSubPath
|
||||
userName = opts.userName
|
||||
nextflowImage = opts.nextflowImage ?: "nextflow/nextflow:${BuildInfo.version}"
|
||||
schedulerInterval = opts.schedulerInterval as Duration ?: new Duration(10, TimeUnit.SECONDS)
|
||||
recordTaskRuntimes = opts.recordTaskRuntimes as boolean ?: false
|
||||
|
||||
launchDir = opts.launchDir ?: "${storageMountPath}/${getUserName()}"
|
||||
projectDir = opts.projectDir ?: "${storageMountPath}/projects"
|
||||
workDir = opts.workDir ?: "${launchDir}/work"
|
||||
runtimeRecordPath = opts.runtimeRecordPath as String ?: "${workDir}/runtimes.csv"
|
||||
|
||||
schedulingStrategy = opts.schedulingStrategy as String ?: "Hash"
|
||||
runtimeEstimator = opts.runtimeEstimator as String ?: "LinearFit"
|
||||
noiseRuntimeEstimatorNoiseMagnitude = opts.noiseRuntimeEstimatorNoiseMagnitude as Duration ?: new Duration(30, TimeUnit.SECONDS)
|
||||
runtimeComparisonEpsilon = opts.runtimeComparisonEpsilon as Duration ?: new Duration(10, TimeUnit.SECONDS)
|
||||
dvfsSchedulingNumTopRuntimes = opts.dvfsSchedulingTopRuntimes as int ?: 3
|
||||
|
||||
// -- shortcut to pod image pull-policy
|
||||
if( imagePullPolicy )
|
||||
pod.imagePullPolicy = imagePullPolicy
|
||||
|
||||
// -- shortcut to pod volume claim
|
||||
if( storageClaimName ) {
|
||||
final volumeClaim = new PodVolumeClaim(storageClaimName, storageMountPath, storageSubPath)
|
||||
pod.volumeClaims.add(volumeClaim)
|
||||
}
|
||||
|
||||
// -- shortcut to pod security context
|
||||
if( runAsUser )
|
||||
pod.securityContext = new PodSecurityContext(runAsUser)
|
||||
else if( securityContext )
|
||||
pod.securityContext = new PodSecurityContext(securityContext)
|
||||
|
||||
nodeInit = new K8sNodeInitConfig(opts.nodeInit as Map ?: Collections.emptyMap())
|
||||
}
|
||||
|
||||
private PodOptions createPodOptions( value ) {
|
||||
if( value instanceof List )
|
||||
return new PodOptions( value as List )
|
||||
|
||||
if( value instanceof Map )
|
||||
return new PodOptions( [(Map)value] )
|
||||
|
||||
if( value == null )
|
||||
return new PodOptions()
|
||||
|
||||
throw new IllegalArgumentException("Not a valid pod setting: $value")
|
||||
}
|
||||
|
||||
Map<String,String> getLabels() {
|
||||
pod.getLabels()
|
||||
}
|
||||
|
||||
Map<String,String> getAnnotations() {
|
||||
pod.getAnnotations()
|
||||
}
|
||||
|
||||
boolean getCleanup(boolean defValue=true) {
|
||||
cleanup == null ? defValue : cleanup
|
||||
}
|
||||
|
||||
String getUserName() {
|
||||
userName ?: System.properties.get('user.name')
|
||||
}
|
||||
|
||||
Map<String,?> fuseDevicePlugin() {
|
||||
fuseDevicePlugin
|
||||
}
|
||||
|
||||
Map<String,?> parseFuseDevicePlugin(Object value) {
|
||||
if( value instanceof Map && value.size()==1 )
|
||||
return value as Map<String,?>
|
||||
if( value != null )
|
||||
log.warn1 "Setting 'k8s.fuseDevicePlugin' should be a map containing exactly one entry - offending value: $value"
|
||||
return DEFAULT_FUSE_PLUGIN
|
||||
}
|
||||
|
||||
/**
|
||||
* Whenever the pod should honour the entrypoint defined by the image (default: false)
|
||||
*
|
||||
* @return When {@code false} the launcher script is run by using pod `command` attributes which
|
||||
* overrides the entrypoint point defined by the image.
|
||||
*
|
||||
* When {@code true} the launcher is run via the pod `args` attribute, without altering the
|
||||
* container entrypoint (it does however require to have a bash shell as the image entrypoint)
|
||||
*
|
||||
*/
|
||||
boolean entrypointOverride() {
|
||||
return ContainerHelper.entrypointOverride()
|
||||
}
|
||||
|
||||
boolean useJobResource() { ResourceType.Job.name() == computeResourceType }
|
||||
|
||||
String getNextflowImageName() {
|
||||
return nextflowImage
|
||||
}
|
||||
|
||||
PodOptions getPodOptions() {
|
||||
pod
|
||||
}
|
||||
|
||||
boolean fetchNodeName() {
|
||||
fetchNodeName
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the collection of defined volume claim names
|
||||
*/
|
||||
Collection<String> getClaimNames() {
|
||||
pod.volumeClaims.collect { it.claimName }
|
||||
}
|
||||
|
||||
Collection<String> getClaimPaths() {
|
||||
pod.volumeClaims.collect { it.mountPath }
|
||||
}
|
||||
|
||||
boolean cpuLimitsEnabled() {
|
||||
cpuLimits
|
||||
}
|
||||
|
||||
/**
|
||||
* Find a volume claim name given the mount path
|
||||
*
|
||||
* @param path The volume claim mount path
|
||||
* @return The volume claim name for the given mount path
|
||||
*/
|
||||
String findVolumeClaimByPath(String path) {
|
||||
final result = pod.volumeClaims.find { path.startsWith(it.mountPath) }
|
||||
return result ? result.claimName : null
|
||||
}
|
||||
|
||||
ClientConfig getClient() {
|
||||
final result = client != null
|
||||
? clientFromNextflow(client, namespace, serviceAccount)
|
||||
: clientDiscovery(context, namespace, serviceAccount)
|
||||
|
||||
if( httpConnectTimeout )
|
||||
result.httpConnectTimeout = httpConnectTimeout
|
||||
|
||||
if( httpReadTimeout )
|
||||
result.httpReadTimeout = httpReadTimeout
|
||||
|
||||
if( retryPolicy )
|
||||
result.retryConfig = retryPolicy
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the K8s client config from the declaration made in the Nextflow config file
|
||||
*
|
||||
* @param map
|
||||
* A map representing the clint configuration options define in the nextflow
|
||||
* config file
|
||||
* @param namespace
|
||||
* The K8s namespace to be used. If omitted {@code default} is used.
|
||||
* @param serviceAccount
|
||||
* The K8s service account to be used. If omitted {@code default} is used.
|
||||
* @return
|
||||
* The Kubernetes {@link ClientConfig} object
|
||||
*/
|
||||
@PackageScope ClientConfig clientFromNextflow(Map map, @Nullable String namespace, @Nullable String serviceAccount ) {
|
||||
ClientConfig.fromNextflowConfig(map,namespace,serviceAccount)
|
||||
}
|
||||
|
||||
/**
|
||||
* Discover the K8s client config from the execution environment
|
||||
* that can be either a `.kube/config` file or service meta file
|
||||
* when running in a pod.
|
||||
*
|
||||
* @param contextName
|
||||
* The name of the configuration context to be used
|
||||
* @param namespace
|
||||
* The Kubernetes namespace to be used
|
||||
* @param serviceAccount
|
||||
* The Kubernetes serviceAccount to be used
|
||||
* @return
|
||||
* The discovered Kube {@link ClientConfig} object
|
||||
*/
|
||||
@PackageScope ClientConfig clientDiscovery(String contextName, String namespace, String serviceAccount) {
|
||||
ClientConfig.discover(contextName, namespace, serviceAccount)
|
||||
}
|
||||
|
||||
void checkStorageAndPaths(K8sClient client) {
|
||||
if( !storageClaimName )
|
||||
throw new AbortOperationException("Missing K8s storage volume claim -- The name of a persistence volume claim needs to be provided in the nextflow configuration file")
|
||||
|
||||
log.debug "Kubernetes workDir=$workDir; projectDir=$projectDir; volumeClaims=${getClaimNames()}"
|
||||
|
||||
for( String name : getClaimNames() ) {
|
||||
try {
|
||||
client.volumeClaimRead(name)
|
||||
}
|
||||
catch (K8sResponseException e) {
|
||||
if( e.response.code == 404 ) {
|
||||
throw new AbortOperationException("Unknown volume claim: $name -- make sure a persistent volume claim with the specified name is defined in your K8s cluster")
|
||||
}
|
||||
else throw e
|
||||
}
|
||||
}
|
||||
|
||||
if( !findVolumeClaimByPath(launchDir) )
|
||||
throw new AbortOperationException("Kubernetes `launchDir` must be a path mounted as a persistent volume -- launchDir=$launchDir; volumes=${getClaimPaths().join(', ')}")
|
||||
|
||||
if( !findVolumeClaimByPath(workDir) )
|
||||
throw new AbortOperationException("Kubernetes `workDir` must be a path mounted as a persistent volume -- workDir=$workDir; volumes=${getClaimPaths().join(', ')}")
|
||||
|
||||
if( !findVolumeClaimByPath(projectDir) )
|
||||
throw new AbortOperationException("Kubernetes `projectDir` must be a path mounted as a persistent volume -- projectDir=$projectDir; volumes=${getClaimPaths().join(', ')}")
|
||||
|
||||
}
|
||||
|
||||
static class K8sDebug implements ConfigScope {
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Save the pod spec for each task to `.command.yaml` in the task directory (default: `false`).
|
||||
""")
|
||||
final boolean yaml
|
||||
|
||||
K8sDebug(Map opts) {
|
||||
yaml = opts.yaml as boolean
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
package recreationaltech.plugin
|
||||
|
||||
import groovy.util.logging.Slf4j
|
||||
|
||||
import java.net.http.HttpClient
|
||||
import java.net.http.HttpRequest
|
||||
import java.net.http.HttpResponse
|
||||
import java.nio.channels.UnresolvedAddressException
|
||||
|
||||
/**
|
||||
* The DVFS client uses the deployed DVFS agents to control the operating frequency of the worker nodes.
|
||||
*/
|
||||
@Slf4j
|
||||
class K8sDVFSClient {
|
||||
private HttpClient httpClient
|
||||
private Map<String, String> ipTable
|
||||
|
||||
K8sDVFSClient(String[] nodes, String[] ips) {
|
||||
assert(nodes.length == ips.length)
|
||||
this.httpClient = HttpClient.newBuilder().build()
|
||||
this.ipTable = new HashMap<>()
|
||||
for (int i = 0; i < nodes.length; i++) {
|
||||
this.ipTable.put(nodes[i], ips[i])
|
||||
}
|
||||
}
|
||||
|
||||
OptionalLong getNodeCurrentFrequency(String node) {
|
||||
log.debug("Getting current frequency of node ${node}")
|
||||
return getNodeFrequency(node, "current")
|
||||
}
|
||||
|
||||
OptionalLong getNodeMaxFrequency(String node) {
|
||||
log.debug("Getting max frequency of node ${node}")
|
||||
return getNodeFrequency(node, "max")
|
||||
}
|
||||
|
||||
OptionalLong getNodeMinFrequency(String node) {
|
||||
log.debug("Getting min frequency of node ${node}")
|
||||
return getNodeFrequency(node, "min")
|
||||
}
|
||||
|
||||
private OptionalLong getNodeFrequency(String node, String endpoint) {
|
||||
return getLong(HttpRequest.newBuilder()
|
||||
.uri(new URI("http://${agentAddress(node)}/cpu/frequency/${endpoint}"))
|
||||
.GET()
|
||||
.build())
|
||||
}
|
||||
|
||||
boolean setNodeFrequency(String node, int frequency) {
|
||||
String body = String.valueOf(frequency)
|
||||
|
||||
HttpRequest request = HttpRequest.newBuilder()
|
||||
.uri(new URI("http://${agentAddress(node)}/cpu/frequency/current"))
|
||||
.PUT(HttpRequest.BodyPublishers.ofString(body))
|
||||
.build()
|
||||
HttpResponse<Void> response = httpClient.send(request, HttpResponse.BodyHandlers.discarding())
|
||||
if (response.statusCode() != 200) {
|
||||
log.error("Request PUT ${request.uri().toString()} returned ${response.statusCode()}")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
OptionalLong getCPUCount(String node) {
|
||||
return getLong(HttpRequest.newBuilder()
|
||||
.uri(new URI("http://${agentAddress(node)}/cpu/count"))
|
||||
.GET()
|
||||
.build())
|
||||
}
|
||||
|
||||
OptionalLong getMemoryAmount(String node) {
|
||||
return getLong(HttpRequest.newBuilder()
|
||||
.uri(new URI("http://${agentAddress(node)}/mem/amount"))
|
||||
.GET()
|
||||
.build())
|
||||
}
|
||||
|
||||
private OptionalLong getLong(HttpRequest request) {
|
||||
try {
|
||||
HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString())
|
||||
if (response.statusCode() != 200) {
|
||||
log.error("Request GET ${request.uri().toString()} returned ${response.statusCode()}")
|
||||
return OptionalLong.empty()
|
||||
}
|
||||
long parsed = Long.parseLong(response.body())
|
||||
return OptionalLong.of(parsed)
|
||||
} catch (NumberFormatException ex) {
|
||||
log.error("Unexpected response ${response.body()} - ${ex.message}")
|
||||
return OptionalLong.empty()
|
||||
} catch (ConnectException ex) {
|
||||
log.error("Request failed ${request.uri().toString()}: ${ex.message}")
|
||||
return OptionalLong.empty()
|
||||
} catch (UnresolvedAddressException ex) {
|
||||
log.error("Failed to resolve address ${request.uri().toString()}: ${ex.message}")
|
||||
return OptionalLong.empty()
|
||||
}
|
||||
}
|
||||
|
||||
private String agentAddress(String node) {
|
||||
return ipTable.get(node) + ":8080"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,718 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin
|
||||
|
||||
import groovy.transform.MapConstructor
|
||||
|
||||
import java.lang.reflect.Field
|
||||
import java.nio.file.NoSuchFileException
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
|
||||
import com.beust.jcommander.DynamicParameter
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.google.common.hash.Hashing
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.cli.CmdKubeRun
|
||||
import nextflow.cli.CmdRun
|
||||
import nextflow.config.ConfigBuilder
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.file.FileHelper
|
||||
import recreationaltech.plugin.client.K8sClient
|
||||
import recreationaltech.plugin.client.K8sResponseException
|
||||
import recreationaltech.plugin.model.PodEnv
|
||||
import recreationaltech.plugin.model.PodMountConfig
|
||||
import recreationaltech.plugin.model.PodSpecBuilder
|
||||
import recreationaltech.plugin.model.ResourceType
|
||||
import nextflow.scm.AssetManager
|
||||
import nextflow.scm.ProviderConfig
|
||||
import nextflow.util.ConfigHelper
|
||||
import nextflow.util.Escape
|
||||
import org.codehaus.groovy.runtime.MethodClosure
|
||||
/**
|
||||
* Configure and submit the execution of pod running the Nextflow main application
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@MapConstructor(includeFields = true)
|
||||
class K8sDriverLauncher {
|
||||
|
||||
/**
|
||||
* Either a Pod or Job
|
||||
*/
|
||||
private ResourceType resourceType = ResourceType.Pod
|
||||
|
||||
/**
|
||||
* Container image to be used for the Nextflow driver pod
|
||||
*/
|
||||
private String headImage
|
||||
|
||||
/**
|
||||
* Request CPUs to be used for the Nextflow driver pod
|
||||
*/
|
||||
private int headCpus
|
||||
|
||||
/**
|
||||
* Request memory to be used for the Nextflow driver pod
|
||||
*/
|
||||
private String headMemory
|
||||
|
||||
/**
|
||||
* Pre-script to run before nextflow
|
||||
*/
|
||||
private String headPreScript
|
||||
|
||||
/**
|
||||
* Nextflow execution run name
|
||||
*/
|
||||
private String runName
|
||||
|
||||
/**
|
||||
* Workflow project to launch
|
||||
*/
|
||||
private String pipelineName
|
||||
|
||||
/**
|
||||
* Command run options
|
||||
*/
|
||||
private CmdKubeRun cmd
|
||||
|
||||
/**
|
||||
* Kubernetes client
|
||||
*/
|
||||
private K8sClient k8sClient
|
||||
|
||||
/**
|
||||
* Nextflow resolved config object
|
||||
*/
|
||||
private ConfigObject config
|
||||
|
||||
/**
|
||||
* Name of the config map used to propagate the nextflow
|
||||
* setting in the container
|
||||
*/
|
||||
private String configMapName
|
||||
|
||||
/**
|
||||
* Kubernetes specific config settings
|
||||
*/
|
||||
private K8sConfig k8sConfig
|
||||
|
||||
private String paramsFile
|
||||
|
||||
private boolean interactive
|
||||
|
||||
/**
|
||||
* Runs in background mode
|
||||
*/
|
||||
private boolean background
|
||||
|
||||
/**
|
||||
* Workflow script positional parameters
|
||||
*/
|
||||
private List<String> args
|
||||
|
||||
/**
|
||||
* Plugins to run the workflow
|
||||
*/
|
||||
private String plugins
|
||||
|
||||
private K8sNodeInitDeployer initDeployer
|
||||
|
||||
/**
|
||||
* Launcher entry point. Set-up the environment and create a pod that run the Nextflow
|
||||
* application (which in turns executed each task as a pod)
|
||||
*
|
||||
* @param name Workflow project entry name
|
||||
* @param args Workflow script positional parameters
|
||||
*/
|
||||
void run(String name, List<String> args) {
|
||||
this.args = args
|
||||
this.pipelineName = name
|
||||
this.interactive = name == 'login'
|
||||
if( background && interactive )
|
||||
throw new AbortOperationException("Option -bg conflicts with interactive mode")
|
||||
this.config = makeConfig(pipelineName)
|
||||
this.k8sConfig = makeK8sConfig(config.toMap())
|
||||
this.k8sClient = makeK8sClient(k8sConfig)
|
||||
this.k8sConfig.checkStorageAndPaths(k8sClient)
|
||||
this.initDeployer = new K8sNodeInitDeployer(k8sClient, k8sConfig)
|
||||
|
||||
createK8sConfigMap()
|
||||
|
||||
initDeployer.deploy()
|
||||
createK8sLauncherPod()
|
||||
waitPodStart()
|
||||
// login into container session
|
||||
if( interactive )
|
||||
launchLogin()
|
||||
// dump pod output
|
||||
else if( !background )
|
||||
printK8sPodOutput()
|
||||
else
|
||||
log.debug "Nextflow driver launched in background mode -- pod: $runName"
|
||||
waitPodEnd()
|
||||
}
|
||||
|
||||
int shutdown() {
|
||||
if( background )
|
||||
return 0
|
||||
// fetch the container exit status
|
||||
final exitCode = waitPodTermination()
|
||||
// cleanup the config map if OK
|
||||
def deleteOnSuccessByDefault = exitCode==0
|
||||
if( k8sConfig.getCleanup(deleteOnSuccessByDefault) ) {
|
||||
deleteConfigMap()
|
||||
}
|
||||
// cleanup pre-workflow pods
|
||||
initDeployer.cleanup()
|
||||
|
||||
return exitCode
|
||||
}
|
||||
|
||||
protected void waitPodEnd() {
|
||||
if( background )
|
||||
return
|
||||
final currentState = k8sConfig.useJobResource() ? k8sClient.jobState(runName) : k8sClient.podState(runName)
|
||||
if (currentState && currentState?.running instanceof Map) {
|
||||
final name = runName
|
||||
println "${resourceType} running: $name ... waiting for ${resourceType.lower()} to stop running"
|
||||
try {
|
||||
while( true ) {
|
||||
sleep 10000
|
||||
final state = k8sConfig.useJobResource() ? k8sClient.jobState(name) : k8sClient.podState(name)
|
||||
if ( state && !(state?.running instanceof Map) ) {
|
||||
println "${resourceType} $name has changed from running state $state"
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
catch( Exception e ) {
|
||||
log.warn "Caught exception while waiting for ${resourceType.lower()} to stop running"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected boolean isWaitTimedOut(long time) {
|
||||
System.currentTimeMillis()-time > 90_000
|
||||
}
|
||||
|
||||
protected int waitPodTermination() {
|
||||
log.debug "Wait for ${resourceType.lower()} termination name=$runName"
|
||||
final rnd = new Random()
|
||||
final time = System.currentTimeMillis()
|
||||
Map state = null
|
||||
try {
|
||||
while( true ) {
|
||||
sleep rnd.nextInt(500)
|
||||
state = k8sConfig.useJobResource() ? k8sClient.jobState(runName) : k8sClient.podState(runName)
|
||||
if( state?.terminated instanceof Map )
|
||||
return state.terminated.exitCode as int
|
||||
|
||||
else if( isWaitTimedOut(time) )
|
||||
throw new IllegalStateException("Timeout waiting for ${resourceType.lower()} terminated state=$state")
|
||||
}
|
||||
}
|
||||
catch( Exception e ) {
|
||||
log.warn "Unable to fetch ${resourceType.lower()} exit status -- ${resourceType.lower()}=$runName state=$state"
|
||||
return 127
|
||||
}
|
||||
}
|
||||
|
||||
protected void deleteConfigMap() {
|
||||
try {
|
||||
k8sClient.configDelete(configMapName)
|
||||
log.debug "Deleted K8s configMap with name: $configMapName"
|
||||
}
|
||||
catch ( Exception e ) {
|
||||
log.warn "Unable to delete configMap: $configMapName", e
|
||||
}
|
||||
}
|
||||
|
||||
protected void waitPodStart() {
|
||||
final name = runName
|
||||
print "${resourceType} submitted: $name .. waiting to start"
|
||||
while( true ) {
|
||||
sleep 1000
|
||||
final state = k8sConfig.useJobResource() ? k8sClient.jobState(name) : k8sClient.podState(name)
|
||||
if( state && !state.containsKey('waiting') ) {
|
||||
break
|
||||
}
|
||||
}
|
||||
print "\33[2K\r"
|
||||
println "${resourceType} started: $name"
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for the driver pod creation and prints the log to the
|
||||
* console standard output
|
||||
*/
|
||||
protected void printK8sPodOutput() {
|
||||
if ( k8sConfig.useJobResource() )
|
||||
k8sClient.jobLog(runName, follow:true).eachLine { println it }
|
||||
else
|
||||
k8sClient.podLog(runName, follow:true).eachLine { println it }
|
||||
}
|
||||
|
||||
protected ConfigObject loadConfig( String pipelineName ) {
|
||||
|
||||
// -- load local config if available
|
||||
final builder = new ConfigBuilder()
|
||||
.setShowClosures(true)
|
||||
.setOptions(cmd.launcher.options)
|
||||
.setProfile(cmd.profile)
|
||||
.setCmdRun(cmd)
|
||||
|
||||
if( !interactive && !pipelineName.startsWith('/') && !cmd.remoteProfile && !cmd.runRemoteConfig ) {
|
||||
// -- check and parse project remote config
|
||||
final pipelineConfig = new AssetManager(pipelineName, cmd.revision, cmd.mainScript, cmd).getConfigFile()
|
||||
builder.setUserConfigFiles(pipelineConfig)
|
||||
}
|
||||
|
||||
return builder.buildConfigObject()
|
||||
}
|
||||
|
||||
protected K8sConfig makeK8sConfig(Map config) {
|
||||
config.k8s instanceof Map ? new K8sConfig(config.k8s as Map) : new K8sConfig()
|
||||
}
|
||||
|
||||
protected makeK8sClient( K8sConfig k8sConfig ) {
|
||||
new K8sClient(k8sConfig.getClient())
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve the workflow configuration and merge with the current local one.
|
||||
*
|
||||
* @param pipelineName Workflow project name
|
||||
* @return A {@link Map} modeling the execution configuration settings
|
||||
*/
|
||||
protected ConfigObject makeConfig(String pipelineName) {
|
||||
|
||||
def file = new File(pipelineName)
|
||||
if( !interactive && file.exists() ) {
|
||||
def message = "The k8s executor cannot run local ${file.directory ? 'project' : 'script'}: $pipelineName"
|
||||
message += " -- provide the absolute path of a project available in the Kubernetes cluster or the URL of a project hosted in a Git repository"
|
||||
throw new AbortOperationException(message)
|
||||
}
|
||||
|
||||
def config = loadConfig(pipelineName)
|
||||
|
||||
// normalize pod entries
|
||||
def k8s = config.k8s
|
||||
|
||||
if( !k8s.isSet('pod') )
|
||||
k8s.pod = []
|
||||
else if( k8s.pod instanceof Map ) {
|
||||
k8s.pod = [ k8s.pod ]
|
||||
}
|
||||
else if( !(k8s.pod instanceof List) )
|
||||
throw new IllegalArgumentException("Illegal k8s.pod configuratun value: ${k8s.pod}")
|
||||
|
||||
// -- use the volume claims specified in the command line
|
||||
// to populate the pod config
|
||||
for( int i=0; i<cmd.volMounts?.size(); i++ ){
|
||||
def entry = cmd.volMounts.get(i)
|
||||
def parts = entry.tokenize(':')
|
||||
def name = parts[0]
|
||||
def path = parts[1]
|
||||
if( i==0 ) {
|
||||
k8s.storageClaimName = name
|
||||
k8s.storageMountPath = path
|
||||
}
|
||||
else {
|
||||
k8s.pod.add( [volumeClaim: name, mountPath: path] )
|
||||
}
|
||||
}
|
||||
|
||||
// -- backward compatibility
|
||||
if( k8s.isSet('volumeClaims') ) {
|
||||
log.warn "Config setting k8s.volumeClaims has been deprecated -- Use k8s.storageClaimName and k8s.storageMountPath instead"
|
||||
k8s.volumeClaims.each { k,v ->
|
||||
def name = k as String
|
||||
def path = v instanceof Map ? v.mountPath : v.toString()
|
||||
if( !k8s.isSet('storageClaimName') ) {
|
||||
k8s.storageClaimName = name
|
||||
k8s.storageMountPath = path
|
||||
}
|
||||
else if( !cmd.volMounts ) {
|
||||
k8s.pod.add( [volumeClaim: name, mountPath: path] )
|
||||
}
|
||||
}
|
||||
// remove it
|
||||
k8s.remove('volumeClaims')
|
||||
}
|
||||
|
||||
// -- set k8s executor
|
||||
config.process.executor = 'k8s'
|
||||
|
||||
// -- strip default work dir
|
||||
if( config.workDir == 'work' )
|
||||
config.remove('workDir')
|
||||
|
||||
// -- check work dir
|
||||
if( cmd?.workDir )
|
||||
k8s.workDir = cmd.workDir
|
||||
else if( !k8s.isSet('workDir') && config.workDir )
|
||||
k8s.workDir = config.workDir
|
||||
|
||||
if ( plugins ) {
|
||||
LinkedList<String> plugins = config.plugins ?: []
|
||||
plugins.addAll( this.plugins.tokenize(',') )
|
||||
config.plugins = plugins
|
||||
}
|
||||
|
||||
// -- some cleanup
|
||||
if( !k8s.pod )
|
||||
k8s.remove('pod')
|
||||
|
||||
if( !k8s.storageClaimName )
|
||||
k8s.remove('storageClaimName')
|
||||
if( !k8s.storageMountPath )
|
||||
k8s.remove('storageMountPath')
|
||||
|
||||
if( !config.libDir )
|
||||
config.remove('libDir')
|
||||
|
||||
log.trace "K8s config object:\n${ConfigHelper.toCanonicalString(config).indent(' ')}"
|
||||
return config
|
||||
}
|
||||
|
||||
|
||||
private Field getField(CmdRun cmd, String name) {
|
||||
def clazz = cmd.class
|
||||
while( clazz != CmdRun ) {
|
||||
clazz = cmd.class.getSuperclass()
|
||||
}
|
||||
clazz.getDeclaredField(name)
|
||||
}
|
||||
|
||||
private void checkUnsupportedOption(String name) {
|
||||
def field = getField(cmd,name)
|
||||
if( !field ) {
|
||||
log.warn "Unknown command-line option to check: $name"
|
||||
return
|
||||
}
|
||||
field.setAccessible(true)
|
||||
if( field.get(cmd) ) {
|
||||
def param = field.getAnnotation(Parameter)
|
||||
def opt = param.names() ? param.names()[0] : "-$name"
|
||||
abort(opt)
|
||||
}
|
||||
}
|
||||
|
||||
private void abort(String opt) {
|
||||
throw new AbortOperationException("Option `$opt` not supported with Kubernetes deployment")
|
||||
}
|
||||
|
||||
private void unsupportedCliOptions(MethodClosure... fields) {
|
||||
unsupportedCliOptions( fields.collect { it.getMethod()} )
|
||||
}
|
||||
|
||||
private void unsupportedCliOptions(List<String> names) {
|
||||
for( String x : names ) {
|
||||
checkUnsupportedOption(x)
|
||||
}
|
||||
}
|
||||
|
||||
private void addOption(List result, MethodClosure m, Closure eval=null) {
|
||||
def name = m.getMethod()
|
||||
def field = getField(cmd,name)
|
||||
field.setAccessible(true)
|
||||
def val = field.get(cmd)
|
||||
if( ( eval ? eval(val) : val ) ) {
|
||||
def param = field.getAnnotation(Parameter)
|
||||
if( param ) {
|
||||
result << "${param.names()[0]} ${Escape.wildcards(String.valueOf(val))}"
|
||||
return
|
||||
}
|
||||
|
||||
param = field.getAnnotation(DynamicParameter)
|
||||
if( param && val instanceof Map ) {
|
||||
val.each { k,v ->
|
||||
result << "${param.names()[0]}$k ${Escape.wildcards(String.valueOf(v))}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return The nextflow driver command line
|
||||
*/
|
||||
protected String getLaunchCli() {
|
||||
assert cmd
|
||||
assert pipelineName
|
||||
|
||||
if( interactive ) {
|
||||
return "tail -f /dev/null"
|
||||
}
|
||||
|
||||
def result = []
|
||||
// -- configure NF command line
|
||||
result << "nextflow"
|
||||
|
||||
if( cmd.launcher.options.trace )
|
||||
result << "-trace ${cmd.launcher.options.trace.join(',')}"
|
||||
if( cmd.launcher.options.debug )
|
||||
result << "-debug ${cmd.launcher.options.debug.join(',')}"
|
||||
if( cmd.launcher.options.jvmOpts )
|
||||
cmd.launcher.options.jvmOpts.each { k,v -> result << "-D$k=$v" }
|
||||
|
||||
result << "run"
|
||||
result << pipelineName
|
||||
|
||||
if( runName )
|
||||
result << '-name' << runName
|
||||
|
||||
addOption(result, cmd.&cacheable, { it==false } )
|
||||
addOption(result, cmd.&resume )
|
||||
addOption(result, cmd.&poolSize )
|
||||
addOption(result, cmd.&pollInterval )
|
||||
addOption(result, cmd.&queueSize)
|
||||
addOption(result, cmd.&revision )
|
||||
addOption(result, cmd.&latest )
|
||||
addOption(result, cmd.&withTrace )
|
||||
addOption(result, cmd.&withTimeline )
|
||||
addOption(result, cmd.&withDag )
|
||||
addOption(result, cmd.&dumpHashes )
|
||||
addOption(result, cmd.&dumpChannels )
|
||||
addOption(result, cmd.&env )
|
||||
addOption(result, cmd.&process )
|
||||
addOption(result, cmd.¶ms )
|
||||
addOption(result, cmd.&entryName )
|
||||
|
||||
if( paramsFile ) {
|
||||
result << "-params-file $paramsFile"
|
||||
}
|
||||
|
||||
if ( cmd.runRemoteConfig )
|
||||
cmd.runRemoteConfig.forEach { result << "-config $it" }
|
||||
|
||||
if ( cmd.remoteProfile )
|
||||
result << "-profile ${cmd.remoteProfile}"
|
||||
|
||||
if( cmd.process?.executor )
|
||||
abort('process.executor')
|
||||
|
||||
unsupportedCliOptions(
|
||||
cmd.&libPath,
|
||||
cmd.&test,
|
||||
cmd.&executorOptions,
|
||||
cmd.&stdin,
|
||||
cmd.&withSingularity,
|
||||
cmd.&withApptainer,
|
||||
cmd.&withDocker,
|
||||
cmd.&withoutDocker,
|
||||
cmd.&withMpi,
|
||||
cmd.&clusterOptions,
|
||||
cmd.&exportSysEnv
|
||||
)
|
||||
|
||||
if( args )
|
||||
result.add(args)
|
||||
|
||||
return result.join(' ')
|
||||
}
|
||||
|
||||
/**
|
||||
* @return A {@link Map} modeling driver pod specification
|
||||
*/
|
||||
protected Map makeLauncherSpec() {
|
||||
assert runName
|
||||
assert k8sClient
|
||||
|
||||
// -- setup config file
|
||||
String cmd = "source /etc/nextflow/init.sh; ${getLaunchCli()}" +
|
||||
"; if [ -x /etc/nextflow/node-init-cleanup.sh ]; then /etc/nextflow/node-init-cleanup.sh || true; fi; " +
|
||||
"exit \$status"
|
||||
|
||||
// create the launcher pod
|
||||
PodSpecBuilder builder = new PodSpecBuilder()
|
||||
.withPodName(runName)
|
||||
.withImageName(headImage ?: k8sConfig.getNextflowImageName())
|
||||
.withCommand(['/bin/bash', '-c', cmd])
|
||||
.withLabels([ app: 'nextflow', runName: runName ])
|
||||
.withNamespace(k8sClient.config.namespace)
|
||||
.withServiceAccount(k8sClient.config.serviceAccount)
|
||||
.withPodOptions(k8sConfig.getPodOptions())
|
||||
.withEnv( PodEnv.value('NXF_WORK', k8sConfig.getWorkDir()) )
|
||||
.withEnv( PodEnv.value('NXF_ASSETS', k8sConfig.getProjectDir()) )
|
||||
.withEnv( PodEnv.value('NXF_EXECUTOR', 'k8s'))
|
||||
.withEnv( PodEnv.value('NXF_ANSI_LOG', 'false'))
|
||||
.withMemory(headMemory?:"")
|
||||
.withCpus(headCpus)
|
||||
.withCpuLimits(k8sConfig.cpuLimitsEnabled())
|
||||
|
||||
if ( k8sConfig.useJobResource()) {
|
||||
this.resourceType = ResourceType.Job
|
||||
return builder.buildAsJob()
|
||||
}
|
||||
else {
|
||||
return builder.build()
|
||||
}
|
||||
|
||||
// note: do *not* set the work directory because it may need to be created by the init script
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates and executes the nextflow driver pod
|
||||
* @return A {@link recreationaltech.plugin.client.K8sResponseJson} response object
|
||||
*/
|
||||
protected createK8sLauncherPod() {
|
||||
final spec = makeLauncherSpec()
|
||||
if ( k8sConfig.useJobResource() ) {
|
||||
k8sClient.jobCreate(spec, yamlDebugPath())
|
||||
} else {
|
||||
k8sClient.podCreate(spec, yamlDebugPath())
|
||||
}
|
||||
}
|
||||
|
||||
protected Path yamlDebugPath() {
|
||||
boolean debug = config.k8s.debug?.yaml?.toString() == 'true'
|
||||
final result = debug ? Paths.get(".nextflow.${resourceType.lower()}.yaml") : null
|
||||
if( result )
|
||||
log.info "Launcher ${resourceType.lower()} spec file: $result"
|
||||
return result
|
||||
}
|
||||
|
||||
protected Path getScmFile() {
|
||||
ProviderConfig.getScmConfigPath()
|
||||
}
|
||||
|
||||
String getPodImage() {
|
||||
return podImage
|
||||
}
|
||||
|
||||
int getHeadCpus() {
|
||||
return headCpus
|
||||
}
|
||||
|
||||
String getHeadMemory() {
|
||||
return headMemory
|
||||
}
|
||||
|
||||
String getRunName() {
|
||||
return runName
|
||||
}
|
||||
|
||||
CmdKubeRun getCmd() {
|
||||
return cmd
|
||||
}
|
||||
|
||||
protected String getPipelineName() {
|
||||
return pipelineName
|
||||
}
|
||||
|
||||
protected boolean getInteractive() {
|
||||
return interactive
|
||||
}
|
||||
|
||||
protected ConfigObject getConfig() {
|
||||
return config
|
||||
}
|
||||
|
||||
protected K8sConfig getK8sConfig() {
|
||||
return k8sConfig
|
||||
}
|
||||
|
||||
protected K8sClient getK8sClient() {
|
||||
return k8sClient
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a K8s ConfigMap to share the nextflow configuration in the K8s cluster
|
||||
*/
|
||||
protected void createK8sConfigMap() {
|
||||
Map<String,String> configMap = [:]
|
||||
|
||||
final launchDir = k8sConfig.getLaunchDir()
|
||||
// init file
|
||||
String initScript = ''
|
||||
initScript += "mkdir -p '$launchDir'; if [ -d '$launchDir' ]; then cd '$launchDir'; else echo 'Cannot create directory: $launchDir'; exit 1; fi; "
|
||||
initScript += '[ -f /etc/nextflow/scm ] && ln -s /etc/nextflow/scm $NXF_HOME/scm; '
|
||||
initScript += '[ -f /etc/nextflow/nextflow.config ] && cp /etc/nextflow/nextflow.config $PWD/nextflow.config; '
|
||||
if( headPreScript )
|
||||
initScript += "[ -f '$headPreScript' ] && '$headPreScript'; "
|
||||
configMap['init.sh'] = initScript
|
||||
|
||||
// nextflow config file
|
||||
if( this.config ) {
|
||||
configMap['nextflow.config'] = ConfigHelper.toCanonicalString( this.config )
|
||||
}
|
||||
|
||||
// scm config file
|
||||
final scmFile = getScmFile()
|
||||
if( scmFile.exists() ) {
|
||||
configMap['scm'] = scmFile.text
|
||||
}
|
||||
|
||||
// params file
|
||||
if( cmd.paramsFile ) {
|
||||
final file = FileHelper.asPath(cmd.paramsFile)
|
||||
if( !file.exists() ) throw new NoSuchFileException("Params file does not exist: $file")
|
||||
configMap[ file.getName() ] = file.text
|
||||
paramsFile = "/etc/nextflow/$file.name"
|
||||
}
|
||||
|
||||
// pre-workflow pod cleanup
|
||||
if ( background )
|
||||
configMap['node-init-cleanup.sh'] = initDeployer.buildCleanupScript()
|
||||
|
||||
// create the config map
|
||||
configMapName = makeConfigMapName(configMap)
|
||||
tryCreateConfigMap(configMapName, configMap)
|
||||
log.debug "Created K8s configMap with name: $configMapName"
|
||||
k8sConfig.getPodOptions().getMountConfigMaps().add( new PodMountConfig(configMapName, '/etc/nextflow') )
|
||||
}
|
||||
|
||||
protected void tryCreateConfigMap(String name, Map data) {
|
||||
try {
|
||||
k8sClient.configCreate(name, data)
|
||||
}
|
||||
catch( K8sResponseException e ) {
|
||||
if( e.response.reason != 'AlreadyExists' )
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
protected String makeConfigMapName( Map configMap ) {
|
||||
"nf-config-${hash(configMap.values())}"
|
||||
}
|
||||
|
||||
protected String hash(Collection<String> text) {
|
||||
def hasher = Hashing .murmur3_32() .newHasher()
|
||||
def itr = text.iterator()
|
||||
while( itr.hasNext() ) {
|
||||
hasher.putUnencodedChars(itr.next())
|
||||
}
|
||||
|
||||
return hasher.hash().toString()
|
||||
}
|
||||
|
||||
protected void launchLogin() {
|
||||
def launchDir = k8sConfig.getLaunchDir()
|
||||
def cmd = "kubectl -n ${k8sClient.config.namespace} exec -it $runName -- /bin/bash -c 'cd $launchDir; exec bash --login -i'"
|
||||
def proc = new ProcessBuilder().command('bash','-c',cmd).inheritIO().start()
|
||||
def result = proc.waitFor()
|
||||
if( result == 0 ) {
|
||||
if ( k8sConfig.useJobResource() )
|
||||
k8sClient.jobDelete(runName)
|
||||
else
|
||||
k8sClient.podDelete(runName)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin
|
||||
|
||||
import groovy.transform.CompileDynamic
|
||||
import recreationaltech.plugin.strategies.K8sDVFSSchedulingStrategy
|
||||
import recreationaltech.plugin.strategies.K8sHashSchedulingStrategy
|
||||
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
import com.google.common.cache.Cache
|
||||
import com.google.common.cache.CacheBuilder
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.Memoized
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.executor.Executor
|
||||
import nextflow.fusion.FusionHelper
|
||||
import recreationaltech.plugin.client.K8sClient
|
||||
import nextflow.processor.TaskHandler
|
||||
import nextflow.processor.TaskMonitor
|
||||
import nextflow.processor.TaskPollingMonitor
|
||||
import nextflow.processor.TaskRun
|
||||
import nextflow.util.Duration
|
||||
import nextflow.util.ServiceName
|
||||
import org.pf4j.ExtensionPoint
|
||||
|
||||
/**
|
||||
* Implement the Kubernetes executor
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
@ServiceName('k8s-dvfs')
|
||||
class K8sExecutor extends Executor implements ExtensionPoint {
|
||||
|
||||
/**
|
||||
* Cache for the Kubernetes HTTP client. The client is refreshed periodically
|
||||
* so that the service account token is re-read when it expires.
|
||||
*/
|
||||
private Cache<String, K8sClient> clientCache
|
||||
|
||||
private K8sTaskScheduler taskScheduler
|
||||
private Thread schedulerThread
|
||||
|
||||
K8sRuntimeRecorder runtimeRecorder
|
||||
K8sRuntimeEstimator runtimeEstimator
|
||||
|
||||
/**
|
||||
* @return The Kubernetes HTTP client. Delegates to a Guava cache that refreshes
|
||||
* the client (including the service account token) when the configured interval expires.
|
||||
*/
|
||||
K8sClient getClient() {
|
||||
clientCache.get('client', () -> new K8sClient(k8sConfig.getClient()))
|
||||
}
|
||||
|
||||
protected K8sTaskScheduler getTaskScheduler() {
|
||||
assert taskScheduler != null
|
||||
return taskScheduler
|
||||
}
|
||||
|
||||
/**
|
||||
* @return The `k8s` configuration scope in the nextflow configuration object
|
||||
*/
|
||||
@Memoized
|
||||
protected K8sConfig getK8sConfig() {
|
||||
new K8sConfig( (Map<String,Object>)session.config.k8s )
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialise the executor setting-up the kubernetes client configuration
|
||||
*/
|
||||
@Override
|
||||
protected void register() {
|
||||
super.register()
|
||||
final k8sConfig = getK8sConfig()
|
||||
final refreshInterval = k8sConfig.clientRefreshInterval
|
||||
this.clientCache = CacheBuilder.newBuilder()
|
||||
.expireAfterWrite(refreshInterval.toMillis(), TimeUnit.MILLISECONDS)
|
||||
.build()
|
||||
final client = getClient()
|
||||
|
||||
log.debug "[K8s] config=$k8sConfig; API client config=$client.config"
|
||||
|
||||
this.runtimeRecorder = new K8sRuntimeRecorder(k8sConfig.recordTaskRuntimes, k8sConfig.runtimeRecordPath)
|
||||
|
||||
if (k8sConfig.runtimeEstimator == "LinearFit") {
|
||||
this.runtimeEstimator = new K8sLinearFitRuntimeEstimator(k8sConfig.runtimeRecordPath)
|
||||
} else if (k8sConfig.runtimeEstimator == "Noise") {
|
||||
this.runtimeEstimator = new K8sNoiseRuntimeEstimator(k8sConfig.runtimeEstimator, k8sConfig.noiseRuntimeEstimatorNoiseMagnitude)
|
||||
} else {
|
||||
log.error "[K8s] invalid runtime estimator ${k8sConfig.runtimeEstimator} - falling back on \"LinearFit\""
|
||||
this.runtimeEstimator = new K8sLinearFitRuntimeEstimator(k8sConfig.runtimeRecordPath)
|
||||
}
|
||||
|
||||
String[] nodes = getNodeList()
|
||||
|
||||
K8sSchedulingStrategy strategy = null
|
||||
if (k8sConfig.schedulingStrategy == "Hash") {
|
||||
strategy = new K8sHashSchedulingStrategy()
|
||||
} else if (k8sConfig.schedulingStrategy == "DVFS" || k8sConfig.schedulingStrategy == "DVFS-SPEED") {
|
||||
String[] ips = new String[nodes.length]
|
||||
for (int i = 0; i < nodes.length; i++) {
|
||||
ips[i] = client.getPodIpAddress(K8sNodeInitDeployer.buildPodName(nodes[i]))
|
||||
log.info "[K8s] node ${nodes[i]} -> ${ips[i]}"
|
||||
}
|
||||
strategy = new K8sDVFSSchedulingStrategy(this.runtimeEstimator,
|
||||
new K8sDVFSClient(nodes, ips),
|
||||
() -> getClient(),
|
||||
k8sConfig.runtimeComparisonEpsilon,
|
||||
k8sConfig.dvfsSchedulingNumTopRuntimes)
|
||||
strategy.fullSpeedMode = k8sConfig.schedulingStrategy == "DVFS-SPEED"
|
||||
} else {
|
||||
log.error "[K8s] invalid scheduling strategy $k8sConfig.schedulingStrategy, falling back on \"Hash\""
|
||||
strategy = new K8sHashSchedulingStrategy()
|
||||
}
|
||||
|
||||
this.taskScheduler = new K8sTaskScheduler(nodes, strategy, k8sConfig.schedulerInterval)
|
||||
this.schedulerThread = new Thread(this.taskScheduler)
|
||||
this.schedulerThread.start()
|
||||
}
|
||||
|
||||
@CompileDynamic
|
||||
private String[] getNodeList() {
|
||||
final resp = getClient().nodeList()
|
||||
ArrayList<String> nodes = new ArrayList<String>()
|
||||
for ( Map item : resp.items ) {
|
||||
nodes.add(item.metadata.name as String)
|
||||
}
|
||||
return nodes.toArray()
|
||||
}
|
||||
|
||||
@Override
|
||||
void shutdown() {
|
||||
this.runtimeRecorder.write()
|
||||
this.taskScheduler.stop()
|
||||
this.schedulerThread.join()
|
||||
}
|
||||
|
||||
/**
|
||||
* @return {@code true} since containerised execution is managed by Kubernetes
|
||||
*/
|
||||
boolean isContainerNative() {
|
||||
return true
|
||||
}
|
||||
|
||||
@Override
|
||||
String containerConfigEngine() {
|
||||
return 'docker'
|
||||
}
|
||||
|
||||
/**
|
||||
* @return A {@link TaskMonitor} associated to this executor type
|
||||
*/
|
||||
@Override
|
||||
protected TaskMonitor createTaskMonitor() {
|
||||
TaskPollingMonitor.create(session, config, name, 100, Duration.of('5 sec'))
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a {@link TaskHandler} for the given {@link TaskRun} instance
|
||||
*
|
||||
* @param task A {@link TaskRun} instance representing a process task to be executed
|
||||
* @return A {@link K8sTaskHandler} instance modeling the execution in the K8s cluster
|
||||
*/
|
||||
@Override
|
||||
TaskHandler createTaskHandler(TaskRun task) {
|
||||
assert task
|
||||
assert task.workDir
|
||||
log.trace "[K8s] launching process > ${task.name} -- work folder: ${task.workDirStr}"
|
||||
new K8sTaskHandler(task,this)
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean isFusionEnabled() {
|
||||
return FusionHelper.isFusionEnabled(session)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package recreationaltech.plugin
|
||||
|
||||
import groovy.util.logging.Slf4j
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
|
||||
/**
|
||||
* Provides runtime estimates for tasks based on task-name and input file size
|
||||
*/
|
||||
@Slf4j
|
||||
class K8sLinearFitRuntimeEstimator extends K8sRuntimeEstimator {
|
||||
|
||||
class Function {
|
||||
private double m
|
||||
private double n
|
||||
|
||||
Function(double m, double n) {
|
||||
this.m = m
|
||||
this.n = n
|
||||
}
|
||||
|
||||
double estimate(long x) {
|
||||
return m * (double)x + n
|
||||
}
|
||||
}
|
||||
|
||||
HashMap<String, Function> estimators;
|
||||
|
||||
/**
|
||||
* Initialize the estimator with data recorded by K8sRuntimeRecorder
|
||||
* @param dataFilePath
|
||||
*/
|
||||
K8sLinearFitRuntimeEstimator(String dataFilePath) {
|
||||
def data = parseDataFile(dataFilePath)
|
||||
fit(data)
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes the runtime estimator with statically known data.
|
||||
* @param data Map from task name to list of recordings, where each recording is a tuple (input-size, runtime-in-ms)
|
||||
*/
|
||||
K8sLinearFitRuntimeEstimator(HashMap<String, ArrayList<Tuple2<Long, Long>>> data) {
|
||||
fit(data)
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns an estimation of the task runtime in milliseconds
|
||||
* @param taskName the name of the task
|
||||
* @param inputSize the total input size in bytes
|
||||
* @return estimated task runtime in milliseconds OR infinity if the task is unknown.
|
||||
*/
|
||||
double estimate(String taskName, long inputSize) {
|
||||
Function estimator = estimators.get(taskName)
|
||||
if (estimator == null) {
|
||||
//log.warn "[K8s] Unable to estimate take ${taskName}: unknown task"
|
||||
return Double.POSITIVE_INFINITY
|
||||
}
|
||||
return estimator.estimate(inputSize)
|
||||
}
|
||||
|
||||
private void fit(HashMap<String, ArrayList<Tuple2<Long, Long>>> data) {
|
||||
estimators = new HashMap<>()
|
||||
for (Map.Entry<String, ArrayList<Tuple2<Long, Long>>> t : data) {
|
||||
Function f = fit(t.value)
|
||||
estimators.put(t.key, f)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Uses linear regression to fit a linear function (y = m * x + n) to the provided observations
|
||||
* @param observations list of tuples (input size, runtime in ms)
|
||||
* @return linear function fitted to the input
|
||||
*/
|
||||
private Function fit(ArrayList<Tuple2<Long, Long>> observations) throws IllegalArgumentException {
|
||||
int n = observations.size()
|
||||
if (n > 1) {
|
||||
double sumX = 0, sumY = 0, sumXY = 0, sumX2 = 0
|
||||
for (Tuple2<Long, Long> o : observations) {
|
||||
double x = (double) o.get(0)
|
||||
double y = (double) o.get(1)
|
||||
sumX += x
|
||||
sumY += y
|
||||
sumXY += x * y
|
||||
sumX2 += x * x
|
||||
}
|
||||
double m = (n * sumXY - sumX * sumY) / (n * sumX2 - sumX * sumX)
|
||||
return new Function(
|
||||
m,
|
||||
(sumY - m * sumX) / n
|
||||
)
|
||||
} else if (n == 1) {
|
||||
// Special case: We only have 1 measurement. We will just assume that the runtime is constant,
|
||||
// because in our observed data, it is.
|
||||
return new Function(0.0, (double) observations[0].get(1))
|
||||
}
|
||||
throw new IllegalArgumentException("requires at least 1 observation")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package recreationaltech.plugin
|
||||
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.config.scopes.Config
|
||||
import nextflow.config.spec.ConfigScope
|
||||
import nextflow.config.spec.ScopeName
|
||||
import nextflow.config.spec.ConfigOption
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.script.dsl.Description
|
||||
|
||||
@CompileStatic
|
||||
@Slf4j
|
||||
@ScopeName("nodeInit")
|
||||
@Description("The nodeInit scope contains options for the pre-workflow execution initialization of nodes")
|
||||
class K8sNodeInitConfig implements ConfigScope {
|
||||
@ConfigOption
|
||||
@Description("enables the pre-workflow execution deployment of pods")
|
||||
final boolean enabled;
|
||||
|
||||
@ConfigOption
|
||||
@Description("the used image")
|
||||
final String image;
|
||||
|
||||
@ConfigOption
|
||||
@Description("the start-command")
|
||||
final List<String> command;
|
||||
|
||||
@ConfigOption
|
||||
@Description("the pod state to wait on")
|
||||
final String wait;
|
||||
|
||||
@ConfigOption
|
||||
@Description("enables cleanup of pre-workflow nodes.")
|
||||
final boolean cleanup;
|
||||
|
||||
K8sNodeInitConfig() {
|
||||
this(Collections.emptyMap())
|
||||
}
|
||||
|
||||
K8sNodeInitConfig(Map opts) {
|
||||
enabled = opts.enabled as boolean
|
||||
image = opts.image as String
|
||||
command = opts.command as List<String>
|
||||
wait = opts.wait as String
|
||||
cleanup = opts.cleanup as boolean
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
package recreationaltech.plugin
|
||||
|
||||
import groovy.util.logging.Slf4j
|
||||
import recreationaltech.plugin.client.K8sClient
|
||||
import recreationaltech.plugin.model.PodHostMount
|
||||
import recreationaltech.plugin.model.PodSpecBuilder
|
||||
|
||||
@Slf4j
|
||||
class K8sNodeInitDeployer {
|
||||
private K8sClient client
|
||||
private K8sConfig config
|
||||
|
||||
K8sNodeInitDeployer(K8sClient client, K8sConfig config) {
|
||||
this.client = client
|
||||
this.config = config
|
||||
}
|
||||
|
||||
void deploy() {
|
||||
final init = config.nodeInit
|
||||
if ( !init?.enabled )
|
||||
return
|
||||
|
||||
log.info("deploying init pods")
|
||||
|
||||
final nodes = getNodes()
|
||||
for ( String nodeName : nodes ) {
|
||||
log.info(" ... deploying to " + nodeName)
|
||||
final spec = makePodSpec(init, nodeName)
|
||||
client.podCreate(spec)
|
||||
}
|
||||
|
||||
log.info("waiting for init pods")
|
||||
waitForPods(nodes)
|
||||
}
|
||||
|
||||
void cleanup() {
|
||||
final init = config.nodeInit
|
||||
if ( !init?.enabled || !init?.cleanup )
|
||||
return
|
||||
|
||||
final nodes = getNodes()
|
||||
for ( String nodeName : nodes ) {
|
||||
final podName = buildPodName(nodeName)
|
||||
client.podDelete(podName)
|
||||
}
|
||||
}
|
||||
|
||||
String buildCleanupScript() {
|
||||
if ( !config.nodeInit?.enabled || !config.nodeInit?.cleanup )
|
||||
return "#!/usr/bin/env/bash\nexit 0"
|
||||
|
||||
final nodes = getNodes()
|
||||
final podNames = nodes.collect {buildPodName(it)}
|
||||
|
||||
String script = '''
|
||||
#!/usr/bin/env bash
|
||||
set -u
|
||||
|
||||
PODS=('''
|
||||
for ( String podName : podNames ) {
|
||||
script += "\"${podName}\"\n"
|
||||
}
|
||||
script += '''
|
||||
)
|
||||
|
||||
SERVICE_ACCOUNT_DIR="/var/run/secrets/kubernetes.io/serviceaccount"
|
||||
TOKEN="$(cat "${SERVICE_ACCOUNT_DIR}/token")"
|
||||
NAMESPACE="$(cat "${SERVICE_ACCOUNT_DIR}/namespace")"
|
||||
CA_CERT="${SERVICE_ACCOUNT_DIR}/ca.crt"
|
||||
|
||||
KUBE_API="https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS:-443}"
|
||||
|
||||
for POD in "${PODS[@]}"; do
|
||||
echo "Deleting pod: ${POD}"
|
||||
|
||||
HTTP_CODE="$(
|
||||
curl \\
|
||||
--silent \\
|
||||
--show-error \\
|
||||
--output /tmp/delete-pod-response.json \\
|
||||
--write-out "%{http_code}" \\
|
||||
--request DELETE \\
|
||||
--cacert "${CA_CERT}" \\
|
||||
--header "Authorization: Bearer ${TOKEN}" \\
|
||||
--header "Accept: application/json" \\
|
||||
"${KUBE_API}/api/v1/namespaces/${NAMESPACE}/pods/${POD}"
|
||||
)"
|
||||
|
||||
case "${HTTP_CODE}" in
|
||||
200|202)
|
||||
echo "Deleted pod: ${POD}"
|
||||
;;
|
||||
404)
|
||||
echo "Pod already absent: ${POD}"
|
||||
;;
|
||||
*)
|
||||
echo "Failed to delete pod: ${POD}; HTTP ${HTTP_CODE}" >&2
|
||||
cat /tmp/delete-pod-response.json >&2 || true
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
exit 0
|
||||
'''
|
||||
return script
|
||||
}
|
||||
|
||||
private List<String> getNodes() {
|
||||
final resp = client.nodeList()
|
||||
ArrayList<String> nodes = new ArrayList<String>()
|
||||
for ( Map item: resp.items ) {
|
||||
nodes.add(item.metadata.name as String)
|
||||
}
|
||||
return nodes
|
||||
}
|
||||
|
||||
public static String buildPodName(String nodeName) {
|
||||
// TODO: Remove anything that is not lowercase alpha-numeric or dash
|
||||
String sanitizedNodeName = nodeName.toLowerCase()
|
||||
String name = "nf-init-${sanitizedNodeName}"
|
||||
if ( name.length() > 63 )
|
||||
name = name.substring(0, 63)
|
||||
return name
|
||||
}
|
||||
|
||||
private Map makePodSpec(K8sNodeInitConfig config, String nodeName) {
|
||||
ArrayList<PodHostMount> mounts = new ArrayList<PodHostMount>()
|
||||
mounts.add(new PodHostMount("/sys", "/sys"))
|
||||
mounts.add(new PodHostMount("/dev", "/dev"))
|
||||
mounts.add(new PodHostMount("/lib/modules", "/lib/modules"))
|
||||
|
||||
PodSpecBuilder builder = new PodSpecBuilder()
|
||||
return builder.withNodeName(nodeName)
|
||||
.withImageName(config.image)
|
||||
.withCommand(config.command)
|
||||
.withPrivileged(true)
|
||||
.withHostMounts(mounts)
|
||||
.withPodName(buildPodName(nodeName))
|
||||
.withPort(8080)
|
||||
.build()
|
||||
}
|
||||
|
||||
private void waitForPods(List<String> nodes) {
|
||||
if ( config.nodeInit.wait == 'Running' ) {
|
||||
for (String nodeName : nodes) {
|
||||
String podName = buildPodName(nodeName)
|
||||
while (true) {
|
||||
sleep 1000
|
||||
final state = client.podState(podName)
|
||||
if (state && !state.containsKey('waiting')) {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if ( config.nodeInit.wait == 'Succeeded' ) {
|
||||
for ( String nodeName : nodes ) {
|
||||
final String podName = buildPodName(nodeName)
|
||||
final currentState = client.podState(podName)
|
||||
if ( currentState && currentState?.running instanceof Map ) {
|
||||
try {
|
||||
while (true) {
|
||||
sleep 10000
|
||||
final state = client.podState(podName)
|
||||
if (state && !(state?.running instanceof Map)) {
|
||||
println "$podName has changed from running state $state"
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception e) {
|
||||
println "Caught exception while waiting for ${podName} to stop running"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package recreationaltech.plugin
|
||||
|
||||
import nextflow.util.Duration
|
||||
|
||||
class K8sNoiseRuntimeEstimator extends K8sRuntimeEstimator {
|
||||
// Magnitude of the noise we add
|
||||
private long noiseMag
|
||||
|
||||
private HashMap<String, ArrayList<Tuple2<Long, Long>>> data;
|
||||
private synchronized Random rnd;
|
||||
|
||||
/**
|
||||
* Initialize the estimator with data recorded by K8sRuntimeRecorder
|
||||
* @param dataFilePath
|
||||
* @param noiseMagnitude magnitude of the noise added to recordings
|
||||
*/
|
||||
K8sNoiseRuntimeEstimator(String dataFilePath, Duration noiseMagnitude) {
|
||||
this.data = parseDataFile(dataFilePath)
|
||||
this.noiseMag = noiseMagnitude.toMillis()
|
||||
this.rnd = new Random()
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes the runtime estimator with statically known data.
|
||||
* @param data Map from task name to list of recordings, where each recording is a tuple (input-size, runtime-in-ms)
|
||||
*/
|
||||
K8sNoiseRuntimeEstimator(HashMap<String, ArrayList<Tuple2<Long, Long>>> data, Duration noiseMagnitude) {
|
||||
this.data = data;
|
||||
this.noiseMag = noiseMagnitude.toMillis()
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns an estimation of the task runtime in milliseconds
|
||||
* @param taskName the name of the task
|
||||
* @param inputSize the total input size in bytes
|
||||
* @return estimated task runtime in milliseconds OR infinity if the task is unknown.
|
||||
*/
|
||||
@Override
|
||||
double estimate(String taskName, long inputSize) {
|
||||
ArrayList recordings = data.get(taskName)
|
||||
for (Tuple2<Long, Long> recording : recordings) {
|
||||
if (recording.get(0).longValue() == inputSize) {
|
||||
double noise = rnd.nextDouble(-1.0, 1.0) * (double)noiseMag
|
||||
long ms = recording.get(1).longValue()
|
||||
return (double)ms + noise
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright 2025, Seqera Labs
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -21,12 +21,14 @@ import nextflow.plugin.BasePlugin
|
||||
import org.pf4j.PluginWrapper
|
||||
|
||||
/**
|
||||
* The plugin entry point
|
||||
* Kubernetes plugin entry point
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class K8sDvfsPlugin extends BasePlugin {
|
||||
class K8sPlugin extends BasePlugin {
|
||||
|
||||
K8sDvfsPlugin(PluginWrapper wrapper) {
|
||||
K8sPlugin(PluginWrapper wrapper) {
|
||||
super(wrapper)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package recreationaltech.plugin
|
||||
|
||||
import groovy.util.logging.Slf4j
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
|
||||
/**
|
||||
* Provides runtime estimates for tasks based on task-name and input file size
|
||||
*/
|
||||
@Slf4j
|
||||
abstract class K8sRuntimeEstimator {
|
||||
/**
|
||||
* Returns an estimation of the task runtime in milliseconds
|
||||
* @param handler the task handler
|
||||
* @return estimated task runtime in milliseconds OR infinity if the task is unknown.
|
||||
*/
|
||||
double estimate(K8sTaskHandler handler) {
|
||||
long x = getTaskHandlerInputSize(handler)
|
||||
return estimate(handler.task.processor.name, x)
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns an estimation of the task runtime in milliseconds
|
||||
* @param taskName the name of the task
|
||||
* @param inputSize the total input size in bytes
|
||||
* @return estimated task runtime in milliseconds OR infinity if the task is unknown.
|
||||
*/
|
||||
abstract double estimate(String taskName, long inputSize);
|
||||
|
||||
protected long getTaskHandlerInputSize(K8sTaskHandler handler) {
|
||||
long inputSizeSum = 0
|
||||
// File input
|
||||
def inputFiles = handler.task.getInputFilesMap()
|
||||
for (Map.Entry<String, Path> f : inputFiles) {
|
||||
try {
|
||||
inputSizeSum += Files.size(f.value)
|
||||
} catch (IOException ex) {
|
||||
log.error "[K8s] failed to get size of input file ${f.value} of task ${task.task.name}: ${ex.message}"
|
||||
}
|
||||
}
|
||||
// Non file input
|
||||
def inputs = handler.task.getInputs()
|
||||
for (Map.Entry i : inputs) {
|
||||
inputSizeSum += i.value.toString().length()
|
||||
}
|
||||
|
||||
return inputSizeSum
|
||||
}
|
||||
|
||||
/// @brief Parses the runtime recording data file
|
||||
/// @return Map from task name to list of recordings, where each recording is a tuple (input-size, runtime-in-ms)
|
||||
protected HashMap<String, ArrayList<Tuple2<Long, Long>>> parseDataFile(String dataFilePath) {
|
||||
HashMap<String, ArrayList<Tuple2<Long, Long>>> data = new HashMap<>();
|
||||
try {
|
||||
BufferedReader reader = new BufferedReader(new FileReader(dataFilePath))
|
||||
String line = reader.readLine()
|
||||
while (line != null) {
|
||||
// <task-name>,<input-size>,<runtime-ms>
|
||||
String[] elems = line.split(",")
|
||||
if (elems.length != 3) {
|
||||
log.warn "[K8s] ${dataFilePath}: Unexpected line ${line}"
|
||||
continue
|
||||
}
|
||||
|
||||
ArrayList<Tuple2<Long, Long>> taskData = data.get(elems[0])
|
||||
if (taskData == null) {
|
||||
taskData = new ArrayList<>()
|
||||
data.put(elems[0], taskData)
|
||||
}
|
||||
try {
|
||||
taskData.add(new Tuple2(Long.parseLong(elems[1]), Long.parseLong(elems[2])))
|
||||
} catch (NumberFormatException ex) {
|
||||
log.error "[K8s] ${dataFilePath} invalid data: ${ex.message}"
|
||||
}
|
||||
line = reader.readLine()
|
||||
}
|
||||
reader.close()
|
||||
} catch (IOException ex) {
|
||||
log.error "[K8s] Failed to load ${dataFilePath}: ${ex.message}"
|
||||
}
|
||||
return data
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package recreationaltech.plugin
|
||||
|
||||
record K8sRuntimeRecord(
|
||||
String taskName,
|
||||
long inputSize,
|
||||
long runtimeMillis
|
||||
) {}
|
||||
@@ -0,0 +1,67 @@
|
||||
package recreationaltech.plugin
|
||||
|
||||
import groovy.util.logging.Slf4j
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
|
||||
@Slf4j
|
||||
class K8sRuntimeRecorder {
|
||||
private final boolean enabled
|
||||
private final String recordPath
|
||||
|
||||
private ArrayList<K8sRuntimeRecord> records
|
||||
|
||||
K8sRuntimeRecorder(boolean enabled, String recordPath) {
|
||||
this.enabled = enabled
|
||||
this.recordPath = recordPath
|
||||
this.records = new ArrayList<>()
|
||||
}
|
||||
|
||||
/**
|
||||
* Records the runtime of a task.
|
||||
* This should be called for a finished task, which has its
|
||||
* start and end timestamps set.
|
||||
* @param task
|
||||
*/
|
||||
synchronized void record(K8sTaskHandler task) {
|
||||
if (!enabled)
|
||||
return;
|
||||
long runtimeMilis = task.completeTimeMillis - task.startTimeMillis
|
||||
long inputSizeSum = 0
|
||||
|
||||
// Input files
|
||||
def inputFiles = task.task.getInputFilesMap()
|
||||
for (Map.Entry<String, Path> f : inputFiles) {
|
||||
try {
|
||||
inputSizeSum += Files.size(f.value)
|
||||
} catch (IOException ex) {
|
||||
log.error "[K8s] failed to get size of input file ${f.value} of task ${task.task.name}: ${ex.message}"
|
||||
}
|
||||
}
|
||||
|
||||
// Non file input
|
||||
def inputs = task.task.getInputs()
|
||||
for (Map.Entry i : inputs) {
|
||||
inputSizeSum += i.value.toString().length()
|
||||
}
|
||||
|
||||
log.info "[K8s] task ${task.task.processor.name} - input ${inputSizeSum} bytes - ran ${runtimeMilis} ms (${task.startTimeMillis} -> ${task.completeTimeMillis})"
|
||||
records.add(new K8sRuntimeRecord(task.task.processor.name, inputSizeSum, runtimeMilis))
|
||||
}
|
||||
|
||||
void write() {
|
||||
if (!enabled)
|
||||
return;
|
||||
try {
|
||||
FileWriter out = new FileWriter(recordPath)
|
||||
for (K8sRuntimeRecord record : records) {
|
||||
out.write("${record.taskName},${record.inputSize},${record.runtimeMillis}\n")
|
||||
}
|
||||
out.close()
|
||||
log.info "[K8s] written runtime recording to ${recordPath}"
|
||||
} catch (IOException err) {
|
||||
log.error "[K8s] failed to write runtime recording ${recordPath}: ${err.message}"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package recreationaltech.plugin
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
|
||||
@CompileStatic
|
||||
class K8sSchedulingDecision {
|
||||
final String nodeName
|
||||
final K8sSchedulingRequest request
|
||||
|
||||
K8sSchedulingDecision(K8sSchedulingRequest request, String nodeName) {
|
||||
this.request = request
|
||||
this.nodeName = nodeName
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package recreationaltech.plugin
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.processor.TaskRun
|
||||
|
||||
@CompileStatic
|
||||
class K8sSchedulingRequest {
|
||||
final K8sTaskHandler handler
|
||||
final TaskRun task
|
||||
final long submitTimeMillis
|
||||
|
||||
K8sSchedulingRequest(K8sTaskHandler handler) {
|
||||
this.handler = handler
|
||||
this.task = handler.task
|
||||
this.submitTimeMillis = System.currentTimeMillis()
|
||||
}
|
||||
|
||||
String getTaskName() {
|
||||
return task.processor.name
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package recreationaltech.plugin
|
||||
|
||||
interface K8sSchedulingStrategy {
|
||||
/**
|
||||
* Selects the next task to run from the given task queue.
|
||||
* If the scheduler should wait, returns {@code null} instead
|
||||
*
|
||||
* @param scheduler the calling scheduler object
|
||||
* @param queue Pending scheduling requests
|
||||
* @return A launch decision, or {@code null} when no task should be launched now
|
||||
*/
|
||||
K8sSchedulingDecision schedule(K8sTaskScheduler scheduler, List<K8sSchedulingRequest> queue)
|
||||
|
||||
/**
|
||||
* Decides if the scheduler should immediately invoke the @ref schedule method of the strategy
|
||||
* @param scheduler the calling scheduler object
|
||||
* @param queue pending scheduling requests
|
||||
* @return {@code true} if @ref schedule should immediately be called
|
||||
*/
|
||||
boolean scheduleImmediately(K8sTaskScheduler scheduler, List<K8sSchedulingRequest> queue)
|
||||
|
||||
/**
|
||||
* Called when a task has finished execution.
|
||||
* @param task the task
|
||||
*/
|
||||
void taskFinished(K8sTaskHandler task)
|
||||
}
|
||||
@@ -0,0 +1,586 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin
|
||||
|
||||
import java.nio.file.FileAlreadyExistsException
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.time.Instant
|
||||
import java.time.format.DateTimeFormatter
|
||||
|
||||
import groovy.transform.CompileDynamic
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.SysEnv
|
||||
import nextflow.container.ContainerHelper
|
||||
import nextflow.container.DockerBuilder
|
||||
import nextflow.exception.NodeTerminationException
|
||||
import recreationaltech.plugin.client.PodUnschedulableException
|
||||
import nextflow.exception.ProcessSubmitException
|
||||
import nextflow.executor.BashWrapperBuilder
|
||||
import nextflow.fusion.FusionAwareTask
|
||||
import recreationaltech.plugin.client.K8sClient
|
||||
import recreationaltech.plugin.client.K8sResponseException
|
||||
import recreationaltech.plugin.model.PodEnv
|
||||
import recreationaltech.plugin.model.PodOptions
|
||||
import recreationaltech.plugin.model.PodSpecBuilder
|
||||
import recreationaltech.plugin.model.ResourceType
|
||||
import nextflow.processor.TaskHandler
|
||||
import nextflow.processor.TaskRun
|
||||
import nextflow.processor.TaskStatus
|
||||
import nextflow.trace.TraceRecord
|
||||
import nextflow.util.Escape
|
||||
import nextflow.util.PathTrie
|
||||
import nextflow.util.TestOnly
|
||||
/**
|
||||
* Implements the {@link TaskHandler} interface for Kubernetes pods
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class K8sTaskHandler extends TaskHandler implements FusionAwareTask {
|
||||
|
||||
@Lazy
|
||||
static private final String OWNER = {
|
||||
if( System.getenv('NXF_OWNER') ) {
|
||||
return System.getenv('NXF_OWNER')
|
||||
}
|
||||
else {
|
||||
def p = ['bash','-c','echo -n $(id -u):$(id -g)'].execute();
|
||||
p.waitFor()
|
||||
return p.text
|
||||
}
|
||||
|
||||
} ()
|
||||
|
||||
private ResourceType resourceType = ResourceType.Pod
|
||||
|
||||
private K8sClient client
|
||||
|
||||
private String podName
|
||||
|
||||
private BashWrapperBuilder builder
|
||||
|
||||
private Path outputFile
|
||||
|
||||
private Path errorFile
|
||||
|
||||
private Path exitFile
|
||||
|
||||
private Map state
|
||||
|
||||
private long timestamp
|
||||
|
||||
private K8sExecutor executor
|
||||
|
||||
private String runsOnNode = null
|
||||
|
||||
K8sTaskHandler( TaskRun task, K8sExecutor executor ) {
|
||||
super(task)
|
||||
this.executor = executor
|
||||
this.client = executor.getClient()
|
||||
this.outputFile = task.workDir.resolve(TaskRun.CMD_OUTFILE)
|
||||
this.errorFile = task.workDir.resolve(TaskRun.CMD_ERRFILE)
|
||||
this.exitFile = task.workDir.resolve(TaskRun.CMD_EXIT)
|
||||
this.resourceType = executor.k8sConfig.useJobResource() ? ResourceType.Job : ResourceType.Pod
|
||||
}
|
||||
|
||||
@TestOnly
|
||||
protected K8sTaskHandler() {}
|
||||
|
||||
/**
|
||||
* @return The workflow execution unique run name
|
||||
*/
|
||||
protected String getRunName() {
|
||||
executor.session.runName
|
||||
}
|
||||
|
||||
protected String getPodName() {
|
||||
return podName
|
||||
}
|
||||
|
||||
protected K8sConfig getK8sConfig() { executor.getK8sConfig() }
|
||||
|
||||
protected boolean useJobResource() { resourceType==ResourceType.Job }
|
||||
|
||||
protected List<String> getContainerMounts() {
|
||||
|
||||
if( !k8sConfig.getAutoMountHostPaths() ) {
|
||||
return Collections.<String>emptyList()
|
||||
}
|
||||
|
||||
// get input files paths
|
||||
final List<Path> paths = DockerBuilder.inputFilesToPaths(builder.getInputFiles())
|
||||
final List<Path> binDirs = builder.binDirs
|
||||
final Path workDir = builder.workDir
|
||||
// add standard paths
|
||||
if( binDirs )
|
||||
paths.addAll(binDirs)
|
||||
if( workDir )
|
||||
paths << workDir
|
||||
|
||||
def trie = new PathTrie()
|
||||
paths.each { trie.add(it) }
|
||||
|
||||
// defines the mounts
|
||||
trie.longest()
|
||||
}
|
||||
|
||||
protected BashWrapperBuilder createBashWrapper(TaskRun task) {
|
||||
return fusionEnabled()
|
||||
? fusionLauncher()
|
||||
: new K8sWrapperBuilder(task)
|
||||
}
|
||||
|
||||
protected List<String> classicSubmitCli(TaskRun task) {
|
||||
final workDir = Escape.path(task.workDir)
|
||||
|
||||
final result = new ArrayList(BashWrapperBuilder.BASH)
|
||||
result.add('-o')
|
||||
result.add('pipefail')
|
||||
result.add('-c')
|
||||
result.add("bash ${workDir}/${TaskRun.CMD_RUN} 2>&1 | tee ${workDir}/${TaskRun.CMD_LOG}")
|
||||
return result
|
||||
}
|
||||
|
||||
protected List<String> getSubmitCommand(TaskRun task) {
|
||||
return fusionEnabled()
|
||||
? fusionSubmitCli()
|
||||
: classicSubmitCli(task)
|
||||
}
|
||||
|
||||
protected String getSyntheticPodName(TaskRun task) {
|
||||
final suffix = System.currentTimeMillis().toString().md5()[-5..-1]
|
||||
return "nf-${task.hash}-${suffix}"
|
||||
}
|
||||
|
||||
protected String getOwner() { OWNER }
|
||||
|
||||
protected Boolean fixOwnership() {
|
||||
ContainerHelper.fixOwnership(task.containerConfig)
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a Pod specification that executed that specified task
|
||||
*
|
||||
* @param task A {@link TaskRun} instance representing the task to execute
|
||||
* @return A {@link Map} object modeling a Pod specification
|
||||
*/
|
||||
protected Map newSubmitRequest(TaskRun task) {
|
||||
return newSubmitRequest(task, null)
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a Pod specification that executed that specified task
|
||||
*
|
||||
* @param task A {@link TaskRun} instance representing the task to execute
|
||||
* @param nodeName The kubernetes node on which the task should run or {@code null}, if no specific node is requested
|
||||
* @return A {@link Map} object modeling a Pod specification
|
||||
*/
|
||||
protected Map newSubmitRequest(TaskRun task, String nodeName) {
|
||||
def imageName = task.container
|
||||
if( !imageName )
|
||||
throw new ProcessSubmitException("Missing container image for process `$task.processor.name`")
|
||||
|
||||
try {
|
||||
newSubmitRequest0(task, imageName, nodeName)
|
||||
}
|
||||
catch( Throwable e ) {
|
||||
throw new ProcessSubmitException("Failed to submit K8s ${resourceType.lower()} -- Cause: ${e.message ?: e}", e)
|
||||
}
|
||||
}
|
||||
|
||||
protected boolean entrypointOverride() {
|
||||
return executor.getK8sConfig().entrypointOverride()
|
||||
}
|
||||
|
||||
protected boolean cpuLimitsEnabled() {
|
||||
return executor.getK8sConfig().cpuLimitsEnabled()
|
||||
}
|
||||
|
||||
protected Map newSubmitRequest0(TaskRun task, String imageName, String nodeName) {
|
||||
|
||||
final launcher = getSubmitCommand(task)
|
||||
final taskCfg = task.getConfig()
|
||||
|
||||
final clientConfig = client.config
|
||||
final builder = new PodSpecBuilder()
|
||||
.withImageName(imageName)
|
||||
.withPodName(getSyntheticPodName(task))
|
||||
.withNamespace(clientConfig.namespace)
|
||||
.withServiceAccount(clientConfig.serviceAccount)
|
||||
.withLabels(getLabels(task))
|
||||
.withAnnotations(getAnnotations())
|
||||
.withPodOptions(getPodOptions())
|
||||
.withCpuLimits(cpuLimitsEnabled())
|
||||
|
||||
// when `entrypointOverride` is false the launcher is run via `args` instead of `command`
|
||||
// to not override the container entrypoint
|
||||
if( !entrypointOverride() ) {
|
||||
builder.withArgs(launcher)
|
||||
}
|
||||
else {
|
||||
builder.withCommand(launcher)
|
||||
}
|
||||
|
||||
if( nodeName )
|
||||
builder.withNodeName(nodeName)
|
||||
|
||||
// note: task environment is managed by the task bash wrapper
|
||||
// do not add here -- see also #680
|
||||
if( fixOwnership() )
|
||||
builder.withEnv(PodEnv.value('NXF_OWNER', getOwner()))
|
||||
|
||||
if( SysEnv.containsKey('NXF_DEBUG') )
|
||||
builder.withEnv(PodEnv.value('NXF_DEBUG', SysEnv.get('NXF_DEBUG')))
|
||||
|
||||
// add computing resources
|
||||
final cpus = taskCfg.getCpus()
|
||||
final mem = taskCfg.getMemory()
|
||||
final disk = taskCfg.getDisk()
|
||||
final acc = taskCfg.getAccelerator()
|
||||
if( cpus )
|
||||
builder.withCpus(cpus)
|
||||
if( mem )
|
||||
builder.withMemory(mem)
|
||||
if( disk )
|
||||
builder.withDisk(disk)
|
||||
if( acc )
|
||||
builder.withAccelerator(acc)
|
||||
|
||||
final List<String> hostMounts = getContainerMounts()
|
||||
for( String mount : hostMounts ) {
|
||||
builder.withHostMount(mount,mount)
|
||||
}
|
||||
|
||||
if ( taskCfg.time ) {
|
||||
final duration = taskCfg.getTime()
|
||||
builder.withActiveDeadline(duration.toSeconds() as int)
|
||||
}
|
||||
|
||||
if ( fusionEnabled() ) {
|
||||
if( fusionConfig().privileged() )
|
||||
builder.withPrivileged(true)
|
||||
else {
|
||||
final device= k8sConfig.fuseDevicePlugin()
|
||||
builder.withResourcesLimits(device)
|
||||
}
|
||||
|
||||
final env = fusionLauncher().fusionEnv()
|
||||
for( Map.Entry<String,String> it : env )
|
||||
builder.withEnv(PodEnv.value(it.key, it.value))
|
||||
}
|
||||
|
||||
return useJobResource()
|
||||
? builder.buildAsJob()
|
||||
: builder.build()
|
||||
}
|
||||
|
||||
protected PodOptions getPodOptions() {
|
||||
// merge the pod options provided in the k8s config
|
||||
// with the ones in process config
|
||||
def opt1 = k8sConfig.getPodOptions()
|
||||
def opt2 = taskPodOptions()
|
||||
return opt1 + opt2
|
||||
}
|
||||
|
||||
protected PodOptions taskPodOptions() {
|
||||
new PodOptions((List)task.getConfig().get('pod'))
|
||||
}
|
||||
|
||||
protected Map<String,String> getLabels(TaskRun task) {
|
||||
final result = new LinkedHashMap<String,String>(10)
|
||||
final labels = k8sConfig.getLabels()
|
||||
if( labels ) {
|
||||
result.putAll(labels)
|
||||
}
|
||||
final resLabels = task.config.getResourceLabels()
|
||||
if( resLabels )
|
||||
result.putAll(resLabels)
|
||||
result.'nextflow.io/app' = 'nextflow'
|
||||
result.'nextflow.io/runName' = getRunName()
|
||||
result.'nextflow.io/taskName' = task.getName()
|
||||
result.'nextflow.io/processName' = task.getProcessor().getName()
|
||||
result.'nextflow.io/sessionId' = "uuid-${executor.getSession().uniqueId}" as String
|
||||
if( task.config.queue )
|
||||
result.'nextflow.io/queue' = task.config.queue
|
||||
return result
|
||||
}
|
||||
|
||||
protected Map getAnnotations() {
|
||||
k8sConfig.getAnnotations()
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepares the task execution and enqueues it at the scheduler
|
||||
*/
|
||||
@Override
|
||||
@CompileDynamic
|
||||
void submit() {
|
||||
builder = createBashWrapper(task)
|
||||
builder.build()
|
||||
log.info "[K8s] submitting task ${this.task.name}"
|
||||
executor.taskScheduler.submit(this)
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new K8s pod executing the associated task
|
||||
*/
|
||||
@CompileDynamic
|
||||
void submitNow(String nodeName) {
|
||||
final req = newSubmitRequest(task, nodeName)
|
||||
final resp = useJobResource()
|
||||
? client.jobCreate(req, yamlDebugPath())
|
||||
: client.podCreate(req, yamlDebugPath())
|
||||
|
||||
if( !resp.metadata?.name )
|
||||
throw new K8sResponseException("Missing created ${resourceType.lower()} name", resp)
|
||||
this.podName = resp.metadata.name
|
||||
this.status = TaskStatus.SUBMITTED
|
||||
}
|
||||
|
||||
@CompileDynamic
|
||||
protected Path yamlDebugPath() {
|
||||
boolean debug = k8sConfig.getDebug().getYaml()
|
||||
return debug ? task.workDir.resolve('.command.yaml') : null
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Retrieve the submitted pod state
|
||||
*/
|
||||
protected Map getState() {
|
||||
final now = System.currentTimeMillis()
|
||||
try {
|
||||
final delta = now - timestamp;
|
||||
if( !state || delta >= 1_000) {
|
||||
def newState = useJobResource()
|
||||
? client.jobState(podName)
|
||||
: client.podState(podName)
|
||||
if( newState ) {
|
||||
log.trace "[K8s] Get ${resourceType.lower()}=$podName state=$newState"
|
||||
state = newState
|
||||
timestamp = now
|
||||
}
|
||||
}
|
||||
return state
|
||||
}
|
||||
catch (NodeTerminationException | PodUnschedulableException e) {
|
||||
// create a synthetic `state` object adding an extra `nodeTermination`
|
||||
// attribute to return the error to the caller method
|
||||
final instant = Instant.now()
|
||||
final result = new HashMap(10)
|
||||
result.terminated = [startedAt:instant.toString(), finishedAt:instant.toString()]
|
||||
result.nodeTermination = e
|
||||
timestamp = now
|
||||
state = result
|
||||
return state
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean checkIfRunning() {
|
||||
if( !podName )
|
||||
return false
|
||||
if(isSubmitted()) {
|
||||
def state = getState()
|
||||
// include `terminated` state to allow the handler status to progress
|
||||
if (state && (state.running != null || state.terminated)) {
|
||||
status = TaskStatus.RUNNING
|
||||
determineNode()
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
long getEpochMilli(String timeString) {
|
||||
final time = DateTimeFormatter.ISO_INSTANT.parse(timeString)
|
||||
return Instant.from(time).toEpochMilli()
|
||||
}
|
||||
|
||||
/**
|
||||
* Update task start and end times based on pod timestamps.
|
||||
* We update timestamps because it's possible for a task to run so quickly
|
||||
* (less than 1 second) that it skips right over the RUNNING status.
|
||||
* If this happens, the startTimeMillis never gets set and remains equal to 0.
|
||||
* To make sure startTimeMillis is non-zero we update it with the pod start time.
|
||||
* We update completeTimeMillis from the same pod info to be consistent.
|
||||
*/
|
||||
void updateTimestamps(Map terminated) {
|
||||
try {
|
||||
startTimeMillis = getEpochMilli(terminated.startedAt as String)
|
||||
completeTimeMillis = getEpochMilli(terminated.finishedAt as String)
|
||||
} catch( Exception e ) {
|
||||
log.debug "Failed updating timestamps '${terminated.toString()}'", e
|
||||
// Only update if startTimeMillis hasn't already been set.
|
||||
// If startTimeMillis _has_ been set, then both startTimeMillis
|
||||
// and completeTimeMillis will have been set with the normal
|
||||
// TaskHandler mechanism, so there's no need to reset them here.
|
||||
if (!startTimeMillis) {
|
||||
startTimeMillis = System.currentTimeMillis()
|
||||
completeTimeMillis = System.currentTimeMillis()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean checkIfCompleted() {
|
||||
if( !podName )
|
||||
return false
|
||||
|
||||
final state = getState()
|
||||
if( state && state.terminated ) {
|
||||
if( state.nodeTermination instanceof NodeTerminationException ||
|
||||
state.nodeTermination instanceof PodUnschedulableException ) {
|
||||
// keep track of the node termination error
|
||||
task.error = (Throwable) state.nodeTermination
|
||||
// mark the task as ABORTED since thr failure is caused by a node failure
|
||||
task.aborted = true
|
||||
}
|
||||
else {
|
||||
// finalize the task
|
||||
// read the exit code from the K8s container terminated state, if missing
|
||||
// take the exit code from the `.exitcode` file created by nextflow
|
||||
// the rationale is that in case of error (e.g. OOMKilled, pod eviction), the exit code from
|
||||
// the K8s API is more reliable because the container may terminate before the exit file is written
|
||||
// See https://github.com/nextflow-io/nextflow/issues/6436
|
||||
// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#containerstateterminated-v1-core
|
||||
log.trace("[k8s] Container Terminated state ${state.terminated}")
|
||||
final k8sExitCode = (state.terminated as Map)?.exitCode as Integer
|
||||
task.exitStatus = k8sExitCode != null ? k8sExitCode : readExitFile()
|
||||
task.stdout = outputFile
|
||||
task.stderr = errorFile
|
||||
}
|
||||
status = TaskStatus.COMPLETED
|
||||
saveJobLogOnError(task)
|
||||
deleteJobIfSuccessful(task)
|
||||
updateTimestamps(state.terminated as Map)
|
||||
determineNode()
|
||||
|
||||
// Signal the scheduler that this task has finished running
|
||||
if (executor != null) {
|
||||
executor.taskScheduler.taskFinished(this)
|
||||
executor.runtimeRecorder.record(this)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
protected void saveJobLogOnError(TaskRun task) {
|
||||
if( task.isSuccess() )
|
||||
return
|
||||
|
||||
if( errorFile && !errorFile.empty() )
|
||||
return
|
||||
|
||||
final session = executor.getSession()
|
||||
if( session.isAborted() || session.isCancelled() || session.isTerminated() )
|
||||
return
|
||||
|
||||
try {
|
||||
final stream = useJobResource()
|
||||
? client.jobLog(podName)
|
||||
: client.podLog(podName)
|
||||
Files.copy(stream, task.workDir.resolve(TaskRun.CMD_LOG))
|
||||
}
|
||||
catch( FileAlreadyExistsException e ) {
|
||||
log.debug "Log file already exists for ${resourceType.lower()} $podName", e
|
||||
}
|
||||
catch( Exception e ) {
|
||||
log.warn "Failed to copy log for ${resourceType.lower()} $podName", e
|
||||
}
|
||||
}
|
||||
|
||||
protected int readExitFile() {
|
||||
try {
|
||||
exitFile.text as Integer
|
||||
}
|
||||
catch( Exception e ) {
|
||||
log.debug "[K8s] Cannot read exitstatus for task: `$task.name` | ${e.message}"
|
||||
return Integer.MAX_VALUE
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Terminates the current task execution
|
||||
*/
|
||||
@Override
|
||||
protected void killTask() {
|
||||
if( !podName )
|
||||
return
|
||||
|
||||
if( cleanupDisabled() )
|
||||
return
|
||||
|
||||
log.trace "[K8s] deleting ${resourceType.lower()} name=$podName"
|
||||
delete0(podName)
|
||||
}
|
||||
|
||||
protected boolean cleanupDisabled() {
|
||||
!k8sConfig.getCleanup()
|
||||
}
|
||||
|
||||
protected void deleteJobIfSuccessful(TaskRun task) {
|
||||
if( !podName )
|
||||
return
|
||||
|
||||
if( cleanupDisabled() )
|
||||
return
|
||||
|
||||
// preserve failed pods for debugging purposes
|
||||
if( !task.isSuccess() )
|
||||
return
|
||||
|
||||
// k8s cluster will cleanup job on its own if TTL is set
|
||||
if( useJobResource() && getPodOptions().getTtlSecondsAfterFinished() != null )
|
||||
return
|
||||
|
||||
delete0(podName)
|
||||
}
|
||||
|
||||
private void delete0(String podName) {
|
||||
try {
|
||||
if ( useJobResource() )
|
||||
client.jobDelete(podName)
|
||||
else
|
||||
client.podDelete(podName)
|
||||
}
|
||||
catch( Exception e ) {
|
||||
log.warn "Unable to delete ${resourceType.lower()}: $podName -- see the log file for details", e
|
||||
}
|
||||
}
|
||||
|
||||
private void determineNode() {
|
||||
try {
|
||||
if ( k8sConfig.fetchNodeName() && !runsOnNode )
|
||||
runsOnNode = client.getNodeOfPod( podName )
|
||||
} catch ( Exception e ) {
|
||||
log.warn ("Unable to get the node name of pod $podName -- see the log file for details", e)
|
||||
}
|
||||
}
|
||||
|
||||
TraceRecord getTraceRecord() {
|
||||
final result = super.getTraceRecord()
|
||||
result.put('native_id', podName)
|
||||
result.put( 'hostname', runsOnNode )
|
||||
return result
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package recreationaltech.plugin
|
||||
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.util.Duration
|
||||
import java.util.concurrent.LinkedBlockingQueue
|
||||
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class K8sTaskScheduler implements Runnable {
|
||||
private final Duration interval
|
||||
private final K8sSchedulingStrategy strategy
|
||||
private final LinkedBlockingQueue<K8sSchedulingRequest> queue = new LinkedBlockingQueue<>()
|
||||
private String[] nodes
|
||||
|
||||
private synchronized boolean shouldStop
|
||||
|
||||
K8sTaskScheduler(String[] nodes, K8sSchedulingStrategy strategy, Duration interval) {
|
||||
this.interval = interval
|
||||
this.strategy = strategy
|
||||
this.nodes = nodes
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a task to the queue of outstanding tasks
|
||||
* @param handler
|
||||
*/
|
||||
void submit(K8sTaskHandler handler) {
|
||||
log.info "[K8s] received queued task ${handler.task.name}"
|
||||
queue.add(new K8sSchedulingRequest(handler))
|
||||
final pending = new ArrayList<K8sSchedulingRequest>(queue)
|
||||
if (strategy.scheduleImmediately(this, pending))
|
||||
schedule()
|
||||
}
|
||||
|
||||
/**
|
||||
* Notify the scheduler that a task has finished execution
|
||||
* @param handler
|
||||
*/
|
||||
void taskFinished(K8sTaskHandler handler) {
|
||||
strategy.taskFinished(handler)
|
||||
|
||||
/* If we have pending tasks, now would be a good time to schedule a new one.
|
||||
* Because resources were freed right now */
|
||||
if (queue.size() > 0)
|
||||
schedule()
|
||||
}
|
||||
|
||||
protected synchronized void schedule() {
|
||||
final pending = new ArrayList<K8sSchedulingRequest>(queue)
|
||||
final decision = strategy.schedule(this, pending)
|
||||
|
||||
if (!decision)
|
||||
return
|
||||
|
||||
if (!queue.remove(decision.request)) {
|
||||
log.warn "[K8s] failed to remove selected task from queue ${decision.request.task.name}"
|
||||
return
|
||||
}
|
||||
|
||||
log.info "[K8s] launching queued task ${decision.request.task.name} on node: ${decision.nodeName}"
|
||||
decision.request.handler.submitNow(decision.nodeName)
|
||||
}
|
||||
|
||||
/* Scheduling Strategy Interface */
|
||||
List<String> getNodes() {
|
||||
return nodes.toList()
|
||||
}
|
||||
|
||||
/**
|
||||
* Run is the scheduler threads main function
|
||||
* */
|
||||
void run() {
|
||||
this.shouldStop = false
|
||||
final interval = this.interval
|
||||
if (interval.toMillis() == 0L) {
|
||||
log.info("[K8s] scheduler loop disabled (interval is 0)")
|
||||
return
|
||||
}
|
||||
log.info("[K8s] launched scheduler loop (${interval.toString()} interval)")
|
||||
while (!shouldStop) {
|
||||
sleep(interval.toMillis())
|
||||
schedule()
|
||||
}
|
||||
log.info("[K8s] terminated scheduler loop")
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop terminates the scheduler thread
|
||||
*/
|
||||
void stop() {
|
||||
log.info("[K8s] stopping scheduler loop")
|
||||
this.shouldStop = true
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright 2025, Seqera Labs
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -16,21 +16,23 @@
|
||||
|
||||
package recreationaltech.plugin
|
||||
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.Session
|
||||
import nextflow.trace.TraceObserver
|
||||
import nextflow.trace.TraceObserverFactory
|
||||
import nextflow.executor.BashWrapperBuilder
|
||||
import nextflow.processor.TaskRun
|
||||
import nextflow.util.Escape
|
||||
|
||||
/**
|
||||
* Implements a factory object required to create
|
||||
* the {@link K8sDvfsObserver} instance.
|
||||
* Implements a BASH wrapper for tasks executed by kubernetes cluster
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class K8sDvfsFactory implements TraceObserverFactory {
|
||||
class K8sWrapperBuilder extends BashWrapperBuilder {
|
||||
|
||||
@Override
|
||||
Collection<TraceObserver> create(Session session) {
|
||||
return List.<TraceObserver>of(new K8sDvfsObserver())
|
||||
K8sWrapperBuilder(TaskRun task) {
|
||||
super(task)
|
||||
this.headerScript = "NXF_CHDIR=${Escape.path(task.workDir)}"
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.cli
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.cli.CmdKubeRun
|
||||
import recreationaltech.plugin.K8sDriverLauncher
|
||||
|
||||
/**
|
||||
* Kuberun command implementation logic
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class KubeCommandImpl implements CmdKubeRun.KubeCommand {
|
||||
|
||||
@Override
|
||||
int run(CmdKubeRun cmd, String pipeline, List<String> args) {
|
||||
// create
|
||||
final driver = new K8sDriverLauncher(
|
||||
cmd: cmd,
|
||||
runName: cmd.runName,
|
||||
headImage: cmd.headImage,
|
||||
background: cmd.background(),
|
||||
headCpus: cmd.headCpus,
|
||||
headMemory: cmd.headMemory,
|
||||
headPreScript: cmd.headPreScript,
|
||||
plugins: cmd.plugins )
|
||||
// run it
|
||||
driver.run(pipeline, args)
|
||||
// return exit code
|
||||
return driver.shutdown()
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.client
|
||||
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.util.Duration
|
||||
|
||||
import javax.net.ssl.KeyManager
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
/**
|
||||
* Models the kubernetes cluster client configuration settings
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@EqualsAndHashCode
|
||||
@CompileStatic
|
||||
@Slf4j
|
||||
class ClientConfig {
|
||||
|
||||
boolean verifySsl
|
||||
|
||||
String server
|
||||
|
||||
String namespace
|
||||
|
||||
/**
|
||||
* k8s service account name
|
||||
* https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/
|
||||
*/
|
||||
String serviceAccount
|
||||
|
||||
String token
|
||||
|
||||
byte[] sslCert
|
||||
|
||||
byte[] clientCert
|
||||
|
||||
byte[] clientKey
|
||||
|
||||
KeyManager[] keyManagers
|
||||
|
||||
K8sRetryConfig retryConfig
|
||||
|
||||
/**
|
||||
* Timeout when reading from Input stream when a connection is established to a resource.
|
||||
* If the timeout expires before there is data available for read, a {@link java.net.SocketTimeoutException} is raised
|
||||
*/
|
||||
Duration httpReadTimeout
|
||||
|
||||
/**
|
||||
* Timeout when opening a communications link to the resource referenced by K8sClient request connection
|
||||
* If the timeout expires before there is data available for read, a {@link java.net.SocketTimeoutException} is raised
|
||||
*/
|
||||
Duration httpConnectTimeout
|
||||
|
||||
/**
|
||||
* When true signal that the configuration was retrieved from within a K8s cluster
|
||||
*/
|
||||
boolean isFromCluster
|
||||
|
||||
String getNamespace() { namespace ?: 'default' }
|
||||
|
||||
ClientConfig() {
|
||||
retryConfig = new K8sRetryConfig()
|
||||
}
|
||||
|
||||
String toString() {
|
||||
"${this.class.getSimpleName()}[ server=$server, namespace=$namespace, serviceAccount=$serviceAccount, token=${cut(token)}, sslCert=${cut(sslCert)}, clientCert=${cut(clientCert)}, clientKey=${cut(clientKey)}, verifySsl=$verifySsl, fromFile=$isFromCluster, httpReadTimeout=$httpReadTimeout, httpConnectTimeout=$httpConnectTimeout, retryConfig=$retryConfig ]"
|
||||
}
|
||||
|
||||
private String cut(String str) {
|
||||
if( !str ) return '-'
|
||||
return str.size()<10 ? str : str[0..10].toString() + '..'
|
||||
}
|
||||
|
||||
private String cut(byte[] bytes) {
|
||||
if( !bytes ) return '-'
|
||||
cut(bytes.encodeBase64().toString())
|
||||
}
|
||||
|
||||
static ClientConfig discover(String context, String namespace, String serviceAccount) {
|
||||
new ConfigDiscovery().discover(context, namespace, serviceAccount)
|
||||
}
|
||||
|
||||
static ClientConfig fromNextflowConfig(Map opts, String namespace, String serviceAccount) {
|
||||
final result = new ClientConfig()
|
||||
|
||||
if( opts.server )
|
||||
result.server = opts.server
|
||||
|
||||
if( opts.token )
|
||||
result.token = opts.token
|
||||
else if( opts.tokenFile )
|
||||
result.token = Paths.get(opts.tokenFile.toString()).getText('UTF-8')
|
||||
|
||||
result.namespace = namespace ?: opts.namespace ?: 'default'
|
||||
|
||||
result.serviceAccount = serviceAccount ?: 'default'
|
||||
|
||||
if( opts.verifySsl )
|
||||
result.verifySsl = opts.verifySsl as boolean
|
||||
|
||||
if( opts.sslCert )
|
||||
result.sslCert = opts.sslCert.toString().decodeBase64()
|
||||
else if( opts.sslCertFile )
|
||||
result.sslCert = Paths.get(opts.sslCertFile.toString()).bytes
|
||||
|
||||
if( opts.clientCert )
|
||||
result.clientCert = opts.clientCert.toString().decodeBase64()
|
||||
else if( opts.clientCertFile )
|
||||
result.clientCert = Paths.get(opts.clientCertFile.toString()).bytes
|
||||
|
||||
if( opts.clientKey )
|
||||
result.clientKey = opts.clientKey.toString().decodeBase64()
|
||||
else if( opts.clientKeyFile )
|
||||
result.clientKey = Paths.get(opts.clientKeyFile.toString()).bytes
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
static ClientConfig fromUserAndCluster(Map user, Map cluster, Path location) {
|
||||
final base = location.isDirectory() ? location : location.parent
|
||||
final result = new ClientConfig()
|
||||
if( user.token )
|
||||
result.token = user.token
|
||||
|
||||
else if( user.tokenFile ) {
|
||||
result.token = Paths.get(user.tokenFile.toString()).getText('UTF-8')
|
||||
}
|
||||
|
||||
if( user."client-certificate" )
|
||||
result.clientCert = base.resolve(user."client-certificate".toString()).bytes
|
||||
|
||||
else if( user."client-certificate-data" )
|
||||
result.clientCert = user."client-certificate-data".toString().decodeBase64()
|
||||
|
||||
if( user."client-key" )
|
||||
result.clientKey = base.resolve(user."client-key".toString()).bytes
|
||||
|
||||
else if( user."client-key-data" )
|
||||
result.clientKey = user."client-key-data".toString().decodeBase64()
|
||||
|
||||
// -- cluster settings
|
||||
|
||||
if( cluster.server )
|
||||
result.server = cluster.server
|
||||
|
||||
if( cluster."certificate-authority-data" )
|
||||
result.sslCert = cluster."certificate-authority-data".toString().decodeBase64()
|
||||
|
||||
else if( cluster."certificate-authority" )
|
||||
result.sslCert = base.resolve(cluster."certificate-authority".toString()).bytes
|
||||
|
||||
result.verifySsl = cluster."insecure-skip-tls-verify" != true
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.client
|
||||
|
||||
import javax.net.ssl.KeyManager
|
||||
import javax.net.ssl.KeyManagerFactory
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
import java.security.KeyStore
|
||||
import static nextflow.util.StringUtils.formatHostName
|
||||
|
||||
import groovy.util.logging.Slf4j
|
||||
import org.yaml.snakeyaml.Yaml
|
||||
/**
|
||||
* Discover Kubernetes configuration from system environment
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
class ConfigDiscovery {
|
||||
|
||||
private Map<String,String> env = System.getenv()
|
||||
|
||||
ConfigDiscovery() { }
|
||||
|
||||
/**
|
||||
* Discover Kubernetes client configuration from current environment using
|
||||
* either the .kube/config file or the pod service service account virtual
|
||||
* file system when running in a pod.
|
||||
*
|
||||
* @param contextName The K8s config context name.
|
||||
* @param namespace The K8s cluster namespace
|
||||
* @param serviceAccount The K8s cluster service account
|
||||
* @return The e
|
||||
*/
|
||||
ClientConfig discover(String contextName, String namespace, String serviceAccount) {
|
||||
|
||||
// Note: System.getProperty('user.home') may not report the correct home path when
|
||||
// running in a container. Use env HOME instead.
|
||||
def home = System.getenv('HOME')
|
||||
def kubeConfig = env.get('KUBECONFIG') ? env.get('KUBECONFIG') : "$home/.kube/config"
|
||||
def configFile = Paths.get(kubeConfig)
|
||||
|
||||
// determine the Kubernetes client configuration via the `.kube/config` file
|
||||
if( configFile.exists() ) {
|
||||
return fromKubeConfig(configFile, contextName, namespace, serviceAccount)
|
||||
}
|
||||
else {
|
||||
log.debug "K8s config file does not exist: $configFile"
|
||||
}
|
||||
|
||||
// determine the Kubernetes client configuration via the pod environment
|
||||
if( env.get('KUBERNETES_SERVICE_HOST') ) {
|
||||
return fromCluster(env, namespace, serviceAccount)
|
||||
}
|
||||
else {
|
||||
log.debug "K8s env variable KUBERNETES_SERVICE_HOST is not defined"
|
||||
}
|
||||
|
||||
throw new IllegalStateException("Unable to lookup Kubernetes cluster configuration")
|
||||
}
|
||||
|
||||
protected ClientConfig fromCluster(Map<String,String> env, String cfgNamespace, String serviceAccount) {
|
||||
|
||||
// See https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/#accessing-the-api-from-a-pod
|
||||
|
||||
final host = env.get('KUBERNETES_SERVICE_HOST')
|
||||
final port = env.get('KUBERNETES_SERVICE_PORT')
|
||||
final server = formatHostName(host, port)
|
||||
|
||||
final cert = path('/var/run/secrets/kubernetes.io/serviceaccount/ca.crt').bytes
|
||||
final token = path('/var/run/secrets/kubernetes.io/serviceaccount/token').text
|
||||
final namespace = path('/var/run/secrets/kubernetes.io/serviceaccount/namespace').text
|
||||
|
||||
return new ClientConfig(
|
||||
server: server,
|
||||
token: token,
|
||||
namespace: cfgNamespace ?: namespace,
|
||||
serviceAccount: serviceAccount,
|
||||
sslCert: cert,
|
||||
isFromCluster: true )
|
||||
}
|
||||
|
||||
protected Path path(String path) {
|
||||
Paths.get(path)
|
||||
}
|
||||
|
||||
protected ClientConfig fromKubeConfig(Path path, String contextName, String namespace, String serviceAccount) {
|
||||
def yaml = (Map)new Yaml().load(Files.newInputStream(path))
|
||||
|
||||
contextName ?= yaml."current-context" as String
|
||||
|
||||
final allContext = yaml.contexts as List
|
||||
final allClusters = yaml.clusters as List
|
||||
final allUsers = yaml.users as List
|
||||
final context = allContext.find { Map it -> it.name == contextName } ?.context
|
||||
if( !context )
|
||||
throw new IllegalArgumentException("Unknown Kubernetes context: $contextName -- check config file: $path")
|
||||
final userName = context?.user
|
||||
final clusterName = context?.cluster
|
||||
final user = allUsers.find{ Map it -> it.name == userName } ?.user ?: [:]
|
||||
final cluster = allClusters.find{ Map it -> it.name == clusterName } ?.cluster ?: [:]
|
||||
|
||||
final config = ClientConfig.fromUserAndCluster(user, cluster, path)
|
||||
|
||||
// the namespace provided should have priority over the context current namespace
|
||||
config.namespace = namespace ?: context?.namespace ?: 'default'
|
||||
|
||||
config.serviceAccount = serviceAccount ?: 'default'
|
||||
|
||||
if( config.clientCert && config.clientKey ) {
|
||||
config.keyManagers = createKeyManagers(config.clientCert, config.clientKey)
|
||||
}
|
||||
else if( !config.token ) {
|
||||
config.token = discoverAuthToken(contextName, config.namespace, config.serviceAccount)
|
||||
}
|
||||
|
||||
return config
|
||||
}
|
||||
|
||||
protected KeyStore createKeyStore0(byte[] clientCert, byte[] clientKey, char[] passphrase, String alg) {
|
||||
def cert = new ByteArrayInputStream(clientCert)
|
||||
def key = new ByteArrayInputStream(clientKey)
|
||||
return SSLUtils.createKeyStore(cert, key, alg, passphrase, null, null)
|
||||
}
|
||||
|
||||
protected KeyStore createKeyStore(byte[] clientCert, byte[] clientKey, char[] passphrase) {
|
||||
try {
|
||||
// try first RSA algorithm
|
||||
return createKeyStore0(clientCert, clientKey, passphrase, "RSA")
|
||||
}
|
||||
catch (Exception e1) {
|
||||
// fallback to EC algorithm
|
||||
try {
|
||||
return createKeyStore0(clientCert, clientKey, passphrase, "EC")
|
||||
}
|
||||
catch (Exception e2) {
|
||||
// if still fails, throws the first exception
|
||||
throw e1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected KeyManager[] createKeyManagers(byte[] clientCert, byte[] clientKey) {
|
||||
final passphrase = "".toCharArray()
|
||||
final keyStore = createKeyStore(clientCert, clientKey, passphrase)
|
||||
final kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
|
||||
kmf.init(keyStore, passphrase);
|
||||
return kmf.getKeyManagers();
|
||||
}
|
||||
|
||||
String discoverAuthToken(String context, String namespace, String serviceAccount) {
|
||||
context ?= 'default'
|
||||
namespace ?= 'default'
|
||||
serviceAccount ?= 'default'
|
||||
|
||||
final cmd = "kubectl --context $context -n ${namespace} get secret -o=jsonpath='{.items[?(@.metadata.annotations.kubernetes\\.io/service-account\\.name==\"$serviceAccount\")].data.token}'"
|
||||
final proc = new ProcessBuilder('bash','-o','pipefail','-c', cmd).start()
|
||||
final status = proc.waitFor()
|
||||
final text = proc.inputStream?.text
|
||||
if( status==0 && text ) {
|
||||
try {
|
||||
return new String(text.trim().decodeBase64())
|
||||
}
|
||||
catch( Exception e ) {
|
||||
log.warn "Unable to decode K8s cluster auth token '$text' -- cause: ${e.message}"
|
||||
}
|
||||
}
|
||||
else {
|
||||
final cause = proc.errorStream?.text ?: text
|
||||
final msg = cause ? "\n- cmd : $cmd\n- exit : $status\n- cause:\n${cause.indent(' ')}" : ''
|
||||
log.warn "[K8s] unable to fetch auth token ${msg}"
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,54 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.client
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
|
||||
/**
|
||||
* Model a Kubernetes API response
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class K8sResponseApi {
|
||||
|
||||
private int code
|
||||
|
||||
private InputStream stream
|
||||
|
||||
private String text
|
||||
|
||||
K8sResponseApi(int code, InputStream stream) {
|
||||
this.code = code
|
||||
this.stream = stream
|
||||
}
|
||||
|
||||
String toString() {
|
||||
"code=$code; stream=$stream"
|
||||
}
|
||||
|
||||
int getCode() { code }
|
||||
|
||||
InputStream getStream() { stream }
|
||||
|
||||
String getText() {
|
||||
if( text == null ) {
|
||||
text = stream?.text
|
||||
}
|
||||
return text
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.client
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
/**
|
||||
* Model a kubernetes invalid response
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class K8sResponseException extends Exception {
|
||||
|
||||
K8sResponseJson response
|
||||
|
||||
K8sResponseException(K8sResponseJson response) {
|
||||
super(msg0(response))
|
||||
this.response = response
|
||||
}
|
||||
|
||||
K8sResponseException(String message, K8sResponseJson response) {
|
||||
super(msg1(message,response))
|
||||
this.response = response
|
||||
}
|
||||
|
||||
K8sResponseException(String message, InputStream response) {
|
||||
this(message, new K8sResponseJson(fetch(response)))
|
||||
}
|
||||
|
||||
static private String msg1( String msg, K8sResponseJson resp ) {
|
||||
if( !msg && resp==null )
|
||||
return null
|
||||
|
||||
if( msg && resp != null ) {
|
||||
def sep = resp.isRawText() ? ' -- ' : '\n'
|
||||
return "${msg}${sep}${msg0(resp)}"
|
||||
}
|
||||
else if( msg ) {
|
||||
return msg
|
||||
}
|
||||
else {
|
||||
return msg0(resp)
|
||||
}
|
||||
}
|
||||
|
||||
static private String msg0( K8sResponseJson response ) {
|
||||
if( response == null )
|
||||
return null
|
||||
|
||||
if( response.isRawText() )
|
||||
response.getRawText()
|
||||
else
|
||||
"\n${response.toString().indent(' ')}"
|
||||
}
|
||||
|
||||
static private String fetch(InputStream stream) {
|
||||
try {
|
||||
return stream?.text
|
||||
}
|
||||
catch( Exception e ) {
|
||||
log.debug "Unable to fetch response text -- Cause: ${e.message ?: e}"
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.client
|
||||
|
||||
import groovy.json.JsonOutput
|
||||
import groovy.json.JsonSlurper
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
|
||||
/**
|
||||
* Model the response of a kubernetes api request
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class K8sResponseJson implements Map {
|
||||
|
||||
@Delegate
|
||||
private Map response
|
||||
|
||||
private String rawText
|
||||
|
||||
K8sResponseJson(Map response) {
|
||||
this.response = response
|
||||
}
|
||||
|
||||
K8sResponseJson(String response) {
|
||||
this.response = toJson(response)
|
||||
this.rawText = response
|
||||
}
|
||||
|
||||
boolean isRawText() { !response && rawText }
|
||||
|
||||
String getRawText() { rawText }
|
||||
|
||||
static private Map toJson(String raw) {
|
||||
try {
|
||||
return (Map)new JsonSlurper().parseText(raw)
|
||||
}
|
||||
catch( Exception e ) {
|
||||
log.trace "[K8s] cannot parse response to json -- raw: ${raw? '\n'+raw.indent(' ') :'null'}"
|
||||
return Collections.emptyMap()
|
||||
}
|
||||
}
|
||||
|
||||
static private String prettyPrint(String json) {
|
||||
try {
|
||||
JsonOutput.prettyPrint(json)
|
||||
}
|
||||
catch( Exception e ) {
|
||||
return json
|
||||
}
|
||||
}
|
||||
|
||||
String toString() {
|
||||
response ? prettyPrint(JsonOutput.toJson(response)) : rawText
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.client
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import groovy.transform.ToString
|
||||
import nextflow.config.spec.ConfigOption
|
||||
import nextflow.config.spec.ConfigScope
|
||||
import nextflow.script.dsl.Description
|
||||
import nextflow.util.Duration
|
||||
|
||||
/**
|
||||
* Model retry policy configuration
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@ToString(includePackage = false, includeNames = true)
|
||||
@EqualsAndHashCode
|
||||
@CompileStatic
|
||||
class K8sRetryConfig implements ConfigScope {
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Delay when retrying failed API requests (default: `250ms`).
|
||||
""")
|
||||
Duration delay = Duration.of('250ms')
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Max delay when retrying failed API requests (default: `90s`).
|
||||
""")
|
||||
Duration maxDelay = Duration.of('90s')
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Max attempts when retrying failed API requests (default: `4`).
|
||||
""")
|
||||
int maxAttempts = 4
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Jitter value when retrying failed API requests (default: `0.25`).
|
||||
""")
|
||||
double jitter = 0.25
|
||||
|
||||
K8sRetryConfig() {
|
||||
this(Collections.emptyMap())
|
||||
}
|
||||
|
||||
K8sRetryConfig(Map config) {
|
||||
if( config.delay )
|
||||
delay = config.delay as Duration
|
||||
if( config.maxDelay )
|
||||
maxDelay = config.maxDelay as Duration
|
||||
if( config.maxAttempts )
|
||||
maxAttempts = config.maxAttempts as int
|
||||
if( config.jitter )
|
||||
jitter = config.jitter as double
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright 2025, Seqera Labs
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -14,32 +14,24 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin
|
||||
package recreationaltech.plugin.client
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.Session
|
||||
import nextflow.plugin.extension.Function
|
||||
import nextflow.plugin.extension.PluginExtensionPoint
|
||||
import nextflow.exception.ProcessException
|
||||
import nextflow.exception.ShowOnlyExceptionMessage
|
||||
|
||||
/**
|
||||
* Implements a custom function which can be imported by
|
||||
* Nextflow scripts.
|
||||
* Exception raised when a pod cannot be scheduled because
|
||||
* e.g. the container image cannot be pulled, required resources
|
||||
* cannot be fulfilled, etc.
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class K8sDvfsExtension extends PluginExtensionPoint {
|
||||
class PodUnschedulableException extends ProcessException implements ShowOnlyExceptionMessage {
|
||||
|
||||
@Override
|
||||
protected void init(Session session) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Say hello to the given target.
|
||||
*
|
||||
* @param target
|
||||
*/
|
||||
@Function
|
||||
void sayHello(String target) {
|
||||
println "Hello, ${target}!"
|
||||
PodUnschedulableException(String message, Throwable cause) {
|
||||
super(message,cause)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,318 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.client;
|
||||
|
||||
/**
|
||||
* This file is derived from
|
||||
* https://github.com/kubernetes-client/java/blob/master/util/src/main/java/io/kubernetes/client/util/SSLUtils.java
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.math.BigInteger;
|
||||
import java.security.KeyFactory;
|
||||
import java.security.KeyStore;
|
||||
import java.security.KeyStoreException;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.security.PrivateKey;
|
||||
import java.security.Security;
|
||||
import java.security.UnrecoverableKeyException;
|
||||
import java.security.cert.Certificate;
|
||||
import java.security.cert.CertificateException;
|
||||
import java.security.cert.CertificateFactory;
|
||||
import java.security.cert.X509Certificate;
|
||||
import java.security.spec.InvalidKeySpecException;
|
||||
import java.security.spec.PKCS8EncodedKeySpec;
|
||||
import java.security.spec.RSAPrivateCrtKeySpec;
|
||||
import javax.net.ssl.KeyManager;
|
||||
import javax.net.ssl.KeyManagerFactory;
|
||||
|
||||
import org.apache.commons.codec.binary.Base64;
|
||||
import org.bouncycastle.asn1.pkcs.PrivateKeyInfo;
|
||||
import org.bouncycastle.openssl.PEMKeyPair;
|
||||
import org.bouncycastle.openssl.PEMParser;
|
||||
import org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter;
|
||||
|
||||
|
||||
public class SSLUtils {
|
||||
|
||||
public static boolean isNotNullOrEmpty(String val) {
|
||||
return val != null && val.length() > 0;
|
||||
}
|
||||
|
||||
public static KeyManager[] keyManagers(String certData, String certFile, String keyData, String keyFile,
|
||||
String algo, String passphrase, String keyStoreFile, String keyStorePassphrase)
|
||||
throws NoSuchAlgorithmException, UnrecoverableKeyException, KeyStoreException, CertificateException,
|
||||
InvalidKeySpecException, IOException {
|
||||
KeyManager[] keyManagers = null;
|
||||
if ((isNotNullOrEmpty(certData) || isNotNullOrEmpty(certFile))
|
||||
&& (isNotNullOrEmpty(keyData) || isNotNullOrEmpty(keyFile))) {
|
||||
KeyStore keyStore = createKeyStore(certData, certFile, keyData, keyFile, algo, passphrase, keyStoreFile,
|
||||
keyStorePassphrase);
|
||||
KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
|
||||
kmf.init(keyStore, passphrase.toCharArray());
|
||||
keyManagers = kmf.getKeyManagers();
|
||||
}
|
||||
return keyManagers;
|
||||
}
|
||||
|
||||
|
||||
public static KeyStore createKeyStore(String clientCertData, String clientCertFile, String clientKeyData,
|
||||
String clientKeyFile, String clientKeyAlgo, String clientKeyPassphrase, String keyStoreFile,
|
||||
String keyStorePassphrase) throws IOException, CertificateException, NoSuchAlgorithmException,
|
||||
InvalidKeySpecException, KeyStoreException {
|
||||
try (InputStream certInputStream = getInputStreamFromDataOrFile(clientCertData, clientCertFile);
|
||||
InputStream keyInputStream = getInputStreamFromDataOrFile(clientKeyData, clientKeyFile)) {
|
||||
return createKeyStore(certInputStream, keyInputStream, clientKeyAlgo,
|
||||
clientKeyPassphrase != null ? clientKeyPassphrase.toCharArray() : null,
|
||||
keyStoreFile, getKeyStorePassphrase(keyStorePassphrase));
|
||||
}
|
||||
}
|
||||
|
||||
static private PrivateKey generateEcKey(InputStream keyInputStream) throws IOException {
|
||||
PrivateKey privateKey=null;
|
||||
Security.addProvider(new org.bouncycastle.jce.provider.BouncyCastleProvider());
|
||||
Object object = new PEMParser(new InputStreamReader(keyInputStream)).readObject();
|
||||
if (object instanceof PEMKeyPair) {
|
||||
PEMKeyPair keys = (PEMKeyPair) object;
|
||||
privateKey = new JcaPEMKeyConverter().getKeyPair(keys).getPrivate();
|
||||
}
|
||||
if( object instanceof PrivateKeyInfo) {
|
||||
PrivateKeyInfo privateKeyInfo = (PrivateKeyInfo)object;
|
||||
privateKey = new JcaPEMKeyConverter().getPrivateKey(privateKeyInfo);
|
||||
}
|
||||
if( privateKey == null) {
|
||||
throw new IOException("Unsupported EC algorithm");
|
||||
}
|
||||
return privateKey;
|
||||
}
|
||||
|
||||
static private PrivateKey generateStdKey(InputStream keyInputStream, String clientKeyAlgo) throws IOException, NoSuchAlgorithmException, InvalidKeySpecException {
|
||||
byte[] keyBytes = decodePem(keyInputStream);
|
||||
|
||||
KeyFactory keyFactory = KeyFactory.getInstance(clientKeyAlgo);
|
||||
try {
|
||||
// First let's try PKCS8
|
||||
return keyFactory.generatePrivate(new PKCS8EncodedKeySpec(keyBytes));
|
||||
}
|
||||
catch (InvalidKeySpecException e) {
|
||||
// Otherwise try PKCS1
|
||||
RSAPrivateCrtKeySpec keySpec = decodePKCS1(keyBytes);
|
||||
return keyFactory.generatePrivate(keySpec);
|
||||
}
|
||||
}
|
||||
|
||||
public static KeyStore createKeyStore(InputStream certInputStream, InputStream keyInputStream, String clientKeyAlgo,
|
||||
char[] clientKeyPassphrase, String keyStoreFile, char[] keyStorePassphrase) throws IOException,
|
||||
CertificateException, NoSuchAlgorithmException, InvalidKeySpecException, KeyStoreException {
|
||||
CertificateFactory certFactory = CertificateFactory.getInstance("X509");
|
||||
X509Certificate cert = (X509Certificate) certFactory.generateCertificate(certInputStream);
|
||||
|
||||
PrivateKey privateKey = clientKeyAlgo.equals("EC")
|
||||
? generateEcKey(keyInputStream)
|
||||
: generateStdKey(keyInputStream, clientKeyAlgo);
|
||||
|
||||
KeyStore keyStore = KeyStore.getInstance("JKS");
|
||||
if (keyStoreFile != null && keyStoreFile.length() > 0) {
|
||||
keyStore.load(new FileInputStream(keyStoreFile), keyStorePassphrase);
|
||||
} else {
|
||||
loadDefaultKeyStoreFile(keyStore, keyStorePassphrase);
|
||||
}
|
||||
|
||||
String alias = cert.getSubjectX500Principal().getName();
|
||||
keyStore.setKeyEntry(alias, privateKey, clientKeyPassphrase, new Certificate[] { cert });
|
||||
|
||||
return keyStore;
|
||||
}
|
||||
|
||||
// This method is inspired and partly taken over from
|
||||
// http://oauth.googlecode.com/svn/code/java/
|
||||
// All credits to belong to them.
|
||||
private static byte[] decodePem(InputStream keyInputStream) throws IOException {
|
||||
BufferedReader reader = new BufferedReader(new InputStreamReader(keyInputStream));
|
||||
try {
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
if (line.contains("-----BEGIN ")) {
|
||||
return readBytes(reader, line.trim().replace("BEGIN", "END"));
|
||||
}
|
||||
}
|
||||
throw new IOException("PEM is invalid: no begin marker");
|
||||
} finally {
|
||||
reader.close();
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] readBytes(BufferedReader reader, String endMarker) throws IOException {
|
||||
String line;
|
||||
StringBuffer buf = new StringBuffer();
|
||||
|
||||
while ((line = reader.readLine()) != null) {
|
||||
if (line.indexOf(endMarker) != -1) {
|
||||
return Base64.decodeBase64(buf.toString());
|
||||
}
|
||||
buf.append(line.trim());
|
||||
}
|
||||
throw new IOException("PEM is invalid : No end marker");
|
||||
}
|
||||
|
||||
public static RSAPrivateCrtKeySpec decodePKCS1(byte[] keyBytes) throws IOException {
|
||||
DerParser parser = new DerParser(keyBytes);
|
||||
Asn1Object sequence = parser.read();
|
||||
sequence.validateSequence();
|
||||
parser = new DerParser(sequence.getValue());
|
||||
parser.read();
|
||||
|
||||
return new RSAPrivateCrtKeySpec(next(parser), next(parser), next(parser), next(parser), next(parser),
|
||||
next(parser), next(parser), next(parser));
|
||||
}
|
||||
|
||||
private static BigInteger next(DerParser parser) throws IOException {
|
||||
return parser.read().getInteger();
|
||||
}
|
||||
|
||||
static class DerParser {
|
||||
|
||||
private InputStream in;
|
||||
|
||||
DerParser(byte[] bytes) throws IOException {
|
||||
this.in = new ByteArrayInputStream(bytes);
|
||||
}
|
||||
|
||||
Asn1Object read() throws IOException {
|
||||
int tag = in.read();
|
||||
|
||||
if (tag == -1) {
|
||||
throw new IOException("Invalid DER: stream too short, missing tag");
|
||||
}
|
||||
|
||||
int length = getLength();
|
||||
byte[] value = new byte[length];
|
||||
if (in.read(value) < length) {
|
||||
throw new IOException("Invalid DER: stream too short, missing value");
|
||||
}
|
||||
|
||||
return new Asn1Object(tag, value);
|
||||
}
|
||||
|
||||
private int getLength() throws IOException {
|
||||
int i = in.read();
|
||||
if (i == -1) {
|
||||
throw new IOException("Invalid DER: length missing");
|
||||
}
|
||||
|
||||
if ((i & ~0x7F) == 0) {
|
||||
return i;
|
||||
}
|
||||
|
||||
int num = i & 0x7F;
|
||||
if (i >= 0xFF || num > 4) {
|
||||
throw new IOException("Invalid DER: length field too big (" + i + ")");
|
||||
}
|
||||
|
||||
byte[] bytes = new byte[num];
|
||||
if (in.read(bytes) < num) {
|
||||
throw new IOException("Invalid DER: length too short");
|
||||
}
|
||||
|
||||
return new BigInteger(1, bytes).intValue();
|
||||
}
|
||||
}
|
||||
|
||||
static class Asn1Object {
|
||||
|
||||
private final int type;
|
||||
private final byte[] value;
|
||||
private final int tag;
|
||||
|
||||
public Asn1Object(int tag, byte[] value) {
|
||||
this.tag = tag;
|
||||
this.type = tag & 0x1F;
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
public byte[] getValue() {
|
||||
return value;
|
||||
}
|
||||
|
||||
BigInteger getInteger() throws IOException {
|
||||
if (type != 0x02) {
|
||||
throw new IOException("Invalid DER: object is not integer"); //$NON-NLS-1$
|
||||
}
|
||||
return new BigInteger(value);
|
||||
}
|
||||
|
||||
void validateSequence() throws IOException {
|
||||
if (type != 0x10) {
|
||||
throw new IOException("Invalid DER: not a sequence");
|
||||
}
|
||||
if ((tag & 0x20) != 0x20) {
|
||||
throw new IOException("Invalid DER: can't parse primitive entity");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void loadDefaultKeyStoreFile(KeyStore keyStore, char[] keyStorePassphrase)
|
||||
throws CertificateException, NoSuchAlgorithmException, IOException {
|
||||
|
||||
String keyStorePath = System.getProperty("javax.net.ssl.keyStore");
|
||||
if (keyStorePath != null && keyStorePath.length() > 0) {
|
||||
File keyStoreFile = new File(keyStorePath);
|
||||
if (loadDefaultStoreFile(keyStore, keyStoreFile, keyStorePassphrase)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
keyStore.load(null);
|
||||
}
|
||||
|
||||
private static boolean loadDefaultStoreFile(KeyStore keyStore, File fileToLoad, char[] passphrase)
|
||||
throws CertificateException, NoSuchAlgorithmException, IOException {
|
||||
if (fileToLoad.exists() && fileToLoad.isFile() && fileToLoad.length() > 0) {
|
||||
keyStore.load(new FileInputStream(fileToLoad), passphrase);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public static InputStream getInputStreamFromDataOrFile(String data, String file) throws FileNotFoundException {
|
||||
if (data != null) {
|
||||
byte[] bytes = Base64.decodeBase64(data);
|
||||
// TODO handle non-base64 here?
|
||||
return new ByteArrayInputStream(bytes);
|
||||
}
|
||||
if (file != null) {
|
||||
return new FileInputStream(file);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static char[] getKeyStorePassphrase(String keyStorePassphrase) {
|
||||
if (keyStorePassphrase == null || keyStorePassphrase.length() == 0) {
|
||||
return System.getProperty("javax.net.ssl.keyStorePassword", "changeit").toCharArray();
|
||||
}
|
||||
return keyStorePassphrase.toCharArray();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.model
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import groovy.transform.ToString
|
||||
|
||||
/**
|
||||
* Model a K8s pod environment variable definition
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
@ToString(includeNames = true)
|
||||
@EqualsAndHashCode(includeFields = true)
|
||||
class PodEnv {
|
||||
|
||||
private Map spec
|
||||
|
||||
private PodEnv(Map spec) {
|
||||
this.spec = spec
|
||||
}
|
||||
|
||||
static PodEnv value(String env, String value) {
|
||||
new PodEnv([name:env, value:value])
|
||||
}
|
||||
|
||||
static PodEnv fieldPath(String env, String fieldPath) {
|
||||
new PodEnv([ name: env, valueFrom: [fieldRef:[fieldPath: fieldPath]]])
|
||||
}
|
||||
|
||||
static PodEnv config(String env, String config) {
|
||||
final tokens = config.tokenize('/')
|
||||
if( tokens.size() > 2 )
|
||||
throw new IllegalArgumentException("K8s invalid pod env file: $config -- Secret must be specified as <config-name>/<config-key>")
|
||||
|
||||
final name = tokens[0]
|
||||
final key = tokens[1]
|
||||
|
||||
assert env, 'Missing pod env variable name'
|
||||
assert name, 'Missing pod env config name'
|
||||
|
||||
final ref = [ name: name, key: (key ?: env) ]
|
||||
new PodEnv([ name: env, valueFrom: [configMapKeyRef: ref]])
|
||||
}
|
||||
|
||||
static PodEnv secret(String env, String secret) {
|
||||
|
||||
final tokens = secret.tokenize('/')
|
||||
if( tokens.size() > 2 )
|
||||
throw new IllegalArgumentException("K8s invalid pod env secret: $secret -- Secret must be specified as <secret-name>/<secret-key>")
|
||||
|
||||
final name = tokens[0]
|
||||
final key = tokens[1]
|
||||
|
||||
final ref = [ name: name, key: (key ?: env) ]
|
||||
new PodEnv([ name: env, valueFrom: [secretKeyRef: ref]])
|
||||
}
|
||||
|
||||
|
||||
Map toSpec() { spec }
|
||||
|
||||
String toString() {
|
||||
"PodEnv[ ${spec?.toString()} ]"
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright 2025, Seqera Labs
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -14,28 +14,28 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin
|
||||
package recreationaltech.plugin.model
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.Session
|
||||
import nextflow.trace.TraceObserver
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import groovy.transform.ToString
|
||||
|
||||
/**
|
||||
* Implements an observer that allows implementing custom
|
||||
* logic on nextflow execution events.
|
||||
* Model a K8s pod host mount definition
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@EqualsAndHashCode
|
||||
@ToString(includeNames = true)
|
||||
@CompileStatic
|
||||
class K8sDvfsObserver implements TraceObserver {
|
||||
class PodHostMount {
|
||||
|
||||
@Override
|
||||
void onFlowCreate(Session session) {
|
||||
println "Pipeline is starting! 🚀"
|
||||
}
|
||||
String hostPath
|
||||
|
||||
@Override
|
||||
void onFlowComplete() {
|
||||
println "Pipeline complete! 👋"
|
||||
String mountPath
|
||||
|
||||
PodHostMount(String host, String container) {
|
||||
this.hostPath = host
|
||||
this.mountPath = container
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.model
|
||||
|
||||
import java.nio.file.Paths
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import groovy.transform.ToString
|
||||
|
||||
/**
|
||||
* Model a K8s pod ConfigMap mount
|
||||
*
|
||||
* See also https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
@ToString(includeNames = true)
|
||||
@EqualsAndHashCode
|
||||
class PodMountConfig {
|
||||
|
||||
String mountPath
|
||||
|
||||
String fileName
|
||||
|
||||
String configName
|
||||
|
||||
String configKey
|
||||
|
||||
PodMountConfig( String config, String mount ) {
|
||||
assert config
|
||||
assert mount
|
||||
|
||||
final path = Paths.get(mount)
|
||||
final tokens = config.tokenize('/')
|
||||
configName = tokens[0].trim()
|
||||
configKey = tokens.size()>1 ? tokens[1].trim() : null
|
||||
if( configKey ) {
|
||||
mountPath = path.parent.toString()
|
||||
fileName = path.fileName.toString()
|
||||
}
|
||||
else {
|
||||
mountPath = path.toString()
|
||||
}
|
||||
}
|
||||
|
||||
PodMountConfig( Map entry ) {
|
||||
this(entry.config as String, entry.mountPath as String)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.model
|
||||
|
||||
import java.nio.file.Paths
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import groovy.transform.ToString
|
||||
|
||||
/**
|
||||
* Model a K8s pod CSI ephemeral volume mount
|
||||
*
|
||||
* See also https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/#csi-ephemeral-volumes
|
||||
*
|
||||
* @author Ben Sherman <bentshermann@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
@ToString(includeNames = true)
|
||||
@EqualsAndHashCode
|
||||
class PodMountCsiEphemeral {
|
||||
|
||||
String mountPath
|
||||
|
||||
Map csi
|
||||
|
||||
PodMountCsiEphemeral( Map csi, String mountPath ) {
|
||||
assert csi
|
||||
assert mountPath
|
||||
|
||||
this.csi = csi
|
||||
this.mountPath = mountPath
|
||||
}
|
||||
|
||||
PodMountCsiEphemeral( Map entry ) {
|
||||
this(entry.csi as Map, entry.mountPath as String)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.model
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import groovy.transform.ToString
|
||||
|
||||
/**
|
||||
* Model a K8s pod emptyDir mount
|
||||
*
|
||||
* See also https://kubernetes.io/docs/concepts/storage/volumes/#emptydir
|
||||
*
|
||||
* @author Ben Sherman <bentshermann@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
@ToString(includeNames = true)
|
||||
@EqualsAndHashCode
|
||||
class PodMountEmptyDir {
|
||||
|
||||
String mountPath
|
||||
|
||||
Map emptyDir
|
||||
|
||||
PodMountEmptyDir( Map emptyDir, String mountPath ) {
|
||||
assert mountPath
|
||||
|
||||
this.emptyDir = emptyDir
|
||||
this.mountPath = mountPath
|
||||
}
|
||||
|
||||
PodMountEmptyDir( Map entry ) {
|
||||
this(entry.emptyDir as Map, entry.mountPath as String)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.model
|
||||
|
||||
import java.nio.file.Paths
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import groovy.transform.ToString
|
||||
|
||||
/**
|
||||
* Model a K8s Secret file mount
|
||||
*
|
||||
* https://kubernetes.io/docs/concepts/configuration/secret/
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
@ToString(includeNames = true)
|
||||
@EqualsAndHashCode
|
||||
class PodMountSecret {
|
||||
|
||||
String mountPath
|
||||
|
||||
String fileName
|
||||
|
||||
String secretName
|
||||
|
||||
String secretKey
|
||||
|
||||
PodMountSecret(String secret, String mount) {
|
||||
assert secret
|
||||
assert mount
|
||||
|
||||
final path = Paths.get(mount)
|
||||
final tokens = secret.tokenize('/')
|
||||
secretName = tokens[0].trim()
|
||||
secretKey = tokens.size()>1 ? tokens[1].trim() : null
|
||||
if( secretKey ) {
|
||||
mountPath = path.parent.toString()
|
||||
fileName = path.fileName.toString()
|
||||
}
|
||||
else {
|
||||
mountPath = path.toString()
|
||||
}
|
||||
}
|
||||
|
||||
PodMountSecret(Map entry) {
|
||||
this(entry.secret as String, entry.mountPath as String)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.model
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import groovy.transform.ToString
|
||||
|
||||
/**
|
||||
* Model a Pod nodeSelector spec
|
||||
*
|
||||
* https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
@ToString(includeNames = true)
|
||||
@EqualsAndHashCode(includeFields = true)
|
||||
class PodNodeSelector {
|
||||
|
||||
private Map spec = [:]
|
||||
|
||||
PodNodeSelector(selector) {
|
||||
if( selector instanceof CharSequence )
|
||||
createWithString(selector.toString())
|
||||
|
||||
else if( selector instanceof Map )
|
||||
createWithMap(selector)
|
||||
|
||||
else if( selector != null )
|
||||
throw new IllegalArgumentException("K8s invalid pod nodeSelector value: $selector [${selector.getClass().getName()}]")
|
||||
}
|
||||
|
||||
private createWithMap(Map selection ) {
|
||||
if(selection) {
|
||||
for( Map.Entry entry : selection.entrySet() ) {
|
||||
spec.put(entry.key.toString(), entry.value?.toString())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param selector
|
||||
* A string representing a comma separated list of pairs
|
||||
* e.g. foo=1,bar=2
|
||||
*
|
||||
*/
|
||||
private createWithString( String selector ) {
|
||||
if(!selector) return
|
||||
def entries = selector.tokenize(',')
|
||||
for( String item : entries ) {
|
||||
def pair = item.tokenize('=')
|
||||
spec.put( trim(pair[0]), trim(pair[1]) ?: 'true' )
|
||||
}
|
||||
}
|
||||
|
||||
private String trim(String v) {
|
||||
v?.trim()
|
||||
}
|
||||
|
||||
Map<String,String> toSpec() { spec }
|
||||
|
||||
String toString() {
|
||||
"PodNodeSelector[ ${spec?.toString()} ]"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,323 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.model
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import groovy.transform.PackageScope
|
||||
import groovy.transform.ToString
|
||||
|
||||
/**
|
||||
* Model K8s pod options such as environment variables,
|
||||
* secret and config-maps
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
@ToString(includeNames = true)
|
||||
@EqualsAndHashCode(includeFields = true)
|
||||
class PodOptions {
|
||||
|
||||
private String imagePullPolicy
|
||||
|
||||
private String imagePullSecret
|
||||
|
||||
private Collection<PodEnv> envVars
|
||||
|
||||
private Collection<PodMountConfig> mountConfigMaps
|
||||
|
||||
private Collection<PodMountCsiEphemeral> mountCsiEphemerals
|
||||
|
||||
private Collection<PodMountEmptyDir> mountEmptyDirs
|
||||
|
||||
private Collection<PodMountSecret> mountSecrets
|
||||
|
||||
private Collection<PodVolumeClaim> mountClaims
|
||||
|
||||
private Collection<PodHostMount> mountHostPaths
|
||||
|
||||
private Map<String,String> labels = [:]
|
||||
|
||||
private Map<String,String> annotations = [:]
|
||||
|
||||
private PodNodeSelector nodeSelector
|
||||
|
||||
private Map affinity
|
||||
|
||||
private PodSecurityContext securityContext
|
||||
|
||||
private boolean automountServiceAccountToken
|
||||
|
||||
private String priorityClassName
|
||||
|
||||
private List<Map> tolerations
|
||||
|
||||
private Boolean privileged
|
||||
|
||||
private String schedulerName
|
||||
|
||||
private Integer ttlSecondsAfterFinished
|
||||
|
||||
private String runtimeClassName
|
||||
|
||||
PodOptions( List<Map> options=null ) {
|
||||
int size = options ? options.size() : 0
|
||||
envVars = new HashSet<>(size)
|
||||
mountConfigMaps = new HashSet<>(size)
|
||||
mountCsiEphemerals = new HashSet<>(size)
|
||||
mountEmptyDirs = new HashSet<>(size)
|
||||
mountSecrets = new HashSet<>(size)
|
||||
mountClaims = new HashSet<>(size)
|
||||
mountHostPaths = new HashSet<>(10)
|
||||
automountServiceAccountToken = true
|
||||
tolerations = new ArrayList<Map>(size)
|
||||
init(options)
|
||||
}
|
||||
|
||||
@PackageScope void init(List<Map> options) {
|
||||
if( !options ) return
|
||||
for( Map entry : options ) {
|
||||
create(entry)
|
||||
}
|
||||
}
|
||||
|
||||
@PackageScope void create(Map<String,String> entry) {
|
||||
if( entry.env && entry.value ) {
|
||||
envVars << PodEnv.value(entry.env, entry.value)
|
||||
}
|
||||
else if( entry.env && entry.fieldPath ) {
|
||||
envVars << PodEnv.fieldPath(entry.env, entry.fieldPath)
|
||||
}
|
||||
else if( entry.env && entry.secret ) {
|
||||
envVars << PodEnv.secret(entry.env, entry.secret)
|
||||
}
|
||||
else if( entry.env && entry.config ) {
|
||||
envVars << PodEnv.config(entry.env, entry.config)
|
||||
}
|
||||
else if( entry.mountPath && entry.secret ) {
|
||||
mountSecrets << new PodMountSecret(entry)
|
||||
}
|
||||
else if( entry.mountPath && entry.config ) {
|
||||
mountConfigMaps << new PodMountConfig(entry)
|
||||
}
|
||||
else if( entry.mountPath && entry.csi ) {
|
||||
mountCsiEphemerals << new PodMountCsiEphemeral(entry)
|
||||
}
|
||||
else if( entry.mountPath && entry.emptyDir != null ) {
|
||||
mountEmptyDirs << new PodMountEmptyDir(entry)
|
||||
}
|
||||
else if( entry.mountPath && entry.volumeClaim ) {
|
||||
mountClaims << new PodVolumeClaim(entry)
|
||||
}
|
||||
else if( entry.mountPath && entry.hostPath instanceof CharSequence ) {
|
||||
mountHostPaths << new PodHostMount(entry.hostPath, entry.mountPath)
|
||||
}
|
||||
else if( entry.pullPolicy || entry.imagePullPolicy ) {
|
||||
this.imagePullPolicy = entry.pullPolicy ?: entry.imagePullPolicy as String
|
||||
}
|
||||
else if( entry.imagePullSecret || entry.imagePullSecrets ) {
|
||||
this.imagePullSecret = entry.imagePullSecret ?: entry.imagePullSecrets
|
||||
}
|
||||
else if( entry.label && entry.value ) {
|
||||
this.labels.put(entry.label as String, entry.value as String)
|
||||
}
|
||||
else if( entry.runAsUser != null ) {
|
||||
this.securityContext = new PodSecurityContext(entry.runAsUser)
|
||||
}
|
||||
else if( entry.securityContext instanceof Map ) {
|
||||
this.securityContext = new PodSecurityContext(entry.securityContext as Map)
|
||||
}
|
||||
else if( entry.nodeSelector ) {
|
||||
this.nodeSelector = new PodNodeSelector(entry.nodeSelector)
|
||||
}
|
||||
else if( entry.affinity instanceof Map ) {
|
||||
this.affinity = entry.affinity as Map
|
||||
}
|
||||
else if( entry.annotation && entry.value ) {
|
||||
this.annotations.put(entry.annotation as String, entry.value as String)
|
||||
}
|
||||
else if( entry.automountServiceAccountToken instanceof Boolean ) {
|
||||
this.automountServiceAccountToken = entry.automountServiceAccountToken as Boolean
|
||||
}
|
||||
else if( entry.priorityClassName ) {
|
||||
this.priorityClassName = entry.priorityClassName
|
||||
}
|
||||
else if( entry.toleration instanceof Map ) {
|
||||
tolerations << (entry.toleration as Map)
|
||||
}
|
||||
else if( entry.privileged instanceof Boolean ) {
|
||||
this.privileged = entry.privileged as Boolean
|
||||
}
|
||||
else if( entry.schedulerName ) {
|
||||
this.schedulerName = entry.schedulerName
|
||||
}
|
||||
else if( entry.ttlSecondsAfterFinished instanceof Integer ) {
|
||||
this.ttlSecondsAfterFinished = entry.ttlSecondsAfterFinished as Integer
|
||||
}
|
||||
else if( entry.runtimeClassName ) {
|
||||
this.runtimeClassName = entry.runtimeClassName
|
||||
}
|
||||
else
|
||||
throw new IllegalArgumentException("Unknown pod options: $entry")
|
||||
}
|
||||
|
||||
|
||||
Collection<PodEnv> getEnvVars() { envVars }
|
||||
|
||||
Collection<PodMountConfig> getMountConfigMaps() { mountConfigMaps }
|
||||
|
||||
Collection<PodMountCsiEphemeral> getMountCsiEphemerals() { mountCsiEphemerals }
|
||||
|
||||
Collection<PodMountEmptyDir> getMountEmptyDirs() { mountEmptyDirs }
|
||||
|
||||
Collection<PodMountSecret> getMountSecrets() { mountSecrets }
|
||||
|
||||
Collection<PodHostMount> getMountHostPaths() { mountHostPaths }
|
||||
|
||||
Collection<PodVolumeClaim> getVolumeClaims() { mountClaims }
|
||||
|
||||
Map<String,String> getLabels() { labels }
|
||||
|
||||
Map<String,String> getAnnotations() { annotations }
|
||||
|
||||
PodNodeSelector getNodeSelector() { nodeSelector }
|
||||
|
||||
PodOptions setNodeSelector( PodNodeSelector sel ) {
|
||||
nodeSelector = sel
|
||||
return this
|
||||
}
|
||||
|
||||
Map getAffinity() { affinity }
|
||||
|
||||
PodSecurityContext getSecurityContext() { securityContext }
|
||||
|
||||
PodOptions setSecurityContext( PodSecurityContext ctx ) {
|
||||
this.securityContext = ctx
|
||||
return this
|
||||
}
|
||||
|
||||
String getImagePullSecret() { imagePullSecret }
|
||||
|
||||
PodOptions setImagePullSecret( String secret ) {
|
||||
this.imagePullSecret = secret
|
||||
return this
|
||||
}
|
||||
|
||||
String getImagePullPolicy() { imagePullPolicy }
|
||||
|
||||
PodOptions setImagePullPolicy( String policy ) {
|
||||
this.imagePullPolicy = policy
|
||||
return this
|
||||
}
|
||||
|
||||
boolean getAutomountServiceAccountToken() { automountServiceAccountToken }
|
||||
|
||||
PodOptions setAutomountServiceAccountToken( boolean mount ) {
|
||||
this.automountServiceAccountToken = mount
|
||||
return this
|
||||
}
|
||||
|
||||
String getPriorityClassName() { priorityClassName }
|
||||
|
||||
String getSchedulerName() { schedulerName }
|
||||
|
||||
List<Map> getTolerations() { tolerations }
|
||||
|
||||
Boolean getPrivileged() { privileged }
|
||||
|
||||
Integer getTtlSecondsAfterFinished() { ttlSecondsAfterFinished }
|
||||
|
||||
String getRuntimeClassName() { runtimeClassName }
|
||||
|
||||
PodOptions plus( PodOptions other ) {
|
||||
def result = new PodOptions()
|
||||
|
||||
// env vars
|
||||
result.envVars.addAll(envVars)
|
||||
result.envVars.addAll( other.envVars )
|
||||
|
||||
// config maps
|
||||
result.mountConfigMaps.addAll( mountConfigMaps )
|
||||
result.mountConfigMaps.addAll( other.mountConfigMaps )
|
||||
|
||||
// csi ephemeral volumes
|
||||
result.mountCsiEphemerals.addAll( mountCsiEphemerals )
|
||||
result.mountCsiEphemerals.addAll( other.mountCsiEphemerals )
|
||||
|
||||
// empty dirs
|
||||
result.mountEmptyDirs.addAll( mountEmptyDirs )
|
||||
result.mountEmptyDirs.addAll( other.mountEmptyDirs )
|
||||
|
||||
// host paths
|
||||
result.mountHostPaths.addAll( mountHostPaths )
|
||||
result.mountHostPaths.addAll( other.mountHostPaths )
|
||||
|
||||
// secrets
|
||||
result.mountSecrets.addAll( mountSecrets )
|
||||
result.mountSecrets.addAll( other.mountSecrets )
|
||||
|
||||
// volume claims
|
||||
result.volumeClaims.addAll( volumeClaims )
|
||||
result.volumeClaims.addAll( other.volumeClaims )
|
||||
|
||||
// sec context
|
||||
result.securityContext = other.securityContext ?: this.securityContext
|
||||
|
||||
// node selector
|
||||
result.nodeSelector = other.nodeSelector ?: this.nodeSelector
|
||||
|
||||
// affinity
|
||||
result.affinity = other.affinity ?: this.affinity
|
||||
|
||||
// pull policy
|
||||
result.imagePullPolicy = other.imagePullPolicy ?: this.imagePullPolicy
|
||||
|
||||
// image secret
|
||||
result.imagePullSecret = other.imagePullSecret ?: this.imagePullSecret
|
||||
|
||||
// labels
|
||||
result.labels.putAll(labels)
|
||||
result.labels.putAll(other.labels)
|
||||
|
||||
// annotations
|
||||
result.annotations.putAll(annotations)
|
||||
result.annotations.putAll(other.annotations)
|
||||
|
||||
// automount service account token
|
||||
result.automountServiceAccountToken = other.automountServiceAccountToken & this.automountServiceAccountToken
|
||||
|
||||
// priority class name
|
||||
result.priorityClassName = other.priorityClassName ?: this.priorityClassName
|
||||
|
||||
// tolerations
|
||||
result.tolerations = other.tolerations ?: this.tolerations
|
||||
|
||||
// privileged execution
|
||||
result.privileged = other.privileged!=null ? other.privileged : this.privileged
|
||||
|
||||
// scheduler name
|
||||
result.schedulerName = other.schedulerName ?: this.schedulerName
|
||||
|
||||
// ttl seconds after finished (job)
|
||||
result.ttlSecondsAfterFinished = other.ttlSecondsAfterFinished!=null ? other.ttlSecondsAfterFinished : this.ttlSecondsAfterFinished
|
||||
|
||||
// runtime class name
|
||||
result.runtimeClassName = other.runtimeClassName ?: this.runtimeClassName
|
||||
|
||||
return result
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.model
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import groovy.transform.ToString
|
||||
|
||||
/**
|
||||
* Models K8s pod security context
|
||||
*
|
||||
* See
|
||||
* https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
@ToString(includeNames = true)
|
||||
@EqualsAndHashCode(includeFields = true)
|
||||
class PodSecurityContext {
|
||||
|
||||
private Map spec
|
||||
|
||||
PodSecurityContext(def user) {
|
||||
spec = [runAsUser: user]
|
||||
}
|
||||
|
||||
PodSecurityContext(Map ctx) {
|
||||
assert ctx
|
||||
spec = ctx
|
||||
}
|
||||
|
||||
Map toSpec() { spec }
|
||||
|
||||
String toString() {
|
||||
"PodSecurityContext[ ${spec?.toString()} ]"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,814 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.model
|
||||
|
||||
import java.nio.file.Path
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
import groovy.transform.CompileDynamic
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.PackageScope
|
||||
import nextflow.executor.res.AcceleratorResource
|
||||
import nextflow.util.MemoryUnit
|
||||
import groovy.util.logging.Slf4j
|
||||
|
||||
/**
|
||||
* Object build for a K8s pod specification
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
@Slf4j
|
||||
class PodSpecBuilder {
|
||||
|
||||
static enum MetaType { LABEL, ANNOTATION }
|
||||
|
||||
static enum SegmentType {
|
||||
PREFIX (253),
|
||||
NAME (63),
|
||||
VALUE (63)
|
||||
|
||||
private final int maxSize;
|
||||
SegmentType(int maxSize) {
|
||||
this.maxSize = maxSize;
|
||||
}
|
||||
}
|
||||
|
||||
static @PackageScope AtomicInteger VOLUMES = new AtomicInteger()
|
||||
|
||||
String podName
|
||||
|
||||
String imageName
|
||||
|
||||
String imagePullPolicy
|
||||
|
||||
String imagePullSecret
|
||||
|
||||
List<String> command = []
|
||||
|
||||
List<String> args = new ArrayList<>()
|
||||
|
||||
Map<String,String> labels = [:]
|
||||
|
||||
Map<String,String> annotations = [:]
|
||||
|
||||
String namespace
|
||||
|
||||
String restart
|
||||
|
||||
List<PodEnv> envVars = []
|
||||
|
||||
String workDir
|
||||
|
||||
Integer cpus
|
||||
|
||||
boolean cpuLimits
|
||||
|
||||
String memory
|
||||
|
||||
String disk
|
||||
|
||||
String serviceAccount
|
||||
|
||||
boolean automountServiceAccountToken = true
|
||||
|
||||
AcceleratorResource accelerator
|
||||
|
||||
Collection<PodMountConfig> configMaps = []
|
||||
|
||||
Collection<PodMountCsiEphemeral> csiEphemerals = []
|
||||
|
||||
Collection<PodMountEmptyDir> emptyDirs = []
|
||||
|
||||
Collection<PodMountSecret> secrets = []
|
||||
|
||||
Collection<PodHostMount> hostMounts = []
|
||||
|
||||
Collection<PodVolumeClaim> volumeClaims = []
|
||||
|
||||
PodSecurityContext securityContext
|
||||
|
||||
PodNodeSelector nodeSelector
|
||||
|
||||
Map affinity
|
||||
|
||||
String priorityClassName
|
||||
|
||||
List<Map> tolerations = []
|
||||
|
||||
boolean privileged
|
||||
|
||||
int activeDeadlineSeconds
|
||||
|
||||
Map<String,List<String>> capabilities
|
||||
|
||||
List<String> devices
|
||||
|
||||
Map<String,?> resourcesLimits
|
||||
|
||||
String schedulerName
|
||||
|
||||
Integer ttlSecondsAfterFinished
|
||||
|
||||
String runtimeClassName
|
||||
|
||||
String nodeName
|
||||
|
||||
Integer port = null
|
||||
|
||||
/**
|
||||
* @return A sequential volume unique identifier
|
||||
*/
|
||||
static protected String nextVolName() {
|
||||
"vol-${VOLUMES.incrementAndGet()}".toString()
|
||||
}
|
||||
|
||||
PodSpecBuilder withPodName(String name) {
|
||||
this.podName = name
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withImageName(String name) {
|
||||
this.imageName = name
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withImagePullPolicy(String policy) {
|
||||
this.imagePullPolicy = policy
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withWorkDir( String path ) {
|
||||
this.workDir = path
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withWorkDir(Path path ) {
|
||||
this.workDir = path.toString()
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withNamespace(String name) {
|
||||
this.namespace = name
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withServiceAccount(String name) {
|
||||
this.serviceAccount = name
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withCommand( cmd ) {
|
||||
if( cmd==null ) return this
|
||||
assert cmd instanceof List || cmd instanceof CharSequence, "Missing or invalid K8s command parameter: $cmd"
|
||||
this.command = cmd instanceof List ? cmd as List<String> : ['/bin/bash','-c', cmd.toString()]
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withArgs( args ) {
|
||||
if( args==null ) return this
|
||||
assert args instanceof List || args instanceof CharSequence, "Missing or invalid K8s args parameter: $args"
|
||||
this.args = args instanceof List ? args as List<String> : ['/bin/bash','-c', args.toString()]
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withCpus( Integer cpus ) {
|
||||
this.cpus = cpus
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withCpuLimits(boolean cpuLimits) {
|
||||
this.cpuLimits = cpuLimits
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withMemory(String mem) {
|
||||
this.memory = mem
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withMemory(MemoryUnit mem) {
|
||||
this.memory = "${mem.mega}Mi".toString()
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withDisk(String disk) {
|
||||
this.disk = disk
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withDisk(MemoryUnit disk) {
|
||||
this.disk = "${disk.mega}Mi".toString()
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withAccelerator(AcceleratorResource acc) {
|
||||
this.accelerator = acc
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withLabel( String name, String value ) {
|
||||
this.labels.put(name, value)
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withLabels(Map labels) {
|
||||
this.labels.putAll(labels)
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withAnnotation( String name, String value ) {
|
||||
this.annotations.put(name, value)
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withAnnotations(Map annotations) {
|
||||
this.annotations.putAll(annotations)
|
||||
return this
|
||||
}
|
||||
|
||||
|
||||
PodSpecBuilder withEnv( PodEnv env ) {
|
||||
envVars.add(env)
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withEnv( Collection envs ) {
|
||||
envVars.addAll(envs)
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withVolumeClaim( PodVolumeClaim claim ) {
|
||||
volumeClaims.add(claim)
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withVolumeClaims( Collection<PodVolumeClaim> claims ) {
|
||||
volumeClaims.addAll(claims)
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withConfigMaps( Collection<PodMountConfig> configMaps ) {
|
||||
this.configMaps.addAll(configMaps)
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withConfigMap( PodMountConfig configMap ) {
|
||||
this.configMaps.add(configMap)
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withCsiEphemerals( Collection<PodMountCsiEphemeral> csiEphemerals ) {
|
||||
this.csiEphemerals.addAll(csiEphemerals)
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withCsiEphemeral( PodMountCsiEphemeral csiEphemeral ) {
|
||||
this.csiEphemerals.add(csiEphemeral)
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withEmptyDirs( Collection<PodMountEmptyDir> emptyDirs ) {
|
||||
this.emptyDirs.addAll(emptyDirs)
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withEmptyDir( PodMountEmptyDir emptyDir ) {
|
||||
this.emptyDirs.add(emptyDir)
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withSecrets( Collection<PodMountSecret> secrets ) {
|
||||
this.secrets.addAll(secrets)
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withSecret( PodMountSecret secret ) {
|
||||
this.secrets.add(secret)
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withHostMounts( Collection<PodHostMount> mounts ) {
|
||||
this.hostMounts.addAll(mounts)
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withHostMount( String host, String mount ) {
|
||||
this.hostMounts.add( new PodHostMount(host, mount))
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withPrivileged(boolean value) {
|
||||
this.privileged = value
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withCapabilities(Map<String,List<String>> cap) {
|
||||
this.capabilities = cap
|
||||
for( String it : cap.keySet() ) {
|
||||
if( it !in ['add','drop']) throw new IllegalArgumentException("K8s capability action can be either 'add' or 'drop' - offending value '$it'")
|
||||
}
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withActiveDeadline(int seconds) {
|
||||
this.activeDeadlineSeconds = seconds
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withResourcesLimits(Map<String,?> limits) {
|
||||
this.resourcesLimits = limits
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withPodOptions(PodOptions opts) {
|
||||
// -- pull policy
|
||||
if( opts.imagePullPolicy )
|
||||
imagePullPolicy = opts.imagePullPolicy
|
||||
if( opts.imagePullSecret )
|
||||
imagePullSecret = opts.imagePullSecret
|
||||
// -- env vars
|
||||
if( opts.getEnvVars() )
|
||||
envVars.addAll( opts.getEnvVars() )
|
||||
// -- configMaps
|
||||
if( opts.getMountConfigMaps() )
|
||||
configMaps.addAll( opts.getMountConfigMaps() )
|
||||
// -- csi ephemeral volumes
|
||||
if( opts.getMountCsiEphemerals() )
|
||||
csiEphemerals.addAll( opts.getMountCsiEphemerals() )
|
||||
// -- emptyDirs
|
||||
if( opts.getMountEmptyDirs() )
|
||||
emptyDirs.addAll( opts.getMountEmptyDirs() )
|
||||
// -- host paths
|
||||
if( opts.getMountHostPaths() )
|
||||
hostMounts.addAll( opts.getMountHostPaths() )
|
||||
// -- secrets
|
||||
if( opts.getMountSecrets() )
|
||||
secrets.addAll( opts.getMountSecrets() )
|
||||
// -- volume claims
|
||||
if( opts.getVolumeClaims() )
|
||||
volumeClaims.addAll( opts.getVolumeClaims() )
|
||||
// -- labels
|
||||
if( opts.labels ) {
|
||||
def keys = opts.labels.keySet()
|
||||
if( 'app' in keys ) throw new IllegalArgumentException("Invalid pod label -- `app` is a reserved label")
|
||||
if( 'runName' in keys ) throw new IllegalArgumentException("Invalid pod label -- `runName` is a reserved label")
|
||||
labels.putAll( opts.labels )
|
||||
}
|
||||
// - annotations
|
||||
if( opts.annotations ) {
|
||||
annotations.putAll( opts.annotations )
|
||||
}
|
||||
// -- security context
|
||||
if( opts.securityContext )
|
||||
securityContext = opts.securityContext
|
||||
// -- node selector
|
||||
if( opts.nodeSelector )
|
||||
nodeSelector = opts.nodeSelector
|
||||
// -- affinity
|
||||
if( opts.affinity )
|
||||
affinity = opts.affinity
|
||||
// -- automount service account token
|
||||
automountServiceAccountToken = opts.automountServiceAccountToken
|
||||
// -- priority class name
|
||||
priorityClassName = opts.priorityClassName
|
||||
// -- tolerations
|
||||
if( opts.tolerations )
|
||||
tolerations.addAll(opts.tolerations)
|
||||
// -- privileged
|
||||
privileged = opts.privileged
|
||||
// -- scheduler name
|
||||
schedulerName = opts.schedulerName
|
||||
// -- ttl seconds after finished (job)
|
||||
if( opts.ttlSecondsAfterFinished != null )
|
||||
ttlSecondsAfterFinished = opts.ttlSecondsAfterFinished
|
||||
// runtime class name
|
||||
if( opts.runtimeClassName != null )
|
||||
runtimeClassName = opts.runtimeClassName
|
||||
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withNodeName(String value) {
|
||||
this.nodeName = value
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withPort(int value) {
|
||||
this.port = new Integer(value)
|
||||
return this
|
||||
}
|
||||
|
||||
@PackageScope List<Map> createPullSecret() {
|
||||
def result = new ArrayList(1)
|
||||
def entry = new LinkedHashMap(1)
|
||||
entry.name = imagePullSecret
|
||||
result.add(entry)
|
||||
return result
|
||||
}
|
||||
|
||||
Map build() {
|
||||
assert this.podName, 'Missing K8s podName parameter'
|
||||
assert this.imageName, 'Missing K8s imageName parameter'
|
||||
assert this.command || this.args, 'Missing K8s command parameter'
|
||||
|
||||
final restart = this.restart ?: 'Never'
|
||||
|
||||
final metadata = new LinkedHashMap<String,Object>()
|
||||
metadata.name = podName
|
||||
metadata.namespace = namespace ?: 'default'
|
||||
|
||||
final labels = this.labels ?: [:]
|
||||
final env = []
|
||||
for( PodEnv entry : this.envVars ) {
|
||||
env.add(entry.toSpec())
|
||||
}
|
||||
|
||||
final container = [ name: this.podName, image: this.imageName ]
|
||||
if( this.command )
|
||||
container.command = this.command
|
||||
if( this.args )
|
||||
container.args = args
|
||||
|
||||
if( this.workDir )
|
||||
container.put('workingDir', workDir)
|
||||
|
||||
if( imagePullPolicy )
|
||||
container.imagePullPolicy = imagePullPolicy
|
||||
|
||||
if( this.port ) {
|
||||
List<Map> ports = []
|
||||
ports << [containerPort: this.port.intValue()]
|
||||
container.ports = ports
|
||||
}
|
||||
|
||||
final secContext = new LinkedHashMap(10)
|
||||
if( privileged ) {
|
||||
// note: privileged flag needs to be defined in the *container* securityContext
|
||||
// not the 'spec' securityContext (see below)
|
||||
secContext.privileged =true
|
||||
}
|
||||
if( capabilities ) {
|
||||
secContext.capabilities = capabilities
|
||||
}
|
||||
if( secContext ) {
|
||||
container.securityContext = secContext
|
||||
}
|
||||
|
||||
final spec = [
|
||||
restartPolicy: restart,
|
||||
containers: [ container ],
|
||||
]
|
||||
|
||||
if( nodeSelector )
|
||||
spec.nodeSelector = nodeSelector.toSpec()
|
||||
|
||||
if( schedulerName )
|
||||
spec.schedulerName = schedulerName
|
||||
|
||||
if( affinity )
|
||||
spec.affinity = affinity
|
||||
|
||||
if( this.serviceAccount )
|
||||
spec.serviceAccountName = this.serviceAccount
|
||||
|
||||
if( ! this.automountServiceAccountToken )
|
||||
spec.automountServiceAccountToken = false
|
||||
|
||||
if( securityContext )
|
||||
spec.securityContext = securityContext.toSpec()
|
||||
|
||||
if( imagePullSecret )
|
||||
spec.imagePullSecrets = createPullSecret()
|
||||
|
||||
if( priorityClassName )
|
||||
spec.priorityClassName = priorityClassName
|
||||
|
||||
// tolerations
|
||||
if( this.tolerations )
|
||||
spec.tolerations = this.tolerations
|
||||
|
||||
// add labels
|
||||
if( labels )
|
||||
metadata.labels = sanitize(labels, MetaType.LABEL)
|
||||
|
||||
if( annotations )
|
||||
metadata.annotations = sanitize(annotations, MetaType.ANNOTATION)
|
||||
|
||||
// time directive
|
||||
if ( activeDeadlineSeconds > 0)
|
||||
spec.activeDeadlineSeconds = activeDeadlineSeconds
|
||||
|
||||
if ( runtimeClassName )
|
||||
spec.runtimeClassName = runtimeClassName
|
||||
|
||||
if ( nodeName )
|
||||
spec.nodeName = nodeName
|
||||
|
||||
final pod = [
|
||||
apiVersion: 'v1',
|
||||
kind: 'Pod',
|
||||
metadata: metadata,
|
||||
spec: spec
|
||||
]
|
||||
|
||||
// add environment
|
||||
if( env )
|
||||
container.env = env
|
||||
|
||||
// add resources
|
||||
if( this.cpus ) {
|
||||
container.resources = addCpuResources(this.cpus, container.resources as Map)
|
||||
}
|
||||
|
||||
if( this.memory ) {
|
||||
container.resources = addMemoryResources(this.memory, container.resources as Map)
|
||||
}
|
||||
|
||||
if( this.accelerator ) {
|
||||
container.resources = addAcceleratorResources(this.accelerator, container.resources as Map)
|
||||
}
|
||||
|
||||
if( this.disk ) {
|
||||
container.resources = addDiskResources(this.disk, container.resources as Map)
|
||||
}
|
||||
|
||||
if( this.resourcesLimits ) {
|
||||
container.resources = addResourcesLimits(this.resourcesLimits, container.resources as Map)
|
||||
}
|
||||
|
||||
// add storage definitions ie. volumes and mounts
|
||||
final List<Map> mounts = []
|
||||
final List<Map> volumes = []
|
||||
final namesMap = [:]
|
||||
|
||||
// creates a volume name for each unique claim name
|
||||
for( String claimName : volumeClaims.collect { it.claimName }.unique() ) {
|
||||
final volName = nextVolName()
|
||||
namesMap[claimName] = volName
|
||||
volumes << [name: volName, persistentVolumeClaim: [claimName: claimName]]
|
||||
}
|
||||
|
||||
// -- persistent volume claims
|
||||
for( PodVolumeClaim entry : volumeClaims ) {
|
||||
//check if we already have a volume for the pvc
|
||||
final name = namesMap.get(entry.claimName)
|
||||
final claim = [name: name, mountPath: entry.mountPath ]
|
||||
if( entry.subPath )
|
||||
claim.subPath = entry.subPath
|
||||
if( entry.readOnly )
|
||||
claim.readOnly = entry.readOnly
|
||||
mounts << claim
|
||||
}
|
||||
|
||||
// -- configMap volumes
|
||||
for( PodMountConfig entry : configMaps ) {
|
||||
final name = nextVolName()
|
||||
configMapToSpec(name, entry, mounts, volumes)
|
||||
}
|
||||
|
||||
// -- csi ephemeral volumes
|
||||
for( PodMountCsiEphemeral entry : csiEphemerals ) {
|
||||
final name = nextVolName()
|
||||
mounts << [name: name, mountPath: entry.mountPath, readOnly: entry.csi.readOnly ?: false]
|
||||
volumes << [name: name, csi: entry.csi]
|
||||
}
|
||||
|
||||
// -- emptyDir volumes
|
||||
for( PodMountEmptyDir entry : emptyDirs ) {
|
||||
final name = nextVolName()
|
||||
mounts << [name: name, mountPath: entry.mountPath]
|
||||
volumes << [name: name, emptyDir: entry.emptyDir]
|
||||
}
|
||||
|
||||
// -- secret volumes
|
||||
for( PodMountSecret entry : secrets ) {
|
||||
final name = nextVolName()
|
||||
secretToSpec(name, entry, mounts, volumes)
|
||||
}
|
||||
|
||||
// -- host path volumes
|
||||
for( PodHostMount entry : hostMounts ) {
|
||||
final name = nextVolName()
|
||||
mounts << [name: name, mountPath: entry.mountPath]
|
||||
volumes << [name: name, hostPath: [path: entry.hostPath]]
|
||||
}
|
||||
|
||||
|
||||
if( volumes )
|
||||
spec.volumes = volumes
|
||||
if( mounts )
|
||||
container.volumeMounts = mounts
|
||||
|
||||
return pod
|
||||
}
|
||||
|
||||
Map buildAsJob() {
|
||||
final pod = build()
|
||||
final spec = [
|
||||
backoffLimit: 0,
|
||||
template: [
|
||||
metadata: pod.metadata,
|
||||
spec: pod.spec
|
||||
]
|
||||
]
|
||||
|
||||
if( ttlSecondsAfterFinished != null )
|
||||
spec.ttlSecondsAfterFinished = ttlSecondsAfterFinished
|
||||
|
||||
return [
|
||||
apiVersion: 'batch/v1',
|
||||
kind: 'Job',
|
||||
metadata: pod.metadata,
|
||||
spec: spec
|
||||
]
|
||||
}
|
||||
|
||||
@PackageScope
|
||||
Map addResourcesLimits(Map limits, Map result) {
|
||||
if( result == null )
|
||||
result = new LinkedHashMap(2)
|
||||
|
||||
final limits0 = result.limits as Map ?: new LinkedHashMap(10)
|
||||
limits0.putAll( limits )
|
||||
result.limits = limits0
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
@PackageScope
|
||||
Map addCpuResources(Integer cpus, Map res) {
|
||||
if( res == null )
|
||||
res = new LinkedHashMap(2)
|
||||
|
||||
final requests0 = res.requests as Map ?: new LinkedHashMap<>(10)
|
||||
requests0.cpu = cpus
|
||||
res.requests = requests0
|
||||
|
||||
if( cpuLimits ) {
|
||||
final limits0 = res.limits as Map ?: new LinkedHashMap(10)
|
||||
limits0.cpu = cpus
|
||||
res.limits = limits0
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
@PackageScope
|
||||
Map addMemoryResources(String memory, Map res) {
|
||||
if( res == null )
|
||||
res = new LinkedHashMap(2)
|
||||
|
||||
final req = res.requests as Map ?: new LinkedHashMap(10)
|
||||
req.memory = memory
|
||||
res.requests = req
|
||||
|
||||
final lim = res.limits as Map ?: new LinkedHashMap(10)
|
||||
lim.memory = memory
|
||||
res.limits = lim
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
@PackageScope
|
||||
Map addDiskResources(String diskRequest, Map res) {
|
||||
if( res == null )
|
||||
res = new LinkedHashMap(2)
|
||||
|
||||
final req = res.requests as Map ?: new LinkedHashMap(10)
|
||||
req.'ephemeral-storage' = diskRequest
|
||||
res.requests = req
|
||||
|
||||
final lim = res.limits as Map ?: new LinkedHashMap(10)
|
||||
lim.'ephemeral-storage' = diskRequest
|
||||
res.limits = lim
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
@PackageScope
|
||||
String getAcceleratorType(AcceleratorResource accelerator) {
|
||||
|
||||
def type = accelerator.type ?: 'nvidia.com'
|
||||
|
||||
if ( type.contains('/') )
|
||||
// Assume the user has fully specified the resource type.
|
||||
return type
|
||||
|
||||
// Assume we're using GPU and update as necessary.
|
||||
if( !type.contains('.') ) type += '.com'
|
||||
type += '/gpu'
|
||||
|
||||
return type
|
||||
}
|
||||
|
||||
|
||||
@PackageScope
|
||||
Map addAcceleratorResources(AcceleratorResource accelerator, Map res) {
|
||||
|
||||
if( res == null )
|
||||
res = new LinkedHashMap(2)
|
||||
|
||||
def type = getAcceleratorType(accelerator)
|
||||
|
||||
if( accelerator.request ) {
|
||||
final req = res.requests as Map ?: new LinkedHashMap<>(2)
|
||||
req.put(type, accelerator.request)
|
||||
res.requests = req
|
||||
}
|
||||
if( accelerator.limit ) {
|
||||
final lim = res.limits as Map ?: new LinkedHashMap<>(2)
|
||||
lim.put(type, accelerator.limit)
|
||||
res.limits = lim
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
@PackageScope
|
||||
@CompileDynamic
|
||||
static void secretToSpec(String volName, PodMountSecret entry, List mounts, List volumes ) {
|
||||
assert entry
|
||||
|
||||
final secret = [secretName: entry.secretName]
|
||||
if( entry.secretKey ) {
|
||||
secret.items = [ [key: entry.secretKey, path: entry.fileName ] ]
|
||||
}
|
||||
|
||||
mounts << [name: volName, mountPath: entry.mountPath]
|
||||
volumes << [name: volName, secret: secret ]
|
||||
}
|
||||
|
||||
@PackageScope
|
||||
@CompileDynamic
|
||||
static void configMapToSpec(String volName, PodMountConfig entry, List<Map> mounts, List<Map> volumes ) {
|
||||
assert entry
|
||||
|
||||
final config = [name: entry.configName]
|
||||
if( entry.configKey ) {
|
||||
config.items = [ [key: entry.configKey, path: entry.fileName ] ]
|
||||
}
|
||||
|
||||
mounts << [name: volName, mountPath: entry.mountPath]
|
||||
volumes << [name: volName, configMap: config ]
|
||||
}
|
||||
|
||||
protected Map sanitize(Map map, MetaType kind) {
|
||||
final result = new HashMap(map.size())
|
||||
for( Map.Entry entry : map ) {
|
||||
final key = sanitizeKey(entry.key as String, kind)
|
||||
final value = (kind == MetaType.LABEL)
|
||||
? sanitizeValue(entry.value, kind, SegmentType.VALUE)
|
||||
: entry.value
|
||||
|
||||
result.put(key, value)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
protected String sanitizeKey(String value, MetaType kind) {
|
||||
final parts = value.tokenize('/')
|
||||
|
||||
if (parts.size() == 2) {
|
||||
return "${sanitizeValue(parts[0], kind, SegmentType.PREFIX)}/${sanitizeValue(parts[1], kind, SegmentType.NAME)}"
|
||||
}
|
||||
if( parts.size() == 1 ) {
|
||||
return sanitizeValue(parts[0], kind, SegmentType.NAME)
|
||||
}
|
||||
else {
|
||||
throw new IllegalArgumentException("Invalid key in pod ${kind.toString().toLowerCase()} -- Key can only contain exactly one '/' character")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Sanitize a string value to contain only alphanumeric characters, '-', '_' or '.',
|
||||
* and to start and end with an alphanumeric character.
|
||||
*/
|
||||
protected String sanitizeValue(value, MetaType kind, SegmentType segment) {
|
||||
def str = String.valueOf(value)
|
||||
if( str.length() > segment.maxSize ) {
|
||||
log.debug "K8s $kind $segment exceeds allowed size: $segment.maxSize -- offending str=$str"
|
||||
str = str.substring(0,segment.maxSize)
|
||||
}
|
||||
str = str.replaceAll(/[^a-zA-Z0-9\.\_\-]+/, '_')
|
||||
str = str.replaceAll(/^[^a-zA-Z0-9]+/, '')
|
||||
str = str.replaceAll(/[^a-zA-Z0-9]+$/, '')
|
||||
return str
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.model
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import groovy.transform.ToString
|
||||
|
||||
/**
|
||||
* Model a K8s pod persistent volume claim mount
|
||||
*
|
||||
* See https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
@ToString(includeNames = true)
|
||||
@EqualsAndHashCode
|
||||
class PodVolumeClaim {
|
||||
|
||||
String claimName
|
||||
|
||||
String mountPath
|
||||
|
||||
String subPath
|
||||
|
||||
boolean readOnly
|
||||
|
||||
PodVolumeClaim(String name, String mount, String subPath=null, boolean readOnly=false) {
|
||||
assert name
|
||||
assert mount
|
||||
this.claimName = name
|
||||
this.mountPath = sanitize(mount)
|
||||
this.subPath = subPath
|
||||
this.readOnly = readOnly
|
||||
validate(mountPath)
|
||||
}
|
||||
|
||||
PodVolumeClaim(Map entry) {
|
||||
assert entry.volumeClaim
|
||||
assert entry.mountPath
|
||||
this.claimName = entry.volumeClaim
|
||||
this.mountPath = sanitize(entry.mountPath)
|
||||
this.subPath = entry.subPath
|
||||
this.readOnly = entry.readOnly ?: false
|
||||
validate(mountPath)
|
||||
}
|
||||
|
||||
private static validate(String path) {
|
||||
if( !path.startsWith('/') )
|
||||
throw new IllegalArgumentException("K8s volume claim path must be an absolute path: $path")
|
||||
}
|
||||
|
||||
static private String sanitize(path) {
|
||||
if( !path ) return null
|
||||
def result = path.toString().trim()
|
||||
while( result.endsWith('/') && result.size()>1 )
|
||||
result = result.substring(0,result.size()-1)
|
||||
return result
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.model
|
||||
|
||||
/**
|
||||
* Model the resource type to be used to run nextflow tasks
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
enum ResourceType {
|
||||
Pod, Job;
|
||||
|
||||
String lower() {
|
||||
return this.name().toLowerCase()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,384 @@
|
||||
package recreationaltech.plugin.strategies
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import recreationaltech.plugin.K8sDVFSClient
|
||||
import recreationaltech.plugin.K8sRuntimeEstimator
|
||||
import recreationaltech.plugin.K8sSchedulingDecision
|
||||
import recreationaltech.plugin.K8sSchedulingRequest
|
||||
import recreationaltech.plugin.K8sSchedulingStrategy
|
||||
import recreationaltech.plugin.K8sTaskHandler
|
||||
import recreationaltech.plugin.K8sTaskScheduler
|
||||
import recreationaltech.plugin.client.K8sClient
|
||||
import nextflow.processor.TaskRun
|
||||
import nextflow.util.Duration
|
||||
|
||||
/**
|
||||
* Implements a scheduling strategy utilizing dvfs to reduce the energy consumption
|
||||
* of workflow execution, while attempting to maintain the same makespan.
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class K8sDVFSSchedulingStrategy implements K8sSchedulingStrategy {
|
||||
|
||||
/** Used for passing K8sExecutor.getClient.
|
||||
* We cannot pass the client directly, becaue it can be refreshed
|
||||
* during workflow execution.
|
||||
*/
|
||||
public interface K8sClientGetter {
|
||||
K8sClient getClient()
|
||||
}
|
||||
|
||||
private static long getTaskMemoryRequirment(TaskRun task) {
|
||||
return task.config.getMemory() ? task.config.getMemory().bytes : 64 * 1024 * 1024
|
||||
}
|
||||
|
||||
private static int getTaskCPURequirement(TaskRun task) {
|
||||
return task.config.hasCpus() ? task.config.getCpus() * 1000 : 1000
|
||||
}
|
||||
|
||||
@Slf4j
|
||||
private class WorkerNode {
|
||||
private class AssignedTask {
|
||||
TaskRun task
|
||||
long frequency
|
||||
|
||||
AssignedTask(TaskRun t, long f) {
|
||||
this.task = t
|
||||
this.frequency = f
|
||||
}
|
||||
}
|
||||
|
||||
String name
|
||||
|
||||
long maxFrequency
|
||||
long minFrequency
|
||||
long currentFrequency
|
||||
|
||||
ArrayList<AssignedTask> tasks
|
||||
|
||||
WorkerNode(String name, long maxF, long minF, long curF, K8sClientGetter clientGetter) {
|
||||
this.name = name
|
||||
this.maxFrequency = maxF
|
||||
this.minFrequency = minF
|
||||
this.currentFrequency = curF
|
||||
this.tasks = new ArrayList<>()
|
||||
}
|
||||
|
||||
/* Return the number of available (unoccupied) bytes */
|
||||
long getAvailableMemory() {
|
||||
def k8sClient = clientGetter.getClient()
|
||||
Long available = k8sClient.getNodeMemoryAvailableBytes(this.name)
|
||||
if (available == null) {
|
||||
log.warn "[K8s] failed to retrieve available memory for node ${name}"
|
||||
// Fallback: try capacity - allocated
|
||||
Long capacity = k8sClient.getNodeMemoryCapacityBytes(this.name)
|
||||
Long allocated = k8sClient.getNodeMemoryAllocatedBytes(this.name)
|
||||
if (capacity != null && allocated != null) {
|
||||
return capacity - allocated
|
||||
}
|
||||
return 0
|
||||
}
|
||||
return available
|
||||
}
|
||||
|
||||
/* Return the number of available (unoccupied) cpu cores */
|
||||
long getAvailableCPUs() {
|
||||
def k8sClient = clientGetter.getClient()
|
||||
Long available = k8sClient.getNodeCpuAvailableMillis(this.name)
|
||||
if (available == null) {
|
||||
log.warn "[K8s] failed to retrieve available CPU for node ${name}"
|
||||
// Fallback: try capacity - allocated
|
||||
Long capacity = k8sClient.getNodeCpuCapacityMillis(this.name)
|
||||
Long allocated = k8sClient.getNodeCpuAllocatedMillis(this.name)
|
||||
if (capacity != null && allocated != null) {
|
||||
return capacity - allocated
|
||||
}
|
||||
return 0
|
||||
}
|
||||
return available
|
||||
}
|
||||
|
||||
/* Return the total amount of installed memory */
|
||||
long getMemoryAmount() {
|
||||
def k8sClient = clientGetter.getClient()
|
||||
Long capacity = k8sClient.getNodeMemoryCapacityBytes(this.name)
|
||||
return capacity != null ? capacity.longValue() : 0
|
||||
}
|
||||
|
||||
/* Return the total number of installed cpu cores */
|
||||
long getCPUCount() {
|
||||
def k8sClient = clientGetter.getClient()
|
||||
Long capacity = k8sClient.getNodeCpuCapacityMillis(this.name)
|
||||
return capacity != null ? capacity.longValue(): 0
|
||||
}
|
||||
|
||||
// Sets the frequency to the max. requested frequency of all currently running tasks.
|
||||
private void updateFrequency(K8sDVFSClient dvfsClient) {
|
||||
if (tasks.size() == 0)
|
||||
return
|
||||
long max = Long.MIN_VALUE
|
||||
for (AssignedTask t : tasks) {
|
||||
max = Long.max(t.frequency, max)
|
||||
}
|
||||
log.info "[K8s] node ${name} running at ${max} Hz / ${maxFrequency} Hz ${((double)max/(double)maxFrequency) * 100.0}%"
|
||||
dvfsClient.setNodeFrequency(name, (int)max)
|
||||
this.currentFrequency = max
|
||||
}
|
||||
|
||||
void assignTask(TaskRun task, long frequency, K8sDVFSClient dvfsClient) {
|
||||
log.info "[K8s] node ${name}: task ${task.name} assigned with ${frequency}/${maxFrequency}"
|
||||
this.tasks.add(new AssignedTask(task, frequency))
|
||||
updateFrequency(dvfsClient)
|
||||
}
|
||||
|
||||
void taskFinished(TaskRun task, K8sDVFSClient dvfsClient) {
|
||||
log.info "[K8s] node ${name}: task ${task.name} finished"
|
||||
this.tasks.removeIf {it.task == task}
|
||||
updateFrequency(dvfsClient)
|
||||
}
|
||||
}
|
||||
|
||||
class SchedulingRequestComparator implements Comparator<K8sSchedulingRequest> {
|
||||
K8sRuntimeEstimator runtimeEstimator
|
||||
long currentTime
|
||||
double epsilon
|
||||
|
||||
@Override
|
||||
int compare(K8sSchedulingRequest o1, K8sSchedulingRequest o2) {
|
||||
// First, check if one of the tasks is (estimated to be) on the critical path
|
||||
double t1 = runtimeEstimator.estimate(o1.handler)
|
||||
double t2 = runtimeEstimator.estimate(o2.handler)
|
||||
|
||||
if (t1 > t2 + epsilon)
|
||||
return -1
|
||||
else if (t2 > t1 + epsilon)
|
||||
return 1
|
||||
|
||||
// Both are not on the critical path. Sort based on the time they spent in the queue
|
||||
long w1 = currentTime - o1.submitTimeMillis
|
||||
long w2 = currentTime - o2.submitTimeMillis
|
||||
if (w1 > w2)
|
||||
return -1
|
||||
else if (w2 > w1)
|
||||
return 1
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
private K8sRuntimeEstimator runtimeEstimator
|
||||
private K8sDVFSClient dvfsClient
|
||||
|
||||
private ArrayList<WorkerNode> nodes
|
||||
private HashMap<String, WorkerNode> taskToNode
|
||||
private long globalMaxFrequency
|
||||
private long globalMinFrequency
|
||||
|
||||
private K8sClientGetter clientGetter
|
||||
|
||||
private double comparisonEpsilonMillis
|
||||
|
||||
private double[] topRuntimes
|
||||
private double averageRuntime
|
||||
private double finishedTaskCount
|
||||
|
||||
boolean fullSpeedMode
|
||||
|
||||
K8sDVFSSchedulingStrategy(K8sRuntimeEstimator runtimeEstimator,
|
||||
K8sDVFSClient dvfsClient,
|
||||
K8sClientGetter clientGetter,
|
||||
Duration runtimeComparisonEpsilon,
|
||||
int topRuntimeCount) {
|
||||
this.runtimeEstimator = runtimeEstimator
|
||||
this.dvfsClient = dvfsClient
|
||||
this.nodes = new ArrayList<>()
|
||||
this.taskToNode = new HashMap<>();
|
||||
this.clientGetter = clientGetter
|
||||
this.comparisonEpsilonMillis = (double)runtimeComparisonEpsilon.toMillis()
|
||||
this.topRuntimes = new double[topRuntimeCount]
|
||||
for (int i = 0; i < topRuntimeCount; i++) {
|
||||
this.topRuntimes[i] = 0.0
|
||||
}
|
||||
this.averageRuntime = 0.0
|
||||
this.finishedTaskCount = 0.0
|
||||
}
|
||||
|
||||
private boolean isInTopRuntimes(double rt) {
|
||||
for (int i = 0; i < topRuntimes.size(); i++) {
|
||||
if (rt >= topRuntimes[i])
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
private void updateTopRuntimes(double rt) {
|
||||
for (int i = 0; i < topRuntimes.size(); i++) {
|
||||
if (rt > topRuntimes[i]) {
|
||||
/* Move all one down */
|
||||
for (int j = topRuntimes.size() - 1; j > i; j--) {
|
||||
topRuntimes[j] = topRuntimes[j - 1];
|
||||
}
|
||||
topRuntimes[i] = rt
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
K8sSchedulingDecision schedule(K8sTaskScheduler scheduler, List<K8sSchedulingRequest> queue) {
|
||||
if (nodes.isEmpty()) {
|
||||
if (!initNodes(scheduler))
|
||||
return null
|
||||
}
|
||||
|
||||
/* Step 1: Sort by task priority. We will attempt to schedule tasks "in order", so that the
|
||||
* highest priority tasks are assigned to nodes as soon as possible.
|
||||
*
|
||||
* Priority is based on a) the tasks estimated runtime and b) the wait time of the task.
|
||||
*/
|
||||
SchedulingRequestComparator comparator = new SchedulingRequestComparator()
|
||||
comparator.runtimeEstimator = runtimeEstimator
|
||||
comparator.epsilon = comparisonEpsilonMillis
|
||||
comparator.currentTime = System.currentTimeMillis()
|
||||
queue.sort(comparator)
|
||||
|
||||
/* Step 2: For each task attempt to schedule it onto a node */
|
||||
for (K8sSchedulingRequest req : queue) {
|
||||
/* Step 2.1: Determine if the task is on the critical path.
|
||||
* If yes, we just schedule it at max frequency on the node with the highest available
|
||||
* frequency. If not, we determine a frequency (see below).
|
||||
*/
|
||||
final double taskEstimation = runtimeEstimator.estimate(req.handler)
|
||||
final boolean isCriticalPath = isInTopRuntimes(taskEstimation)
|
||||
long frequency = globalMaxFrequency
|
||||
if (!isCriticalPath && !fullSpeedMode) {
|
||||
/* Set frequency so that we expect the runtime to be close to the mean runtime. */
|
||||
frequency = (long)Math.floor((taskEstimation * globalMaxFrequency) / averageRuntime)
|
||||
frequency = Math.max(frequency, globalMinFrequency)
|
||||
frequency = Math.min(frequency, globalMaxFrequency)
|
||||
}
|
||||
|
||||
/* Step 2.2: Filter nodes based on task requirements */
|
||||
ArrayList<WorkerNode> suitableNodes = filterNodes(req.task)
|
||||
if (suitableNodes.size() == 0) {
|
||||
if (!anyNode(req.task)) {
|
||||
log.error "[K8s] unable to schedule task ${req.task} - no node satisfies resource requirements ${getTaskMemoryRequirment(req.task)} bytes ${getTaskCPURequirement(req.task)} cpus"
|
||||
return null
|
||||
}
|
||||
/* No node can currently execute this task, but it should be possible in the future */
|
||||
/* log.info "[K8s] ${req.task} can not be scheduled: ${getTaskMemoryRequirment(req.task)} bytes ${getTaskCPURequirement(req.task)} CPUs"
|
||||
for (WorkerNode n : this.nodes) {
|
||||
log.info "[K8s] node ${n.name} - ${n.availableMemory}, ${n.availableCPUs}"
|
||||
} */
|
||||
continue
|
||||
}
|
||||
|
||||
/* Step 2.3: Assign to node based on "best fit" - current node frequency is closest to determined frequency */
|
||||
long minDist = Math.abs(suitableNodes[0].currentFrequency - frequency)
|
||||
WorkerNode closest = suitableNodes[0]
|
||||
for (WorkerNode node : suitableNodes) {
|
||||
long dist = Math.abs(node.currentFrequency - frequency)
|
||||
if (dist < minDist) {
|
||||
closest = node
|
||||
minDist = dist
|
||||
}
|
||||
}
|
||||
|
||||
closest.assignTask(req.task, frequency, dvfsClient)
|
||||
taskToNode.put(req.task.hash.toString(), closest)
|
||||
log.info "[K8s] DVFS: Assigned task ${req.task} to node ${closest.name} - ${taskToNode.size()} assigned tasks"
|
||||
return new K8sSchedulingDecision(req, closest.name)
|
||||
}
|
||||
|
||||
log.info "[K8s] unable to schedule any task. The queue contains ${queue.size()} tasks."
|
||||
return null
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean scheduleImmediately(K8sTaskScheduler scheduler, List<K8sSchedulingRequest> queue) {
|
||||
if (nodes.isEmpty()) {
|
||||
if (!initNodes(scheduler))
|
||||
return false
|
||||
}
|
||||
/* We want to schedule immediately if there are unoccupied nodes */
|
||||
boolean doIt = queue != null && queue.size() > 0 && taskToNode.size() < nodes.size()
|
||||
log.info "[K8s] scheduleImmediately: ${queue.size()} tasks in queue, ${taskToNode.size()} tasks running on ${nodes.size()} nodes: ${doIt}"
|
||||
return doIt
|
||||
}
|
||||
|
||||
@Override
|
||||
synchronized void taskFinished(K8sTaskHandler task) {
|
||||
/* TODO: This just uses elapsed wall-clock time, regardless of the frequency used to execute the task.
|
||||
* This will skew the average towards longer runtimes, which is undesirable, because it will lead to more
|
||||
* tasks classified as "critical path".
|
||||
* A simple (rough) solution could be to keep track of the tasks "relative" frequency and just scale the
|
||||
* elapsed time based on that.
|
||||
*/
|
||||
double runtime = (double)(task.getCompleteTimeMillis() - task.getStartTimeMillis())
|
||||
averageRuntime = (runtime + finishedTaskCount * averageRuntime) / (finishedTaskCount + 1.0)
|
||||
finishedTaskCount += 1.0
|
||||
updateTopRuntimes(runtime)
|
||||
|
||||
/* Free resources allocated by this task */
|
||||
WorkerNode node = taskToNode.get(task.task.hash.toString())
|
||||
if (node != null) {
|
||||
node.taskFinished(task.task, dvfsClient)
|
||||
taskToNode.remove(task.task.hash.toString())
|
||||
} else {
|
||||
log.warn "[K8s] no node recorded for task ${task.toString()}"
|
||||
}
|
||||
|
||||
log.info "[K8s] task ${task.toString()} finished - ${taskToNode.size()} tasks running"
|
||||
if (node != null) {
|
||||
log.info "[K8s] task ran on node ${node.name} - ${node.availableMemory} bytes ${node.availableCPUs}"
|
||||
}
|
||||
}
|
||||
|
||||
private synchronized boolean initNodes(K8sTaskScheduler scheduler) {
|
||||
this.globalMaxFrequency = Long.MAX_VALUE
|
||||
final nodes = scheduler.getNodes()
|
||||
for (String node : nodes) {
|
||||
final cur = dvfsClient.getNodeCurrentFrequency(node)
|
||||
final min = dvfsClient.getNodeMinFrequency(node)
|
||||
final max = dvfsClient.getNodeMaxFrequency(node)
|
||||
final cpus = dvfsClient.getCPUCount(node)
|
||||
final mem = dvfsClient.getMemoryAmount(node)
|
||||
|
||||
if (cur.empty || min.empty || max.empty || cpus.empty || mem.empty) {
|
||||
log.error "[K8s] failed to query node $node information"
|
||||
continue
|
||||
}
|
||||
globalMaxFrequency = Long.min(globalMaxFrequency, max.asLong)
|
||||
globalMinFrequency = Long.max(globalMinFrequency, min.asLong)
|
||||
|
||||
log.info "[K8s] node ${node}: ${cpus.asLong} CPUs, ${mem.asLong} bytes RAM ${min.asLong} Hz - ${max.asLong} Hz current ${cur.asLong}"
|
||||
this.nodes.add(new WorkerNode(node, max.asLong, min.asLong, cur.asLong, clientGetter))
|
||||
}
|
||||
return !this.nodes.isEmpty()
|
||||
}
|
||||
|
||||
/* Returns a list of nodes that fulfill the tasks resource requirements
|
||||
*/
|
||||
private ArrayList<WorkerNode> filterNodes(TaskRun task) {
|
||||
final long reqBytes = getTaskMemoryRequirment(task)
|
||||
final int reqCPUs = getTaskCPURequirement(task)
|
||||
ArrayList<WorkerNode> suitableNodes = new ArrayList<>()
|
||||
for (WorkerNode n : nodes) {
|
||||
if (n.availableMemory >= reqBytes && n.availableCPUs >= reqCPUs) {
|
||||
log.info "[K8s] task ${task.name}: ${reqBytes} bytes ${reqCPUs} cpus: node ${n.name} has ${n.availableMemory} bytes, ${n.availableCPUs} cpus"
|
||||
suitableNodes.add(n)
|
||||
}
|
||||
}
|
||||
return suitableNodes
|
||||
}
|
||||
|
||||
private boolean anyNode(TaskRun task) {
|
||||
final long reqBytes = getTaskMemoryRequirment(task)
|
||||
final int reqCPUs = getTaskCPURequirement(task)
|
||||
for (WorkerNode n : nodes) {
|
||||
if (n.memoryAmount >= reqBytes && n.CPUCount >= reqCPUs)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package recreationaltech.plugin.strategies
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import recreationaltech.plugin.K8sSchedulingDecision
|
||||
import recreationaltech.plugin.K8sSchedulingRequest
|
||||
import recreationaltech.plugin.K8sSchedulingStrategy
|
||||
import recreationaltech.plugin.K8sTaskHandler
|
||||
import recreationaltech.plugin.K8sTaskScheduler
|
||||
|
||||
@CompileStatic
|
||||
class K8sHashSchedulingStrategy implements K8sSchedulingStrategy {
|
||||
|
||||
@Override
|
||||
K8sSchedulingDecision schedule(K8sTaskScheduler scheduler, List<K8sSchedulingRequest> queue) {
|
||||
if (!queue)
|
||||
return null
|
||||
|
||||
final freeNodes = scheduler.nodes
|
||||
if (freeNodes.size() == 0)
|
||||
return null
|
||||
|
||||
final request = queue[0]
|
||||
final index = Math.floorMod(request.task.hash.asInt(), freeNodes.size())
|
||||
return new K8sSchedulingDecision(request, freeNodes[index])
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean scheduleImmediately(K8sTaskScheduler scheduler, List<K8sSchedulingRequest> queue) {
|
||||
return false
|
||||
}
|
||||
|
||||
@Override
|
||||
void taskFinished(K8sTaskHandler task) { /* nop */ }
|
||||
}
|
||||
Reference in New Issue
Block a user