kevin/ma
1
0
Fork 0
Code Issues 1 Packages Activity

various changes

Browse Source
This commit is contained in:
Kevin
2026-09-08 10:35:18 +02:00
parent 4188bea061
commit dfae78af71
2871 changed files with 280 additions and 732763 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
k8s-dvfs/build.gradle
View File

@@ -5,7 +5,7 @@ plugins {
version = '0.1.0'
nextflowPlugin {
nextflowVersion = '26.04.0'
nextflowVersion = '26.04.6'
provider = 'recreational.tech'
className = 'recreationaltech.plugin.K8sPlugin'
@@ -36,9 +36,10 @@ dependencies {
//compileOnly project(':nextflow')
compileOnly 'org.slf4j:slf4j-api:2.0.17'
compileOnly 'org.pf4j:pf4j:3.14.1'
compileOnly 'dev.failsafe:failsafe:3.3.2'
api 'org.bouncycastle:bcprov-ext-jdk18on:1.78.1'
api 'org.bouncycastle:bcpkix-jdk18on:1.84'
implementation 'org.bouncycastle:bcprov-ext-jdk18on:1.78.1'
implementation 'org.bouncycastle:bcpkix-jdk18on:1.84'
//testImplementation(testFixtures(project(":nextflow")))
testImplementation "org.apache.groovy:groovy:4.0.31"

17
k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sConfig.groovy
View File

@@ -272,6 +272,12 @@ class K8sConfig implements ConfigScope {
""")
final int dvfsSchedulingNumTopRuntimes
@ConfigOption
@Description("""
If not empty, only nodes with the given label will be used.
""")
final String nodeLabel
/* required by extension point -- do not remove */
K8sConfig() {
this(Collections.emptyMap())
@@ -314,7 +320,10 @@ class K8sConfig implements ConfigScope {
runtimeEstimator = opts.runtimeEstimator as String ?: "LinearFit"
noiseRuntimeEstimatorNoiseMagnitude = opts.noiseRuntimeEstimatorNoiseMagnitude as Duration ?: new Duration(30, TimeUnit.SECONDS)
runtimeComparisonEpsilon = opts.runtimeComparisonEpsilon as Duration ?: new Duration(10, TimeUnit.SECONDS)
dvfsSchedulingNumTopRuntimes = opts.dvfsSchedulingTopRuntimes as int ?: 3
def numTop = opts.dvfsSchedulingNumTopRuntimes
dvfsSchedulingNumTopRuntimes = numTop != null && numTop.toString().trim() != '' ? numTop as int : 3
nodeLabel = opts.nodeLabel as String ?: ""
// -- shortcut to pod image pull-policy
if( imagePullPolicy )
@@ -376,6 +385,12 @@ class K8sConfig implements ConfigScope {
return DEFAULT_FUSE_PLUGIN
}
String[] getNodeLabelFilter() {
if (nodeLabel.empty)
return null
return nodeLabel.split('=', 2)
}
/**
* Whenever the pod should honour the entrypoint defined by the image (default: false)
*

3
k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sDriverLauncher.groovy
View File

@@ -132,8 +132,6 @@ class K8sDriverLauncher {
*/
private String plugins
private K8sNodeInitDeployer initDeployer
/**
* Launcher entry point. Set-up the environment and create a pod that run the Nextflow
* application (which in turns executed each task as a pod)
@@ -155,7 +153,6 @@ class K8sDriverLauncher {
createK8sConfigMap()
initDeployer.deploy()
createK8sLauncherPod()
waitPodStart()
// login into container session

37
k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sExecutor.groovy
View File

@@ -57,6 +57,8 @@ class K8sExecutor extends Executor implements ExtensionPoint {
private K8sTaskScheduler taskScheduler
private Thread schedulerThread
private K8sNodeInitDeployer initDeployer
K8sRuntimeRecorder runtimeRecorder
K8sRuntimeEstimator runtimeEstimator
@@ -96,6 +98,9 @@ class K8sExecutor extends Executor implements ExtensionPoint {
log.debug "[K8s] config=$k8sConfig; API client config=$client.config"
this.initDeployer = new K8sNodeInitDeployer(client, k8sConfig)
initDeployer.deploy()
this.runtimeRecorder = new K8sRuntimeRecorder(k8sConfig.recordTaskRuntimes, k8sConfig.runtimeRecordPath)
if (k8sConfig.runtimeEstimator == "LinearFit") {
@@ -115,7 +120,8 @@ class K8sExecutor extends Executor implements ExtensionPoint {
} else if (k8sConfig.schedulingStrategy == "DVFS" || k8sConfig.schedulingStrategy == "DVFS-SPEED") {
String[] ips = new String[nodes.length]
for (int i = 0; i < nodes.length; i++) {
ips[i] = client.getPodIpAddress(K8sNodeInitDeployer.buildPodName(nodes[i]))
// Use node external IP since pods use hostNetwork
ips[i] = client.getNodeExternalIp(nodes[i])
log.info "[K8s] node ${nodes[i]} -> ${ips[i]}"
}
strategy = new K8sDVFSSchedulingStrategy(this.runtimeEstimator,
@@ -124,6 +130,19 @@ class K8sExecutor extends Executor implements ExtensionPoint {
k8sConfig.runtimeComparisonEpsilon,
k8sConfig.dvfsSchedulingNumTopRuntimes)
strategy.fullSpeedMode = k8sConfig.schedulingStrategy == "DVFS-SPEED"
} else if (k8sConfig.schedulingStrategy == "DVFS-BAND") {
String[] ips = new String[nodes.length]
for (int i = 0; i < nodes.length; i++) {
// Use node external IP since pods use hostNetwork
ips[i] = client.getNodeExternalIp(nodes[i])
log.info "[K8s] node ${nodes[i]} -> ${ips[i]}"
}
strategy = new K8sDVFSSchedulingStrategy(this.runtimeEstimator,
new K8sDVFSClient(nodes, ips),
() -> getClient(),
k8sConfig.runtimeComparisonEpsilon,
k8sConfig.dvfsSchedulingNumTopRuntimes)
strategy.fullSpeedMode = false
} else {
log.error "[K8s] invalid scheduling strategy $k8sConfig.schedulingStrategy, falling back on \"Hash\""
strategy = new K8sHashSchedulingStrategy()
@@ -136,10 +155,19 @@ class K8sExecutor extends Executor implements ExtensionPoint {
@CompileDynamic
private String[] getNodeList() {
final resp = getClient().nodeList()
final resp = client.nodeList()
ArrayList<String> nodes = new ArrayList<String>()
for ( Map item : resp.items ) {
nodes.add(item.metadata.name as String)
String[] filter = getK8sConfig().getNodeLabelFilter()
if (filter == null) {
for (Map item : resp.items) {
nodes.add(item.metadata.name as String)
}
} else {
for (Map item : resp.items) {
Map<String, String> labels = item.metadata.labels as Map<String, String>
if (labels.get(filter[0]) == filter[1])
nodes.add(item.metadata.name as String)
}
}
return nodes.toArray()
}
@@ -149,6 +177,7 @@ class K8sExecutor extends Executor implements ExtensionPoint {
this.runtimeRecorder.write()
this.taskScheduler.stop()
this.schedulerThread.join()
this.initDeployer.cleanup()
}
/**

20
k8s-dvfs/src/main/groovy/recreationaltech/plugin/K8sNodeInitDeployer.groovy
View File

@@ -19,13 +19,14 @@ class K8sNodeInitDeployer {
final init = config.nodeInit
if ( !init?.enabled )
return
final namespace = config.namespace
log.info("deploying init pods")
final nodes = getNodes()
for ( String nodeName : nodes ) {
log.info(" ... deploying to " + nodeName)
final spec = makePodSpec(init, nodeName)
final spec = makePodSpec(init, nodeName, namespace)
client.podCreate(spec)
}
@@ -108,8 +109,17 @@ exit 0
private List<String> getNodes() {
final resp = client.nodeList()
ArrayList<String> nodes = new ArrayList<String>()
for ( Map item: resp.items ) {
nodes.add(item.metadata.name as String)
String[] filter = config.getNodeLabelFilter()
if (filter == null) {
for (Map item : resp.items) {
nodes.add(item.metadata.name as String)
}
} else {
for (Map item : resp.items) {
Map<String, String> labels = item.metadata.labels as Map<String, String>
if (labels.get(filter[0]) == filter[1])
nodes.add(item.metadata.name as String)
}
}
return nodes
}
@@ -123,7 +133,7 @@ exit 0
return name
}
private Map makePodSpec(K8sNodeInitConfig config, String nodeName) {
private Map makePodSpec(K8sNodeInitConfig config, String nodeName, String namespace) {
ArrayList<PodHostMount> mounts = new ArrayList<PodHostMount>()
mounts.add(new PodHostMount("/sys", "/sys"))
mounts.add(new PodHostMount("/dev", "/dev"))
@@ -137,6 +147,8 @@ exit 0
.withHostMounts(mounts)
.withPodName(buildPodName(nodeName))
.withPort(8080)
.withHostNetwork(true)
.withNamespace(namespace)
.build()
}

54
k8s-dvfs/src/main/groovy/recreationaltech/plugin/client/K8sClient.groovy
View File

@@ -21,6 +21,7 @@ import dev.failsafe.FailsafeException
import dev.failsafe.RetryPolicy
import dev.failsafe.event.EventListener
import dev.failsafe.event.ExecutionAttemptedEvent
import dev.failsafe.function.CheckedPredicate
import dev.failsafe.function.CheckedSupplier
import nextflow.exception.K8sOutOfCpuException
import nextflow.exception.K8sOutOfMemoryException
@@ -371,6 +372,32 @@ class K8sClient {
(resp?.spec as Map)?.nodeName as String
}
/**
* Get the external IP address of a node
* @param nodeName The node name
* @return The external IP address of the node, or internal IP if external is not available
*/
String getNodeExternalIp(String nodeName) {
assert nodeName
final K8sResponseJson resp = nodeDescribe(nodeName)
final status = resp.status as Map
final addresses = status?.addresses as List<Map>
// Try to find ExternalIP first
for (Map address : addresses) {
if (address.type == 'ExternalIP') {
return address.address as String
}
}
// Fall back to InternalIP
for (Map address : addresses) {
if (address.type == 'InternalIP') {
return address.address as String
}
}
return null
}
/**
* Get pod current state object
*
@@ -870,10 +897,17 @@ class K8sClient {
trace('GET', action, resp.text)
final podList = new K8sResponseJson(resp.text)
final items = podList.items as List<Map>
long totalAllocated = 0L
if (items) {
for (Map pod : items) {
// Skip terminated/succeeded pods
final status = pod.status as Map
final phase = status?.phase as String
if (phase in ['Succeeded', 'Failed', 'Unknown']) {
continue
}
final spec = pod.spec as Map
if (spec) {
final containers = spec.containers as List<Map>
@@ -936,6 +970,13 @@ class K8sClient {
long totalAllocated = 0L
if (items) {
for (Map pod : items) {
// Skip terminated/succeeded pods
final status = pod.status as Map
final phase = status?.phase as String
if (phase in ['Succeeded', 'Failed', 'Unknown']) {
continue
}
final spec = pod.spec as Map
if (spec) {
final containers = spec.containers as List<Map>
@@ -1192,12 +1233,12 @@ class K8sClient {
* @param cond A predicate that determines when a retry should be triggered
* @return The {@link dev.failsafe.RetryPolicy} instance
*/
protected <T> RetryPolicy<T> retryPolicy(Predicate<? extends Throwable> cond) {
protected <T> RetryPolicy<T> retryPolicy(CheckedPredicate<? extends Throwable> cond) {
final cfg = config.retryConfig
final listener = new EventListener<ExecutionAttemptedEvent<T>>() {
@Override
void accept(ExecutionAttemptedEvent<T> event) throws Throwable {
log.debug("K8s response error - attempt: ${event.attemptCount}; reason: ${event.lastFailure.message}")
log.debug("K8s response error - attempt: ${event.attemptCount}; reason: ${event.lastException?.message ?: 'unknown'}")
}
}
return RetryPolicy.<T>builder()
@@ -1219,9 +1260,8 @@ class K8sClient {
*/
protected <T> T apply(CheckedSupplier<T> action) {
// define the retry condition
final cond = new Predicate<? extends Throwable>() {
@Override
boolean test(Throwable t) {
final cond = [
test: { Throwable t ->
if ( t instanceof K8sResponseException && t.response.code in RETRY_CODES )
return true
if( t instanceof SocketException || t.cause instanceof SocketException )
@@ -1230,7 +1270,7 @@ class K8sClient {
return true
return false
}
}
] as CheckedPredicate<Throwable>
// create the retry policy object
final policy = retryPolicy(cond)
// apply the action with and throw the original cause

10
k8s-dvfs/src/main/groovy/recreationaltech/plugin/model/PodSpecBuilder.groovy
View File

@@ -130,6 +130,8 @@ class PodSpecBuilder {
Integer port = null
boolean hostNetwork = false
/**
* @return A sequential volume unique identifier
*/
@@ -413,6 +415,11 @@ class PodSpecBuilder {
return this
}
PodSpecBuilder withHostNetwork(boolean value) {
this.hostNetwork = value
return this
}
@PackageScope List<Map> createPullSecret() {
def result = new ArrayList(1)
def entry = new LinkedHashMap(1)
@@ -519,6 +526,9 @@ class PodSpecBuilder {
if ( nodeName )
spec.nodeName = nodeName
if ( hostNetwork )
spec.hostNetwork = true
final pod = [
apiVersion: 'v1',
kind: 'Pod',

395
k8s-dvfs/src/main/groovy/recreationaltech/plugin/strategies/K8sDVFSFrequencyBandsSchedulingStrategy.groovy Normal file
View File

@@ -0,0 +1,395 @@
package recreationaltech.plugin.strategies
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.processor.TaskRun
import nextflow.util.Duration
import recreationaltech.plugin.*
import recreationaltech.plugin.client.K8sClient
/**
* Implements a scheduling strategy utilizing dvfs to reduce the energy consumption
* of workflow execution, while attempting to maintain the same makespan.
*
* This variant divides available nodes into three classes:
* - High frequency (100%)
* - Middle frequency (66%)
* - Low frequency (33%)
*
* Tasks are assigned to frequency classes according to their
* estimated runtime t:
* - t is in top-k runtimes => High
* - t is at most 1/2 top runtime => Middle
* - t is below 1/2 top runtime => Low
*
* Nodes are assigned to the classes via round robin,
* starting with high.
*/
@Slf4j
@CompileStatic
class K8sDVFSFrequencyBandsSchedulingStrategy implements K8sSchedulingStrategy {
/** Used for passing K8sExecutor.getClient.
* We cannot pass the client directly, becaue it can be refreshed
* during workflow execution.
*/
public interface K8sClientGetter {
K8sClient getClient()
}
private static long getTaskMemoryRequirment(TaskRun task) {
return task.config.getMemory() ? task.config.getMemory().bytes : 64 * 1024 * 1024
}
private static int getTaskCPURequirement(TaskRun task) {
return task.config.hasCpus() ? task.config.getCpus() * 1000 : 1000
}
@Slf4j
private class WorkerNode {
private class AssignedTask {
TaskRun task
AssignedTask(TaskRun t) {
this.task = t
}
}
String name
long frequency
ArrayList<AssignedTask> tasks
WorkerNode(String name, long frequency, K8sClientGetter clientGetter) {
this.name = name
this.frequency = frequency
this.tasks = new ArrayList<>()
}
/* Return the number of available (unoccupied) bytes */
long getAvailableMemory() {
def k8sClient = clientGetter.getClient()
Long available = k8sClient.getNodeMemoryAvailableBytes(this.name)
if (available == null) {
log.warn "[K8s] failed to retrieve available memory for node ${name}"
// Fallback: try capacity - allocated
Long capacity = k8sClient.getNodeMemoryCapacityBytes(this.name)
Long allocated = k8sClient.getNodeMemoryAllocatedBytes(this.name)
if (capacity != null && allocated != null) {
return capacity - allocated
}
return 0
}
return available
}
/* Return the number of available (unoccupied) cpu cores */
long getAvailableCPUs() {
def k8sClient = clientGetter.getClient()
Long available = k8sClient.getNodeCpuAvailableMillis(this.name)
if (available == null) {
log.warn "[K8s] failed to retrieve available CPU for node ${name}"
// Fallback: try capacity - allocated
Long capacity = k8sClient.getNodeCpuCapacityMillis(this.name)
Long allocated = k8sClient.getNodeCpuAllocatedMillis(this.name)
if (capacity != null && allocated != null) {
return capacity - allocated
}
return 0
}
return available
}
/* Return the total amount of installed memory */
long getMemoryAmount() {
def k8sClient = clientGetter.getClient()
Long capacity = k8sClient.getNodeMemoryCapacityBytes(this.name)
return capacity != null ? capacity.longValue() : 0
}
/* Return the total number of installed cpu cores */
long getCPUCount() {
def k8sClient = clientGetter.getClient()
Long capacity = k8sClient.getNodeCpuCapacityMillis(this.name)
return capacity != null ? capacity.longValue(): 0
}
// Resets the frequency
private void updateFrequency(K8sDVFSClient dvfsClient) {
if (tasks.size() == 0)
return
long maxFrequency = dvfsClient.getNodeMaxFrequency(this.name).orElse(this.frequency)
log.info "[K8s] node ${name} running at ${frequency} Hz / ${maxFrequency} Hz ${((double)frequency/(double)maxFrequency) * 100.0}%"
dvfsClient.setNodeFrequency(name, (int)frequency)
}
void assignTask(TaskRun task, K8sDVFSClient dvfsClient) {
log.info "[K8s] node ${name}: task ${task.name} assigned with ${this.frequency}"
this.tasks.add(new AssignedTask(task))
updateFrequency(dvfsClient)
}
void taskFinished(TaskRun task, K8sDVFSClient dvfsClient) {
log.info "[K8s] node ${name}: task ${task.name} finished"
this.tasks.removeIf {it.task == task}
updateFrequency(dvfsClient)
}
}
class SchedulingRequestComparator implements Comparator<K8sSchedulingRequest> {
K8sRuntimeEstimator runtimeEstimator
long currentTime
double epsilon
@Override
int compare(K8sSchedulingRequest o1, K8sSchedulingRequest o2) {
// First, check if one of the tasks is (estimated to be) on the critical path
double t1 = runtimeEstimator.estimate(o1.handler)
double t2 = runtimeEstimator.estimate(o2.handler)
if (t1 > t2 + epsilon)
return -1
else if (t2 > t1 + epsilon)
return 1
// Both are not on the critical path. Sort based on the time they spent in the queue
long w1 = currentTime - o1.submitTimeMillis
long w2 = currentTime - o2.submitTimeMillis
if (w1 > w2)
return -1
else if (w2 > w1)
return 1
return 0
}
}
private K8sRuntimeEstimator runtimeEstimator
private K8sDVFSClient dvfsClient
private ArrayList<WorkerNode> nodes
private HashMap<String, WorkerNode> taskToNode
private long globalMaxFrequency
private long globalMinFrequency
private long[] frequencies
private K8sClientGetter clientGetter
private double comparisonEpsilonMillis
private double[] topRuntimes
private double averageRuntime
private double finishedTaskCount
K8sDVFSFrequencyBandsSchedulingStrategy(K8sRuntimeEstimator runtimeEstimator,
K8sDVFSClient dvfsClient,
K8sClientGetter clientGetter,
Duration runtimeComparisonEpsilon,
int topRuntimeCount) {
this.runtimeEstimator = runtimeEstimator
this.dvfsClient = dvfsClient
this.nodes = new ArrayList<>()
this.taskToNode = new HashMap<>();
this.clientGetter = clientGetter
this.comparisonEpsilonMillis = (double)runtimeComparisonEpsilon.toMillis()
this.topRuntimes = new double[topRuntimeCount]
for (int i = 0; i < topRuntimeCount; i++) {
this.topRuntimes[i] = 0.0
}
this.averageRuntime = 0.0
this.finishedTaskCount = 0.0
}
private boolean isInTopRuntimes(double rt) {
for (int i = 0; i < topRuntimes.size(); i++) {
if (rt >= topRuntimes[i])
return true
}
return false
}
private void updateTopRuntimes(double rt) {
for (int i = 0; i < topRuntimes.size(); i++) {
if (rt > topRuntimes[i]) {
/* Move all one down */
for (int j = topRuntimes.size() - 1; j > i; j--) {
topRuntimes[j] = topRuntimes[j - 1];
}
topRuntimes[i] = rt
break
}
}
}
@Override
K8sSchedulingDecision schedule(K8sTaskScheduler scheduler, List<K8sSchedulingRequest> queue) {
if (nodes.isEmpty()) {
if (!initNodes(scheduler))
return null
}
/* Step 1: Sort by task priority. We will attempt to schedule tasks "in order", so that the
* highest priority tasks are assigned to nodes as soon as possible.
*
* Priority is based on a) the tasks estimated runtime and b) the wait time of the task.
*/
SchedulingRequestComparator comparator = new SchedulingRequestComparator()
comparator.runtimeEstimator = runtimeEstimator
comparator.epsilon = comparisonEpsilonMillis
comparator.currentTime = System.currentTimeMillis()
queue.sort(comparator)
/* Step 2: For each task attempt to schedule it onto a node */
for (K8sSchedulingRequest req : queue) {
/* Step 2.1: Determine if the task is on the critical path.
* If yes, we just schedule it at max frequency on the node with the highest available
* frequency. If not, we determine a frequency (see below).
*/
final double taskEstimation = runtimeEstimator.estimate(req.handler)
final boolean isCriticalPath = isInTopRuntimes(taskEstimation)
long frequency = this.frequencies[0]
if (!isCriticalPath) {
if (taskEstimation >= topRuntimes[0] * .5)
frequency = this.frequencies[1]
else
frequency = this.frequencies[2]
}
/* Step 2.2: Filter nodes based on task requirements */
ArrayList<WorkerNode> suitableNodes = filterNodes(req.task)
if (suitableNodes.size() == 0) {
if (!anyNode(req.task)) {
log.error "[K8s] unable to schedule task ${req.task} - no node satisfies resource requirements ${getTaskMemoryRequirment(req.task)} bytes ${getTaskCPURequirement(req.task)} cpus"
return null
}
/* No node can currently execute this task, but it should be possible in the future */
/* log.info "[K8s] ${req.task} can not be scheduled: ${getTaskMemoryRequirment(req.task)} bytes ${getTaskCPURequirement(req.task)} CPUs"
for (WorkerNode n : this.nodes) {
log.info "[K8s] node ${n.name} - ${n.availableMemory}, ${n.availableCPUs}"
} */
continue
}
/* Step 2.3: Assign to node based on "best fit" - current node frequency is closest to determined frequency */
long minDist = Math.abs(suitableNodes[0].frequency - frequency)
WorkerNode closest = suitableNodes[0]
for (WorkerNode node : suitableNodes) {
long dist = Math.abs(node.frequency - frequency)
if (dist < minDist) {
closest = node
minDist = dist
}
}
closest.assignTask(req.task, dvfsClient)
taskToNode.put(req.task.hash.toString(), closest)
log.info "[K8s] DVFS: Assigned task ${req.task} to node ${closest.name} - ${taskToNode.size()} assigned tasks"
return new K8sSchedulingDecision(req, closest.name)
}
log.info "[K8s] unable to schedule any task. The queue contains ${queue.size()} tasks."
return null
}
@Override
boolean scheduleImmediately(K8sTaskScheduler scheduler, List<K8sSchedulingRequest> queue) {
if (nodes.isEmpty()) {
if (!initNodes(scheduler))
return false
}
/* We want to schedule immediately if there are unoccupied nodes */
boolean doIt = queue != null && queue.size() > 0 && taskToNode.size() < nodes.size()
log.info "[K8s] scheduleImmediately: ${queue.size()} tasks in queue, ${taskToNode.size()} tasks running on ${nodes.size()} nodes: ${doIt}"
return doIt
}
@Override
synchronized void taskFinished(K8sTaskHandler task) {
/* TODO: This just uses elapsed wall-clock time, regardless of the frequency used to execute the task.
* This will skew the average towards longer runtimes, which is undesirable, because it will lead to more
* tasks classified as "critical path".
* A simple (rough) solution could be to keep track of the tasks "relative" frequency and just scale the
* elapsed time based on that.
*/
double runtime = (double)(task.getCompleteTimeMillis() - task.getStartTimeMillis())
averageRuntime = (runtime + finishedTaskCount * averageRuntime) / (finishedTaskCount + 1.0)
finishedTaskCount += 1.0
updateTopRuntimes(runtime)
/* Free resources allocated by this task */
WorkerNode node = taskToNode.get(task.task.hash.toString())
if (node != null) {
node.taskFinished(task.task, dvfsClient)
taskToNode.remove(task.task.hash.toString())
} else {
log.warn "[K8s] no node recorded for task ${task.toString()}"
}
log.info "[K8s] task ${task.toString()} finished - ${taskToNode.size()} tasks running"
if (node != null) {
log.info "[K8s] task ran on node ${node.name} - ${node.availableMemory} bytes ${node.availableCPUs}"
}
}
private synchronized boolean initNodes(K8sTaskScheduler scheduler) {
this.globalMaxFrequency = Long.MAX_VALUE
this.globalMinFrequency = Long.MIN_VALUE
final nodes = scheduler.getNodes()
for (String node : nodes) {
final min = dvfsClient.getNodeMinFrequency(node)
final max = dvfsClient.getNodeMaxFrequency(node)
if (min.empty || max.empty) {
log.error "[K8s] failed to query node $node information"
continue
}
globalMaxFrequency = Long.min(globalMaxFrequency, max.asLong)
globalMinFrequency = Long.max(globalMinFrequency, min.asLong)
log.info "[K8s] node ${node}: ${min.asLong} Hz - ${max.asLong} Hz"
}
long diff = globalMaxFrequency - globalMinFrequency
long diff3 = diff.intdiv(3)
this.frequencies = [
globalMaxFrequency,
globalMinFrequency + diff3 * 2,
globalMinFrequency + diff3,
]
int next = 0
for (String node : nodes) {
this.nodes.add(new WorkerNode(node, this.frequencies[next], this.clientGetter))
log.info "[K8s] node ${node}: set to ${this.frequencies[next]} Hz"
next = (next + 1) % this.frequencies.length
}
return !this.nodes.isEmpty()
}
/* Returns a list of nodes that fulfill the tasks resource requirements
*/
private ArrayList<WorkerNode> filterNodes(TaskRun task) {
final long reqBytes = getTaskMemoryRequirment(task)
final int reqCPUs = getTaskCPURequirement(task)
ArrayList<WorkerNode> suitableNodes = new ArrayList<>()
for (WorkerNode n : nodes) {
if (n.availableMemory >= reqBytes && n.availableCPUs >= reqCPUs) {
log.info "[K8s] task ${task.name}: ${reqBytes} bytes ${reqCPUs} cpus: node ${n.name} has ${n.availableMemory} bytes, ${n.availableCPUs} cpus"
suitableNodes.add(n)
}
}
return suitableNodes
}
private boolean anyNode(TaskRun task) {
final long reqBytes = getTaskMemoryRequirment(task)
final int reqCPUs = getTaskCPURequirement(task)
for (WorkerNode n : nodes) {
if (n.memoryAmount >= reqBytes && n.CPUCount >= reqCPUs)
return true
}
return false
}
}

42
k8s-dvfs/src/main/groovy/recreationaltech/plugin/strategies/K8sDVFSSchedulingStrategy.groovy
View File

@@ -132,10 +132,17 @@ class K8sDVFSSchedulingStrategy implements K8sSchedulingStrategy {
updateFrequency(dvfsClient)
}
void taskFinished(TaskRun task, K8sDVFSClient dvfsClient) {
// Returns the frequency that was assigned to the task
long taskFinished(TaskRun task, K8sDVFSClient dvfsClient) {
long f = Long.MAX_VALUE
log.info "[K8s] node ${name}: task ${task.name} finished"
this.tasks.removeIf {it.task == task}
updateFrequency(dvfsClient)
AssignedTask t = this.tasks.find { it.task == task }
if (t != null) {
f = t.frequency
this.tasks.remove(t)
updateFrequency(dvfsClient)
}
return f
}
}
@@ -266,10 +273,10 @@ class K8sDVFSSchedulingStrategy implements K8sSchedulingStrategy {
return null
}
/* No node can currently execute this task, but it should be possible in the future */
/* log.info "[K8s] ${req.task} can not be scheduled: ${getTaskMemoryRequirment(req.task)} bytes ${getTaskCPURequirement(req.task)} CPUs"
log.info "[K8s] ${req.task} can not be scheduled: ${getTaskMemoryRequirment(req.task)} bytes ${getTaskCPURequirement(req.task)} CPUs"
for (WorkerNode n : this.nodes) {
log.info "[K8s] node ${n.name} - ${n.availableMemory}, ${n.availableCPUs}"
} */
}
continue
}
@@ -308,21 +315,11 @@ class K8sDVFSSchedulingStrategy implements K8sSchedulingStrategy {
@Override
synchronized void taskFinished(K8sTaskHandler task) {
/* TODO: This just uses elapsed wall-clock time, regardless of the frequency used to execute the task.
* This will skew the average towards longer runtimes, which is undesirable, because it will lead to more
* tasks classified as "critical path".
* A simple (rough) solution could be to keep track of the tasks "relative" frequency and just scale the
* elapsed time based on that.
*/
double runtime = (double)(task.getCompleteTimeMillis() - task.getStartTimeMillis())
averageRuntime = (runtime + finishedTaskCount * averageRuntime) / (finishedTaskCount + 1.0)
finishedTaskCount += 1.0
updateTopRuntimes(runtime)
long freq = globalMaxFrequency
/* Free resources allocated by this task */
WorkerNode node = taskToNode.get(task.task.hash.toString())
if (node != null) {
node.taskFinished(task.task, dvfsClient)
freq = node.taskFinished(task.task, dvfsClient)
taskToNode.remove(task.task.hash.toString())
} else {
log.warn "[K8s] no node recorded for task ${task.toString()}"
@@ -330,8 +327,17 @@ class K8sDVFSSchedulingStrategy implements K8sSchedulingStrategy {
log.info "[K8s] task ${task.toString()} finished - ${taskToNode.size()} tasks running"
if (node != null) {
log.info "[K8s] task ran on node ${node.name} - ${node.availableMemory} bytes ${node.availableCPUs}"
log.info "[K8s] task ran on node ${node.name} - ${node.availableMemory} bytes ${node.availableCPUs} at ${freq}/${globalMaxFrequency} Hz (${(double)freq / (double)globalMaxFrequency}%)"
}
/* We scale the runtime by the tasks relative frequency to avoid skewing the average runtime towards
* longer runtimes. This is obviously only a rough approximation.
*/
double runtime = (double)(task.getCompleteTimeMillis() - task.getStartTimeMillis())
runtime *= (double)freq / (double)globalMaxFrequency
averageRuntime = (runtime + finishedTaskCount * averageRuntime) / (finishedTaskCount + 1.0)
finishedTaskCount += 1.0
updateTopRuntimes(runtime)
}
private synchronized boolean initNodes(K8sTaskScheduler scheduler) {

1
k8s-dvfs/src/test/groovy/recreationaltech/plugin/K8sNodeInitDeployerTest.groovy
View File

@@ -52,6 +52,7 @@ class K8sNodeInitDeployerTest extends Specification {
spec.metadata.namespace == 'default'
spec.spec.nodeName == 'node-a'
spec.spec.restartPolicy == 'Never'
spec.spec.hostNetwork == true
def container = spec.spec.containers[0]
container.name == 'nf-init-node-a'

55
k8s-dvfs/src/test/groovy/recreationaltech/plugin/K8sRuntimeEstimatorTest.groovy
View File

@@ -1,55 +0,0 @@
package recreationaltech.plugin
import spock.lang.Specification
class K8sRuntimeEstimatorTest extends Specification {
def 'does extract a function' () {
given:
def observations = [a: [new Tuple2(0, 0), new Tuple2(100, 100)]]
def estimator = new K8sRuntimeEstimator(observations)
when:
def estimation = estimator.estimate("a", 1234)
then:
estimation != Double.POSITIVE_INFINITY
estimation == 1234.0
}
def 'can extract a function from single measurement' () {
given:
def observations = [a: [new Tuple2(100, 100)]]
def estimator = new K8sRuntimeEstimator(observations)
when:
def estimation = estimator.estimate("a", 1234)
then:
estimation == 100.0 /* Single measurement implies constant runtime */
}
def 'unknown function returns positive infinity' () {
given:
def observations = [] as HashMap<String, ArrayList>
def estimator = new K8sRuntimeEstimator(observations)
when:
def estimation = estimator.estimate("b", 1234)
then:
estimation == Double.POSITIVE_INFINITY
}
def 'extracts function from multiple measurements' () {
given:
def observations = [a: [new Tuple2(10, 15), new Tuple2(20, 34), new Tuple2(30, 46)]]
def estimator = new K8sRuntimeEstimator(observations)
when:
def estimation = estimator.estimate("a", 40)
then:
estimation != Double.POSITIVE_INFINITY
}
}

55
k8s-dvfs/src/test/groovy/recreationaltech/plugin/client/K8sClientTest.groovy
View File

@@ -1139,6 +1139,61 @@ class K8sClientTest extends Specification {
result.metadata.name == 'test-node'
}
def 'should get node external IP with ExternalIP address' () {
given:
def JSON = '''
{
"kind": "Node",
"apiVersion": "v1",
"metadata": {
"name": "test-node"
},
"status": {
"addresses": [
{"type": "InternalIP", "address": "192.168.1.100"},
{"type": "ExternalIP", "address": "203.0.113.10"}
]
}
}
'''
def client = Spy(K8sClient)
final NODE_NAME = 'test-node'
when:
def result = client.getNodeExternalIp(NODE_NAME)
then:
1 * client.nodeDescribe(NODE_NAME) >> new K8sResponseJson(JSON)
result == '203.0.113.10'
}
def 'should get node external IP fallback to InternalIP' () {
given:
def JSON = '''
{
"kind": "Node",
"apiVersion": "v1",
"metadata": {
"name": "test-node"
},
"status": {
"addresses": [
{"type": "InternalIP", "address": "192.168.1.100"}
]
}
}
'''
def client = Spy(K8sClient)
final NODE_NAME = 'test-node'
when:
def result = client.getNodeExternalIp(NODE_NAME)
then:
1 * client.nodeDescribe(NODE_NAME) >> new K8sResponseJson(JSON)
result == '192.168.1.100'
}
def 'should get node CPU capacity' () {
given:
def JSON = '''