/* * Copyright 2013-2026, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package recreationaltech.plugin.client import nextflow.exception.K8sOutOfCpuException import nextflow.exception.K8sOutOfMemoryException import javax.net.ssl.HttpsURLConnection import nextflow.exception.NodeTerminationException import spock.lang.Specification /** * * @author Paolo Di Tommaso */ class K8sClientTest extends Specification { def 'should create a request' () { given: final TOKEN = '8d09d0ds' final client = Spy(K8sClient) def HTTPS_CONN = Mock(HttpsURLConnection) def HTTP_CONN = Mock(HttpURLConnection) when: client.config.server = 'host.com:443' client.config.token = TOKEN def resp = client.makeRequest('GET', '/foo/bar') then: 1 * client.createConnection0("https://host.com:443/foo/bar") >> HTTPS_CONN 1 * client.setupHttpsConn(HTTPS_CONN) >> null 1 * HTTPS_CONN.setRequestMethod('GET') >> null 1 * HTTPS_CONN.setRequestProperty("Authorization", "Bearer $TOKEN") 1 * HTTPS_CONN.setRequestProperty("Content-Type", "application/json") 1 * HTTPS_CONN.getResponseCode() >> 200 1 * HTTPS_CONN.getInputStream() >> { new ByteArrayInputStream('{"field_x":"hello"}'.bytes) } resp instanceof K8sResponseApi resp.text == '{"field_x":"hello"}' when: client.config.server = 'http://my-server.com' client.config.token = TOKEN client.makeRequest('POST', '/foo/bar') then: 1 * client.createConnection0("http://my-server.com/foo/bar") >> HTTP_CONN 0 * client.setupHttpsConn(_) >> null 1 * HTTP_CONN.setRequestMethod('POST') >> null 1 * HTTP_CONN.getResponseCode() >> 401 1 * HTTP_CONN.getErrorStream() >> { new ByteArrayInputStream('{"field_x":"oops.."}'.bytes) } def e = thrown(K8sResponseException) e.response.field_x == 'oops..' } def 'should make a get request' () { given: def client = Spy(K8sClient) when: client.get('/foo/bar') then: 1 * client.makeRequest('GET', '/foo/bar') >> null } def 'should make a post request' () { given: def client = Spy(K8sClient) when: client.post('/foo/bar', '{ the: body }') then: 1 * client.makeRequest('POST', '/foo/bar', '{ the: body }') >> null } def 'should make a delete request' () { given: def client = Spy(K8sClient) when: client.delete('/foo/bar', '{ the: body }') then: 1 * client.makeRequest('DELETE', '/foo/bar', '{ the: body }') >> null } def 'should delete a pod' () { given: K8sResponseJson result def client = Spy(K8sClient) def RESP = Mock(K8sResponseApi) RESP.getText() >> '{"field":"OK"}' when: result = client.podDelete('foo') then: 1 * client.delete("/api/v1/namespaces/default/pods/foo",null) >> RESP result.field == "OK" when: client.config.namespace = 'bar' result = client.podDelete('foo') then: 1 * client.delete("/api/v1/namespaces/bar/pods/foo",null) >> RESP result.field == "OK" } def 'should list pods' () { given: K8sResponseJson result def client = Spy(K8sClient) def RESP = Mock(K8sResponseApi) RESP.getText() >> '{"response":"hello"}' when: result = client.podList(true) then: 1 * client.get('/api/v1/pods') >> RESP result.response == 'hello' when: result = client.podList() then: 1 * client.get('/api/v1/namespaces/default/pods') >> RESP result.response == 'hello' when: client.config.namespace = 'foo' result = client.podList() then: 1 * client.get('/api/v1/namespaces/foo/pods') >> RESP result.response == 'hello' } def 'should list secrets' () { given: K8sResponseJson result def client = Spy(K8sClient) def RESP = Mock(K8sResponseApi) RESP.getText() >> '{"response":"hello"}' when: result = client.secretesList() then: 1 * client.get("/api/v1/namespaces/default/secrets") >> RESP result.response == 'hello' when: client.config.namespace = 'pippo' result = client.podList() then: 1 * client.get('/api/v1/namespaces/pippo/pods') >> RESP result.response == 'hello' } def 'should describe secret' () { given: K8sResponseJson result def client = Spy(K8sClient) def RESP = Mock(K8sResponseApi) RESP.getText() >> '{"response":"hello"}' when: result = client.secretDescribe('my-secret') then: 1 * client.get("/api/v1/namespaces/default/secrets/my-secret") >> RESP result.response == 'hello' when: client.config.namespace = 'paperino' result = client.secretDescribe('pluto') then: 1 * client.get('/api/v1/namespaces/paperino/secrets/pluto') >> RESP result.response == 'hello' } def 'should log pod' () { given: InputStream result def client = Spy(K8sClient) def STREAM = Mock(InputStream) def RESP = Mock(K8sResponseApi) RESP.getStream() >> STREAM when: result = client.podLog('pod-123') then: 1 * client.get('/api/v1/namespaces/default/pods/pod-123/log') >> RESP result == STREAM when: result = client.podLog('pod-123', follow: true) then: 1 * client.get('/api/v1/namespaces/default/pods/pod-123/log?follow=true') >> RESP result == STREAM when: result = client.podLog('pod-123', follow: true, foo:1, bar: 'x') then: 1 * client.get('/api/v1/namespaces/default/pods/pod-123/log?follow=true&foo=1&bar=x') >> RESP result == STREAM } def 'should create config' () { given: K8sResponseJson result def client = Spy(K8sClient) def RESP = Mock(K8sResponseApi) RESP.getText() >> '{"response":"done"}' def CONFIG = [foo: 'hello', bar:'world'] def JSON1 = '{"apiVersion":"v1","kind":"ConfigMap","metadata":{"name":"foo","namespace":"default"},"data":{"foo":"hello","bar":"world"}}' def JSON2 = '{"apiVersion":"v1","kind":"ConfigMap","metadata":{"name":"foo","namespace":"bar"},"data":{"foo":"hello","bar":"world"}}' when: result = client.configCreate('foo' , CONFIG) then: 1 * client.post("/api/v1/namespaces/default/configmaps", JSON1) >> RESP result.response == 'done' when: client.config.namespace = 'bar' result = client.configCreate('foo' , CONFIG) then: 1 * client.post("/api/v1/namespaces/bar/configmaps", JSON2) >> RESP result.response == 'done' } def 'should delete config' () { given: K8sResponseJson result def client = Spy(K8sClient) def RESP = Mock(K8sResponseApi) RESP.getText() >> '{"response":"done"}' when: result = client.configDelete('foo') then: 1 * client.delete("/api/v1/namespaces/default/configmaps/foo") >> RESP result.response == 'done' when: client.config.namespace = 'ns-1' result = client.configDelete('foo') then: 1 * client.delete("/api/v1/namespaces/ns-1/configmaps/foo") >> RESP result.response == 'done' } def 'should delete all configs' () { given: K8sResponseJson result def client = Spy(K8sClient) def RESP = Mock(K8sResponseApi) RESP.getText() >> '{"response":"done"}' when: result = client.configDeleteAll() then: 1 * client.delete("/api/v1/namespaces/default/configmaps") >> RESP result.response == 'done' when: client.config.namespace = 'ns-1' result = client.configDeleteAll() then: 1 * client.delete("/api/v1/namespaces/ns-1/configmaps") >> RESP result.response == 'done' } def 'should get a pod state' () { given: def JSON = ''' { "kind": "Pod", "apiVersion": "v1", "metadata": { "name": "pod-xyz", "namespace": "default", "selfLink": "/api/v1/namespaces/default/pods/pod-xyz/status", "uid": "33390e2c-f84b-11e7-a89d-025000000001", "resourceVersion": "119932", "creationTimestamp": "2018-01-13T10:19:12Z", "labels": { "app": "nextflow" } }, "status": { "phase": "Succeeded", "conditions": [ { "type": "Initialized", "status": "True", "lastProbeTime": null, "lastTransitionTime": "2018-01-13T10:19:12Z", "reason": "PodCompleted" }, { "type": "Ready", "status": "False", "lastProbeTime": null, "lastTransitionTime": "2018-01-13T10:19:37Z", "reason": "PodCompleted" }, { "type": "PodScheduled", "status": "True", "lastProbeTime": null, "lastTransitionTime": "2018-01-13T10:19:12Z" } ], "hostIP": "192.168.65.3", "podIP": "10.1.0.25", "startTime": "2018-01-13T10:19:12Z", "containerStatuses": [ { "name": "pod-xyz", "state": { "terminated": { "exitCode": 0, "reason": "Completed", "startedAt": "2018-01-13T10:19:16Z", "finishedAt": "2018-01-13T10:19:36Z", "containerID": "docker://90d447d4a2518642b12c8979474aa2bbb5fe8e96ed9e5caf3c979bb3b751519a" } }, "lastState": { }, "ready": false, "restartCount": 0, "image": "debian:latest", "imageID": "docker-pullable://debian@sha256:0a5fcee6f52d5170f557ee2447d7a10a5bdcf715dd7f0250be0b678c556a501b", "containerID": "docker://90d447d4a2518642b12c8979474aa2bbb5fe8e96ed9e5caf3c979bb3b751519a" } ], "qosClass": "BestEffort" } } ''' def client = Spy(K8sClient) final POD_NAME = 'pod-xyz' when: def result = client.podState(POD_NAME) then: 1 * client.podStatus(POD_NAME) >> new K8sResponseJson(JSON) result == [terminated: [exitCode:0, reason: 'Completed', startedAt: "2018-01-13T10:19:16Z", finishedAt: "2018-01-13T10:19:36Z", containerID: "docker://90d447d4a2518642b12c8979474aa2bbb5fe8e96ed9e5caf3c979bb3b751519a"]] } def 'should get a pod its node' () { given: def JSON = ''' { "kind": "Pod", "apiVersion": "v1", "metadata": { "name": "pod-xyz", "namespace": "default", "selfLink": "/api/v1/namespaces/default/pods/pod-xyz/status", "uid": "33390e2c-f84b-11e7-a89d-025000000001", "resourceVersion": "119932", "creationTimestamp": "2018-01-13T10:19:12Z", "labels": { "app": "nextflow" } }, "spec": { "restartPolicy": "Never", "terminationGracePeriodSeconds": 30, "dnsPolicy": "ClusterFirst", "serviceAccountName": "default", "serviceAccount": "default", "nodeName": "gke-bioinformatics-s-pipeline-pool-sm-bbac2e1c-k1tw", "priority": 0, "enableServiceLinks": true, "preemptionPolicy": "PreemptLowerPriority" } } ''' def client = Spy(K8sClient) final POD_NAME = 'pod-xyz' when: def result = client.getNodeOfPod(POD_NAME) then: 1 * client.podStatus(POD_NAME) >> new K8sResponseJson(JSON) result == "gke-bioinformatics-s-pipeline-pool-sm-bbac2e1c-k1tw" } def 'should return undetermined status' () { given: def JSON = ''' { "kind": "Pod", "apiVersion": "v1", "metadata": { "name": "nf-eb853c8010b8e173b23d8d15489d1a31", "namespace": "default", "selfLink": "/api/v1/namespaces/default/pods/nf-eb853c8010b8e173b23d8d15489d1a31/status", "uid": "5ccdeb23-4f69-11e8-89b1-fa163e31bb09", "resourceVersion": "2847182", "creationTimestamp": "2018-05-04T07:04:18Z", "labels": { "app": "nextflow", "processName": "markDuplicates", "runName": "grave-jones", "sessionId": "uuid-f51cd941-c21c-447b-86ca-eaebafa5ad9b", "taskName": "markDuplicates_22028_2_118_1AlignedByCoord.out" } }, "status": { "phase": "Pending", "conditions": [ { "type": "PodScheduled", "status": "True", "lastProbeTime": null, "lastTransitionTime": "2018-05-04T07:04:18Z" } ], "qosClass": "Guaranteed" } } ''' def client = Spy(K8sClient) final POD_NAME = 'nf-eb853c8010b8e173b23d8d15489d1a31' when: def result = client.podState(POD_NAME) then: 1 * client.podStatus(POD_NAME) >> new K8sResponseJson(JSON) result == [:] } def 'should return undetermined status when status conditions are missing' () { given: def JSON = ''' { "kind": "Pod", "apiVersion": "v1", "metadata": { "name": "nf-89b34d7c2d1dc11daad72b1fcf7e0540", "namespace": "default", "selfLink": "/api/v1/namespaces/default/pods/nf-89b34d7c2d1dc11daad72b1fcf7e0540/status", "uid": "02c3a34d-7720-11e9-8e1a-0a8b849038f8", "resourceVersion": "21753960", "creationTimestamp": "2019-05-15T14:44:58Z", "labels": { "app": "nextflow", "processName": "split_input_file", "runName": "hopeful_lalande", "sessionId": "uuid-6b10c771-1a70-4af0-92be-7c0280e0e17f", "taskName": "split_input_file_variants.gord_chr11_72000001-78000000", } }, "status": { "phase": "Pending", "qosClass": "Guaranteed" } } ''' def client = Spy(K8sClient) final POD_NAME = 'nf-89b34d7c2d1dc11daad72b1fcf7e0540' when: def result = client.podState(POD_NAME) then: 1 * client.podStatus(POD_NAME) >> new K8sResponseJson(JSON) result == [:] } def 'should return a process execution on pod not found' () { given: def JSON = ''' { "kind": "Status", "apiVersion": "v1", "metadata": { }, "status": "Failure", "message": "pods \\"nf-7cee928c1dd05b39cd50ab79a3e742f9\\" not found", "reason": "NotFound", "details": { "name": "nf-7cee928c1dd05b39cd50ab79a3e742f9", "kind": "pods" }, "code": 404 } ''' def client = Spy(K8sClient) final POD_NAME = 'pod-xyz' when: client.podState(POD_NAME) then: 1 * client.podStatus(POD_NAME) >> { throw new K8sResponseException("Request GET /api/v1/namespaces/xyz/pods/nf-xyz/status returned an error code=404", new ByteArrayInputStream(JSON.bytes)) } and: thrown(NodeTerminationException) } def 'should fail to get pod state' () { given: def client = Spy(K8sClient) final POD_NAME = 'pod-xyz' final STATE = [foo:1, bar:2] when: def e client.podState(POD_NAME) then: 1 * client.podStatus(POD_NAME) >> new K8sResponseJson([:]) e = thrown(K8sResponseException) e.message.startsWith('K8s undetermined status conditions for pod') when: client.podState(POD_NAME) then: 1 * client.podStatus(POD_NAME) >> new K8sResponseJson([status:[containerStatuses: []]]) e = thrown(K8sResponseException) e.message.startsWith('K8s undetermined status conditions for pod') when: client.podState(POD_NAME) then: 1 * client.podStatus(POD_NAME) >> new K8sResponseJson([status:[containerStatuses: [ [name: 'foo'] ]]]) e = thrown(K8sResponseException) e.message.startsWith('K8s invalid status for pod: pod-xyz (unexpected container name: foo)') when: client.podState(POD_NAME) then: 1 * client.podStatus(POD_NAME) >> new K8sResponseJson([status:[containerStatuses: [ [name: POD_NAME] ]]]) e = thrown(K8sResponseException) e.message.startsWith('K8s invalid status for pod: pod-xyz (missing state object)') when: def result = client.podState(POD_NAME) then: 1 * client.podStatus(POD_NAME) >> new K8sResponseJson([status:[containerStatuses: [ [name: POD_NAME, state: STATE] ]]]) result == STATE } def 'client should throw an exception when container status returns ErrImagePull' () { given: def JSON = ''' { "kind": "Pod", "apiVersion": "v1", "metadata": { "name": "nf-c6e49ee9ebc79486f774a47924a743d7", "namespace": "default", "selfLink": "/api/v1/namespaces/default/pods/nf-c6e49ee9ebc79486f774a47924a743d7/status", "uid": "18954b48-5811-11e8-8e71-025000000001", "resourceVersion": "3615842", "creationTimestamp": "2018-05-15T07:25:08Z", "labels": { "app": "nextflow", "processName": "sayHello", "runName": "lethal-jones", "sessionId": "uuid-b6243a3d-5c07-44f4-97eb-9de499747800", "taskName": "sayHello_2" } }, "spec": { }, "status": { "phase": "Pending", "conditions": [ { "type": "Initialized", "status": "True", "lastProbeTime": null, "lastTransitionTime": "2018-05-15T07:25:08Z" }, { "type": "Ready", "status": "False", "lastProbeTime": null, "lastTransitionTime": "2018-05-15T07:25:08Z", "reason": "ContainersNotReady", "message": "containers with unready status: [nf-c6e49ee9ebc79486f774a47924a743d7]" }, { "type": "PodScheduled", "status": "True", "lastProbeTime": null, "lastTransitionTime": "2018-05-15T07:25:08Z" } ], "hostIP": "192.168.65.3", "podIP": "10.1.3.173", "startTime": "2018-05-15T07:25:08Z", "containerStatuses": [ { "name": "nf-c6e49ee9ebc79486f774a47924a743d7", "state": { "waiting": { "reason": "ErrImagePull", "message": "rpc error: code = Unknown desc = Error response from daemon: pull access denied for nextflow/foo, repository does not exist or may require 'docker login'" } }, "lastState": { }, "ready": false, "restartCount": 0, "image": "nextflow/foo", "imageID": "" } ], "qosClass": "BestEffort" } } ''' def client = Spy(K8sClient) final POD_NAME = 'nf-c6e49ee9ebc79486f774a47924a743d7' when: client.podState(POD_NAME) then: 1 * client.podStatus(POD_NAME) >> new K8sResponseJson(JSON) def e = thrown(PodUnschedulableException) e.message == "K8s pod image cannot be pulled -- rpc error: code = Unknown desc = Error response from daemon: pull access denied for nextflow/foo, repository does not exist or may require 'docker login'" } def 'client should throw an exception when k8s is out of cpu' () { given: def JSON = ''' { "kind": "Pod", "apiVersion": "v1", "metadata": { "name": "nf-3b344812fe0aeb9554424bcf6caa7ffb", "namespace": "default", "uid": "0dd3c071-82a3-4b20-bdd4-0d34ff3d90bd", "resourceVersion": "55320", "creationTimestamp": "2022-09-23T13:43:48Z", "labels": { "app": "nextflow", "processName": "combineFiles", "runName": "insane-kare", "sessionId": "uuid-85951b91-b5bc-4566-8938-fef4256c21c8", "taskName": "combineFiles_1" }, }, "spec": { }, "status": { "phase": "Failed", "message": "Pod Node didn't have enough resource: cpu, requested: 4000, used: 2100, capacity: 6000", "reason": "OutOfcpu", "startTime": "2022-09-23T13:43:48Z" } } ''' def client = Spy(K8sClient) final POD_NAME = 'nf-3b344812fe0aeb9554424bcf6caa7ffb' when: client.podState(POD_NAME) then: 1 * client.podStatus(POD_NAME) >> new K8sResponseJson(JSON) def e = thrown(K8sOutOfCpuException) e.message == "K8s pod 'nf-3b344812fe0aeb9554424bcf6caa7ffb' execution failed - reason: OutOfcpu - message: Pod Node didn't have enough resource: cpu, requested: 4000, used: 2100, capacity: 6000" } def 'client should throw an exception when k8s is out of memory' () { given: def JSON = ''' { "kind": "Pod", "apiVersion": "v1", "metadata": { "name": "nf-3b344812fe0aeb9554424bcf6caa7ffb", "namespace": "default", "uid": "0dd3c071-82a3-4b20-bdd4-0d34ff3d90bd", "resourceVersion": "55320", "creationTimestamp": "2022-09-23T13:43:48Z", "labels": { "app": "nextflow", "processName": "combineFiles", "runName": "insane-kare", "sessionId": "uuid-85951b91-b5bc-4566-8938-fef4256c21c8", "taskName": "combineFiles_2" }, }, "spec": { }, "status": { "phase": "Failed", "message": "Pod Node didn't have enough resource: memory, requested: 16106127360, used: 16158556160, capacity: 16778358784", "reason": "OutOfmemory", "startTime": "2022-09-23T13:42:59Z" } } ''' def client = Spy(K8sClient) final POD_NAME = 'nf-3b344812fe0aeb9554424bcf6caa7ffb' when: client.podState(POD_NAME) then: 1 * client.podStatus(POD_NAME) >> new K8sResponseJson(JSON) def e = thrown(K8sOutOfMemoryException) e.message == "K8s pod 'nf-3b344812fe0aeb9554424bcf6caa7ffb' execution failed - reason: OutOfmemory - message: Pod Node didn't have enough resource: memory, requested: 16106127360, used: 16158556160, capacity: 16778358784" } def 'client should throw an exception when container status returns ImagePullBackOff' () { def JSON = ''' { "kind": "Pod", "apiVersion": "v1", "metadata": { "name": "nf-c6e49ee9ebc79486f774a47924a743d7", "namespace": "default", "selfLink": "/api/v1/namespaces/default/pods/nf-c6e49ee9ebc79486f774a47924a743d7/status", "uid": "18954b48-5811-11e8-8e71-025000000001", "resourceVersion": "3615866", "creationTimestamp": "2018-05-15T07:25:08Z", "labels": { "app": "nextflow", "processName": "sayHello", "runName": "lethal-jones", "sessionId": "uuid-b6243a3d-5c07-44f4-97eb-9de499747800", "taskName": "sayHello_2" } }, "spec": { }, "status": { "phase": "Pending", "conditions": [ { "type": "Initialized", "status": "True", "lastProbeTime": null, "lastTransitionTime": "2018-05-15T07:25:08Z" }, { "type": "Ready", "status": "False", "lastProbeTime": null, "lastTransitionTime": "2018-05-15T07:25:08Z", "reason": "ContainersNotReady", "message": "containers with unready status: [nf-c6e49ee9ebc79486f774a47924a743d7]" }, { "type": "PodScheduled", "status": "True", "lastProbeTime": null, "lastTransitionTime": "2018-05-15T07:25:08Z" } ], "hostIP": "192.168.65.3", "podIP": "10.1.3.173", "startTime": "2018-05-15T07:25:08Z", "containerStatuses": [ { "name": "nf-c6e49ee9ebc79486f774a47924a743d7", "state": { "waiting": { "reason": "ImagePullBackOff", "message": "Back-off pulling image \\"nextflow/foo\\"" } }, "lastState": { }, "ready": false, "restartCount": 0, "image": "nextflow/foo", "imageID": "" } ], "qosClass": "BestEffort" } } ''' def client = Spy(K8sClient) final POD_NAME = 'nf-c6e49ee9ebc79486f774a47924a743d7' when: client.podState(POD_NAME) then: 1 * client.podStatus(POD_NAME) >> new K8sResponseJson(JSON) def e = thrown(PodUnschedulableException) e.message == "K8s pod image cannot be pulled -- Back-off pulling image \"nextflow/foo\"" } def 'client should throw process exception on failed state' () { def JSON = ''' { "kind": "Pod", "apiVersion": "v1", "metadata": { "name": "nf-f34e124e1471736f27c8ef1aa52d02be", "namespace": "tower-nf", "uid": "8ddb536f-da4c-4a99-a5c5-3b5b53df6411", "resourceVersion": "31269576", "creationTimestamp": "2021-08-09T17:58:42Z", "labels": { "app": "nextflow", "processName": "bulk_rnaseq_trim_galore", "runName": "insane_agnesi", "sessionId": "uuid-35a8fe1a-2622-4582-a1b2-1cc21bd85b57", "taskName": "bulk_rnaseq_trim_galore_d0_tdTOM-0701" } }, "spec": { "restartPolicy": "Never", "terminationGracePeriodSeconds": 30, "dnsPolicy": "ClusterFirst", "serviceAccountName": "default", "serviceAccount": "default", "nodeName": "gke-bioinformatics-s-pipeline-pool-sm-bbac2e1c-k1tw", "priority": 0, "enableServiceLinks": true, "preemptionPolicy": "PreemptLowerPriority" }, "status": { "phase": "Failed", "message": "Node is shutting, evicting pods", "reason": "Shutdown", "startTime": "2021-08-09T17:59:31Z" } } ''' def client = Spy(K8sClient) final POD_NAME = 'nf-xyz' when: client.podState(POD_NAME) then: 1 * client.podStatus(POD_NAME) >> new K8sResponseJson(JSON) and: def e = thrown(NodeTerminationException) e.message == "K8s pod 'nf-xyz' execution failed - reason: Shutdown - message: Node is shutting, evicting pods" } def 'client should fail when config fail' () { given: def JSON = ''' { "kind": "Pod", "apiVersion": "v1", "metadata": { "name": "angry-blackwell", "namespace": "default", "selfLink": "/api/v1/namespaces/default/pods/angry-blackwell/status", "uid": "83a35c1e-73b6-11e8-8259-025000000001", "resourceVersion": "465382", "creationTimestamp": "2018-06-19T11:47:16Z", "labels": { "app": "nextflow", "runName": "angry-blackwell" } }, "spec": { }, "status": { "phase": "Pending", "conditions": [ { "type": "Initialized", "status": "True", "lastProbeTime": null, "lastTransitionTime": "2018-06-19T11:47:16Z" }, { "type": "Ready", "status": "False", "lastProbeTime": null, "lastTransitionTime": "2018-06-19T11:47:16Z", "reason": "ContainersNotReady", "message": "containers with unready status: [angry-blackwell]" }, { "type": "PodScheduled", "status": "True", "lastProbeTime": null, "lastTransitionTime": "2018-06-19T11:47:16Z" } ], "hostIP": "192.168.65.3", "podIP": "10.1.4.20", "startTime": "2018-06-19T11:47:16Z", "containerStatuses": [ { "name": "angry-blackwell", "state": { "waiting": { "reason": "CreateContainerConfigError", "message": "secrets \\"my-env\\" not found" } }, "lastState": { }, "ready": false, "restartCount": 0, "image": "nextflow/nextflow:0.30.2", "imageID": "" } ], "qosClass": "BestEffort" } } ''' def client = Spy(K8sClient) final POD_NAME = 'angry-blackwell' when: client.podState(POD_NAME) then: 1 * client.podStatus(POD_NAME) >> new K8sResponseJson(JSON) def e = thrown(PodUnschedulableException) e.message == 'K8s pod configuration failed -- secrets "my-env" not found' } def 'client should throw process exception when initContainer fails' () { // Addresses bug 2428, which is when an initContainer exists and fails // leaving the containerStatuses in "PodInitializing", but the pod // phase is "Failed". def JSON = ''' { "apiVersion": "v1", "kind": "Pod", "metadata": { "creationTimestamp": "2021-11-03T21:37:45Z", "labels": { "app": "nextflow", "magic-version": "v7.5.3", "processName": "some_special_magic", "runName": "lonely_tuckerman", "sessionId": "uuid-41ef6f42-bab5-438f-83e4-85e91a6386ea", "taskName": "some_special_magic_1" }, "name": "nf-45d62fd8462ba390bacfa20ad9065bfe", "namespace": "default", "resourceVersion": "90692", "uid": "280bf29c-4c14-4451-a300-080760ff7a0e" }, "status": { "containerStatuses": [ { "image": "docker/asdf/magictest:7.5.3", "imageID": "", "lastState": {}, "name": "nf-45d62fd8462ba390bacfa20ad9065bfe", "ready": false, "restartCount": 0, "started": false, "state": { "waiting": { "reason": "PodInitializing" } } } ], "hostIP": "10.12.205.49", "initContainerStatuses": [ { "containerID": "containerd://8ac143e547ef8c417f3655da31a81018e9d14db08c218e56ee3d181c70e0b755", "image": "docker/asdf/magic-special-handling:0.4.0", "imageID": "sha256:587f96e98fd0a2b17772e9a505522e4c0e1cf5194a4f08924c339848df42100c", "lastState": {}, "name": "magic-init", "ready": false, "restartCount": 0, "state": { "terminated": { "containerID": "containerd://8ac143e547ef8c417f3655da31a81018e9d14db08c218e56ee3d181c70e0b755", "exitCode": 1, "finishedAt": "2021-11-03T21:37:51Z", "reason": "Error", "startedAt": "2021-11-03T21:37:46Z" } } } ], "phase": "Failed", "podIP": "10.42.0.67", "podIPs": [ { "ip": "10.42.0.67" } ], "qosClass": "Burstable", "startTime": "2021-11-03T21:37:45Z" } } ''' def client = Spy(K8sClient) final POD_NAME = 'nf-45d62fd8462ba390bacfa20ad9065bfe' when: client.podState(POD_NAME) then: 1 * client.podStatus(POD_NAME) >> new K8sResponseJson(JSON) and: def e = thrown(PodUnschedulableException) e.message == "K8s pod in Failed state" } def 'should fallback to job status when pod is gone and not return hardcoded exit code' () { given: def JOB_STATUS_JSON = ''' { "apiVersion": "batch/v1", "kind": "Job", "metadata": { "name": "test-job" }, "status": { "succeeded": 1, "startTime": "2025-01-15T10:00:00Z", "completionTime": "2025-01-15T10:05:00Z", "conditions": [ { "type": "Complete", "status": "True", "lastProbeTime": "2025-01-15T10:05:00Z", "lastTransitionTime": "2025-01-15T10:05:00Z" } ] } } ''' def client = Spy(K8sClient) final JOB_NAME = 'test-job' when: def result = client.jobStateFallback0(JOB_NAME) then: 1 * client.jobStatus(JOB_NAME) >> new K8sResponseJson(JOB_STATUS_JSON) and: result.terminated != null result.terminated.reason == 'Completed' result.terminated.startedAt == '2025-01-15T10:00:00Z' result.terminated.finishedAt == '2025-01-15T10:05:00Z' // The key assertion: exitCode should not be present (null) so fallback to .exitcode file works result.terminated.exitCode == null result.terminated.exitcode == null } def 'should describe a node' () { given: def JSON = ''' { "kind": "Node", "apiVersion": "v1", "metadata": { "name": "test-node", "namespace": "default" }, "status": { "capacity": { "cpu": "4", "memory": "16Gi" }, "allocatable": { "cpu": "3.5", "memory": "14Gi" } } } ''' def client = Spy(K8sClient) final NODE_NAME = 'test-node' def RESP = Mock(K8sResponseApi) RESP.getText() >> JSON when: def result = client.nodeDescribe(NODE_NAME) then: 1 * client.get('/api/v1/nodes/test-node') >> RESP result.kind == 'Node' result.metadata.name == 'test-node' } def 'should get node external IP with ExternalIP address' () { given: def JSON = ''' { "kind": "Node", "apiVersion": "v1", "metadata": { "name": "test-node" }, "status": { "addresses": [ {"type": "InternalIP", "address": "192.168.1.100"}, {"type": "ExternalIP", "address": "203.0.113.10"} ] } } ''' def client = Spy(K8sClient) final NODE_NAME = 'test-node' when: def result = client.getNodeExternalIp(NODE_NAME) then: 1 * client.nodeDescribe(NODE_NAME) >> new K8sResponseJson(JSON) result == '203.0.113.10' } def 'should get node external IP fallback to InternalIP' () { given: def JSON = ''' { "kind": "Node", "apiVersion": "v1", "metadata": { "name": "test-node" }, "status": { "addresses": [ {"type": "InternalIP", "address": "192.168.1.100"} ] } } ''' def client = Spy(K8sClient) final NODE_NAME = 'test-node' when: def result = client.getNodeExternalIp(NODE_NAME) then: 1 * client.nodeDescribe(NODE_NAME) >> new K8sResponseJson(JSON) result == '192.168.1.100' } def 'should get node CPU capacity' () { given: def JSON = ''' { "kind": "Node", "apiVersion": "v1", "metadata": { "name": "test-node" }, "status": { "capacity": { "cpu": "4", "memory": "16Gi" } } } ''' def client = Spy(K8sClient) final NODE_NAME = 'test-node' def RESP = Mock(K8sResponseApi) RESP.getText() >> JSON when: def result = client.getNodeCpuCapacity(NODE_NAME) then: 1 * client.nodeDescribe(NODE_NAME) >> new K8sResponseJson(JSON) result == '4' } def 'should get node memory capacity' () { given: def JSON = ''' { "kind": "Node", "apiVersion": "v1", "metadata": { "name": "test-node" }, "status": { "capacity": { "cpu": "4", "memory": "16Gi" } } } ''' def client = Spy(K8sClient) final NODE_NAME = 'test-node' when: def result = client.getNodeMemoryCapacity(NODE_NAME) then: 1 * client.nodeDescribe(NODE_NAME) >> new K8sResponseJson(JSON) result == '16Gi' } def 'should get node CPU used from metrics server' () { given: def NODE_JSON = ''' { "kind": "Node", "apiVersion": "v1", "metadata": { "name": "test-node" }, "status": { "capacity": { "cpu": "4", "memory": "16Gi" } } } ''' def METRICS_JSON = ''' { "kind": "NodeMetrics", "apiVersion": "metrics.k8s.io/v1beta1", "metadata": { "name": "test-node" }, "usage": { "cpu": "1500m", "memory": "2Gi" } } ''' def client = Spy(K8sClient) final NODE_NAME = 'test-node' def NODE_RESP = Mock(K8sResponseApi) NODE_RESP.getText() >> NODE_JSON def METRICS_RESP = Mock(K8sResponseApi) METRICS_RESP.getText() >> METRICS_JSON when: def result = client.getNodeCpuUsed(NODE_NAME) then: 1 * client.get('/apis/metrics.k8s.io/v1beta1/nodes/test-node') >> METRICS_RESP result == '1500m' } def 'should get node memory used from metrics server' () { given: def NODE_JSON = ''' { "kind": "Node", "apiVersion": "v1", "metadata": { "name": "test-node" }, "status": { "capacity": { "cpu": "4", "memory": "16Gi" } } } ''' def METRICS_JSON = ''' { "kind": "NodeMetrics", "apiVersion": "metrics.k8s.io/v1beta1", "metadata": { "name": "test-node" }, "usage": { "cpu": "1500m", "memory": "2Gi" } } ''' def client = Spy(K8sClient) final NODE_NAME = 'test-node' def METRICS_RESP = Mock(K8sResponseApi) METRICS_RESP.getText() >> METRICS_JSON when: def result = client.getNodeMemoryUsed(NODE_NAME) then: 1 * client.get('/apis/metrics.k8s.io/v1beta1/nodes/test-node') >> METRICS_RESP result == '2Gi' } def 'should fallback when metrics server not available for CPU' () { given: def NODE_JSON = ''' { "kind": "Node", "apiVersion": "v1", "metadata": { "name": "test-node" }, "status": { "capacity": { "cpu": "4", "memory": "16Gi" }, "allocated": { "cpu": "2", "memory": "8Gi" } } } ''' def client = Spy(K8sClient) final NODE_NAME = 'test-node' def NODE_RESP = Mock(K8sResponseApi) NODE_RESP.getText() >> NODE_JSON when: def result = client.getNodeCpuUsed(NODE_NAME) then: // First attempt to metrics server fails 1 * client.get('/apis/metrics.k8s.io/v1beta1/nodes/test-node') >> { throw new K8sResponseException("Metrics not available", new ByteArrayInputStream('{}'.bytes)) } // Fallback to node describe - which calls get internally 1 * client.get('/api/v1/nodes/test-node') >> NODE_RESP result == '2' } def 'should fallback when metrics server not available for memory' () { given: def NODE_JSON = ''' { "kind": "Node", "apiVersion": "v1", "metadata": { "name": "test-node" }, "status": { "capacity": { "cpu": "4", "memory": "16Gi" }, "allocated": { "cpu": "2", "memory": "8Gi" } } } ''' def client = Spy(K8sClient) final NODE_NAME = 'test-node' def NODE_RESP = Mock(K8sResponseApi) NODE_RESP.getText() >> NODE_JSON when: def result = client.getNodeMemoryUsed(NODE_NAME) then: // First attempt to metrics server fails 1 * client.get('/apis/metrics.k8s.io/v1beta1/nodes/test-node') >> { throw new K8sResponseException("Metrics not available", new ByteArrayInputStream('{}'.bytes)) } // Fallback to node describe - which calls get internally 1 * client.get('/api/v1/nodes/test-node') >> NODE_RESP result == '8Gi' } def 'should return null when no allocated resources available' () { given: def NODE_JSON = ''' { "kind": "Node", "apiVersion": "v1", "metadata": { "name": "test-node" }, "status": { "capacity": { "cpu": "4", "memory": "16Gi" } } } ''' def client = Spy(K8sClient) final NODE_NAME = 'test-node' def NODE_RESP = Mock(K8sResponseApi) NODE_RESP.getText() >> NODE_JSON when: def resultCpu = client.getNodeCpuUsed(NODE_NAME) def resultMem = client.getNodeMemoryUsed(NODE_NAME) then: // First attempt to metrics server fails for CPU 1 * client.get('/apis/metrics.k8s.io/v1beta1/nodes/test-node') >> { throw new K8sResponseException("Metrics not available", new ByteArrayInputStream('{}'.bytes)) } // Fallback to node describe for CPU 1 * client.get('/api/v1/nodes/test-node') >> NODE_RESP // First attempt to metrics server fails for Memory 1 * client.get('/apis/metrics.k8s.io/v1beta1/nodes/test-node') >> { throw new K8sResponseException("Metrics not available", new ByteArrayInputStream('{}'.bytes)) } // Fallback to node describe for Memory 1 * client.get('/api/v1/nodes/test-node') >> NODE_RESP resultCpu == null resultMem == null } def 'should parse memory quantity strings' () { expect: K8sClient.parseK8sQuantity(null) == null K8sClient.parseK8sQuantity('') == null K8sClient.parseK8sQuantity('1024') == 1024L K8sClient.parseK8sQuantity('1Ki') == 1024L K8sClient.parseK8sQuantity('1KiB') == 1024L K8sClient.parseK8sQuantity('1Mi') == 1024L * 1024L K8sClient.parseK8sQuantity('1MiB') == 1024L * 1024L K8sClient.parseK8sQuantity('1Gi') == 1024L * 1024L * 1024L K8sClient.parseK8sQuantity('1GiB') == 1024L * 1024L * 1024L K8sClient.parseK8sQuantity('16Gi') == 16L * 1024L * 1024L * 1024L K8sClient.parseK8sQuantity('8192Mi') == 8192L * 1024L * 1024L K8sClient.parseK8sQuantity('0.5Gi') == (long)(0.5 * 1024L * 1024L * 1024L) } def 'should parse CPU quantity strings' () { expect: K8sClient.parseK8sQuantity(null, true) == null K8sClient.parseK8sQuantity('', true) == null K8sClient.parseK8sQuantity('1', true) == 1000L // 1 core = 1000 millicores K8sClient.parseK8sQuantity('2', true) == 2000L // 2 cores = 2000 millicores K8sClient.parseK8sQuantity('0.5', true) == 500L // 0.5 cores = 500 millicores K8sClient.parseK8sQuantity('1500m', true) == 1500L // 1500 millicores K8sClient.parseK8sQuantity('100m', true) == 100L // 100 millicores // Nanocores: 1 core = 1,000,000,000 nanocores, so we divide by 1,000,000 to get millicores K8sClient.parseK8sQuantity('1000000n', true) == 1L // 1,000,000 nanocores = 1 millicore K8sClient.parseK8sQuantity('1000000000n', true) == 1000L // 1,000,000,000 nanocores = 1000 millicores = 1 core K8sClient.parseK8sQuantity('298332671n', true) == 298L // 298,332,671 nanocores ≈ 298.33 millicores K8sClient.parseK8sQuantity('434126984n', true) == 434L // 434,126,984 nanocores ≈ 434.13 millicores } def 'should get node CPU capacity in millicores' () { given: def NODE_JSON = ''' { "kind": "Node", "apiVersion": "v1", "metadata": { "name": "test-node" }, "status": { "capacity": { "cpu": "4", "memory": "16Gi" } } } ''' def client = Spy(K8sClient) final NODE_NAME = 'test-node' when: def result = client.getNodeCpuCapacityMillis(NODE_NAME) then: 1 * client.nodeDescribe(NODE_NAME) >> new K8sResponseJson(NODE_JSON) result == 4000L // 4 cores = 4000 millicores } def 'should get node CPU used in millicores' () { given: def METRICS_JSON = ''' { "kind": "NodeMetrics", "apiVersion": "metrics.k8s.io/v1beta1", "metadata": { "name": "test-node" }, "usage": { "cpu": "1500m", "memory": "2Gi" } } ''' def client = Spy(K8sClient) final NODE_NAME = 'test-node' def METRICS_RESP = Mock(K8sResponseApi) METRICS_RESP.getText() >> METRICS_JSON when: def result = client.getNodeCpuUsedMillis(NODE_NAME) then: 1 * client.get('/apis/metrics.k8s.io/v1beta1/nodes/test-node') >> METRICS_RESP result == 1500L // 1500 millicores } def 'should get node memory capacity in bytes' () { given: def NODE_JSON = ''' { "kind": "Node", "apiVersion": "v1", "metadata": { "name": "test-node" }, "status": { "capacity": { "cpu": "4", "memory": "16Gi" } } } ''' def client = Spy(K8sClient) final NODE_NAME = 'test-node' when: def result = client.getNodeMemoryCapacityBytes(NODE_NAME) then: 1 * client.nodeDescribe(NODE_NAME) >> new K8sResponseJson(NODE_JSON) result == 16L * 1024L * 1024L * 1024L // 16 GiB in bytes } def 'should get node memory used in bytes' () { given: def METRICS_JSON = ''' { "kind": "NodeMetrics", "apiVersion": "metrics.k8s.io/v1beta1", "metadata": { "name": "test-node" }, "usage": { "cpu": "1500m", "memory": "2Gi" } } ''' def client = Spy(K8sClient) final NODE_NAME = 'test-node' def METRICS_RESP = Mock(K8sResponseApi) METRICS_RESP.getText() >> METRICS_JSON when: def result = client.getNodeMemoryUsedBytes(NODE_NAME) then: 1 * client.get('/apis/metrics.k8s.io/v1beta1/nodes/test-node') >> METRICS_RESP result == 2L * 1024L * 1024L * 1024L // 2 GiB in bytes } def 'should handle fractional CPU values' () { expect: K8sClient.parseK8sQuantity('2.5', true) == 2500L // 2.5 cores = 2500 millicores K8sClient.parseK8sQuantity('0.25', true) == 250L // 0.25 cores = 250 millicores } def 'should handle fractional memory values' () { expect: K8sClient.parseK8sQuantity('1.5Gi') == (long)(1.5 * 1024L * 1024L * 1024L) K8sClient.parseK8sQuantity('0.5Mi') == (long)(0.5 * 1024L * 1024L) } def 'should handle decimal suffixes for memory' () { expect: K8sClient.parseK8sQuantity('1K') == 1000L K8sClient.parseK8sQuantity('1M') == 1000L * 1000L K8sClient.parseK8sQuantity('1G') == 1000L * 1000L * 1000L } def 'should return null for invalid quantity strings' () { expect: K8sClient.parseK8sQuantity('invalid') == null K8sClient.parseK8sQuantity('16 GiB') == null // space not allowed K8sClient.parseK8sQuantity('Gi16') == null // suffix before number (doesn't match any suffix pattern) K8sClient.parseK8sQuantity('16.5.5') == null // multiple decimal points K8sClient.parseK8sQuantity('abc123') == null // non-numeric start K8sClient.parseK8sQuantity('123abc') == null // non-numeric end (no valid suffix) } def 'should handle large memory values like 8058776Ki' () { expect: K8sClient.parseK8sQuantity('8058776Ki') == 8058776L * 1024L } def 'should handle large memory values like 8058776ki' () { expect: K8sClient.parseK8sQuantity('8058776ki') == 8058776L * 1024L } def 'should handle memory values with B suffix' () { expect: K8sClient.parseK8sQuantity('1KiB') == 1024L K8sClient.parseK8sQuantity('1MiB') == 1024L * 1024L K8sClient.parseK8sQuantity('1GiB') == 1024L * 1024L * 1024L } def 'should handle decimal memory suffixes' () { expect: K8sClient.parseK8sQuantity('1K') == 1000L K8sClient.parseK8sQuantity('1KB') == 1000L K8sClient.parseK8sQuantity('1M') == 1000L * 1000L K8sClient.parseK8sQuantity('1MB') == 1000L * 1000L K8sClient.parseK8sQuantity('1G') == 1000L * 1000L * 1000L } def 'should handle real world memory value 8058776Ki' () { expect: K8sClient.parseK8sQuantity('8058776Ki') == 8058776L * 1024L K8sClient.parseK8sQuantity('8058776ki') == 8058776L * 1024L K8sClient.parseK8sQuantity('8058776KI') == 8058776L * 1024L } def 'should handle CPU values without suffix' () { expect: K8sClient.parseK8sQuantity('1', true) == 1000L K8sClient.parseK8sQuantity('2', true) == 2000L K8sClient.parseK8sQuantity('0.5', true) == 500L K8sClient.parseK8sQuantity('2.5', true) == 2500L } def 'should handle fractional memory values with suffixes' () { expect: K8sClient.parseK8sQuantity('1.5Gi') == (long)(1.5 * 1024 * 1024 * 1024) K8sClient.parseK8sQuantity('0.5Mi') == (long)(0.5 * 1024 * 1024) K8sClient.parseK8sQuantity('2.25Ki') == (long)(2.25 * 1024) } def 'should handle edge cases' () { expect: K8sClient.parseK8sQuantity('0') == 0L K8sClient.parseK8sQuantity('0Ki') == 0L K8sClient.parseK8sQuantity('0m', true) == 0L K8sClient.parseK8sQuantity('+100') == 100L K8sClient.parseK8sQuantity('-50') == -50L } }