Compare commits
20 Commits
0ec084b720
...
exp/standa
| Author | SHA1 | Date | |
|---|---|---|---|
| 4188bea061 | |||
| f227d054b8 | |||
| b0a5447797 | |||
| fb85e20f44 | |||
| c253d30b5d | |||
| 7b9188ab54 | |||
| ca45b4069f | |||
| 008b8c7cd9 | |||
| e9ff5ecf06 | |||
| 1ff4446081 | |||
| 64bb8c342a | |||
| f663095134 | |||
| 28466b2b54 | |||
| b92feeab5b | |||
| 933611af55 | |||
| 7f18e5aae8 | |||
| 2ae1ac6e38 | |||
| 78fe51a40d | |||
| a8d0af6d77 | |||
| 7a68546afa |
1
.gitignore
vendored
Normal file
1
.gitignore
vendored
Normal file
@@ -0,0 +1 @@
|
||||
cleanup-volume-claim.sh
|
||||
8
.woodpecker/nextflow-k8s.yaml
Normal file
8
.woodpecker/nextflow-k8s.yaml
Normal file
@@ -0,0 +1,8 @@
|
||||
when:
|
||||
- event: push
|
||||
|
||||
steps:
|
||||
- name: Test Kubernetes Plugin
|
||||
image: groovy:4-jdk21
|
||||
commands:
|
||||
- cd nextflow && ./gradlew :plugins:nf-k8s:test
|
||||
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"math"
|
||||
@@ -18,6 +19,11 @@ type cpuHandler struct {
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
type memHandler struct {
|
||||
amount int64
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
// readIntFromFile reads the given file until the end
|
||||
// and converts the contained string into an integer
|
||||
func readIntFromFile(path string) (int, error) {
|
||||
@@ -28,6 +34,25 @@ func readIntFromFile(path string) (int, error) {
|
||||
return strconv.Atoi(strings.TrimSpace(string(b)))
|
||||
}
|
||||
|
||||
func readTotalMemoryKb() (int, error) {
|
||||
b, err := os.ReadFile("/proc/meminfo")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
s := string(b)
|
||||
for line := range strings.Lines(s) {
|
||||
if !strings.Contains(line, "MemTotal:") {
|
||||
continue
|
||||
}
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 2 {
|
||||
return 0, fmt.Errorf("expected line '%s' to have at least 2 fields", line)
|
||||
}
|
||||
return strconv.Atoi(fields[1])
|
||||
}
|
||||
return 0, fmt.Errorf("failed to find MemTotal in /proc/meminfo")
|
||||
}
|
||||
|
||||
func enumerateCPUCores(logger *slog.Logger) ([]string, int, int, error) {
|
||||
files, err := os.ReadDir("/sys/devices/system/cpu/")
|
||||
if err != nil {
|
||||
@@ -135,6 +160,17 @@ func (s *cpuHandler) handleCurrent(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *cpuHandler) handleCount(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != "GET" {
|
||||
s.logger.Error("unexepcted method for cpu count", "method", r.Method)
|
||||
writeResponse("invalid method", http.StatusBadRequest, w, s.logger)
|
||||
return
|
||||
}
|
||||
|
||||
count := len(s.corePaths)
|
||||
writeResponse(strconv.Itoa(count), http.StatusOK, w, s.logger)
|
||||
}
|
||||
|
||||
func (s *cpuHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
requestedPath := getStrippedRequestPath(r.URL, r.Pattern)
|
||||
switch requestedPath {
|
||||
@@ -147,12 +183,25 @@ func (s *cpuHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
case "frequency/current":
|
||||
s.handleCurrent(w, r)
|
||||
return
|
||||
case "count":
|
||||
s.handleCount(w, r)
|
||||
return
|
||||
default:
|
||||
s.logger.Error("unknown resource", "path", requestedPath)
|
||||
writeResponse("not found", http.StatusNotFound, w, s.logger)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *memHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
requestedPath := getStrippedRequestPath(r.URL, r.Pattern)
|
||||
if requestedPath == "amount" {
|
||||
writeResponse(strconv.FormatInt(m.amount, 10), http.StatusOK, w, m.logger)
|
||||
} else {
|
||||
m.logger.Error("unknown resource", "path", requestedPath)
|
||||
writeResponse("not found", http.StatusNotFound, w, m.logger)
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
appEnv := os.Getenv("AGENT_ENV")
|
||||
loggerOpts := &slog.HandlerOptions{
|
||||
@@ -176,6 +225,18 @@ func main() {
|
||||
minFreq: minF,
|
||||
corePaths: freqPaths,
|
||||
}
|
||||
|
||||
memKb, err := readTotalMemoryKb()
|
||||
if err != nil {
|
||||
logger.Error("failed to read memory size", "err", err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
memHandler := &memHandler{
|
||||
amount: int64(memKb) * 1024,
|
||||
logger: logger,
|
||||
}
|
||||
|
||||
http.Handle("/cpu/", logRequests(logger, cpuHandler))
|
||||
http.Handle("/mem/", logRequests(logger, memHandler))
|
||||
logger.Info("exit", "result", http.ListenAndServe(":8080", nil))
|
||||
}
|
||||
|
||||
10
k8s-dvfs/.gitignore
vendored
Normal file
10
k8s-dvfs/.gitignore
vendored
Normal file
@@ -0,0 +1,10 @@
|
||||
# Ignore Gradle project-specific cache directory
|
||||
.gradle
|
||||
.idea
|
||||
.nextflow*
|
||||
|
||||
# Ignore Gradle build output directory
|
||||
build
|
||||
work
|
||||
lib/groovy-6.0.0-beta-1.jar lib/groovy-6.0.0-beta-1-sources.jar lib/groovy-ant-6.0.0-beta-1.jar lib/groovy-ant-6.0.0-beta-1-sources.jar lib/groovy-astbuilder-6.0.0-beta-1.jar lib/groovy-astbuilder-6.0.0-beta-1-sources.jar lib/groovy-cli-commons-6.0.0-beta-1.jar lib/groovy-cli-commons-6.0.0-beta-1-sources.jar lib/groovy-cli-picocli-6.0.0-beta-1.jar lib/groovy-cli-picocli-6.0.0-beta-1-sources.jar lib/groovy-console-6.0.0-beta-1.jar lib/groovy-console-6.0.0-beta-1-sources.jar lib/groovy-contracts-6.0.0-beta-1.jar lib/groovy-contracts-6.0.0-beta-1-sources.jar lib/groovy-csv-6.0.0-beta-1.jar lib/groovy-csv-6.0.0-beta-1-sources.jar lib/groovy-datetime-6.0.0-beta-1.jar lib/groovy-datetime-6.0.0-beta-1-sources.jar lib/groovy-dateutil-6.0.0-beta-1.jar lib/groovy-dateutil-6.0.0-beta-1-sources.jar lib/groovy-docgenerator-6.0.0-beta-1.jar lib/groovy-docgenerator-6.0.0-beta-1-sources.jar lib/groovy-ginq-6.0.0-beta-1.jar lib/groovy-ginq-6.0.0-beta-1-sources.jar lib/groovy-grape-ivy-6.0.0-beta-1.jar lib/groovy-grape-ivy-6.0.0-beta-1-sources.jar lib/groovy-grape-maven-6.0.0-beta-1.jar lib/groovy-grape-maven-6.0.0-beta-1-sources.jar lib/groovy-groovydoc-6.0.0-beta-1.jar lib/groovy-groovydoc-6.0.0-beta-1-sources.jar lib/groovy-groovysh-6.0.0-beta-1.jar lib/groovy-groovysh-6.0.0-beta-1-sources.jar lib/groovy-http-builder-6.0.0-beta-1.jar lib/groovy-http-builder-6.0.0-beta-1-sources.jar lib/groovy-jmx-6.0.0-beta-1.jar lib/groovy-jmx-6.0.0-beta-1-sources.jar lib/groovy-json-6.0.0-beta-1.jar lib/groovy-json-6.0.0-beta-1-sources.jar lib/groovy-jsr223-6.0.0-beta-1.jar lib/groovy-jsr223-6.0.0-beta-1-sources.jar lib/groovy-macro-6.0.0-beta-1.jar lib/groovy-macro-6.0.0-beta-1-sources.jar lib/groovy-macro-library-6.0.0-beta-1.jar lib/groovy-macro-library-6.0.0-beta-1-sources.jar lib/groovy-markdown-6.0.0-beta-1.jar lib/groovy-markdown-6.0.0-beta-1-sources.jar lib/groovy-nio-6.0.0-beta-1.jar lib/groovy-nio-6.0.0-beta-1-sources.jar lib/groovy-reactor-6.0.0-beta-1.jar lib/groovy-reactor-6.0.0-beta-1-sources.jar lib/groovy-rxjava-6.0.0-beta-1.jar lib/groovy-rxjava-6.0.0-beta-1-sources.jar lib/groovy-servlet-6.0.0-beta-1.jar lib/groovy-servlet-6.0.0-beta-1-sources.jar lib/groovy-sql-6.0.0-beta-1.jar lib/groovy-sql-6.0.0-beta-1-sources.jar lib/groovy-swing-6.0.0-beta-1.jar lib/groovy-swing-6.0.0-beta-1-sources.jar lib/groovy-templates-6.0.0-beta-1.jar lib/groovy-templates-6.0.0-beta-1-sources.jar lib/groovy-test-6.0.0-beta-1.jar lib/groovy-test-6.0.0-beta-1-sources.jar lib/groovy-test-junit5-6.0.0-beta-1.jar lib/groovy-test-junit5-6.0.0-beta-1-sources.jar lib/groovy-test-junit6-6.0.0-beta-1.jar lib/groovy-test-junit6-6.0.0-beta-1-sources.jar lib/groovy-testng-6.0.0-beta-1.jar lib/groovy-testng-6.0.0-beta-1-sources.jar lib/groovy-toml-6.0.0-beta-1.jar lib/groovy-toml-6.0.0-beta-1-sources.jar lib/groovy-typecheckers-6.0.0-beta-1.jar lib/groovy-typecheckers-6.0.0-beta-1-sources.jar lib/groovy-xml-6.0.0-beta-1.jar lib/groovy-xml-6.0.0-beta-1-sources.jar lib/groovy-yaml-6.0.0-beta-1.jar lib/groovy-yaml-6.0.0-beta-1-sources.jar
|
||||
lib
|
||||
176
k8s-dvfs/COPYING
Normal file
176
k8s-dvfs/COPYING
Normal file
@@ -0,0 +1,176 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
25
k8s-dvfs/Makefile
Normal file
25
k8s-dvfs/Makefile
Normal file
@@ -0,0 +1,25 @@
|
||||
# Use the Gradle wrapper by default; override with e.g. `make GRADLE=gradle ...`
|
||||
# or `export GRADLE=gradle` (useful in pixi/conda environments).
|
||||
GRADLE ?= ./gradlew
|
||||
|
||||
# Build the plugin
|
||||
assemble:
|
||||
$(GRADLE) assemble
|
||||
|
||||
clean:
|
||||
rm -rf .nextflow*
|
||||
rm -rf work
|
||||
rm -rf build
|
||||
$(GRADLE) clean
|
||||
|
||||
# Run plugin unit tests
|
||||
test:
|
||||
$(GRADLE) test
|
||||
|
||||
# Install the plugin into local nextflow plugins dir
|
||||
install:
|
||||
$(GRADLE) install
|
||||
|
||||
# Publish the plugin
|
||||
release:
|
||||
$(GRADLE) releasePlugin
|
||||
86
k8s-dvfs/README.md
Normal file
86
k8s-dvfs/README.md
Normal file
@@ -0,0 +1,86 @@
|
||||
# k8s-dvfs
|
||||
|
||||
## Summary
|
||||
|
||||
`k8s-dvfs` is a Nextflow plugin scaffolded from the official plugin
|
||||
template. Out of the box it provides:
|
||||
|
||||
- A custom function `sayHello` that can be imported into Nextflow scripts.
|
||||
- A workflow observer that reacts to pipeline lifecycle events (start and
|
||||
completion).
|
||||
|
||||
Replace this section with a description of what your plugin actually does.
|
||||
|
||||
Note: The **Summary**, **Get Started**, **Examples**, and **License** sections are
|
||||
mandatory: they are required by the Nextflow Registry, which uses this
|
||||
file as the plugin description. The **Plugin development** section below is
|
||||
guidance for working on the plugin and can be removed before publishing.
|
||||
|
||||
## Get Started
|
||||
|
||||
Enable the plugin in your pipeline `nextflow.config`:
|
||||
|
||||
```groovy
|
||||
plugins {
|
||||
id 'k8s-dvfs@0.1.0'
|
||||
}
|
||||
```
|
||||
|
||||
Nextflow downloads the plugin from the Nextflow Registry the first time
|
||||
the pipeline runs.
|
||||
|
||||
## Examples
|
||||
|
||||
Import and call the `sayHello` function from a Nextflow script:
|
||||
|
||||
```nextflow
|
||||
include { sayHello } from 'plugin/k8s-dvfs'
|
||||
|
||||
workflow {
|
||||
channel.of('Mundo', 'World').map { target -> sayHello(target) }
|
||||
}
|
||||
```
|
||||
|
||||
The bundled observer prints a message when the pipeline starts and completes,
|
||||
so running any pipeline with the plugin enabled produces:
|
||||
|
||||
```
|
||||
Pipeline is starting! 🚀
|
||||
Pipeline complete! 👋
|
||||
```
|
||||
|
||||
## Plugin development
|
||||
|
||||
This project was created from the [Nextflow plugin template](https://www.nextflow.io/docs/latest/guides/gradle-plugin.html#gradle-plugin-create).
|
||||
|
||||
### Building
|
||||
|
||||
To build the plugin:
|
||||
|
||||
```bash
|
||||
make assemble
|
||||
```
|
||||
|
||||
### Testing with Nextflow
|
||||
|
||||
The plugin can be tested without a local Nextflow installation:
|
||||
|
||||
1. Build and install the plugin to your local Nextflow installation: `make install`
|
||||
2. Run a pipeline with the plugin: `nextflow run hello -plugins k8s-dvfs@0.1.0`
|
||||
|
||||
### Publishing
|
||||
|
||||
Plugins can be published to a central Nextflow registry to make them accessible to the Nextflow community.
|
||||
|
||||
Follow these steps to publish the plugin to the Nextflow Registry:
|
||||
|
||||
1. Create a file named `$HOME/.gradle/gradle.properties`, where `$HOME` is your home directory. Add the following properties:
|
||||
* `npr.apiKey`: Your Nextflow Registry access token.
|
||||
2. Package your plugin and publish it to the registry: `make release`.
|
||||
|
||||
## License
|
||||
|
||||
Apache License 2.0. See the [`COPYING`](COPYING) file for details.
|
||||
|
||||
Note: The above license is given for guidance only; however the Nextflow Registry
|
||||
requires the plugin to include an OSS (open source software) license.
|
||||
50
k8s-dvfs/build.gradle
Normal file
50
k8s-dvfs/build.gradle
Normal file
@@ -0,0 +1,50 @@
|
||||
plugins {
|
||||
id 'io.nextflow.nextflow-plugin' version '1.0.0-beta.15'
|
||||
}
|
||||
|
||||
version = '0.1.0'
|
||||
|
||||
nextflowPlugin {
|
||||
nextflowVersion = '26.04.0'
|
||||
|
||||
provider = 'recreational.tech'
|
||||
className = 'recreationaltech.plugin.K8sPlugin'
|
||||
useDefaultDependencies = true
|
||||
generateSpec = false
|
||||
extensionPoints = [
|
||||
'recreationaltech.plugin.K8sConfig',
|
||||
'recreationaltech.plugin.K8sExecutor',
|
||||
'recreationaltech.plugin.cli.KubeCommandImpl',
|
||||
]
|
||||
}
|
||||
|
||||
sourceSets {
|
||||
main.java.srcDirs = []
|
||||
main.groovy.srcDirs = ['src/main/groovy']
|
||||
main.resources.srcDirs = ['src/resources']
|
||||
test.groovy.srcDirs = ['src/test/groovy']
|
||||
test.java.srcDirs = []
|
||||
test.resources.srcDirs = []
|
||||
}
|
||||
|
||||
configurations {
|
||||
// see https://docs.gradle.org/4.1/userguide/dependency_management.html#sub:exclude_transitive_dependencies
|
||||
runtimeClasspath.exclude group: 'org.slf4j', module: 'slf4j-api'
|
||||
}
|
||||
|
||||
dependencies {
|
||||
//compileOnly project(':nextflow')
|
||||
compileOnly 'org.slf4j:slf4j-api:2.0.17'
|
||||
compileOnly 'org.pf4j:pf4j:3.14.1'
|
||||
|
||||
api 'org.bouncycastle:bcprov-ext-jdk18on:1.78.1'
|
||||
api 'org.bouncycastle:bcpkix-jdk18on:1.84'
|
||||
|
||||
//testImplementation(testFixtures(project(":nextflow")))
|
||||
testImplementation "org.apache.groovy:groovy:4.0.31"
|
||||
testImplementation "org.apache.groovy:groovy-nio:4.0.31"
|
||||
}
|
||||
|
||||
test {
|
||||
useJUnitPlatform()
|
||||
}
|
||||
BIN
k8s-dvfs/gradle/wrapper/gradle-wrapper.jar
vendored
Normal file
BIN
k8s-dvfs/gradle/wrapper/gradle-wrapper.jar
vendored
Normal file
Binary file not shown.
7
k8s-dvfs/gradle/wrapper/gradle-wrapper.properties
vendored
Normal file
7
k8s-dvfs/gradle/wrapper/gradle-wrapper.properties
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
distributionBase=GRADLE_USER_HOME
|
||||
distributionPath=wrapper/dists
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-8.14-bin.zip
|
||||
networkTimeout=10000
|
||||
validateDistributionUrl=true
|
||||
zipStoreBase=GRADLE_USER_HOME
|
||||
zipStorePath=wrapper/dists
|
||||
249
k8s-dvfs/gradlew
vendored
Executable file
249
k8s-dvfs/gradlew
vendored
Executable file
@@ -0,0 +1,249 @@
|
||||
#!/bin/sh
|
||||
|
||||
#
|
||||
# Copyright © 2015-2021 the original authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
##############################################################################
|
||||
#
|
||||
# Gradle start up script for POSIX generated by Gradle.
|
||||
#
|
||||
# Important for running:
|
||||
#
|
||||
# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is
|
||||
# noncompliant, but you have some other compliant shell such as ksh or
|
||||
# bash, then to run this script, type that shell name before the whole
|
||||
# command line, like:
|
||||
#
|
||||
# ksh Gradle
|
||||
#
|
||||
# Busybox and similar reduced shells will NOT work, because this script
|
||||
# requires all of these POSIX shell features:
|
||||
# * functions;
|
||||
# * expansions «$var», «${var}», «${var:-default}», «${var+SET}»,
|
||||
# «${var#prefix}», «${var%suffix}», and «$( cmd )»;
|
||||
# * compound commands having a testable exit status, especially «case»;
|
||||
# * various built-in commands including «command», «set», and «ulimit».
|
||||
#
|
||||
# Important for patching:
|
||||
#
|
||||
# (2) This script targets any POSIX shell, so it avoids extensions provided
|
||||
# by Bash, Ksh, etc; in particular arrays are avoided.
|
||||
#
|
||||
# The "traditional" practice of packing multiple parameters into a
|
||||
# space-separated string is a well documented source of bugs and security
|
||||
# problems, so this is (mostly) avoided, by progressively accumulating
|
||||
# options in "$@", and eventually passing that to Java.
|
||||
#
|
||||
# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS,
|
||||
# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly;
|
||||
# see the in-line comments for details.
|
||||
#
|
||||
# There are tweaks for specific operating systems such as AIX, CygWin,
|
||||
# Darwin, MinGW, and NonStop.
|
||||
#
|
||||
# (3) This script is generated from the Groovy template
|
||||
# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
|
||||
# within the Gradle project.
|
||||
#
|
||||
# You can find Gradle at https://github.com/gradle/gradle/.
|
||||
#
|
||||
##############################################################################
|
||||
|
||||
# Attempt to set APP_HOME
|
||||
|
||||
# Resolve links: $0 may be a link
|
||||
app_path=$0
|
||||
|
||||
# Need this for daisy-chained symlinks.
|
||||
while
|
||||
APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path
|
||||
[ -h "$app_path" ]
|
||||
do
|
||||
ls=$( ls -ld "$app_path" )
|
||||
link=${ls#*' -> '}
|
||||
case $link in #(
|
||||
/*) app_path=$link ;; #(
|
||||
*) app_path=$APP_HOME$link ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# This is normally unused
|
||||
# shellcheck disable=SC2034
|
||||
APP_BASE_NAME=${0##*/}
|
||||
# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
|
||||
APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit
|
||||
|
||||
# Use the maximum available, or set MAX_FD != -1 to use that value.
|
||||
MAX_FD=maximum
|
||||
|
||||
warn () {
|
||||
echo "$*"
|
||||
} >&2
|
||||
|
||||
die () {
|
||||
echo
|
||||
echo "$*"
|
||||
echo
|
||||
exit 1
|
||||
} >&2
|
||||
|
||||
# OS specific support (must be 'true' or 'false').
|
||||
cygwin=false
|
||||
msys=false
|
||||
darwin=false
|
||||
nonstop=false
|
||||
case "$( uname )" in #(
|
||||
CYGWIN* ) cygwin=true ;; #(
|
||||
Darwin* ) darwin=true ;; #(
|
||||
MSYS* | MINGW* ) msys=true ;; #(
|
||||
NONSTOP* ) nonstop=true ;;
|
||||
esac
|
||||
|
||||
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
|
||||
|
||||
|
||||
# Determine the Java command to use to start the JVM.
|
||||
if [ -n "$JAVA_HOME" ] ; then
|
||||
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
|
||||
# IBM's JDK on AIX uses strange locations for the executables
|
||||
JAVACMD=$JAVA_HOME/jre/sh/java
|
||||
else
|
||||
JAVACMD=$JAVA_HOME/bin/java
|
||||
fi
|
||||
if [ ! -x "$JAVACMD" ] ; then
|
||||
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
|
||||
|
||||
Please set the JAVA_HOME variable in your environment to match the
|
||||
location of your Java installation."
|
||||
fi
|
||||
else
|
||||
JAVACMD=java
|
||||
if ! command -v java >/dev/null 2>&1
|
||||
then
|
||||
die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
||||
|
||||
Please set the JAVA_HOME variable in your environment to match the
|
||||
location of your Java installation."
|
||||
fi
|
||||
fi
|
||||
|
||||
# Increase the maximum file descriptors if we can.
|
||||
if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
|
||||
case $MAX_FD in #(
|
||||
max*)
|
||||
# In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
|
||||
# shellcheck disable=SC2039,SC3045
|
||||
MAX_FD=$( ulimit -H -n ) ||
|
||||
warn "Could not query maximum file descriptor limit"
|
||||
esac
|
||||
case $MAX_FD in #(
|
||||
'' | soft) :;; #(
|
||||
*)
|
||||
# In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
|
||||
# shellcheck disable=SC2039,SC3045
|
||||
ulimit -n "$MAX_FD" ||
|
||||
warn "Could not set maximum file descriptor limit to $MAX_FD"
|
||||
esac
|
||||
fi
|
||||
|
||||
# Collect all arguments for the java command, stacking in reverse order:
|
||||
# * args from the command line
|
||||
# * the main class name
|
||||
# * -classpath
|
||||
# * -D...appname settings
|
||||
# * --module-path (only if needed)
|
||||
# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables.
|
||||
|
||||
# For Cygwin or MSYS, switch paths to Windows format before running java
|
||||
if "$cygwin" || "$msys" ; then
|
||||
APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
|
||||
CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" )
|
||||
|
||||
JAVACMD=$( cygpath --unix "$JAVACMD" )
|
||||
|
||||
# Now convert the arguments - kludge to limit ourselves to /bin/sh
|
||||
for arg do
|
||||
if
|
||||
case $arg in #(
|
||||
-*) false ;; # don't mess with options #(
|
||||
/?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath
|
||||
[ -e "$t" ] ;; #(
|
||||
*) false ;;
|
||||
esac
|
||||
then
|
||||
arg=$( cygpath --path --ignore --mixed "$arg" )
|
||||
fi
|
||||
# Roll the args list around exactly as many times as the number of
|
||||
# args, so each arg winds up back in the position where it started, but
|
||||
# possibly modified.
|
||||
#
|
||||
# NB: a `for` loop captures its iteration list before it begins, so
|
||||
# changing the positional parameters here affects neither the number of
|
||||
# iterations, nor the values presented in `arg`.
|
||||
shift # remove old arg
|
||||
set -- "$@" "$arg" # push replacement arg
|
||||
done
|
||||
fi
|
||||
|
||||
|
||||
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
||||
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
|
||||
|
||||
# Collect all arguments for the java command:
|
||||
# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
|
||||
# and any embedded shellness will be escaped.
|
||||
# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
|
||||
# treated as '${Hostname}' itself on the command line.
|
||||
|
||||
set -- \
|
||||
"-Dorg.gradle.appname=$APP_BASE_NAME" \
|
||||
-classpath "$CLASSPATH" \
|
||||
org.gradle.wrapper.GradleWrapperMain \
|
||||
"$@"
|
||||
|
||||
# Stop when "xargs" is not available.
|
||||
if ! command -v xargs >/dev/null 2>&1
|
||||
then
|
||||
die "xargs is not available"
|
||||
fi
|
||||
|
||||
# Use "xargs" to parse quoted args.
|
||||
#
|
||||
# With -n1 it outputs one arg per line, with the quotes and backslashes removed.
|
||||
#
|
||||
# In Bash we could simply go:
|
||||
#
|
||||
# readarray ARGS < <( xargs -n1 <<<"$var" ) &&
|
||||
# set -- "${ARGS[@]}" "$@"
|
||||
#
|
||||
# but POSIX shell has neither arrays nor command substitution, so instead we
|
||||
# post-process each arg (as a line of input to sed) to backslash-escape any
|
||||
# character that might be a shell metacharacter, then use eval to reverse
|
||||
# that process (while maintaining the separation between arguments), and wrap
|
||||
# the whole thing up as a single "set" statement.
|
||||
#
|
||||
# This will of course break if any of these variables contains a newline or
|
||||
# an unmatched quote.
|
||||
#
|
||||
|
||||
eval "set -- $(
|
||||
printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" |
|
||||
xargs -n1 |
|
||||
sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' |
|
||||
tr '\n' ' '
|
||||
)" '"$@"'
|
||||
|
||||
exec "$JAVACMD" "$@"
|
||||
1
k8s-dvfs/settings.gradle
Normal file
1
k8s-dvfs/settings.gradle
Normal file
@@ -0,0 +1 @@
|
||||
rootProject.name = 'k8s-dvfs'
|
||||
@@ -0,0 +1,527 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin
|
||||
|
||||
import nextflow.config.scopes.Config
|
||||
import recreationaltech.plugin.client.K8sRetryConfig
|
||||
|
||||
import javax.annotation.Nullable
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.PackageScope
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.BuildInfo
|
||||
import nextflow.config.spec.ConfigOption
|
||||
import nextflow.config.spec.ConfigScope
|
||||
import nextflow.config.spec.ScopeName
|
||||
import nextflow.container.ContainerHelper
|
||||
import nextflow.script.dsl.Description
|
||||
import nextflow.exception.AbortOperationException
|
||||
import recreationaltech.plugin.client.ClientConfig
|
||||
import recreationaltech.plugin.client.K8sClient
|
||||
import recreationaltech.plugin.client.K8sResponseException
|
||||
import recreationaltech.plugin.model.PodOptions
|
||||
import recreationaltech.plugin.model.PodSecurityContext
|
||||
import recreationaltech.plugin.model.PodVolumeClaim
|
||||
import recreationaltech.plugin.model.ResourceType
|
||||
import nextflow.util.Duration
|
||||
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
/**
|
||||
* Model Kubernetes specific settings defined in the nextflow
|
||||
* configuration file
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@ScopeName("k8s")
|
||||
@Description("""
|
||||
The `k8s` scope controls the deployment and execution of workflow applications in a Kubernetes cluster.
|
||||
""")
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class K8sConfig implements ConfigScope {
|
||||
|
||||
static final private Map<String,?> DEFAULT_FUSE_PLUGIN = Map.of('nextflow.io/fuse', 1)
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Automatically mount host paths into the task pods (default: `false`). Only intended for development purposes when using a single node.
|
||||
""")
|
||||
final boolean autoMountHostPaths
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Whether to use Kubernetes `Pod` or `Job` resource type to carry out Nextflow tasks (default: `Pod`).
|
||||
""")
|
||||
final String computeResourceType
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
When `true`, successful pods are automatically deleted (default: `true`).
|
||||
""")
|
||||
final private Boolean cleanup
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Map of options for the K8s client.
|
||||
|
||||
If this option is specified, it will be used instead of `.kube/config`.
|
||||
""")
|
||||
final Map client
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The interval after which the Kubernetes client configuration is refreshed (default: `50m`).
|
||||
""")
|
||||
final Duration clientRefreshInterval
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The Kubernetes [configuration context](https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/) to use.
|
||||
""")
|
||||
final String context
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
When `true`, set both the pod CPUs `request` and `limit` to the value specified by the `cpus` directive, otherwise set only the `request` (default: `false`).
|
||||
""")
|
||||
final boolean cpuLimits
|
||||
|
||||
final K8sDebug debug
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Include the hostname of each task in the execution trace (default: `false`).
|
||||
""")
|
||||
final boolean fetchNodeName
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The FUSE device plugin to be used when enabling Fusion in unprivileged mode (default: `['nextflow.io/fuse': 1]`).
|
||||
""")
|
||||
final Map fuseDevicePlugin
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The Kubernetes HTTP client request connection timeout e.g. `'60s'`.
|
||||
""")
|
||||
final Duration httpConnectTimeout
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The Kubernetes HTTP client request connection read timeout e.g. `'60s'`.
|
||||
""")
|
||||
final Duration httpReadTimeout
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The strategy for pulling container images. Can be `IfNotPresent`, `Always`, `Never`.
|
||||
|
||||
[Read more](https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy)
|
||||
""")
|
||||
final String imagePullPolicy
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The path where the workflow is launched and the user data is stored (default: `<volume-claim-mount-path>/<user-name>`). Must be a path in a shared K8s persistent volume.
|
||||
""")
|
||||
final String launchDir
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The Kubernetes namespace to use (default: `default`).
|
||||
""")
|
||||
final String namespace
|
||||
|
||||
@ConfigOption(types=[List, Map])
|
||||
@Description("""
|
||||
Additional pod configuration options such as environment variables, config maps, secrets, etc. Allows the same settings as the [pod](https://nextflow.io/docs/latest/process.html#pod) process directive.
|
||||
""")
|
||||
final PodOptions pod
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The path where Nextflow projects are downloaded (default: `<volume-claim-mount-path>/projects`). Must be a path in a shared K8s persistent volume.
|
||||
""")
|
||||
final String projectDir
|
||||
|
||||
@Deprecated
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
""")
|
||||
final String pullPolicy
|
||||
|
||||
final K8sRetryConfig retryPolicy
|
||||
|
||||
@ConfigOption(types=[Integer, String])
|
||||
@Description("""
|
||||
The user ID to be used to run the containers. Shortcut for the `securityContext` option.
|
||||
""")
|
||||
final Object runAsUser
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The [security context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) to use for all pods.
|
||||
""")
|
||||
final Map securityContext
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The Kubernetes [service account name](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/) to use.
|
||||
""")
|
||||
final String serviceAccount
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The name of the persistent volume claim where the shared work directory is stored.
|
||||
""")
|
||||
final String storageClaimName
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The mount path for the persistent volume claim (default: `/workspace`).
|
||||
""")
|
||||
final String storageMountPath
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The path in the persistent volume to be mounted (default: `/`).
|
||||
""")
|
||||
final String storageSubPath
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
""")
|
||||
final String userName
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The path of the shared work directory (default: `<user-dir>/work`). Must be a path in a shared K8s persistent volume.
|
||||
""")
|
||||
final String workDir
|
||||
|
||||
@Description("Node initialization config")
|
||||
final K8sNodeInitConfig nodeInit
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The image name of the nextflow launcher image
|
||||
""")
|
||||
final String nextflowImage
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The run interval of the kubernetes scheduler
|
||||
""")
|
||||
final Duration schedulerInterval
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Enables task runtime recording
|
||||
""")
|
||||
final boolean recordTaskRuntimes
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The runtime recording file
|
||||
""")
|
||||
final String runtimeRecordPath
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Selects a scheduling strategy
|
||||
""")
|
||||
final String schedulingStrategy
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Selects a runtime estimator
|
||||
""")
|
||||
final String runtimeEstimator
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Noise magnitude used for the noise runtime estimator
|
||||
""")
|
||||
final Duration noiseRuntimeEstimatorNoiseMagnitude
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Max. amount of time two runtime estimates can differ to be considered equal.
|
||||
""")
|
||||
final Duration runtimeComparisonEpsilon
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Number of saved top runtimes used to classify a task as critical.
|
||||
""")
|
||||
final int dvfsSchedulingNumTopRuntimes
|
||||
|
||||
/* required by extension point -- do not remove */
|
||||
K8sConfig() {
|
||||
this(Collections.emptyMap())
|
||||
}
|
||||
|
||||
K8sConfig(Map opts) {
|
||||
autoMountHostPaths = opts.autoMountHostPaths as boolean
|
||||
cleanup = opts.cleanup as Boolean
|
||||
client = opts.client as Map
|
||||
clientRefreshInterval = opts.clientRefreshInterval as Duration ?: Duration.of('50m')
|
||||
computeResourceType = opts.computeResourceType
|
||||
context = opts.context
|
||||
cpuLimits = opts.cpuLimits as boolean
|
||||
debug = new K8sDebug(opts.debug as Map ?: Collections.emptyMap())
|
||||
fetchNodeName = opts.fetchNodeName as boolean
|
||||
fuseDevicePlugin = parseFuseDevicePlugin(opts.fuseDevicePlugin)
|
||||
httpConnectTimeout = opts.httpConnectTimeout as Duration
|
||||
httpReadTimeout = opts.httpReadTimeout as Duration
|
||||
imagePullPolicy = opts.pullPolicy ?: opts.imagePullPolicy
|
||||
namespace = opts.namespace
|
||||
pod = createPodOptions(opts.pod)
|
||||
retryPolicy = new K8sRetryConfig(opts.retryPolicy as Map ?: Collections.emptyMap())
|
||||
runAsUser = opts.runAsUser
|
||||
securityContext = opts.securityContext as Map
|
||||
serviceAccount = opts.serviceAccount
|
||||
storageClaimName = opts.storageClaimName
|
||||
storageMountPath = opts.storageMountPath ?: '/workspace'
|
||||
storageSubPath = opts.storageSubPath
|
||||
userName = opts.userName
|
||||
nextflowImage = opts.nextflowImage ?: "nextflow/nextflow:${BuildInfo.version}"
|
||||
schedulerInterval = opts.schedulerInterval as Duration ?: new Duration(10, TimeUnit.SECONDS)
|
||||
recordTaskRuntimes = opts.recordTaskRuntimes as boolean ?: false
|
||||
|
||||
launchDir = opts.launchDir ?: "${storageMountPath}/${getUserName()}"
|
||||
projectDir = opts.projectDir ?: "${storageMountPath}/projects"
|
||||
workDir = opts.workDir ?: "${launchDir}/work"
|
||||
runtimeRecordPath = opts.runtimeRecordPath as String ?: "${workDir}/runtimes.csv"
|
||||
|
||||
schedulingStrategy = opts.schedulingStrategy as String ?: "Hash"
|
||||
runtimeEstimator = opts.runtimeEstimator as String ?: "LinearFit"
|
||||
noiseRuntimeEstimatorNoiseMagnitude = opts.noiseRuntimeEstimatorNoiseMagnitude as Duration ?: new Duration(30, TimeUnit.SECONDS)
|
||||
runtimeComparisonEpsilon = opts.runtimeComparisonEpsilon as Duration ?: new Duration(10, TimeUnit.SECONDS)
|
||||
dvfsSchedulingNumTopRuntimes = opts.dvfsSchedulingTopRuntimes as int ?: 3
|
||||
|
||||
// -- shortcut to pod image pull-policy
|
||||
if( imagePullPolicy )
|
||||
pod.imagePullPolicy = imagePullPolicy
|
||||
|
||||
// -- shortcut to pod volume claim
|
||||
if( storageClaimName ) {
|
||||
final volumeClaim = new PodVolumeClaim(storageClaimName, storageMountPath, storageSubPath)
|
||||
pod.volumeClaims.add(volumeClaim)
|
||||
}
|
||||
|
||||
// -- shortcut to pod security context
|
||||
if( runAsUser )
|
||||
pod.securityContext = new PodSecurityContext(runAsUser)
|
||||
else if( securityContext )
|
||||
pod.securityContext = new PodSecurityContext(securityContext)
|
||||
|
||||
nodeInit = new K8sNodeInitConfig(opts.nodeInit as Map ?: Collections.emptyMap())
|
||||
}
|
||||
|
||||
private PodOptions createPodOptions( value ) {
|
||||
if( value instanceof List )
|
||||
return new PodOptions( value as List )
|
||||
|
||||
if( value instanceof Map )
|
||||
return new PodOptions( [(Map)value] )
|
||||
|
||||
if( value == null )
|
||||
return new PodOptions()
|
||||
|
||||
throw new IllegalArgumentException("Not a valid pod setting: $value")
|
||||
}
|
||||
|
||||
Map<String,String> getLabels() {
|
||||
pod.getLabels()
|
||||
}
|
||||
|
||||
Map<String,String> getAnnotations() {
|
||||
pod.getAnnotations()
|
||||
}
|
||||
|
||||
boolean getCleanup(boolean defValue=true) {
|
||||
cleanup == null ? defValue : cleanup
|
||||
}
|
||||
|
||||
String getUserName() {
|
||||
userName ?: System.properties.get('user.name')
|
||||
}
|
||||
|
||||
Map<String,?> fuseDevicePlugin() {
|
||||
fuseDevicePlugin
|
||||
}
|
||||
|
||||
Map<String,?> parseFuseDevicePlugin(Object value) {
|
||||
if( value instanceof Map && value.size()==1 )
|
||||
return value as Map<String,?>
|
||||
if( value != null )
|
||||
log.warn1 "Setting 'k8s.fuseDevicePlugin' should be a map containing exactly one entry - offending value: $value"
|
||||
return DEFAULT_FUSE_PLUGIN
|
||||
}
|
||||
|
||||
/**
|
||||
* Whenever the pod should honour the entrypoint defined by the image (default: false)
|
||||
*
|
||||
* @return When {@code false} the launcher script is run by using pod `command` attributes which
|
||||
* overrides the entrypoint point defined by the image.
|
||||
*
|
||||
* When {@code true} the launcher is run via the pod `args` attribute, without altering the
|
||||
* container entrypoint (it does however require to have a bash shell as the image entrypoint)
|
||||
*
|
||||
*/
|
||||
boolean entrypointOverride() {
|
||||
return ContainerHelper.entrypointOverride()
|
||||
}
|
||||
|
||||
boolean useJobResource() { ResourceType.Job.name() == computeResourceType }
|
||||
|
||||
String getNextflowImageName() {
|
||||
return nextflowImage
|
||||
}
|
||||
|
||||
PodOptions getPodOptions() {
|
||||
pod
|
||||
}
|
||||
|
||||
boolean fetchNodeName() {
|
||||
fetchNodeName
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the collection of defined volume claim names
|
||||
*/
|
||||
Collection<String> getClaimNames() {
|
||||
pod.volumeClaims.collect { it.claimName }
|
||||
}
|
||||
|
||||
Collection<String> getClaimPaths() {
|
||||
pod.volumeClaims.collect { it.mountPath }
|
||||
}
|
||||
|
||||
boolean cpuLimitsEnabled() {
|
||||
cpuLimits
|
||||
}
|
||||
|
||||
/**
|
||||
* Find a volume claim name given the mount path
|
||||
*
|
||||
* @param path The volume claim mount path
|
||||
* @return The volume claim name for the given mount path
|
||||
*/
|
||||
String findVolumeClaimByPath(String path) {
|
||||
final result = pod.volumeClaims.find { path.startsWith(it.mountPath) }
|
||||
return result ? result.claimName : null
|
||||
}
|
||||
|
||||
ClientConfig getClient() {
|
||||
final result = client != null
|
||||
? clientFromNextflow(client, namespace, serviceAccount)
|
||||
: clientDiscovery(context, namespace, serviceAccount)
|
||||
|
||||
if( httpConnectTimeout )
|
||||
result.httpConnectTimeout = httpConnectTimeout
|
||||
|
||||
if( httpReadTimeout )
|
||||
result.httpReadTimeout = httpReadTimeout
|
||||
|
||||
if( retryPolicy )
|
||||
result.retryConfig = retryPolicy
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the K8s client config from the declaration made in the Nextflow config file
|
||||
*
|
||||
* @param map
|
||||
* A map representing the clint configuration options define in the nextflow
|
||||
* config file
|
||||
* @param namespace
|
||||
* The K8s namespace to be used. If omitted {@code default} is used.
|
||||
* @param serviceAccount
|
||||
* The K8s service account to be used. If omitted {@code default} is used.
|
||||
* @return
|
||||
* The Kubernetes {@link ClientConfig} object
|
||||
*/
|
||||
@PackageScope ClientConfig clientFromNextflow(Map map, @Nullable String namespace, @Nullable String serviceAccount ) {
|
||||
ClientConfig.fromNextflowConfig(map,namespace,serviceAccount)
|
||||
}
|
||||
|
||||
/**
|
||||
* Discover the K8s client config from the execution environment
|
||||
* that can be either a `.kube/config` file or service meta file
|
||||
* when running in a pod.
|
||||
*
|
||||
* @param contextName
|
||||
* The name of the configuration context to be used
|
||||
* @param namespace
|
||||
* The Kubernetes namespace to be used
|
||||
* @param serviceAccount
|
||||
* The Kubernetes serviceAccount to be used
|
||||
* @return
|
||||
* The discovered Kube {@link ClientConfig} object
|
||||
*/
|
||||
@PackageScope ClientConfig clientDiscovery(String contextName, String namespace, String serviceAccount) {
|
||||
ClientConfig.discover(contextName, namespace, serviceAccount)
|
||||
}
|
||||
|
||||
void checkStorageAndPaths(K8sClient client) {
|
||||
if( !storageClaimName )
|
||||
throw new AbortOperationException("Missing K8s storage volume claim -- The name of a persistence volume claim needs to be provided in the nextflow configuration file")
|
||||
|
||||
log.debug "Kubernetes workDir=$workDir; projectDir=$projectDir; volumeClaims=${getClaimNames()}"
|
||||
|
||||
for( String name : getClaimNames() ) {
|
||||
try {
|
||||
client.volumeClaimRead(name)
|
||||
}
|
||||
catch (K8sResponseException e) {
|
||||
if( e.response.code == 404 ) {
|
||||
throw new AbortOperationException("Unknown volume claim: $name -- make sure a persistent volume claim with the specified name is defined in your K8s cluster")
|
||||
}
|
||||
else throw e
|
||||
}
|
||||
}
|
||||
|
||||
if( !findVolumeClaimByPath(launchDir) )
|
||||
throw new AbortOperationException("Kubernetes `launchDir` must be a path mounted as a persistent volume -- launchDir=$launchDir; volumes=${getClaimPaths().join(', ')}")
|
||||
|
||||
if( !findVolumeClaimByPath(workDir) )
|
||||
throw new AbortOperationException("Kubernetes `workDir` must be a path mounted as a persistent volume -- workDir=$workDir; volumes=${getClaimPaths().join(', ')}")
|
||||
|
||||
if( !findVolumeClaimByPath(projectDir) )
|
||||
throw new AbortOperationException("Kubernetes `projectDir` must be a path mounted as a persistent volume -- projectDir=$projectDir; volumes=${getClaimPaths().join(', ')}")
|
||||
|
||||
}
|
||||
|
||||
static class K8sDebug implements ConfigScope {
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Save the pod spec for each task to `.command.yaml` in the task directory (default: `false`).
|
||||
""")
|
||||
final boolean yaml
|
||||
|
||||
K8sDebug(Map opts) {
|
||||
yaml = opts.yaml as boolean
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
package recreationaltech.plugin
|
||||
|
||||
import groovy.util.logging.Slf4j
|
||||
|
||||
import java.net.http.HttpClient
|
||||
import java.net.http.HttpRequest
|
||||
import java.net.http.HttpResponse
|
||||
import java.nio.channels.UnresolvedAddressException
|
||||
|
||||
/**
|
||||
* The DVFS client uses the deployed DVFS agents to control the operating frequency of the worker nodes.
|
||||
*/
|
||||
@Slf4j
|
||||
class K8sDVFSClient {
|
||||
private HttpClient httpClient
|
||||
private Map<String, String> ipTable
|
||||
|
||||
K8sDVFSClient(String[] nodes, String[] ips) {
|
||||
assert(nodes.length == ips.length)
|
||||
this.httpClient = HttpClient.newBuilder().build()
|
||||
this.ipTable = new HashMap<>()
|
||||
for (int i = 0; i < nodes.length; i++) {
|
||||
this.ipTable.put(nodes[i], ips[i])
|
||||
}
|
||||
}
|
||||
|
||||
OptionalLong getNodeCurrentFrequency(String node) {
|
||||
log.debug("Getting current frequency of node ${node}")
|
||||
return getNodeFrequency(node, "current")
|
||||
}
|
||||
|
||||
OptionalLong getNodeMaxFrequency(String node) {
|
||||
log.debug("Getting max frequency of node ${node}")
|
||||
return getNodeFrequency(node, "max")
|
||||
}
|
||||
|
||||
OptionalLong getNodeMinFrequency(String node) {
|
||||
log.debug("Getting min frequency of node ${node}")
|
||||
return getNodeFrequency(node, "min")
|
||||
}
|
||||
|
||||
private OptionalLong getNodeFrequency(String node, String endpoint) {
|
||||
return getLong(HttpRequest.newBuilder()
|
||||
.uri(new URI("http://${agentAddress(node)}/cpu/frequency/${endpoint}"))
|
||||
.GET()
|
||||
.build())
|
||||
}
|
||||
|
||||
boolean setNodeFrequency(String node, int frequency) {
|
||||
String body = String.valueOf(frequency)
|
||||
|
||||
HttpRequest request = HttpRequest.newBuilder()
|
||||
.uri(new URI("http://${agentAddress(node)}/cpu/frequency/current"))
|
||||
.PUT(HttpRequest.BodyPublishers.ofString(body))
|
||||
.build()
|
||||
HttpResponse<Void> response = httpClient.send(request, HttpResponse.BodyHandlers.discarding())
|
||||
if (response.statusCode() != 200) {
|
||||
log.error("Request PUT ${request.uri().toString()} returned ${response.statusCode()}")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
OptionalLong getCPUCount(String node) {
|
||||
return getLong(HttpRequest.newBuilder()
|
||||
.uri(new URI("http://${agentAddress(node)}/cpu/count"))
|
||||
.GET()
|
||||
.build())
|
||||
}
|
||||
|
||||
OptionalLong getMemoryAmount(String node) {
|
||||
return getLong(HttpRequest.newBuilder()
|
||||
.uri(new URI("http://${agentAddress(node)}/mem/amount"))
|
||||
.GET()
|
||||
.build())
|
||||
}
|
||||
|
||||
private OptionalLong getLong(HttpRequest request) {
|
||||
try {
|
||||
HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString())
|
||||
if (response.statusCode() != 200) {
|
||||
log.error("Request GET ${request.uri().toString()} returned ${response.statusCode()}")
|
||||
return OptionalLong.empty()
|
||||
}
|
||||
long parsed = Long.parseLong(response.body())
|
||||
return OptionalLong.of(parsed)
|
||||
} catch (NumberFormatException ex) {
|
||||
log.error("Unexpected response ${response.body()} - ${ex.message}")
|
||||
return OptionalLong.empty()
|
||||
} catch (ConnectException ex) {
|
||||
log.error("Request failed ${request.uri().toString()}: ${ex.message}")
|
||||
return OptionalLong.empty()
|
||||
} catch (UnresolvedAddressException ex) {
|
||||
log.error("Failed to resolve address ${request.uri().toString()}: ${ex.message}")
|
||||
return OptionalLong.empty()
|
||||
}
|
||||
}
|
||||
|
||||
private String agentAddress(String node) {
|
||||
return ipTable.get(node) + ":8080"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,718 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin
|
||||
|
||||
import groovy.transform.MapConstructor
|
||||
|
||||
import java.lang.reflect.Field
|
||||
import java.nio.file.NoSuchFileException
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
|
||||
import com.beust.jcommander.DynamicParameter
|
||||
import com.beust.jcommander.Parameter
|
||||
import com.google.common.hash.Hashing
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.cli.CmdKubeRun
|
||||
import nextflow.cli.CmdRun
|
||||
import nextflow.config.ConfigBuilder
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.file.FileHelper
|
||||
import recreationaltech.plugin.client.K8sClient
|
||||
import recreationaltech.plugin.client.K8sResponseException
|
||||
import recreationaltech.plugin.model.PodEnv
|
||||
import recreationaltech.plugin.model.PodMountConfig
|
||||
import recreationaltech.plugin.model.PodSpecBuilder
|
||||
import recreationaltech.plugin.model.ResourceType
|
||||
import nextflow.scm.AssetManager
|
||||
import nextflow.scm.ProviderConfig
|
||||
import nextflow.util.ConfigHelper
|
||||
import nextflow.util.Escape
|
||||
import org.codehaus.groovy.runtime.MethodClosure
|
||||
/**
|
||||
* Configure and submit the execution of pod running the Nextflow main application
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@MapConstructor(includeFields = true)
|
||||
class K8sDriverLauncher {
|
||||
|
||||
/**
|
||||
* Either a Pod or Job
|
||||
*/
|
||||
private ResourceType resourceType = ResourceType.Pod
|
||||
|
||||
/**
|
||||
* Container image to be used for the Nextflow driver pod
|
||||
*/
|
||||
private String headImage
|
||||
|
||||
/**
|
||||
* Request CPUs to be used for the Nextflow driver pod
|
||||
*/
|
||||
private int headCpus
|
||||
|
||||
/**
|
||||
* Request memory to be used for the Nextflow driver pod
|
||||
*/
|
||||
private String headMemory
|
||||
|
||||
/**
|
||||
* Pre-script to run before nextflow
|
||||
*/
|
||||
private String headPreScript
|
||||
|
||||
/**
|
||||
* Nextflow execution run name
|
||||
*/
|
||||
private String runName
|
||||
|
||||
/**
|
||||
* Workflow project to launch
|
||||
*/
|
||||
private String pipelineName
|
||||
|
||||
/**
|
||||
* Command run options
|
||||
*/
|
||||
private CmdKubeRun cmd
|
||||
|
||||
/**
|
||||
* Kubernetes client
|
||||
*/
|
||||
private K8sClient k8sClient
|
||||
|
||||
/**
|
||||
* Nextflow resolved config object
|
||||
*/
|
||||
private ConfigObject config
|
||||
|
||||
/**
|
||||
* Name of the config map used to propagate the nextflow
|
||||
* setting in the container
|
||||
*/
|
||||
private String configMapName
|
||||
|
||||
/**
|
||||
* Kubernetes specific config settings
|
||||
*/
|
||||
private K8sConfig k8sConfig
|
||||
|
||||
private String paramsFile
|
||||
|
||||
private boolean interactive
|
||||
|
||||
/**
|
||||
* Runs in background mode
|
||||
*/
|
||||
private boolean background
|
||||
|
||||
/**
|
||||
* Workflow script positional parameters
|
||||
*/
|
||||
private List<String> args
|
||||
|
||||
/**
|
||||
* Plugins to run the workflow
|
||||
*/
|
||||
private String plugins
|
||||
|
||||
private K8sNodeInitDeployer initDeployer
|
||||
|
||||
/**
|
||||
* Launcher entry point. Set-up the environment and create a pod that run the Nextflow
|
||||
* application (which in turns executed each task as a pod)
|
||||
*
|
||||
* @param name Workflow project entry name
|
||||
* @param args Workflow script positional parameters
|
||||
*/
|
||||
void run(String name, List<String> args) {
|
||||
this.args = args
|
||||
this.pipelineName = name
|
||||
this.interactive = name == 'login'
|
||||
if( background && interactive )
|
||||
throw new AbortOperationException("Option -bg conflicts with interactive mode")
|
||||
this.config = makeConfig(pipelineName)
|
||||
this.k8sConfig = makeK8sConfig(config.toMap())
|
||||
this.k8sClient = makeK8sClient(k8sConfig)
|
||||
this.k8sConfig.checkStorageAndPaths(k8sClient)
|
||||
this.initDeployer = new K8sNodeInitDeployer(k8sClient, k8sConfig)
|
||||
|
||||
createK8sConfigMap()
|
||||
|
||||
initDeployer.deploy()
|
||||
createK8sLauncherPod()
|
||||
waitPodStart()
|
||||
// login into container session
|
||||
if( interactive )
|
||||
launchLogin()
|
||||
// dump pod output
|
||||
else if( !background )
|
||||
printK8sPodOutput()
|
||||
else
|
||||
log.debug "Nextflow driver launched in background mode -- pod: $runName"
|
||||
waitPodEnd()
|
||||
}
|
||||
|
||||
int shutdown() {
|
||||
if( background )
|
||||
return 0
|
||||
// fetch the container exit status
|
||||
final exitCode = waitPodTermination()
|
||||
// cleanup the config map if OK
|
||||
def deleteOnSuccessByDefault = exitCode==0
|
||||
if( k8sConfig.getCleanup(deleteOnSuccessByDefault) ) {
|
||||
deleteConfigMap()
|
||||
}
|
||||
// cleanup pre-workflow pods
|
||||
initDeployer.cleanup()
|
||||
|
||||
return exitCode
|
||||
}
|
||||
|
||||
protected void waitPodEnd() {
|
||||
if( background )
|
||||
return
|
||||
final currentState = k8sConfig.useJobResource() ? k8sClient.jobState(runName) : k8sClient.podState(runName)
|
||||
if (currentState && currentState?.running instanceof Map) {
|
||||
final name = runName
|
||||
println "${resourceType} running: $name ... waiting for ${resourceType.lower()} to stop running"
|
||||
try {
|
||||
while( true ) {
|
||||
sleep 10000
|
||||
final state = k8sConfig.useJobResource() ? k8sClient.jobState(name) : k8sClient.podState(name)
|
||||
if ( state && !(state?.running instanceof Map) ) {
|
||||
println "${resourceType} $name has changed from running state $state"
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
catch( Exception e ) {
|
||||
log.warn "Caught exception while waiting for ${resourceType.lower()} to stop running"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected boolean isWaitTimedOut(long time) {
|
||||
System.currentTimeMillis()-time > 90_000
|
||||
}
|
||||
|
||||
protected int waitPodTermination() {
|
||||
log.debug "Wait for ${resourceType.lower()} termination name=$runName"
|
||||
final rnd = new Random()
|
||||
final time = System.currentTimeMillis()
|
||||
Map state = null
|
||||
try {
|
||||
while( true ) {
|
||||
sleep rnd.nextInt(500)
|
||||
state = k8sConfig.useJobResource() ? k8sClient.jobState(runName) : k8sClient.podState(runName)
|
||||
if( state?.terminated instanceof Map )
|
||||
return state.terminated.exitCode as int
|
||||
|
||||
else if( isWaitTimedOut(time) )
|
||||
throw new IllegalStateException("Timeout waiting for ${resourceType.lower()} terminated state=$state")
|
||||
}
|
||||
}
|
||||
catch( Exception e ) {
|
||||
log.warn "Unable to fetch ${resourceType.lower()} exit status -- ${resourceType.lower()}=$runName state=$state"
|
||||
return 127
|
||||
}
|
||||
}
|
||||
|
||||
protected void deleteConfigMap() {
|
||||
try {
|
||||
k8sClient.configDelete(configMapName)
|
||||
log.debug "Deleted K8s configMap with name: $configMapName"
|
||||
}
|
||||
catch ( Exception e ) {
|
||||
log.warn "Unable to delete configMap: $configMapName", e
|
||||
}
|
||||
}
|
||||
|
||||
protected void waitPodStart() {
|
||||
final name = runName
|
||||
print "${resourceType} submitted: $name .. waiting to start"
|
||||
while( true ) {
|
||||
sleep 1000
|
||||
final state = k8sConfig.useJobResource() ? k8sClient.jobState(name) : k8sClient.podState(name)
|
||||
if( state && !state.containsKey('waiting') ) {
|
||||
break
|
||||
}
|
||||
}
|
||||
print "\33[2K\r"
|
||||
println "${resourceType} started: $name"
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for the driver pod creation and prints the log to the
|
||||
* console standard output
|
||||
*/
|
||||
protected void printK8sPodOutput() {
|
||||
if ( k8sConfig.useJobResource() )
|
||||
k8sClient.jobLog(runName, follow:true).eachLine { println it }
|
||||
else
|
||||
k8sClient.podLog(runName, follow:true).eachLine { println it }
|
||||
}
|
||||
|
||||
protected ConfigObject loadConfig( String pipelineName ) {
|
||||
|
||||
// -- load local config if available
|
||||
final builder = new ConfigBuilder()
|
||||
.setShowClosures(true)
|
||||
.setOptions(cmd.launcher.options)
|
||||
.setProfile(cmd.profile)
|
||||
.setCmdRun(cmd)
|
||||
|
||||
if( !interactive && !pipelineName.startsWith('/') && !cmd.remoteProfile && !cmd.runRemoteConfig ) {
|
||||
// -- check and parse project remote config
|
||||
final pipelineConfig = new AssetManager(pipelineName, cmd.revision, cmd.mainScript, cmd).getConfigFile()
|
||||
builder.setUserConfigFiles(pipelineConfig)
|
||||
}
|
||||
|
||||
return builder.buildConfigObject()
|
||||
}
|
||||
|
||||
protected K8sConfig makeK8sConfig(Map config) {
|
||||
config.k8s instanceof Map ? new K8sConfig(config.k8s as Map) : new K8sConfig()
|
||||
}
|
||||
|
||||
protected makeK8sClient( K8sConfig k8sConfig ) {
|
||||
new K8sClient(k8sConfig.getClient())
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve the workflow configuration and merge with the current local one.
|
||||
*
|
||||
* @param pipelineName Workflow project name
|
||||
* @return A {@link Map} modeling the execution configuration settings
|
||||
*/
|
||||
protected ConfigObject makeConfig(String pipelineName) {
|
||||
|
||||
def file = new File(pipelineName)
|
||||
if( !interactive && file.exists() ) {
|
||||
def message = "The k8s executor cannot run local ${file.directory ? 'project' : 'script'}: $pipelineName"
|
||||
message += " -- provide the absolute path of a project available in the Kubernetes cluster or the URL of a project hosted in a Git repository"
|
||||
throw new AbortOperationException(message)
|
||||
}
|
||||
|
||||
def config = loadConfig(pipelineName)
|
||||
|
||||
// normalize pod entries
|
||||
def k8s = config.k8s
|
||||
|
||||
if( !k8s.isSet('pod') )
|
||||
k8s.pod = []
|
||||
else if( k8s.pod instanceof Map ) {
|
||||
k8s.pod = [ k8s.pod ]
|
||||
}
|
||||
else if( !(k8s.pod instanceof List) )
|
||||
throw new IllegalArgumentException("Illegal k8s.pod configuratun value: ${k8s.pod}")
|
||||
|
||||
// -- use the volume claims specified in the command line
|
||||
// to populate the pod config
|
||||
for( int i=0; i<cmd.volMounts?.size(); i++ ){
|
||||
def entry = cmd.volMounts.get(i)
|
||||
def parts = entry.tokenize(':')
|
||||
def name = parts[0]
|
||||
def path = parts[1]
|
||||
if( i==0 ) {
|
||||
k8s.storageClaimName = name
|
||||
k8s.storageMountPath = path
|
||||
}
|
||||
else {
|
||||
k8s.pod.add( [volumeClaim: name, mountPath: path] )
|
||||
}
|
||||
}
|
||||
|
||||
// -- backward compatibility
|
||||
if( k8s.isSet('volumeClaims') ) {
|
||||
log.warn "Config setting k8s.volumeClaims has been deprecated -- Use k8s.storageClaimName and k8s.storageMountPath instead"
|
||||
k8s.volumeClaims.each { k,v ->
|
||||
def name = k as String
|
||||
def path = v instanceof Map ? v.mountPath : v.toString()
|
||||
if( !k8s.isSet('storageClaimName') ) {
|
||||
k8s.storageClaimName = name
|
||||
k8s.storageMountPath = path
|
||||
}
|
||||
else if( !cmd.volMounts ) {
|
||||
k8s.pod.add( [volumeClaim: name, mountPath: path] )
|
||||
}
|
||||
}
|
||||
// remove it
|
||||
k8s.remove('volumeClaims')
|
||||
}
|
||||
|
||||
// -- set k8s executor
|
||||
config.process.executor = 'k8s'
|
||||
|
||||
// -- strip default work dir
|
||||
if( config.workDir == 'work' )
|
||||
config.remove('workDir')
|
||||
|
||||
// -- check work dir
|
||||
if( cmd?.workDir )
|
||||
k8s.workDir = cmd.workDir
|
||||
else if( !k8s.isSet('workDir') && config.workDir )
|
||||
k8s.workDir = config.workDir
|
||||
|
||||
if ( plugins ) {
|
||||
LinkedList<String> plugins = config.plugins ?: []
|
||||
plugins.addAll( this.plugins.tokenize(',') )
|
||||
config.plugins = plugins
|
||||
}
|
||||
|
||||
// -- some cleanup
|
||||
if( !k8s.pod )
|
||||
k8s.remove('pod')
|
||||
|
||||
if( !k8s.storageClaimName )
|
||||
k8s.remove('storageClaimName')
|
||||
if( !k8s.storageMountPath )
|
||||
k8s.remove('storageMountPath')
|
||||
|
||||
if( !config.libDir )
|
||||
config.remove('libDir')
|
||||
|
||||
log.trace "K8s config object:\n${ConfigHelper.toCanonicalString(config).indent(' ')}"
|
||||
return config
|
||||
}
|
||||
|
||||
|
||||
private Field getField(CmdRun cmd, String name) {
|
||||
def clazz = cmd.class
|
||||
while( clazz != CmdRun ) {
|
||||
clazz = cmd.class.getSuperclass()
|
||||
}
|
||||
clazz.getDeclaredField(name)
|
||||
}
|
||||
|
||||
private void checkUnsupportedOption(String name) {
|
||||
def field = getField(cmd,name)
|
||||
if( !field ) {
|
||||
log.warn "Unknown command-line option to check: $name"
|
||||
return
|
||||
}
|
||||
field.setAccessible(true)
|
||||
if( field.get(cmd) ) {
|
||||
def param = field.getAnnotation(Parameter)
|
||||
def opt = param.names() ? param.names()[0] : "-$name"
|
||||
abort(opt)
|
||||
}
|
||||
}
|
||||
|
||||
private void abort(String opt) {
|
||||
throw new AbortOperationException("Option `$opt` not supported with Kubernetes deployment")
|
||||
}
|
||||
|
||||
private void unsupportedCliOptions(MethodClosure... fields) {
|
||||
unsupportedCliOptions( fields.collect { it.getMethod()} )
|
||||
}
|
||||
|
||||
private void unsupportedCliOptions(List<String> names) {
|
||||
for( String x : names ) {
|
||||
checkUnsupportedOption(x)
|
||||
}
|
||||
}
|
||||
|
||||
private void addOption(List result, MethodClosure m, Closure eval=null) {
|
||||
def name = m.getMethod()
|
||||
def field = getField(cmd,name)
|
||||
field.setAccessible(true)
|
||||
def val = field.get(cmd)
|
||||
if( ( eval ? eval(val) : val ) ) {
|
||||
def param = field.getAnnotation(Parameter)
|
||||
if( param ) {
|
||||
result << "${param.names()[0]} ${Escape.wildcards(String.valueOf(val))}"
|
||||
return
|
||||
}
|
||||
|
||||
param = field.getAnnotation(DynamicParameter)
|
||||
if( param && val instanceof Map ) {
|
||||
val.each { k,v ->
|
||||
result << "${param.names()[0]}$k ${Escape.wildcards(String.valueOf(v))}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return The nextflow driver command line
|
||||
*/
|
||||
protected String getLaunchCli() {
|
||||
assert cmd
|
||||
assert pipelineName
|
||||
|
||||
if( interactive ) {
|
||||
return "tail -f /dev/null"
|
||||
}
|
||||
|
||||
def result = []
|
||||
// -- configure NF command line
|
||||
result << "nextflow"
|
||||
|
||||
if( cmd.launcher.options.trace )
|
||||
result << "-trace ${cmd.launcher.options.trace.join(',')}"
|
||||
if( cmd.launcher.options.debug )
|
||||
result << "-debug ${cmd.launcher.options.debug.join(',')}"
|
||||
if( cmd.launcher.options.jvmOpts )
|
||||
cmd.launcher.options.jvmOpts.each { k,v -> result << "-D$k=$v" }
|
||||
|
||||
result << "run"
|
||||
result << pipelineName
|
||||
|
||||
if( runName )
|
||||
result << '-name' << runName
|
||||
|
||||
addOption(result, cmd.&cacheable, { it==false } )
|
||||
addOption(result, cmd.&resume )
|
||||
addOption(result, cmd.&poolSize )
|
||||
addOption(result, cmd.&pollInterval )
|
||||
addOption(result, cmd.&queueSize)
|
||||
addOption(result, cmd.&revision )
|
||||
addOption(result, cmd.&latest )
|
||||
addOption(result, cmd.&withTrace )
|
||||
addOption(result, cmd.&withTimeline )
|
||||
addOption(result, cmd.&withDag )
|
||||
addOption(result, cmd.&dumpHashes )
|
||||
addOption(result, cmd.&dumpChannels )
|
||||
addOption(result, cmd.&env )
|
||||
addOption(result, cmd.&process )
|
||||
addOption(result, cmd.¶ms )
|
||||
addOption(result, cmd.&entryName )
|
||||
|
||||
if( paramsFile ) {
|
||||
result << "-params-file $paramsFile"
|
||||
}
|
||||
|
||||
if ( cmd.runRemoteConfig )
|
||||
cmd.runRemoteConfig.forEach { result << "-config $it" }
|
||||
|
||||
if ( cmd.remoteProfile )
|
||||
result << "-profile ${cmd.remoteProfile}"
|
||||
|
||||
if( cmd.process?.executor )
|
||||
abort('process.executor')
|
||||
|
||||
unsupportedCliOptions(
|
||||
cmd.&libPath,
|
||||
cmd.&test,
|
||||
cmd.&executorOptions,
|
||||
cmd.&stdin,
|
||||
cmd.&withSingularity,
|
||||
cmd.&withApptainer,
|
||||
cmd.&withDocker,
|
||||
cmd.&withoutDocker,
|
||||
cmd.&withMpi,
|
||||
cmd.&clusterOptions,
|
||||
cmd.&exportSysEnv
|
||||
)
|
||||
|
||||
if( args )
|
||||
result.add(args)
|
||||
|
||||
return result.join(' ')
|
||||
}
|
||||
|
||||
/**
|
||||
* @return A {@link Map} modeling driver pod specification
|
||||
*/
|
||||
protected Map makeLauncherSpec() {
|
||||
assert runName
|
||||
assert k8sClient
|
||||
|
||||
// -- setup config file
|
||||
String cmd = "source /etc/nextflow/init.sh; ${getLaunchCli()}" +
|
||||
"; if [ -x /etc/nextflow/node-init-cleanup.sh ]; then /etc/nextflow/node-init-cleanup.sh || true; fi; " +
|
||||
"exit \$status"
|
||||
|
||||
// create the launcher pod
|
||||
PodSpecBuilder builder = new PodSpecBuilder()
|
||||
.withPodName(runName)
|
||||
.withImageName(headImage ?: k8sConfig.getNextflowImageName())
|
||||
.withCommand(['/bin/bash', '-c', cmd])
|
||||
.withLabels([ app: 'nextflow', runName: runName ])
|
||||
.withNamespace(k8sClient.config.namespace)
|
||||
.withServiceAccount(k8sClient.config.serviceAccount)
|
||||
.withPodOptions(k8sConfig.getPodOptions())
|
||||
.withEnv( PodEnv.value('NXF_WORK', k8sConfig.getWorkDir()) )
|
||||
.withEnv( PodEnv.value('NXF_ASSETS', k8sConfig.getProjectDir()) )
|
||||
.withEnv( PodEnv.value('NXF_EXECUTOR', 'k8s'))
|
||||
.withEnv( PodEnv.value('NXF_ANSI_LOG', 'false'))
|
||||
.withMemory(headMemory?:"")
|
||||
.withCpus(headCpus)
|
||||
.withCpuLimits(k8sConfig.cpuLimitsEnabled())
|
||||
|
||||
if ( k8sConfig.useJobResource()) {
|
||||
this.resourceType = ResourceType.Job
|
||||
return builder.buildAsJob()
|
||||
}
|
||||
else {
|
||||
return builder.build()
|
||||
}
|
||||
|
||||
// note: do *not* set the work directory because it may need to be created by the init script
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates and executes the nextflow driver pod
|
||||
* @return A {@link recreationaltech.plugin.client.K8sResponseJson} response object
|
||||
*/
|
||||
protected createK8sLauncherPod() {
|
||||
final spec = makeLauncherSpec()
|
||||
if ( k8sConfig.useJobResource() ) {
|
||||
k8sClient.jobCreate(spec, yamlDebugPath())
|
||||
} else {
|
||||
k8sClient.podCreate(spec, yamlDebugPath())
|
||||
}
|
||||
}
|
||||
|
||||
protected Path yamlDebugPath() {
|
||||
boolean debug = config.k8s.debug?.yaml?.toString() == 'true'
|
||||
final result = debug ? Paths.get(".nextflow.${resourceType.lower()}.yaml") : null
|
||||
if( result )
|
||||
log.info "Launcher ${resourceType.lower()} spec file: $result"
|
||||
return result
|
||||
}
|
||||
|
||||
protected Path getScmFile() {
|
||||
ProviderConfig.getScmConfigPath()
|
||||
}
|
||||
|
||||
String getPodImage() {
|
||||
return podImage
|
||||
}
|
||||
|
||||
int getHeadCpus() {
|
||||
return headCpus
|
||||
}
|
||||
|
||||
String getHeadMemory() {
|
||||
return headMemory
|
||||
}
|
||||
|
||||
String getRunName() {
|
||||
return runName
|
||||
}
|
||||
|
||||
CmdKubeRun getCmd() {
|
||||
return cmd
|
||||
}
|
||||
|
||||
protected String getPipelineName() {
|
||||
return pipelineName
|
||||
}
|
||||
|
||||
protected boolean getInteractive() {
|
||||
return interactive
|
||||
}
|
||||
|
||||
protected ConfigObject getConfig() {
|
||||
return config
|
||||
}
|
||||
|
||||
protected K8sConfig getK8sConfig() {
|
||||
return k8sConfig
|
||||
}
|
||||
|
||||
protected K8sClient getK8sClient() {
|
||||
return k8sClient
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a K8s ConfigMap to share the nextflow configuration in the K8s cluster
|
||||
*/
|
||||
protected void createK8sConfigMap() {
|
||||
Map<String,String> configMap = [:]
|
||||
|
||||
final launchDir = k8sConfig.getLaunchDir()
|
||||
// init file
|
||||
String initScript = ''
|
||||
initScript += "mkdir -p '$launchDir'; if [ -d '$launchDir' ]; then cd '$launchDir'; else echo 'Cannot create directory: $launchDir'; exit 1; fi; "
|
||||
initScript += '[ -f /etc/nextflow/scm ] && ln -s /etc/nextflow/scm $NXF_HOME/scm; '
|
||||
initScript += '[ -f /etc/nextflow/nextflow.config ] && cp /etc/nextflow/nextflow.config $PWD/nextflow.config; '
|
||||
if( headPreScript )
|
||||
initScript += "[ -f '$headPreScript' ] && '$headPreScript'; "
|
||||
configMap['init.sh'] = initScript
|
||||
|
||||
// nextflow config file
|
||||
if( this.config ) {
|
||||
configMap['nextflow.config'] = ConfigHelper.toCanonicalString( this.config )
|
||||
}
|
||||
|
||||
// scm config file
|
||||
final scmFile = getScmFile()
|
||||
if( scmFile.exists() ) {
|
||||
configMap['scm'] = scmFile.text
|
||||
}
|
||||
|
||||
// params file
|
||||
if( cmd.paramsFile ) {
|
||||
final file = FileHelper.asPath(cmd.paramsFile)
|
||||
if( !file.exists() ) throw new NoSuchFileException("Params file does not exist: $file")
|
||||
configMap[ file.getName() ] = file.text
|
||||
paramsFile = "/etc/nextflow/$file.name"
|
||||
}
|
||||
|
||||
// pre-workflow pod cleanup
|
||||
if ( background )
|
||||
configMap['node-init-cleanup.sh'] = initDeployer.buildCleanupScript()
|
||||
|
||||
// create the config map
|
||||
configMapName = makeConfigMapName(configMap)
|
||||
tryCreateConfigMap(configMapName, configMap)
|
||||
log.debug "Created K8s configMap with name: $configMapName"
|
||||
k8sConfig.getPodOptions().getMountConfigMaps().add( new PodMountConfig(configMapName, '/etc/nextflow') )
|
||||
}
|
||||
|
||||
protected void tryCreateConfigMap(String name, Map data) {
|
||||
try {
|
||||
k8sClient.configCreate(name, data)
|
||||
}
|
||||
catch( K8sResponseException e ) {
|
||||
if( e.response.reason != 'AlreadyExists' )
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
protected String makeConfigMapName( Map configMap ) {
|
||||
"nf-config-${hash(configMap.values())}"
|
||||
}
|
||||
|
||||
protected String hash(Collection<String> text) {
|
||||
def hasher = Hashing .murmur3_32() .newHasher()
|
||||
def itr = text.iterator()
|
||||
while( itr.hasNext() ) {
|
||||
hasher.putUnencodedChars(itr.next())
|
||||
}
|
||||
|
||||
return hasher.hash().toString()
|
||||
}
|
||||
|
||||
protected void launchLogin() {
|
||||
def launchDir = k8sConfig.getLaunchDir()
|
||||
def cmd = "kubectl -n ${k8sClient.config.namespace} exec -it $runName -- /bin/bash -c 'cd $launchDir; exec bash --login -i'"
|
||||
def proc = new ProcessBuilder().command('bash','-c',cmd).inheritIO().start()
|
||||
def result = proc.waitFor()
|
||||
if( result == 0 ) {
|
||||
if ( k8sConfig.useJobResource() )
|
||||
k8sClient.jobDelete(runName)
|
||||
else
|
||||
k8sClient.podDelete(runName)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin
|
||||
|
||||
import groovy.transform.CompileDynamic
|
||||
import recreationaltech.plugin.strategies.K8sDVFSSchedulingStrategy
|
||||
import recreationaltech.plugin.strategies.K8sHashSchedulingStrategy
|
||||
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
import com.google.common.cache.Cache
|
||||
import com.google.common.cache.CacheBuilder
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.Memoized
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.executor.Executor
|
||||
import nextflow.fusion.FusionHelper
|
||||
import recreationaltech.plugin.client.K8sClient
|
||||
import nextflow.processor.TaskHandler
|
||||
import nextflow.processor.TaskMonitor
|
||||
import nextflow.processor.TaskPollingMonitor
|
||||
import nextflow.processor.TaskRun
|
||||
import nextflow.util.Duration
|
||||
import nextflow.util.ServiceName
|
||||
import org.pf4j.ExtensionPoint
|
||||
|
||||
/**
|
||||
* Implement the Kubernetes executor
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
@ServiceName('k8s-dvfs')
|
||||
class K8sExecutor extends Executor implements ExtensionPoint {
|
||||
|
||||
/**
|
||||
* Cache for the Kubernetes HTTP client. The client is refreshed periodically
|
||||
* so that the service account token is re-read when it expires.
|
||||
*/
|
||||
private Cache<String, K8sClient> clientCache
|
||||
|
||||
private K8sTaskScheduler taskScheduler
|
||||
private Thread schedulerThread
|
||||
|
||||
K8sRuntimeRecorder runtimeRecorder
|
||||
K8sRuntimeEstimator runtimeEstimator
|
||||
|
||||
/**
|
||||
* @return The Kubernetes HTTP client. Delegates to a Guava cache that refreshes
|
||||
* the client (including the service account token) when the configured interval expires.
|
||||
*/
|
||||
K8sClient getClient() {
|
||||
clientCache.get('client', () -> new K8sClient(k8sConfig.getClient()))
|
||||
}
|
||||
|
||||
protected K8sTaskScheduler getTaskScheduler() {
|
||||
assert taskScheduler != null
|
||||
return taskScheduler
|
||||
}
|
||||
|
||||
/**
|
||||
* @return The `k8s` configuration scope in the nextflow configuration object
|
||||
*/
|
||||
@Memoized
|
||||
protected K8sConfig getK8sConfig() {
|
||||
new K8sConfig( (Map<String,Object>)session.config.k8s )
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialise the executor setting-up the kubernetes client configuration
|
||||
*/
|
||||
@Override
|
||||
protected void register() {
|
||||
super.register()
|
||||
final k8sConfig = getK8sConfig()
|
||||
final refreshInterval = k8sConfig.clientRefreshInterval
|
||||
this.clientCache = CacheBuilder.newBuilder()
|
||||
.expireAfterWrite(refreshInterval.toMillis(), TimeUnit.MILLISECONDS)
|
||||
.build()
|
||||
final client = getClient()
|
||||
|
||||
log.debug "[K8s] config=$k8sConfig; API client config=$client.config"
|
||||
|
||||
this.runtimeRecorder = new K8sRuntimeRecorder(k8sConfig.recordTaskRuntimes, k8sConfig.runtimeRecordPath)
|
||||
|
||||
if (k8sConfig.runtimeEstimator == "LinearFit") {
|
||||
this.runtimeEstimator = new K8sLinearFitRuntimeEstimator(k8sConfig.runtimeRecordPath)
|
||||
} else if (k8sConfig.runtimeEstimator == "Noise") {
|
||||
this.runtimeEstimator = new K8sNoiseRuntimeEstimator(k8sConfig.runtimeEstimator, k8sConfig.noiseRuntimeEstimatorNoiseMagnitude)
|
||||
} else {
|
||||
log.error "[K8s] invalid runtime estimator ${k8sConfig.runtimeEstimator} - falling back on \"LinearFit\""
|
||||
this.runtimeEstimator = new K8sLinearFitRuntimeEstimator(k8sConfig.runtimeRecordPath)
|
||||
}
|
||||
|
||||
String[] nodes = getNodeList()
|
||||
|
||||
K8sSchedulingStrategy strategy = null
|
||||
if (k8sConfig.schedulingStrategy == "Hash") {
|
||||
strategy = new K8sHashSchedulingStrategy()
|
||||
} else if (k8sConfig.schedulingStrategy == "DVFS" || k8sConfig.schedulingStrategy == "DVFS-SPEED") {
|
||||
String[] ips = new String[nodes.length]
|
||||
for (int i = 0; i < nodes.length; i++) {
|
||||
ips[i] = client.getPodIpAddress(K8sNodeInitDeployer.buildPodName(nodes[i]))
|
||||
log.info "[K8s] node ${nodes[i]} -> ${ips[i]}"
|
||||
}
|
||||
strategy = new K8sDVFSSchedulingStrategy(this.runtimeEstimator,
|
||||
new K8sDVFSClient(nodes, ips),
|
||||
() -> getClient(),
|
||||
k8sConfig.runtimeComparisonEpsilon,
|
||||
k8sConfig.dvfsSchedulingNumTopRuntimes)
|
||||
strategy.fullSpeedMode = k8sConfig.schedulingStrategy == "DVFS-SPEED"
|
||||
} else {
|
||||
log.error "[K8s] invalid scheduling strategy $k8sConfig.schedulingStrategy, falling back on \"Hash\""
|
||||
strategy = new K8sHashSchedulingStrategy()
|
||||
}
|
||||
|
||||
this.taskScheduler = new K8sTaskScheduler(nodes, strategy, k8sConfig.schedulerInterval)
|
||||
this.schedulerThread = new Thread(this.taskScheduler)
|
||||
this.schedulerThread.start()
|
||||
}
|
||||
|
||||
@CompileDynamic
|
||||
private String[] getNodeList() {
|
||||
final resp = getClient().nodeList()
|
||||
ArrayList<String> nodes = new ArrayList<String>()
|
||||
for ( Map item : resp.items ) {
|
||||
nodes.add(item.metadata.name as String)
|
||||
}
|
||||
return nodes.toArray()
|
||||
}
|
||||
|
||||
@Override
|
||||
void shutdown() {
|
||||
this.runtimeRecorder.write()
|
||||
this.taskScheduler.stop()
|
||||
this.schedulerThread.join()
|
||||
}
|
||||
|
||||
/**
|
||||
* @return {@code true} since containerised execution is managed by Kubernetes
|
||||
*/
|
||||
boolean isContainerNative() {
|
||||
return true
|
||||
}
|
||||
|
||||
@Override
|
||||
String containerConfigEngine() {
|
||||
return 'docker'
|
||||
}
|
||||
|
||||
/**
|
||||
* @return A {@link TaskMonitor} associated to this executor type
|
||||
*/
|
||||
@Override
|
||||
protected TaskMonitor createTaskMonitor() {
|
||||
TaskPollingMonitor.create(session, config, name, 100, Duration.of('5 sec'))
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a {@link TaskHandler} for the given {@link TaskRun} instance
|
||||
*
|
||||
* @param task A {@link TaskRun} instance representing a process task to be executed
|
||||
* @return A {@link K8sTaskHandler} instance modeling the execution in the K8s cluster
|
||||
*/
|
||||
@Override
|
||||
TaskHandler createTaskHandler(TaskRun task) {
|
||||
assert task
|
||||
assert task.workDir
|
||||
log.trace "[K8s] launching process > ${task.name} -- work folder: ${task.workDirStr}"
|
||||
new K8sTaskHandler(task,this)
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean isFusionEnabled() {
|
||||
return FusionHelper.isFusionEnabled(session)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package recreationaltech.plugin
|
||||
|
||||
import groovy.util.logging.Slf4j
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
|
||||
/**
|
||||
* Provides runtime estimates for tasks based on task-name and input file size
|
||||
*/
|
||||
@Slf4j
|
||||
class K8sLinearFitRuntimeEstimator extends K8sRuntimeEstimator {
|
||||
|
||||
class Function {
|
||||
private double m
|
||||
private double n
|
||||
|
||||
Function(double m, double n) {
|
||||
this.m = m
|
||||
this.n = n
|
||||
}
|
||||
|
||||
double estimate(long x) {
|
||||
return m * (double)x + n
|
||||
}
|
||||
}
|
||||
|
||||
HashMap<String, Function> estimators;
|
||||
|
||||
/**
|
||||
* Initialize the estimator with data recorded by K8sRuntimeRecorder
|
||||
* @param dataFilePath
|
||||
*/
|
||||
K8sLinearFitRuntimeEstimator(String dataFilePath) {
|
||||
def data = parseDataFile(dataFilePath)
|
||||
fit(data)
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes the runtime estimator with statically known data.
|
||||
* @param data Map from task name to list of recordings, where each recording is a tuple (input-size, runtime-in-ms)
|
||||
*/
|
||||
K8sLinearFitRuntimeEstimator(HashMap<String, ArrayList<Tuple2<Long, Long>>> data) {
|
||||
fit(data)
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns an estimation of the task runtime in milliseconds
|
||||
* @param taskName the name of the task
|
||||
* @param inputSize the total input size in bytes
|
||||
* @return estimated task runtime in milliseconds OR infinity if the task is unknown.
|
||||
*/
|
||||
double estimate(String taskName, long inputSize) {
|
||||
Function estimator = estimators.get(taskName)
|
||||
if (estimator == null) {
|
||||
//log.warn "[K8s] Unable to estimate take ${taskName}: unknown task"
|
||||
return Double.POSITIVE_INFINITY
|
||||
}
|
||||
return estimator.estimate(inputSize)
|
||||
}
|
||||
|
||||
private void fit(HashMap<String, ArrayList<Tuple2<Long, Long>>> data) {
|
||||
estimators = new HashMap<>()
|
||||
for (Map.Entry<String, ArrayList<Tuple2<Long, Long>>> t : data) {
|
||||
Function f = fit(t.value)
|
||||
estimators.put(t.key, f)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Uses linear regression to fit a linear function (y = m * x + n) to the provided observations
|
||||
* @param observations list of tuples (input size, runtime in ms)
|
||||
* @return linear function fitted to the input
|
||||
*/
|
||||
private Function fit(ArrayList<Tuple2<Long, Long>> observations) throws IllegalArgumentException {
|
||||
int n = observations.size()
|
||||
if (n > 1) {
|
||||
double sumX = 0, sumY = 0, sumXY = 0, sumX2 = 0
|
||||
for (Tuple2<Long, Long> o : observations) {
|
||||
double x = (double) o.get(0)
|
||||
double y = (double) o.get(1)
|
||||
sumX += x
|
||||
sumY += y
|
||||
sumXY += x * y
|
||||
sumX2 += x * x
|
||||
}
|
||||
double m = (n * sumXY - sumX * sumY) / (n * sumX2 - sumX * sumX)
|
||||
return new Function(
|
||||
m,
|
||||
(sumY - m * sumX) / n
|
||||
)
|
||||
} else if (n == 1) {
|
||||
// Special case: We only have 1 measurement. We will just assume that the runtime is constant,
|
||||
// because in our observed data, it is.
|
||||
return new Function(0.0, (double) observations[0].get(1))
|
||||
}
|
||||
throw new IllegalArgumentException("requires at least 1 observation")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package recreationaltech.plugin
|
||||
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.config.scopes.Config
|
||||
import nextflow.config.spec.ConfigScope
|
||||
import nextflow.config.spec.ScopeName
|
||||
import nextflow.config.spec.ConfigOption
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.script.dsl.Description
|
||||
|
||||
@CompileStatic
|
||||
@Slf4j
|
||||
@ScopeName("nodeInit")
|
||||
@Description("The nodeInit scope contains options for the pre-workflow execution initialization of nodes")
|
||||
class K8sNodeInitConfig implements ConfigScope {
|
||||
@ConfigOption
|
||||
@Description("enables the pre-workflow execution deployment of pods")
|
||||
final boolean enabled;
|
||||
|
||||
@ConfigOption
|
||||
@Description("the used image")
|
||||
final String image;
|
||||
|
||||
@ConfigOption
|
||||
@Description("the start-command")
|
||||
final List<String> command;
|
||||
|
||||
@ConfigOption
|
||||
@Description("the pod state to wait on")
|
||||
final String wait;
|
||||
|
||||
@ConfigOption
|
||||
@Description("enables cleanup of pre-workflow nodes.")
|
||||
final boolean cleanup;
|
||||
|
||||
K8sNodeInitConfig() {
|
||||
this(Collections.emptyMap())
|
||||
}
|
||||
|
||||
K8sNodeInitConfig(Map opts) {
|
||||
enabled = opts.enabled as boolean
|
||||
image = opts.image as String
|
||||
command = opts.command as List<String>
|
||||
wait = opts.wait as String
|
||||
cleanup = opts.cleanup as boolean
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
package recreationaltech.plugin
|
||||
|
||||
import groovy.util.logging.Slf4j
|
||||
import recreationaltech.plugin.client.K8sClient
|
||||
import recreationaltech.plugin.model.PodHostMount
|
||||
import recreationaltech.plugin.model.PodSpecBuilder
|
||||
|
||||
@Slf4j
|
||||
class K8sNodeInitDeployer {
|
||||
private K8sClient client
|
||||
private K8sConfig config
|
||||
|
||||
K8sNodeInitDeployer(K8sClient client, K8sConfig config) {
|
||||
this.client = client
|
||||
this.config = config
|
||||
}
|
||||
|
||||
void deploy() {
|
||||
final init = config.nodeInit
|
||||
if ( !init?.enabled )
|
||||
return
|
||||
|
||||
log.info("deploying init pods")
|
||||
|
||||
final nodes = getNodes()
|
||||
for ( String nodeName : nodes ) {
|
||||
log.info(" ... deploying to " + nodeName)
|
||||
final spec = makePodSpec(init, nodeName)
|
||||
client.podCreate(spec)
|
||||
}
|
||||
|
||||
log.info("waiting for init pods")
|
||||
waitForPods(nodes)
|
||||
}
|
||||
|
||||
void cleanup() {
|
||||
final init = config.nodeInit
|
||||
if ( !init?.enabled || !init?.cleanup )
|
||||
return
|
||||
|
||||
final nodes = getNodes()
|
||||
for ( String nodeName : nodes ) {
|
||||
final podName = buildPodName(nodeName)
|
||||
client.podDelete(podName)
|
||||
}
|
||||
}
|
||||
|
||||
String buildCleanupScript() {
|
||||
if ( !config.nodeInit?.enabled || !config.nodeInit?.cleanup )
|
||||
return "#!/usr/bin/env/bash\nexit 0"
|
||||
|
||||
final nodes = getNodes()
|
||||
final podNames = nodes.collect {buildPodName(it)}
|
||||
|
||||
String script = '''
|
||||
#!/usr/bin/env bash
|
||||
set -u
|
||||
|
||||
PODS=('''
|
||||
for ( String podName : podNames ) {
|
||||
script += "\"${podName}\"\n"
|
||||
}
|
||||
script += '''
|
||||
)
|
||||
|
||||
SERVICE_ACCOUNT_DIR="/var/run/secrets/kubernetes.io/serviceaccount"
|
||||
TOKEN="$(cat "${SERVICE_ACCOUNT_DIR}/token")"
|
||||
NAMESPACE="$(cat "${SERVICE_ACCOUNT_DIR}/namespace")"
|
||||
CA_CERT="${SERVICE_ACCOUNT_DIR}/ca.crt"
|
||||
|
||||
KUBE_API="https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS:-443}"
|
||||
|
||||
for POD in "${PODS[@]}"; do
|
||||
echo "Deleting pod: ${POD}"
|
||||
|
||||
HTTP_CODE="$(
|
||||
curl \\
|
||||
--silent \\
|
||||
--show-error \\
|
||||
--output /tmp/delete-pod-response.json \\
|
||||
--write-out "%{http_code}" \\
|
||||
--request DELETE \\
|
||||
--cacert "${CA_CERT}" \\
|
||||
--header "Authorization: Bearer ${TOKEN}" \\
|
||||
--header "Accept: application/json" \\
|
||||
"${KUBE_API}/api/v1/namespaces/${NAMESPACE}/pods/${POD}"
|
||||
)"
|
||||
|
||||
case "${HTTP_CODE}" in
|
||||
200|202)
|
||||
echo "Deleted pod: ${POD}"
|
||||
;;
|
||||
404)
|
||||
echo "Pod already absent: ${POD}"
|
||||
;;
|
||||
*)
|
||||
echo "Failed to delete pod: ${POD}; HTTP ${HTTP_CODE}" >&2
|
||||
cat /tmp/delete-pod-response.json >&2 || true
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
exit 0
|
||||
'''
|
||||
return script
|
||||
}
|
||||
|
||||
private List<String> getNodes() {
|
||||
final resp = client.nodeList()
|
||||
ArrayList<String> nodes = new ArrayList<String>()
|
||||
for ( Map item: resp.items ) {
|
||||
nodes.add(item.metadata.name as String)
|
||||
}
|
||||
return nodes
|
||||
}
|
||||
|
||||
public static String buildPodName(String nodeName) {
|
||||
// TODO: Remove anything that is not lowercase alpha-numeric or dash
|
||||
String sanitizedNodeName = nodeName.toLowerCase()
|
||||
String name = "nf-init-${sanitizedNodeName}"
|
||||
if ( name.length() > 63 )
|
||||
name = name.substring(0, 63)
|
||||
return name
|
||||
}
|
||||
|
||||
private Map makePodSpec(K8sNodeInitConfig config, String nodeName) {
|
||||
ArrayList<PodHostMount> mounts = new ArrayList<PodHostMount>()
|
||||
mounts.add(new PodHostMount("/sys", "/sys"))
|
||||
mounts.add(new PodHostMount("/dev", "/dev"))
|
||||
mounts.add(new PodHostMount("/lib/modules", "/lib/modules"))
|
||||
|
||||
PodSpecBuilder builder = new PodSpecBuilder()
|
||||
return builder.withNodeName(nodeName)
|
||||
.withImageName(config.image)
|
||||
.withCommand(config.command)
|
||||
.withPrivileged(true)
|
||||
.withHostMounts(mounts)
|
||||
.withPodName(buildPodName(nodeName))
|
||||
.withPort(8080)
|
||||
.build()
|
||||
}
|
||||
|
||||
private void waitForPods(List<String> nodes) {
|
||||
if ( config.nodeInit.wait == 'Running' ) {
|
||||
for (String nodeName : nodes) {
|
||||
String podName = buildPodName(nodeName)
|
||||
while (true) {
|
||||
sleep 1000
|
||||
final state = client.podState(podName)
|
||||
if (state && !state.containsKey('waiting')) {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if ( config.nodeInit.wait == 'Succeeded' ) {
|
||||
for ( String nodeName : nodes ) {
|
||||
final String podName = buildPodName(nodeName)
|
||||
final currentState = client.podState(podName)
|
||||
if ( currentState && currentState?.running instanceof Map ) {
|
||||
try {
|
||||
while (true) {
|
||||
sleep 10000
|
||||
final state = client.podState(podName)
|
||||
if (state && !(state?.running instanceof Map)) {
|
||||
println "$podName has changed from running state $state"
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception e) {
|
||||
println "Caught exception while waiting for ${podName} to stop running"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package recreationaltech.plugin
|
||||
|
||||
import nextflow.util.Duration
|
||||
|
||||
class K8sNoiseRuntimeEstimator extends K8sRuntimeEstimator {
|
||||
// Magnitude of the noise we add
|
||||
private long noiseMag
|
||||
|
||||
private HashMap<String, ArrayList<Tuple2<Long, Long>>> data;
|
||||
private synchronized Random rnd;
|
||||
|
||||
/**
|
||||
* Initialize the estimator with data recorded by K8sRuntimeRecorder
|
||||
* @param dataFilePath
|
||||
* @param noiseMagnitude magnitude of the noise added to recordings
|
||||
*/
|
||||
K8sNoiseRuntimeEstimator(String dataFilePath, Duration noiseMagnitude) {
|
||||
this.data = parseDataFile(dataFilePath)
|
||||
this.noiseMag = noiseMagnitude.toMillis()
|
||||
this.rnd = new Random()
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes the runtime estimator with statically known data.
|
||||
* @param data Map from task name to list of recordings, where each recording is a tuple (input-size, runtime-in-ms)
|
||||
*/
|
||||
K8sNoiseRuntimeEstimator(HashMap<String, ArrayList<Tuple2<Long, Long>>> data, Duration noiseMagnitude) {
|
||||
this.data = data;
|
||||
this.noiseMag = noiseMagnitude.toMillis()
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns an estimation of the task runtime in milliseconds
|
||||
* @param taskName the name of the task
|
||||
* @param inputSize the total input size in bytes
|
||||
* @return estimated task runtime in milliseconds OR infinity if the task is unknown.
|
||||
*/
|
||||
@Override
|
||||
double estimate(String taskName, long inputSize) {
|
||||
ArrayList recordings = data.get(taskName)
|
||||
for (Tuple2<Long, Long> recording : recordings) {
|
||||
if (recording.get(0).longValue() == inputSize) {
|
||||
double noise = rnd.nextDouble(-1.0, 1.0) * (double)noiseMag
|
||||
long ms = recording.get(1).longValue()
|
||||
return (double)ms + noise
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.plugin.BasePlugin
|
||||
import org.pf4j.PluginWrapper
|
||||
|
||||
/**
|
||||
* Kubernetes plugin entry point
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class K8sPlugin extends BasePlugin {
|
||||
|
||||
K8sPlugin(PluginWrapper wrapper) {
|
||||
super(wrapper)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package recreationaltech.plugin
|
||||
|
||||
import groovy.util.logging.Slf4j
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
|
||||
/**
|
||||
* Provides runtime estimates for tasks based on task-name and input file size
|
||||
*/
|
||||
@Slf4j
|
||||
abstract class K8sRuntimeEstimator {
|
||||
/**
|
||||
* Returns an estimation of the task runtime in milliseconds
|
||||
* @param handler the task handler
|
||||
* @return estimated task runtime in milliseconds OR infinity if the task is unknown.
|
||||
*/
|
||||
double estimate(K8sTaskHandler handler) {
|
||||
long x = getTaskHandlerInputSize(handler)
|
||||
return estimate(handler.task.processor.name, x)
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns an estimation of the task runtime in milliseconds
|
||||
* @param taskName the name of the task
|
||||
* @param inputSize the total input size in bytes
|
||||
* @return estimated task runtime in milliseconds OR infinity if the task is unknown.
|
||||
*/
|
||||
abstract double estimate(String taskName, long inputSize);
|
||||
|
||||
protected long getTaskHandlerInputSize(K8sTaskHandler handler) {
|
||||
long inputSizeSum = 0
|
||||
// File input
|
||||
def inputFiles = handler.task.getInputFilesMap()
|
||||
for (Map.Entry<String, Path> f : inputFiles) {
|
||||
try {
|
||||
inputSizeSum += Files.size(f.value)
|
||||
} catch (IOException ex) {
|
||||
log.error "[K8s] failed to get size of input file ${f.value} of task ${task.task.name}: ${ex.message}"
|
||||
}
|
||||
}
|
||||
// Non file input
|
||||
def inputs = handler.task.getInputs()
|
||||
for (Map.Entry i : inputs) {
|
||||
inputSizeSum += i.value.toString().length()
|
||||
}
|
||||
|
||||
return inputSizeSum
|
||||
}
|
||||
|
||||
/// @brief Parses the runtime recording data file
|
||||
/// @return Map from task name to list of recordings, where each recording is a tuple (input-size, runtime-in-ms)
|
||||
protected HashMap<String, ArrayList<Tuple2<Long, Long>>> parseDataFile(String dataFilePath) {
|
||||
HashMap<String, ArrayList<Tuple2<Long, Long>>> data = new HashMap<>();
|
||||
try {
|
||||
BufferedReader reader = new BufferedReader(new FileReader(dataFilePath))
|
||||
String line = reader.readLine()
|
||||
while (line != null) {
|
||||
// <task-name>,<input-size>,<runtime-ms>
|
||||
String[] elems = line.split(",")
|
||||
if (elems.length != 3) {
|
||||
log.warn "[K8s] ${dataFilePath}: Unexpected line ${line}"
|
||||
continue
|
||||
}
|
||||
|
||||
ArrayList<Tuple2<Long, Long>> taskData = data.get(elems[0])
|
||||
if (taskData == null) {
|
||||
taskData = new ArrayList<>()
|
||||
data.put(elems[0], taskData)
|
||||
}
|
||||
try {
|
||||
taskData.add(new Tuple2(Long.parseLong(elems[1]), Long.parseLong(elems[2])))
|
||||
} catch (NumberFormatException ex) {
|
||||
log.error "[K8s] ${dataFilePath} invalid data: ${ex.message}"
|
||||
}
|
||||
line = reader.readLine()
|
||||
}
|
||||
reader.close()
|
||||
} catch (IOException ex) {
|
||||
log.error "[K8s] Failed to load ${dataFilePath}: ${ex.message}"
|
||||
}
|
||||
return data
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package recreationaltech.plugin
|
||||
|
||||
record K8sRuntimeRecord(
|
||||
String taskName,
|
||||
long inputSize,
|
||||
long runtimeMillis
|
||||
) {}
|
||||
@@ -0,0 +1,67 @@
|
||||
package recreationaltech.plugin
|
||||
|
||||
import groovy.util.logging.Slf4j
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
|
||||
@Slf4j
|
||||
class K8sRuntimeRecorder {
|
||||
private final boolean enabled
|
||||
private final String recordPath
|
||||
|
||||
private ArrayList<K8sRuntimeRecord> records
|
||||
|
||||
K8sRuntimeRecorder(boolean enabled, String recordPath) {
|
||||
this.enabled = enabled
|
||||
this.recordPath = recordPath
|
||||
this.records = new ArrayList<>()
|
||||
}
|
||||
|
||||
/**
|
||||
* Records the runtime of a task.
|
||||
* This should be called for a finished task, which has its
|
||||
* start and end timestamps set.
|
||||
* @param task
|
||||
*/
|
||||
synchronized void record(K8sTaskHandler task) {
|
||||
if (!enabled)
|
||||
return;
|
||||
long runtimeMilis = task.completeTimeMillis - task.startTimeMillis
|
||||
long inputSizeSum = 0
|
||||
|
||||
// Input files
|
||||
def inputFiles = task.task.getInputFilesMap()
|
||||
for (Map.Entry<String, Path> f : inputFiles) {
|
||||
try {
|
||||
inputSizeSum += Files.size(f.value)
|
||||
} catch (IOException ex) {
|
||||
log.error "[K8s] failed to get size of input file ${f.value} of task ${task.task.name}: ${ex.message}"
|
||||
}
|
||||
}
|
||||
|
||||
// Non file input
|
||||
def inputs = task.task.getInputs()
|
||||
for (Map.Entry i : inputs) {
|
||||
inputSizeSum += i.value.toString().length()
|
||||
}
|
||||
|
||||
log.info "[K8s] task ${task.task.processor.name} - input ${inputSizeSum} bytes - ran ${runtimeMilis} ms (${task.startTimeMillis} -> ${task.completeTimeMillis})"
|
||||
records.add(new K8sRuntimeRecord(task.task.processor.name, inputSizeSum, runtimeMilis))
|
||||
}
|
||||
|
||||
void write() {
|
||||
if (!enabled)
|
||||
return;
|
||||
try {
|
||||
FileWriter out = new FileWriter(recordPath)
|
||||
for (K8sRuntimeRecord record : records) {
|
||||
out.write("${record.taskName},${record.inputSize},${record.runtimeMillis}\n")
|
||||
}
|
||||
out.close()
|
||||
log.info "[K8s] written runtime recording to ${recordPath}"
|
||||
} catch (IOException err) {
|
||||
log.error "[K8s] failed to write runtime recording ${recordPath}: ${err.message}"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package recreationaltech.plugin
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
|
||||
@CompileStatic
|
||||
class K8sSchedulingDecision {
|
||||
final String nodeName
|
||||
final K8sSchedulingRequest request
|
||||
|
||||
K8sSchedulingDecision(K8sSchedulingRequest request, String nodeName) {
|
||||
this.request = request
|
||||
this.nodeName = nodeName
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package recreationaltech.plugin
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.processor.TaskRun
|
||||
|
||||
@CompileStatic
|
||||
class K8sSchedulingRequest {
|
||||
final K8sTaskHandler handler
|
||||
final TaskRun task
|
||||
final long submitTimeMillis
|
||||
|
||||
K8sSchedulingRequest(K8sTaskHandler handler) {
|
||||
this.handler = handler
|
||||
this.task = handler.task
|
||||
this.submitTimeMillis = System.currentTimeMillis()
|
||||
}
|
||||
|
||||
String getTaskName() {
|
||||
return task.processor.name
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package recreationaltech.plugin
|
||||
|
||||
interface K8sSchedulingStrategy {
|
||||
/**
|
||||
* Selects the next task to run from the given task queue.
|
||||
* If the scheduler should wait, returns {@code null} instead
|
||||
*
|
||||
* @param scheduler the calling scheduler object
|
||||
* @param queue Pending scheduling requests
|
||||
* @return A launch decision, or {@code null} when no task should be launched now
|
||||
*/
|
||||
K8sSchedulingDecision schedule(K8sTaskScheduler scheduler, List<K8sSchedulingRequest> queue)
|
||||
|
||||
/**
|
||||
* Decides if the scheduler should immediately invoke the @ref schedule method of the strategy
|
||||
* @param scheduler the calling scheduler object
|
||||
* @param queue pending scheduling requests
|
||||
* @return {@code true} if @ref schedule should immediately be called
|
||||
*/
|
||||
boolean scheduleImmediately(K8sTaskScheduler scheduler, List<K8sSchedulingRequest> queue)
|
||||
|
||||
/**
|
||||
* Called when a task has finished execution.
|
||||
* @param task the task
|
||||
*/
|
||||
void taskFinished(K8sTaskHandler task)
|
||||
}
|
||||
@@ -0,0 +1,586 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin
|
||||
|
||||
import java.nio.file.FileAlreadyExistsException
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.time.Instant
|
||||
import java.time.format.DateTimeFormatter
|
||||
|
||||
import groovy.transform.CompileDynamic
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.SysEnv
|
||||
import nextflow.container.ContainerHelper
|
||||
import nextflow.container.DockerBuilder
|
||||
import nextflow.exception.NodeTerminationException
|
||||
import recreationaltech.plugin.client.PodUnschedulableException
|
||||
import nextflow.exception.ProcessSubmitException
|
||||
import nextflow.executor.BashWrapperBuilder
|
||||
import nextflow.fusion.FusionAwareTask
|
||||
import recreationaltech.plugin.client.K8sClient
|
||||
import recreationaltech.plugin.client.K8sResponseException
|
||||
import recreationaltech.plugin.model.PodEnv
|
||||
import recreationaltech.plugin.model.PodOptions
|
||||
import recreationaltech.plugin.model.PodSpecBuilder
|
||||
import recreationaltech.plugin.model.ResourceType
|
||||
import nextflow.processor.TaskHandler
|
||||
import nextflow.processor.TaskRun
|
||||
import nextflow.processor.TaskStatus
|
||||
import nextflow.trace.TraceRecord
|
||||
import nextflow.util.Escape
|
||||
import nextflow.util.PathTrie
|
||||
import nextflow.util.TestOnly
|
||||
/**
|
||||
* Implements the {@link TaskHandler} interface for Kubernetes pods
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class K8sTaskHandler extends TaskHandler implements FusionAwareTask {
|
||||
|
||||
@Lazy
|
||||
static private final String OWNER = {
|
||||
if( System.getenv('NXF_OWNER') ) {
|
||||
return System.getenv('NXF_OWNER')
|
||||
}
|
||||
else {
|
||||
def p = ['bash','-c','echo -n $(id -u):$(id -g)'].execute();
|
||||
p.waitFor()
|
||||
return p.text
|
||||
}
|
||||
|
||||
} ()
|
||||
|
||||
private ResourceType resourceType = ResourceType.Pod
|
||||
|
||||
private K8sClient client
|
||||
|
||||
private String podName
|
||||
|
||||
private BashWrapperBuilder builder
|
||||
|
||||
private Path outputFile
|
||||
|
||||
private Path errorFile
|
||||
|
||||
private Path exitFile
|
||||
|
||||
private Map state
|
||||
|
||||
private long timestamp
|
||||
|
||||
private K8sExecutor executor
|
||||
|
||||
private String runsOnNode = null
|
||||
|
||||
K8sTaskHandler( TaskRun task, K8sExecutor executor ) {
|
||||
super(task)
|
||||
this.executor = executor
|
||||
this.client = executor.getClient()
|
||||
this.outputFile = task.workDir.resolve(TaskRun.CMD_OUTFILE)
|
||||
this.errorFile = task.workDir.resolve(TaskRun.CMD_ERRFILE)
|
||||
this.exitFile = task.workDir.resolve(TaskRun.CMD_EXIT)
|
||||
this.resourceType = executor.k8sConfig.useJobResource() ? ResourceType.Job : ResourceType.Pod
|
||||
}
|
||||
|
||||
@TestOnly
|
||||
protected K8sTaskHandler() {}
|
||||
|
||||
/**
|
||||
* @return The workflow execution unique run name
|
||||
*/
|
||||
protected String getRunName() {
|
||||
executor.session.runName
|
||||
}
|
||||
|
||||
protected String getPodName() {
|
||||
return podName
|
||||
}
|
||||
|
||||
protected K8sConfig getK8sConfig() { executor.getK8sConfig() }
|
||||
|
||||
protected boolean useJobResource() { resourceType==ResourceType.Job }
|
||||
|
||||
protected List<String> getContainerMounts() {
|
||||
|
||||
if( !k8sConfig.getAutoMountHostPaths() ) {
|
||||
return Collections.<String>emptyList()
|
||||
}
|
||||
|
||||
// get input files paths
|
||||
final List<Path> paths = DockerBuilder.inputFilesToPaths(builder.getInputFiles())
|
||||
final List<Path> binDirs = builder.binDirs
|
||||
final Path workDir = builder.workDir
|
||||
// add standard paths
|
||||
if( binDirs )
|
||||
paths.addAll(binDirs)
|
||||
if( workDir )
|
||||
paths << workDir
|
||||
|
||||
def trie = new PathTrie()
|
||||
paths.each { trie.add(it) }
|
||||
|
||||
// defines the mounts
|
||||
trie.longest()
|
||||
}
|
||||
|
||||
protected BashWrapperBuilder createBashWrapper(TaskRun task) {
|
||||
return fusionEnabled()
|
||||
? fusionLauncher()
|
||||
: new K8sWrapperBuilder(task)
|
||||
}
|
||||
|
||||
protected List<String> classicSubmitCli(TaskRun task) {
|
||||
final workDir = Escape.path(task.workDir)
|
||||
|
||||
final result = new ArrayList(BashWrapperBuilder.BASH)
|
||||
result.add('-o')
|
||||
result.add('pipefail')
|
||||
result.add('-c')
|
||||
result.add("bash ${workDir}/${TaskRun.CMD_RUN} 2>&1 | tee ${workDir}/${TaskRun.CMD_LOG}")
|
||||
return result
|
||||
}
|
||||
|
||||
protected List<String> getSubmitCommand(TaskRun task) {
|
||||
return fusionEnabled()
|
||||
? fusionSubmitCli()
|
||||
: classicSubmitCli(task)
|
||||
}
|
||||
|
||||
protected String getSyntheticPodName(TaskRun task) {
|
||||
final suffix = System.currentTimeMillis().toString().md5()[-5..-1]
|
||||
return "nf-${task.hash}-${suffix}"
|
||||
}
|
||||
|
||||
protected String getOwner() { OWNER }
|
||||
|
||||
protected Boolean fixOwnership() {
|
||||
ContainerHelper.fixOwnership(task.containerConfig)
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a Pod specification that executed that specified task
|
||||
*
|
||||
* @param task A {@link TaskRun} instance representing the task to execute
|
||||
* @return A {@link Map} object modeling a Pod specification
|
||||
*/
|
||||
protected Map newSubmitRequest(TaskRun task) {
|
||||
return newSubmitRequest(task, null)
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a Pod specification that executed that specified task
|
||||
*
|
||||
* @param task A {@link TaskRun} instance representing the task to execute
|
||||
* @param nodeName The kubernetes node on which the task should run or {@code null}, if no specific node is requested
|
||||
* @return A {@link Map} object modeling a Pod specification
|
||||
*/
|
||||
protected Map newSubmitRequest(TaskRun task, String nodeName) {
|
||||
def imageName = task.container
|
||||
if( !imageName )
|
||||
throw new ProcessSubmitException("Missing container image for process `$task.processor.name`")
|
||||
|
||||
try {
|
||||
newSubmitRequest0(task, imageName, nodeName)
|
||||
}
|
||||
catch( Throwable e ) {
|
||||
throw new ProcessSubmitException("Failed to submit K8s ${resourceType.lower()} -- Cause: ${e.message ?: e}", e)
|
||||
}
|
||||
}
|
||||
|
||||
protected boolean entrypointOverride() {
|
||||
return executor.getK8sConfig().entrypointOverride()
|
||||
}
|
||||
|
||||
protected boolean cpuLimitsEnabled() {
|
||||
return executor.getK8sConfig().cpuLimitsEnabled()
|
||||
}
|
||||
|
||||
protected Map newSubmitRequest0(TaskRun task, String imageName, String nodeName) {
|
||||
|
||||
final launcher = getSubmitCommand(task)
|
||||
final taskCfg = task.getConfig()
|
||||
|
||||
final clientConfig = client.config
|
||||
final builder = new PodSpecBuilder()
|
||||
.withImageName(imageName)
|
||||
.withPodName(getSyntheticPodName(task))
|
||||
.withNamespace(clientConfig.namespace)
|
||||
.withServiceAccount(clientConfig.serviceAccount)
|
||||
.withLabels(getLabels(task))
|
||||
.withAnnotations(getAnnotations())
|
||||
.withPodOptions(getPodOptions())
|
||||
.withCpuLimits(cpuLimitsEnabled())
|
||||
|
||||
// when `entrypointOverride` is false the launcher is run via `args` instead of `command`
|
||||
// to not override the container entrypoint
|
||||
if( !entrypointOverride() ) {
|
||||
builder.withArgs(launcher)
|
||||
}
|
||||
else {
|
||||
builder.withCommand(launcher)
|
||||
}
|
||||
|
||||
if( nodeName )
|
||||
builder.withNodeName(nodeName)
|
||||
|
||||
// note: task environment is managed by the task bash wrapper
|
||||
// do not add here -- see also #680
|
||||
if( fixOwnership() )
|
||||
builder.withEnv(PodEnv.value('NXF_OWNER', getOwner()))
|
||||
|
||||
if( SysEnv.containsKey('NXF_DEBUG') )
|
||||
builder.withEnv(PodEnv.value('NXF_DEBUG', SysEnv.get('NXF_DEBUG')))
|
||||
|
||||
// add computing resources
|
||||
final cpus = taskCfg.getCpus()
|
||||
final mem = taskCfg.getMemory()
|
||||
final disk = taskCfg.getDisk()
|
||||
final acc = taskCfg.getAccelerator()
|
||||
if( cpus )
|
||||
builder.withCpus(cpus)
|
||||
if( mem )
|
||||
builder.withMemory(mem)
|
||||
if( disk )
|
||||
builder.withDisk(disk)
|
||||
if( acc )
|
||||
builder.withAccelerator(acc)
|
||||
|
||||
final List<String> hostMounts = getContainerMounts()
|
||||
for( String mount : hostMounts ) {
|
||||
builder.withHostMount(mount,mount)
|
||||
}
|
||||
|
||||
if ( taskCfg.time ) {
|
||||
final duration = taskCfg.getTime()
|
||||
builder.withActiveDeadline(duration.toSeconds() as int)
|
||||
}
|
||||
|
||||
if ( fusionEnabled() ) {
|
||||
if( fusionConfig().privileged() )
|
||||
builder.withPrivileged(true)
|
||||
else {
|
||||
final device= k8sConfig.fuseDevicePlugin()
|
||||
builder.withResourcesLimits(device)
|
||||
}
|
||||
|
||||
final env = fusionLauncher().fusionEnv()
|
||||
for( Map.Entry<String,String> it : env )
|
||||
builder.withEnv(PodEnv.value(it.key, it.value))
|
||||
}
|
||||
|
||||
return useJobResource()
|
||||
? builder.buildAsJob()
|
||||
: builder.build()
|
||||
}
|
||||
|
||||
protected PodOptions getPodOptions() {
|
||||
// merge the pod options provided in the k8s config
|
||||
// with the ones in process config
|
||||
def opt1 = k8sConfig.getPodOptions()
|
||||
def opt2 = taskPodOptions()
|
||||
return opt1 + opt2
|
||||
}
|
||||
|
||||
protected PodOptions taskPodOptions() {
|
||||
new PodOptions((List)task.getConfig().get('pod'))
|
||||
}
|
||||
|
||||
protected Map<String,String> getLabels(TaskRun task) {
|
||||
final result = new LinkedHashMap<String,String>(10)
|
||||
final labels = k8sConfig.getLabels()
|
||||
if( labels ) {
|
||||
result.putAll(labels)
|
||||
}
|
||||
final resLabels = task.config.getResourceLabels()
|
||||
if( resLabels )
|
||||
result.putAll(resLabels)
|
||||
result.'nextflow.io/app' = 'nextflow'
|
||||
result.'nextflow.io/runName' = getRunName()
|
||||
result.'nextflow.io/taskName' = task.getName()
|
||||
result.'nextflow.io/processName' = task.getProcessor().getName()
|
||||
result.'nextflow.io/sessionId' = "uuid-${executor.getSession().uniqueId}" as String
|
||||
if( task.config.queue )
|
||||
result.'nextflow.io/queue' = task.config.queue
|
||||
return result
|
||||
}
|
||||
|
||||
protected Map getAnnotations() {
|
||||
k8sConfig.getAnnotations()
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepares the task execution and enqueues it at the scheduler
|
||||
*/
|
||||
@Override
|
||||
@CompileDynamic
|
||||
void submit() {
|
||||
builder = createBashWrapper(task)
|
||||
builder.build()
|
||||
log.info "[K8s] submitting task ${this.task.name}"
|
||||
executor.taskScheduler.submit(this)
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new K8s pod executing the associated task
|
||||
*/
|
||||
@CompileDynamic
|
||||
void submitNow(String nodeName) {
|
||||
final req = newSubmitRequest(task, nodeName)
|
||||
final resp = useJobResource()
|
||||
? client.jobCreate(req, yamlDebugPath())
|
||||
: client.podCreate(req, yamlDebugPath())
|
||||
|
||||
if( !resp.metadata?.name )
|
||||
throw new K8sResponseException("Missing created ${resourceType.lower()} name", resp)
|
||||
this.podName = resp.metadata.name
|
||||
this.status = TaskStatus.SUBMITTED
|
||||
}
|
||||
|
||||
@CompileDynamic
|
||||
protected Path yamlDebugPath() {
|
||||
boolean debug = k8sConfig.getDebug().getYaml()
|
||||
return debug ? task.workDir.resolve('.command.yaml') : null
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Retrieve the submitted pod state
|
||||
*/
|
||||
protected Map getState() {
|
||||
final now = System.currentTimeMillis()
|
||||
try {
|
||||
final delta = now - timestamp;
|
||||
if( !state || delta >= 1_000) {
|
||||
def newState = useJobResource()
|
||||
? client.jobState(podName)
|
||||
: client.podState(podName)
|
||||
if( newState ) {
|
||||
log.trace "[K8s] Get ${resourceType.lower()}=$podName state=$newState"
|
||||
state = newState
|
||||
timestamp = now
|
||||
}
|
||||
}
|
||||
return state
|
||||
}
|
||||
catch (NodeTerminationException | PodUnschedulableException e) {
|
||||
// create a synthetic `state` object adding an extra `nodeTermination`
|
||||
// attribute to return the error to the caller method
|
||||
final instant = Instant.now()
|
||||
final result = new HashMap(10)
|
||||
result.terminated = [startedAt:instant.toString(), finishedAt:instant.toString()]
|
||||
result.nodeTermination = e
|
||||
timestamp = now
|
||||
state = result
|
||||
return state
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean checkIfRunning() {
|
||||
if( !podName )
|
||||
return false
|
||||
if(isSubmitted()) {
|
||||
def state = getState()
|
||||
// include `terminated` state to allow the handler status to progress
|
||||
if (state && (state.running != null || state.terminated)) {
|
||||
status = TaskStatus.RUNNING
|
||||
determineNode()
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
long getEpochMilli(String timeString) {
|
||||
final time = DateTimeFormatter.ISO_INSTANT.parse(timeString)
|
||||
return Instant.from(time).toEpochMilli()
|
||||
}
|
||||
|
||||
/**
|
||||
* Update task start and end times based on pod timestamps.
|
||||
* We update timestamps because it's possible for a task to run so quickly
|
||||
* (less than 1 second) that it skips right over the RUNNING status.
|
||||
* If this happens, the startTimeMillis never gets set and remains equal to 0.
|
||||
* To make sure startTimeMillis is non-zero we update it with the pod start time.
|
||||
* We update completeTimeMillis from the same pod info to be consistent.
|
||||
*/
|
||||
void updateTimestamps(Map terminated) {
|
||||
try {
|
||||
startTimeMillis = getEpochMilli(terminated.startedAt as String)
|
||||
completeTimeMillis = getEpochMilli(terminated.finishedAt as String)
|
||||
} catch( Exception e ) {
|
||||
log.debug "Failed updating timestamps '${terminated.toString()}'", e
|
||||
// Only update if startTimeMillis hasn't already been set.
|
||||
// If startTimeMillis _has_ been set, then both startTimeMillis
|
||||
// and completeTimeMillis will have been set with the normal
|
||||
// TaskHandler mechanism, so there's no need to reset them here.
|
||||
if (!startTimeMillis) {
|
||||
startTimeMillis = System.currentTimeMillis()
|
||||
completeTimeMillis = System.currentTimeMillis()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean checkIfCompleted() {
|
||||
if( !podName )
|
||||
return false
|
||||
|
||||
final state = getState()
|
||||
if( state && state.terminated ) {
|
||||
if( state.nodeTermination instanceof NodeTerminationException ||
|
||||
state.nodeTermination instanceof PodUnschedulableException ) {
|
||||
// keep track of the node termination error
|
||||
task.error = (Throwable) state.nodeTermination
|
||||
// mark the task as ABORTED since thr failure is caused by a node failure
|
||||
task.aborted = true
|
||||
}
|
||||
else {
|
||||
// finalize the task
|
||||
// read the exit code from the K8s container terminated state, if missing
|
||||
// take the exit code from the `.exitcode` file created by nextflow
|
||||
// the rationale is that in case of error (e.g. OOMKilled, pod eviction), the exit code from
|
||||
// the K8s API is more reliable because the container may terminate before the exit file is written
|
||||
// See https://github.com/nextflow-io/nextflow/issues/6436
|
||||
// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#containerstateterminated-v1-core
|
||||
log.trace("[k8s] Container Terminated state ${state.terminated}")
|
||||
final k8sExitCode = (state.terminated as Map)?.exitCode as Integer
|
||||
task.exitStatus = k8sExitCode != null ? k8sExitCode : readExitFile()
|
||||
task.stdout = outputFile
|
||||
task.stderr = errorFile
|
||||
}
|
||||
status = TaskStatus.COMPLETED
|
||||
saveJobLogOnError(task)
|
||||
deleteJobIfSuccessful(task)
|
||||
updateTimestamps(state.terminated as Map)
|
||||
determineNode()
|
||||
|
||||
// Signal the scheduler that this task has finished running
|
||||
if (executor != null) {
|
||||
executor.taskScheduler.taskFinished(this)
|
||||
executor.runtimeRecorder.record(this)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
protected void saveJobLogOnError(TaskRun task) {
|
||||
if( task.isSuccess() )
|
||||
return
|
||||
|
||||
if( errorFile && !errorFile.empty() )
|
||||
return
|
||||
|
||||
final session = executor.getSession()
|
||||
if( session.isAborted() || session.isCancelled() || session.isTerminated() )
|
||||
return
|
||||
|
||||
try {
|
||||
final stream = useJobResource()
|
||||
? client.jobLog(podName)
|
||||
: client.podLog(podName)
|
||||
Files.copy(stream, task.workDir.resolve(TaskRun.CMD_LOG))
|
||||
}
|
||||
catch( FileAlreadyExistsException e ) {
|
||||
log.debug "Log file already exists for ${resourceType.lower()} $podName", e
|
||||
}
|
||||
catch( Exception e ) {
|
||||
log.warn "Failed to copy log for ${resourceType.lower()} $podName", e
|
||||
}
|
||||
}
|
||||
|
||||
protected int readExitFile() {
|
||||
try {
|
||||
exitFile.text as Integer
|
||||
}
|
||||
catch( Exception e ) {
|
||||
log.debug "[K8s] Cannot read exitstatus for task: `$task.name` | ${e.message}"
|
||||
return Integer.MAX_VALUE
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Terminates the current task execution
|
||||
*/
|
||||
@Override
|
||||
protected void killTask() {
|
||||
if( !podName )
|
||||
return
|
||||
|
||||
if( cleanupDisabled() )
|
||||
return
|
||||
|
||||
log.trace "[K8s] deleting ${resourceType.lower()} name=$podName"
|
||||
delete0(podName)
|
||||
}
|
||||
|
||||
protected boolean cleanupDisabled() {
|
||||
!k8sConfig.getCleanup()
|
||||
}
|
||||
|
||||
protected void deleteJobIfSuccessful(TaskRun task) {
|
||||
if( !podName )
|
||||
return
|
||||
|
||||
if( cleanupDisabled() )
|
||||
return
|
||||
|
||||
// preserve failed pods for debugging purposes
|
||||
if( !task.isSuccess() )
|
||||
return
|
||||
|
||||
// k8s cluster will cleanup job on its own if TTL is set
|
||||
if( useJobResource() && getPodOptions().getTtlSecondsAfterFinished() != null )
|
||||
return
|
||||
|
||||
delete0(podName)
|
||||
}
|
||||
|
||||
private void delete0(String podName) {
|
||||
try {
|
||||
if ( useJobResource() )
|
||||
client.jobDelete(podName)
|
||||
else
|
||||
client.podDelete(podName)
|
||||
}
|
||||
catch( Exception e ) {
|
||||
log.warn "Unable to delete ${resourceType.lower()}: $podName -- see the log file for details", e
|
||||
}
|
||||
}
|
||||
|
||||
private void determineNode() {
|
||||
try {
|
||||
if ( k8sConfig.fetchNodeName() && !runsOnNode )
|
||||
runsOnNode = client.getNodeOfPod( podName )
|
||||
} catch ( Exception e ) {
|
||||
log.warn ("Unable to get the node name of pod $podName -- see the log file for details", e)
|
||||
}
|
||||
}
|
||||
|
||||
TraceRecord getTraceRecord() {
|
||||
final result = super.getTraceRecord()
|
||||
result.put('native_id', podName)
|
||||
result.put( 'hostname', runsOnNode )
|
||||
return result
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package recreationaltech.plugin
|
||||
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.util.Duration
|
||||
import java.util.concurrent.LinkedBlockingQueue
|
||||
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class K8sTaskScheduler implements Runnable {
|
||||
private final Duration interval
|
||||
private final K8sSchedulingStrategy strategy
|
||||
private final LinkedBlockingQueue<K8sSchedulingRequest> queue = new LinkedBlockingQueue<>()
|
||||
private String[] nodes
|
||||
|
||||
private synchronized boolean shouldStop
|
||||
|
||||
K8sTaskScheduler(String[] nodes, K8sSchedulingStrategy strategy, Duration interval) {
|
||||
this.interval = interval
|
||||
this.strategy = strategy
|
||||
this.nodes = nodes
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a task to the queue of outstanding tasks
|
||||
* @param handler
|
||||
*/
|
||||
void submit(K8sTaskHandler handler) {
|
||||
log.info "[K8s] received queued task ${handler.task.name}"
|
||||
queue.add(new K8sSchedulingRequest(handler))
|
||||
final pending = new ArrayList<K8sSchedulingRequest>(queue)
|
||||
if (strategy.scheduleImmediately(this, pending))
|
||||
schedule()
|
||||
}
|
||||
|
||||
/**
|
||||
* Notify the scheduler that a task has finished execution
|
||||
* @param handler
|
||||
*/
|
||||
void taskFinished(K8sTaskHandler handler) {
|
||||
strategy.taskFinished(handler)
|
||||
|
||||
/* If we have pending tasks, now would be a good time to schedule a new one.
|
||||
* Because resources were freed right now */
|
||||
if (queue.size() > 0)
|
||||
schedule()
|
||||
}
|
||||
|
||||
protected synchronized void schedule() {
|
||||
final pending = new ArrayList<K8sSchedulingRequest>(queue)
|
||||
final decision = strategy.schedule(this, pending)
|
||||
|
||||
if (!decision)
|
||||
return
|
||||
|
||||
if (!queue.remove(decision.request)) {
|
||||
log.warn "[K8s] failed to remove selected task from queue ${decision.request.task.name}"
|
||||
return
|
||||
}
|
||||
|
||||
log.info "[K8s] launching queued task ${decision.request.task.name} on node: ${decision.nodeName}"
|
||||
decision.request.handler.submitNow(decision.nodeName)
|
||||
}
|
||||
|
||||
/* Scheduling Strategy Interface */
|
||||
List<String> getNodes() {
|
||||
return nodes.toList()
|
||||
}
|
||||
|
||||
/**
|
||||
* Run is the scheduler threads main function
|
||||
* */
|
||||
void run() {
|
||||
this.shouldStop = false
|
||||
final interval = this.interval
|
||||
if (interval.toMillis() == 0L) {
|
||||
log.info("[K8s] scheduler loop disabled (interval is 0)")
|
||||
return
|
||||
}
|
||||
log.info("[K8s] launched scheduler loop (${interval.toString()} interval)")
|
||||
while (!shouldStop) {
|
||||
sleep(interval.toMillis())
|
||||
schedule()
|
||||
}
|
||||
log.info("[K8s] terminated scheduler loop")
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop terminates the scheduler thread
|
||||
*/
|
||||
void stop() {
|
||||
log.info("[K8s] stopping scheduler loop")
|
||||
this.shouldStop = true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin
|
||||
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.executor.BashWrapperBuilder
|
||||
import nextflow.processor.TaskRun
|
||||
import nextflow.util.Escape
|
||||
|
||||
/**
|
||||
* Implements a BASH wrapper for tasks executed by kubernetes cluster
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class K8sWrapperBuilder extends BashWrapperBuilder {
|
||||
|
||||
K8sWrapperBuilder(TaskRun task) {
|
||||
super(task)
|
||||
this.headerScript = "NXF_CHDIR=${Escape.path(task.workDir)}"
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.cli
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.cli.CmdKubeRun
|
||||
import recreationaltech.plugin.K8sDriverLauncher
|
||||
|
||||
/**
|
||||
* Kuberun command implementation logic
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class KubeCommandImpl implements CmdKubeRun.KubeCommand {
|
||||
|
||||
@Override
|
||||
int run(CmdKubeRun cmd, String pipeline, List<String> args) {
|
||||
// create
|
||||
final driver = new K8sDriverLauncher(
|
||||
cmd: cmd,
|
||||
runName: cmd.runName,
|
||||
headImage: cmd.headImage,
|
||||
background: cmd.background(),
|
||||
headCpus: cmd.headCpus,
|
||||
headMemory: cmd.headMemory,
|
||||
headPreScript: cmd.headPreScript,
|
||||
plugins: cmd.plugins )
|
||||
// run it
|
||||
driver.run(pipeline, args)
|
||||
// return exit code
|
||||
return driver.shutdown()
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.client
|
||||
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.util.Duration
|
||||
|
||||
import javax.net.ssl.KeyManager
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
/**
|
||||
* Models the kubernetes cluster client configuration settings
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@EqualsAndHashCode
|
||||
@CompileStatic
|
||||
@Slf4j
|
||||
class ClientConfig {
|
||||
|
||||
boolean verifySsl
|
||||
|
||||
String server
|
||||
|
||||
String namespace
|
||||
|
||||
/**
|
||||
* k8s service account name
|
||||
* https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/
|
||||
*/
|
||||
String serviceAccount
|
||||
|
||||
String token
|
||||
|
||||
byte[] sslCert
|
||||
|
||||
byte[] clientCert
|
||||
|
||||
byte[] clientKey
|
||||
|
||||
KeyManager[] keyManagers
|
||||
|
||||
K8sRetryConfig retryConfig
|
||||
|
||||
/**
|
||||
* Timeout when reading from Input stream when a connection is established to a resource.
|
||||
* If the timeout expires before there is data available for read, a {@link java.net.SocketTimeoutException} is raised
|
||||
*/
|
||||
Duration httpReadTimeout
|
||||
|
||||
/**
|
||||
* Timeout when opening a communications link to the resource referenced by K8sClient request connection
|
||||
* If the timeout expires before there is data available for read, a {@link java.net.SocketTimeoutException} is raised
|
||||
*/
|
||||
Duration httpConnectTimeout
|
||||
|
||||
/**
|
||||
* When true signal that the configuration was retrieved from within a K8s cluster
|
||||
*/
|
||||
boolean isFromCluster
|
||||
|
||||
String getNamespace() { namespace ?: 'default' }
|
||||
|
||||
ClientConfig() {
|
||||
retryConfig = new K8sRetryConfig()
|
||||
}
|
||||
|
||||
String toString() {
|
||||
"${this.class.getSimpleName()}[ server=$server, namespace=$namespace, serviceAccount=$serviceAccount, token=${cut(token)}, sslCert=${cut(sslCert)}, clientCert=${cut(clientCert)}, clientKey=${cut(clientKey)}, verifySsl=$verifySsl, fromFile=$isFromCluster, httpReadTimeout=$httpReadTimeout, httpConnectTimeout=$httpConnectTimeout, retryConfig=$retryConfig ]"
|
||||
}
|
||||
|
||||
private String cut(String str) {
|
||||
if( !str ) return '-'
|
||||
return str.size()<10 ? str : str[0..10].toString() + '..'
|
||||
}
|
||||
|
||||
private String cut(byte[] bytes) {
|
||||
if( !bytes ) return '-'
|
||||
cut(bytes.encodeBase64().toString())
|
||||
}
|
||||
|
||||
static ClientConfig discover(String context, String namespace, String serviceAccount) {
|
||||
new ConfigDiscovery().discover(context, namespace, serviceAccount)
|
||||
}
|
||||
|
||||
static ClientConfig fromNextflowConfig(Map opts, String namespace, String serviceAccount) {
|
||||
final result = new ClientConfig()
|
||||
|
||||
if( opts.server )
|
||||
result.server = opts.server
|
||||
|
||||
if( opts.token )
|
||||
result.token = opts.token
|
||||
else if( opts.tokenFile )
|
||||
result.token = Paths.get(opts.tokenFile.toString()).getText('UTF-8')
|
||||
|
||||
result.namespace = namespace ?: opts.namespace ?: 'default'
|
||||
|
||||
result.serviceAccount = serviceAccount ?: 'default'
|
||||
|
||||
if( opts.verifySsl )
|
||||
result.verifySsl = opts.verifySsl as boolean
|
||||
|
||||
if( opts.sslCert )
|
||||
result.sslCert = opts.sslCert.toString().decodeBase64()
|
||||
else if( opts.sslCertFile )
|
||||
result.sslCert = Paths.get(opts.sslCertFile.toString()).bytes
|
||||
|
||||
if( opts.clientCert )
|
||||
result.clientCert = opts.clientCert.toString().decodeBase64()
|
||||
else if( opts.clientCertFile )
|
||||
result.clientCert = Paths.get(opts.clientCertFile.toString()).bytes
|
||||
|
||||
if( opts.clientKey )
|
||||
result.clientKey = opts.clientKey.toString().decodeBase64()
|
||||
else if( opts.clientKeyFile )
|
||||
result.clientKey = Paths.get(opts.clientKeyFile.toString()).bytes
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
static ClientConfig fromUserAndCluster(Map user, Map cluster, Path location) {
|
||||
final base = location.isDirectory() ? location : location.parent
|
||||
final result = new ClientConfig()
|
||||
if( user.token )
|
||||
result.token = user.token
|
||||
|
||||
else if( user.tokenFile ) {
|
||||
result.token = Paths.get(user.tokenFile.toString()).getText('UTF-8')
|
||||
}
|
||||
|
||||
if( user."client-certificate" )
|
||||
result.clientCert = base.resolve(user."client-certificate".toString()).bytes
|
||||
|
||||
else if( user."client-certificate-data" )
|
||||
result.clientCert = user."client-certificate-data".toString().decodeBase64()
|
||||
|
||||
if( user."client-key" )
|
||||
result.clientKey = base.resolve(user."client-key".toString()).bytes
|
||||
|
||||
else if( user."client-key-data" )
|
||||
result.clientKey = user."client-key-data".toString().decodeBase64()
|
||||
|
||||
// -- cluster settings
|
||||
|
||||
if( cluster.server )
|
||||
result.server = cluster.server
|
||||
|
||||
if( cluster."certificate-authority-data" )
|
||||
result.sslCert = cluster."certificate-authority-data".toString().decodeBase64()
|
||||
|
||||
else if( cluster."certificate-authority" )
|
||||
result.sslCert = base.resolve(cluster."certificate-authority".toString()).bytes
|
||||
|
||||
result.verifySsl = cluster."insecure-skip-tls-verify" != true
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.client
|
||||
|
||||
import javax.net.ssl.KeyManager
|
||||
import javax.net.ssl.KeyManagerFactory
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
import java.security.KeyStore
|
||||
import static nextflow.util.StringUtils.formatHostName
|
||||
|
||||
import groovy.util.logging.Slf4j
|
||||
import org.yaml.snakeyaml.Yaml
|
||||
/**
|
||||
* Discover Kubernetes configuration from system environment
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
class ConfigDiscovery {
|
||||
|
||||
private Map<String,String> env = System.getenv()
|
||||
|
||||
ConfigDiscovery() { }
|
||||
|
||||
/**
|
||||
* Discover Kubernetes client configuration from current environment using
|
||||
* either the .kube/config file or the pod service service account virtual
|
||||
* file system when running in a pod.
|
||||
*
|
||||
* @param contextName The K8s config context name.
|
||||
* @param namespace The K8s cluster namespace
|
||||
* @param serviceAccount The K8s cluster service account
|
||||
* @return The e
|
||||
*/
|
||||
ClientConfig discover(String contextName, String namespace, String serviceAccount) {
|
||||
|
||||
// Note: System.getProperty('user.home') may not report the correct home path when
|
||||
// running in a container. Use env HOME instead.
|
||||
def home = System.getenv('HOME')
|
||||
def kubeConfig = env.get('KUBECONFIG') ? env.get('KUBECONFIG') : "$home/.kube/config"
|
||||
def configFile = Paths.get(kubeConfig)
|
||||
|
||||
// determine the Kubernetes client configuration via the `.kube/config` file
|
||||
if( configFile.exists() ) {
|
||||
return fromKubeConfig(configFile, contextName, namespace, serviceAccount)
|
||||
}
|
||||
else {
|
||||
log.debug "K8s config file does not exist: $configFile"
|
||||
}
|
||||
|
||||
// determine the Kubernetes client configuration via the pod environment
|
||||
if( env.get('KUBERNETES_SERVICE_HOST') ) {
|
||||
return fromCluster(env, namespace, serviceAccount)
|
||||
}
|
||||
else {
|
||||
log.debug "K8s env variable KUBERNETES_SERVICE_HOST is not defined"
|
||||
}
|
||||
|
||||
throw new IllegalStateException("Unable to lookup Kubernetes cluster configuration")
|
||||
}
|
||||
|
||||
protected ClientConfig fromCluster(Map<String,String> env, String cfgNamespace, String serviceAccount) {
|
||||
|
||||
// See https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/#accessing-the-api-from-a-pod
|
||||
|
||||
final host = env.get('KUBERNETES_SERVICE_HOST')
|
||||
final port = env.get('KUBERNETES_SERVICE_PORT')
|
||||
final server = formatHostName(host, port)
|
||||
|
||||
final cert = path('/var/run/secrets/kubernetes.io/serviceaccount/ca.crt').bytes
|
||||
final token = path('/var/run/secrets/kubernetes.io/serviceaccount/token').text
|
||||
final namespace = path('/var/run/secrets/kubernetes.io/serviceaccount/namespace').text
|
||||
|
||||
return new ClientConfig(
|
||||
server: server,
|
||||
token: token,
|
||||
namespace: cfgNamespace ?: namespace,
|
||||
serviceAccount: serviceAccount,
|
||||
sslCert: cert,
|
||||
isFromCluster: true )
|
||||
}
|
||||
|
||||
protected Path path(String path) {
|
||||
Paths.get(path)
|
||||
}
|
||||
|
||||
protected ClientConfig fromKubeConfig(Path path, String contextName, String namespace, String serviceAccount) {
|
||||
def yaml = (Map)new Yaml().load(Files.newInputStream(path))
|
||||
|
||||
contextName ?= yaml."current-context" as String
|
||||
|
||||
final allContext = yaml.contexts as List
|
||||
final allClusters = yaml.clusters as List
|
||||
final allUsers = yaml.users as List
|
||||
final context = allContext.find { Map it -> it.name == contextName } ?.context
|
||||
if( !context )
|
||||
throw new IllegalArgumentException("Unknown Kubernetes context: $contextName -- check config file: $path")
|
||||
final userName = context?.user
|
||||
final clusterName = context?.cluster
|
||||
final user = allUsers.find{ Map it -> it.name == userName } ?.user ?: [:]
|
||||
final cluster = allClusters.find{ Map it -> it.name == clusterName } ?.cluster ?: [:]
|
||||
|
||||
final config = ClientConfig.fromUserAndCluster(user, cluster, path)
|
||||
|
||||
// the namespace provided should have priority over the context current namespace
|
||||
config.namespace = namespace ?: context?.namespace ?: 'default'
|
||||
|
||||
config.serviceAccount = serviceAccount ?: 'default'
|
||||
|
||||
if( config.clientCert && config.clientKey ) {
|
||||
config.keyManagers = createKeyManagers(config.clientCert, config.clientKey)
|
||||
}
|
||||
else if( !config.token ) {
|
||||
config.token = discoverAuthToken(contextName, config.namespace, config.serviceAccount)
|
||||
}
|
||||
|
||||
return config
|
||||
}
|
||||
|
||||
protected KeyStore createKeyStore0(byte[] clientCert, byte[] clientKey, char[] passphrase, String alg) {
|
||||
def cert = new ByteArrayInputStream(clientCert)
|
||||
def key = new ByteArrayInputStream(clientKey)
|
||||
return SSLUtils.createKeyStore(cert, key, alg, passphrase, null, null)
|
||||
}
|
||||
|
||||
protected KeyStore createKeyStore(byte[] clientCert, byte[] clientKey, char[] passphrase) {
|
||||
try {
|
||||
// try first RSA algorithm
|
||||
return createKeyStore0(clientCert, clientKey, passphrase, "RSA")
|
||||
}
|
||||
catch (Exception e1) {
|
||||
// fallback to EC algorithm
|
||||
try {
|
||||
return createKeyStore0(clientCert, clientKey, passphrase, "EC")
|
||||
}
|
||||
catch (Exception e2) {
|
||||
// if still fails, throws the first exception
|
||||
throw e1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected KeyManager[] createKeyManagers(byte[] clientCert, byte[] clientKey) {
|
||||
final passphrase = "".toCharArray()
|
||||
final keyStore = createKeyStore(clientCert, clientKey, passphrase)
|
||||
final kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
|
||||
kmf.init(keyStore, passphrase);
|
||||
return kmf.getKeyManagers();
|
||||
}
|
||||
|
||||
String discoverAuthToken(String context, String namespace, String serviceAccount) {
|
||||
context ?= 'default'
|
||||
namespace ?= 'default'
|
||||
serviceAccount ?= 'default'
|
||||
|
||||
final cmd = "kubectl --context $context -n ${namespace} get secret -o=jsonpath='{.items[?(@.metadata.annotations.kubernetes\\.io/service-account\\.name==\"$serviceAccount\")].data.token}'"
|
||||
final proc = new ProcessBuilder('bash','-o','pipefail','-c', cmd).start()
|
||||
final status = proc.waitFor()
|
||||
final text = proc.inputStream?.text
|
||||
if( status==0 && text ) {
|
||||
try {
|
||||
return new String(text.trim().decodeBase64())
|
||||
}
|
||||
catch( Exception e ) {
|
||||
log.warn "Unable to decode K8s cluster auth token '$text' -- cause: ${e.message}"
|
||||
}
|
||||
}
|
||||
else {
|
||||
final cause = proc.errorStream?.text ?: text
|
||||
final msg = cause ? "\n- cmd : $cmd\n- exit : $status\n- cause:\n${cause.indent(' ')}" : ''
|
||||
log.warn "[K8s] unable to fetch auth token ${msg}"
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,54 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.client
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
|
||||
/**
|
||||
* Model a Kubernetes API response
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class K8sResponseApi {
|
||||
|
||||
private int code
|
||||
|
||||
private InputStream stream
|
||||
|
||||
private String text
|
||||
|
||||
K8sResponseApi(int code, InputStream stream) {
|
||||
this.code = code
|
||||
this.stream = stream
|
||||
}
|
||||
|
||||
String toString() {
|
||||
"code=$code; stream=$stream"
|
||||
}
|
||||
|
||||
int getCode() { code }
|
||||
|
||||
InputStream getStream() { stream }
|
||||
|
||||
String getText() {
|
||||
if( text == null ) {
|
||||
text = stream?.text
|
||||
}
|
||||
return text
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.client
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
/**
|
||||
* Model a kubernetes invalid response
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class K8sResponseException extends Exception {
|
||||
|
||||
K8sResponseJson response
|
||||
|
||||
K8sResponseException(K8sResponseJson response) {
|
||||
super(msg0(response))
|
||||
this.response = response
|
||||
}
|
||||
|
||||
K8sResponseException(String message, K8sResponseJson response) {
|
||||
super(msg1(message,response))
|
||||
this.response = response
|
||||
}
|
||||
|
||||
K8sResponseException(String message, InputStream response) {
|
||||
this(message, new K8sResponseJson(fetch(response)))
|
||||
}
|
||||
|
||||
static private String msg1( String msg, K8sResponseJson resp ) {
|
||||
if( !msg && resp==null )
|
||||
return null
|
||||
|
||||
if( msg && resp != null ) {
|
||||
def sep = resp.isRawText() ? ' -- ' : '\n'
|
||||
return "${msg}${sep}${msg0(resp)}"
|
||||
}
|
||||
else if( msg ) {
|
||||
return msg
|
||||
}
|
||||
else {
|
||||
return msg0(resp)
|
||||
}
|
||||
}
|
||||
|
||||
static private String msg0( K8sResponseJson response ) {
|
||||
if( response == null )
|
||||
return null
|
||||
|
||||
if( response.isRawText() )
|
||||
response.getRawText()
|
||||
else
|
||||
"\n${response.toString().indent(' ')}"
|
||||
}
|
||||
|
||||
static private String fetch(InputStream stream) {
|
||||
try {
|
||||
return stream?.text
|
||||
}
|
||||
catch( Exception e ) {
|
||||
log.debug "Unable to fetch response text -- Cause: ${e.message ?: e}"
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.client
|
||||
|
||||
import groovy.json.JsonOutput
|
||||
import groovy.json.JsonSlurper
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
|
||||
/**
|
||||
* Model the response of a kubernetes api request
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class K8sResponseJson implements Map {
|
||||
|
||||
@Delegate
|
||||
private Map response
|
||||
|
||||
private String rawText
|
||||
|
||||
K8sResponseJson(Map response) {
|
||||
this.response = response
|
||||
}
|
||||
|
||||
K8sResponseJson(String response) {
|
||||
this.response = toJson(response)
|
||||
this.rawText = response
|
||||
}
|
||||
|
||||
boolean isRawText() { !response && rawText }
|
||||
|
||||
String getRawText() { rawText }
|
||||
|
||||
static private Map toJson(String raw) {
|
||||
try {
|
||||
return (Map)new JsonSlurper().parseText(raw)
|
||||
}
|
||||
catch( Exception e ) {
|
||||
log.trace "[K8s] cannot parse response to json -- raw: ${raw? '\n'+raw.indent(' ') :'null'}"
|
||||
return Collections.emptyMap()
|
||||
}
|
||||
}
|
||||
|
||||
static private String prettyPrint(String json) {
|
||||
try {
|
||||
JsonOutput.prettyPrint(json)
|
||||
}
|
||||
catch( Exception e ) {
|
||||
return json
|
||||
}
|
||||
}
|
||||
|
||||
String toString() {
|
||||
response ? prettyPrint(JsonOutput.toJson(response)) : rawText
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.client
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import groovy.transform.ToString
|
||||
import nextflow.config.spec.ConfigOption
|
||||
import nextflow.config.spec.ConfigScope
|
||||
import nextflow.script.dsl.Description
|
||||
import nextflow.util.Duration
|
||||
|
||||
/**
|
||||
* Model retry policy configuration
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@ToString(includePackage = false, includeNames = true)
|
||||
@EqualsAndHashCode
|
||||
@CompileStatic
|
||||
class K8sRetryConfig implements ConfigScope {
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Delay when retrying failed API requests (default: `250ms`).
|
||||
""")
|
||||
Duration delay = Duration.of('250ms')
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Max delay when retrying failed API requests (default: `90s`).
|
||||
""")
|
||||
Duration maxDelay = Duration.of('90s')
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Max attempts when retrying failed API requests (default: `4`).
|
||||
""")
|
||||
int maxAttempts = 4
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Jitter value when retrying failed API requests (default: `0.25`).
|
||||
""")
|
||||
double jitter = 0.25
|
||||
|
||||
K8sRetryConfig() {
|
||||
this(Collections.emptyMap())
|
||||
}
|
||||
|
||||
K8sRetryConfig(Map config) {
|
||||
if( config.delay )
|
||||
delay = config.delay as Duration
|
||||
if( config.maxDelay )
|
||||
maxDelay = config.maxDelay as Duration
|
||||
if( config.maxAttempts )
|
||||
maxAttempts = config.maxAttempts as int
|
||||
if( config.jitter )
|
||||
jitter = config.jitter as double
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.client
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.exception.ProcessException
|
||||
import nextflow.exception.ShowOnlyExceptionMessage
|
||||
|
||||
/**
|
||||
* Exception raised when a pod cannot be scheduled because
|
||||
* e.g. the container image cannot be pulled, required resources
|
||||
* cannot be fulfilled, etc.
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class PodUnschedulableException extends ProcessException implements ShowOnlyExceptionMessage {
|
||||
|
||||
PodUnschedulableException(String message, Throwable cause) {
|
||||
super(message,cause)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,318 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.client;
|
||||
|
||||
/**
|
||||
* This file is derived from
|
||||
* https://github.com/kubernetes-client/java/blob/master/util/src/main/java/io/kubernetes/client/util/SSLUtils.java
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.math.BigInteger;
|
||||
import java.security.KeyFactory;
|
||||
import java.security.KeyStore;
|
||||
import java.security.KeyStoreException;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.security.PrivateKey;
|
||||
import java.security.Security;
|
||||
import java.security.UnrecoverableKeyException;
|
||||
import java.security.cert.Certificate;
|
||||
import java.security.cert.CertificateException;
|
||||
import java.security.cert.CertificateFactory;
|
||||
import java.security.cert.X509Certificate;
|
||||
import java.security.spec.InvalidKeySpecException;
|
||||
import java.security.spec.PKCS8EncodedKeySpec;
|
||||
import java.security.spec.RSAPrivateCrtKeySpec;
|
||||
import javax.net.ssl.KeyManager;
|
||||
import javax.net.ssl.KeyManagerFactory;
|
||||
|
||||
import org.apache.commons.codec.binary.Base64;
|
||||
import org.bouncycastle.asn1.pkcs.PrivateKeyInfo;
|
||||
import org.bouncycastle.openssl.PEMKeyPair;
|
||||
import org.bouncycastle.openssl.PEMParser;
|
||||
import org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter;
|
||||
|
||||
|
||||
public class SSLUtils {
|
||||
|
||||
public static boolean isNotNullOrEmpty(String val) {
|
||||
return val != null && val.length() > 0;
|
||||
}
|
||||
|
||||
public static KeyManager[] keyManagers(String certData, String certFile, String keyData, String keyFile,
|
||||
String algo, String passphrase, String keyStoreFile, String keyStorePassphrase)
|
||||
throws NoSuchAlgorithmException, UnrecoverableKeyException, KeyStoreException, CertificateException,
|
||||
InvalidKeySpecException, IOException {
|
||||
KeyManager[] keyManagers = null;
|
||||
if ((isNotNullOrEmpty(certData) || isNotNullOrEmpty(certFile))
|
||||
&& (isNotNullOrEmpty(keyData) || isNotNullOrEmpty(keyFile))) {
|
||||
KeyStore keyStore = createKeyStore(certData, certFile, keyData, keyFile, algo, passphrase, keyStoreFile,
|
||||
keyStorePassphrase);
|
||||
KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
|
||||
kmf.init(keyStore, passphrase.toCharArray());
|
||||
keyManagers = kmf.getKeyManagers();
|
||||
}
|
||||
return keyManagers;
|
||||
}
|
||||
|
||||
|
||||
public static KeyStore createKeyStore(String clientCertData, String clientCertFile, String clientKeyData,
|
||||
String clientKeyFile, String clientKeyAlgo, String clientKeyPassphrase, String keyStoreFile,
|
||||
String keyStorePassphrase) throws IOException, CertificateException, NoSuchAlgorithmException,
|
||||
InvalidKeySpecException, KeyStoreException {
|
||||
try (InputStream certInputStream = getInputStreamFromDataOrFile(clientCertData, clientCertFile);
|
||||
InputStream keyInputStream = getInputStreamFromDataOrFile(clientKeyData, clientKeyFile)) {
|
||||
return createKeyStore(certInputStream, keyInputStream, clientKeyAlgo,
|
||||
clientKeyPassphrase != null ? clientKeyPassphrase.toCharArray() : null,
|
||||
keyStoreFile, getKeyStorePassphrase(keyStorePassphrase));
|
||||
}
|
||||
}
|
||||
|
||||
static private PrivateKey generateEcKey(InputStream keyInputStream) throws IOException {
|
||||
PrivateKey privateKey=null;
|
||||
Security.addProvider(new org.bouncycastle.jce.provider.BouncyCastleProvider());
|
||||
Object object = new PEMParser(new InputStreamReader(keyInputStream)).readObject();
|
||||
if (object instanceof PEMKeyPair) {
|
||||
PEMKeyPair keys = (PEMKeyPair) object;
|
||||
privateKey = new JcaPEMKeyConverter().getKeyPair(keys).getPrivate();
|
||||
}
|
||||
if( object instanceof PrivateKeyInfo) {
|
||||
PrivateKeyInfo privateKeyInfo = (PrivateKeyInfo)object;
|
||||
privateKey = new JcaPEMKeyConverter().getPrivateKey(privateKeyInfo);
|
||||
}
|
||||
if( privateKey == null) {
|
||||
throw new IOException("Unsupported EC algorithm");
|
||||
}
|
||||
return privateKey;
|
||||
}
|
||||
|
||||
static private PrivateKey generateStdKey(InputStream keyInputStream, String clientKeyAlgo) throws IOException, NoSuchAlgorithmException, InvalidKeySpecException {
|
||||
byte[] keyBytes = decodePem(keyInputStream);
|
||||
|
||||
KeyFactory keyFactory = KeyFactory.getInstance(clientKeyAlgo);
|
||||
try {
|
||||
// First let's try PKCS8
|
||||
return keyFactory.generatePrivate(new PKCS8EncodedKeySpec(keyBytes));
|
||||
}
|
||||
catch (InvalidKeySpecException e) {
|
||||
// Otherwise try PKCS1
|
||||
RSAPrivateCrtKeySpec keySpec = decodePKCS1(keyBytes);
|
||||
return keyFactory.generatePrivate(keySpec);
|
||||
}
|
||||
}
|
||||
|
||||
public static KeyStore createKeyStore(InputStream certInputStream, InputStream keyInputStream, String clientKeyAlgo,
|
||||
char[] clientKeyPassphrase, String keyStoreFile, char[] keyStorePassphrase) throws IOException,
|
||||
CertificateException, NoSuchAlgorithmException, InvalidKeySpecException, KeyStoreException {
|
||||
CertificateFactory certFactory = CertificateFactory.getInstance("X509");
|
||||
X509Certificate cert = (X509Certificate) certFactory.generateCertificate(certInputStream);
|
||||
|
||||
PrivateKey privateKey = clientKeyAlgo.equals("EC")
|
||||
? generateEcKey(keyInputStream)
|
||||
: generateStdKey(keyInputStream, clientKeyAlgo);
|
||||
|
||||
KeyStore keyStore = KeyStore.getInstance("JKS");
|
||||
if (keyStoreFile != null && keyStoreFile.length() > 0) {
|
||||
keyStore.load(new FileInputStream(keyStoreFile), keyStorePassphrase);
|
||||
} else {
|
||||
loadDefaultKeyStoreFile(keyStore, keyStorePassphrase);
|
||||
}
|
||||
|
||||
String alias = cert.getSubjectX500Principal().getName();
|
||||
keyStore.setKeyEntry(alias, privateKey, clientKeyPassphrase, new Certificate[] { cert });
|
||||
|
||||
return keyStore;
|
||||
}
|
||||
|
||||
// This method is inspired and partly taken over from
|
||||
// http://oauth.googlecode.com/svn/code/java/
|
||||
// All credits to belong to them.
|
||||
private static byte[] decodePem(InputStream keyInputStream) throws IOException {
|
||||
BufferedReader reader = new BufferedReader(new InputStreamReader(keyInputStream));
|
||||
try {
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
if (line.contains("-----BEGIN ")) {
|
||||
return readBytes(reader, line.trim().replace("BEGIN", "END"));
|
||||
}
|
||||
}
|
||||
throw new IOException("PEM is invalid: no begin marker");
|
||||
} finally {
|
||||
reader.close();
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] readBytes(BufferedReader reader, String endMarker) throws IOException {
|
||||
String line;
|
||||
StringBuffer buf = new StringBuffer();
|
||||
|
||||
while ((line = reader.readLine()) != null) {
|
||||
if (line.indexOf(endMarker) != -1) {
|
||||
return Base64.decodeBase64(buf.toString());
|
||||
}
|
||||
buf.append(line.trim());
|
||||
}
|
||||
throw new IOException("PEM is invalid : No end marker");
|
||||
}
|
||||
|
||||
public static RSAPrivateCrtKeySpec decodePKCS1(byte[] keyBytes) throws IOException {
|
||||
DerParser parser = new DerParser(keyBytes);
|
||||
Asn1Object sequence = parser.read();
|
||||
sequence.validateSequence();
|
||||
parser = new DerParser(sequence.getValue());
|
||||
parser.read();
|
||||
|
||||
return new RSAPrivateCrtKeySpec(next(parser), next(parser), next(parser), next(parser), next(parser),
|
||||
next(parser), next(parser), next(parser));
|
||||
}
|
||||
|
||||
private static BigInteger next(DerParser parser) throws IOException {
|
||||
return parser.read().getInteger();
|
||||
}
|
||||
|
||||
static class DerParser {
|
||||
|
||||
private InputStream in;
|
||||
|
||||
DerParser(byte[] bytes) throws IOException {
|
||||
this.in = new ByteArrayInputStream(bytes);
|
||||
}
|
||||
|
||||
Asn1Object read() throws IOException {
|
||||
int tag = in.read();
|
||||
|
||||
if (tag == -1) {
|
||||
throw new IOException("Invalid DER: stream too short, missing tag");
|
||||
}
|
||||
|
||||
int length = getLength();
|
||||
byte[] value = new byte[length];
|
||||
if (in.read(value) < length) {
|
||||
throw new IOException("Invalid DER: stream too short, missing value");
|
||||
}
|
||||
|
||||
return new Asn1Object(tag, value);
|
||||
}
|
||||
|
||||
private int getLength() throws IOException {
|
||||
int i = in.read();
|
||||
if (i == -1) {
|
||||
throw new IOException("Invalid DER: length missing");
|
||||
}
|
||||
|
||||
if ((i & ~0x7F) == 0) {
|
||||
return i;
|
||||
}
|
||||
|
||||
int num = i & 0x7F;
|
||||
if (i >= 0xFF || num > 4) {
|
||||
throw new IOException("Invalid DER: length field too big (" + i + ")");
|
||||
}
|
||||
|
||||
byte[] bytes = new byte[num];
|
||||
if (in.read(bytes) < num) {
|
||||
throw new IOException("Invalid DER: length too short");
|
||||
}
|
||||
|
||||
return new BigInteger(1, bytes).intValue();
|
||||
}
|
||||
}
|
||||
|
||||
static class Asn1Object {
|
||||
|
||||
private final int type;
|
||||
private final byte[] value;
|
||||
private final int tag;
|
||||
|
||||
public Asn1Object(int tag, byte[] value) {
|
||||
this.tag = tag;
|
||||
this.type = tag & 0x1F;
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
public byte[] getValue() {
|
||||
return value;
|
||||
}
|
||||
|
||||
BigInteger getInteger() throws IOException {
|
||||
if (type != 0x02) {
|
||||
throw new IOException("Invalid DER: object is not integer"); //$NON-NLS-1$
|
||||
}
|
||||
return new BigInteger(value);
|
||||
}
|
||||
|
||||
void validateSequence() throws IOException {
|
||||
if (type != 0x10) {
|
||||
throw new IOException("Invalid DER: not a sequence");
|
||||
}
|
||||
if ((tag & 0x20) != 0x20) {
|
||||
throw new IOException("Invalid DER: can't parse primitive entity");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void loadDefaultKeyStoreFile(KeyStore keyStore, char[] keyStorePassphrase)
|
||||
throws CertificateException, NoSuchAlgorithmException, IOException {
|
||||
|
||||
String keyStorePath = System.getProperty("javax.net.ssl.keyStore");
|
||||
if (keyStorePath != null && keyStorePath.length() > 0) {
|
||||
File keyStoreFile = new File(keyStorePath);
|
||||
if (loadDefaultStoreFile(keyStore, keyStoreFile, keyStorePassphrase)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
keyStore.load(null);
|
||||
}
|
||||
|
||||
private static boolean loadDefaultStoreFile(KeyStore keyStore, File fileToLoad, char[] passphrase)
|
||||
throws CertificateException, NoSuchAlgorithmException, IOException {
|
||||
if (fileToLoad.exists() && fileToLoad.isFile() && fileToLoad.length() > 0) {
|
||||
keyStore.load(new FileInputStream(fileToLoad), passphrase);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public static InputStream getInputStreamFromDataOrFile(String data, String file) throws FileNotFoundException {
|
||||
if (data != null) {
|
||||
byte[] bytes = Base64.decodeBase64(data);
|
||||
// TODO handle non-base64 here?
|
||||
return new ByteArrayInputStream(bytes);
|
||||
}
|
||||
if (file != null) {
|
||||
return new FileInputStream(file);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static char[] getKeyStorePassphrase(String keyStorePassphrase) {
|
||||
if (keyStorePassphrase == null || keyStorePassphrase.length() == 0) {
|
||||
return System.getProperty("javax.net.ssl.keyStorePassword", "changeit").toCharArray();
|
||||
}
|
||||
return keyStorePassphrase.toCharArray();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.model
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import groovy.transform.ToString
|
||||
|
||||
/**
|
||||
* Model a K8s pod environment variable definition
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
@ToString(includeNames = true)
|
||||
@EqualsAndHashCode(includeFields = true)
|
||||
class PodEnv {
|
||||
|
||||
private Map spec
|
||||
|
||||
private PodEnv(Map spec) {
|
||||
this.spec = spec
|
||||
}
|
||||
|
||||
static PodEnv value(String env, String value) {
|
||||
new PodEnv([name:env, value:value])
|
||||
}
|
||||
|
||||
static PodEnv fieldPath(String env, String fieldPath) {
|
||||
new PodEnv([ name: env, valueFrom: [fieldRef:[fieldPath: fieldPath]]])
|
||||
}
|
||||
|
||||
static PodEnv config(String env, String config) {
|
||||
final tokens = config.tokenize('/')
|
||||
if( tokens.size() > 2 )
|
||||
throw new IllegalArgumentException("K8s invalid pod env file: $config -- Secret must be specified as <config-name>/<config-key>")
|
||||
|
||||
final name = tokens[0]
|
||||
final key = tokens[1]
|
||||
|
||||
assert env, 'Missing pod env variable name'
|
||||
assert name, 'Missing pod env config name'
|
||||
|
||||
final ref = [ name: name, key: (key ?: env) ]
|
||||
new PodEnv([ name: env, valueFrom: [configMapKeyRef: ref]])
|
||||
}
|
||||
|
||||
static PodEnv secret(String env, String secret) {
|
||||
|
||||
final tokens = secret.tokenize('/')
|
||||
if( tokens.size() > 2 )
|
||||
throw new IllegalArgumentException("K8s invalid pod env secret: $secret -- Secret must be specified as <secret-name>/<secret-key>")
|
||||
|
||||
final name = tokens[0]
|
||||
final key = tokens[1]
|
||||
|
||||
final ref = [ name: name, key: (key ?: env) ]
|
||||
new PodEnv([ name: env, valueFrom: [secretKeyRef: ref]])
|
||||
}
|
||||
|
||||
|
||||
Map toSpec() { spec }
|
||||
|
||||
String toString() {
|
||||
"PodEnv[ ${spec?.toString()} ]"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.model
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import groovy.transform.ToString
|
||||
|
||||
/**
|
||||
* Model a K8s pod host mount definition
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@EqualsAndHashCode
|
||||
@ToString(includeNames = true)
|
||||
@CompileStatic
|
||||
class PodHostMount {
|
||||
|
||||
String hostPath
|
||||
|
||||
String mountPath
|
||||
|
||||
PodHostMount(String host, String container) {
|
||||
this.hostPath = host
|
||||
this.mountPath = container
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.model
|
||||
|
||||
import java.nio.file.Paths
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import groovy.transform.ToString
|
||||
|
||||
/**
|
||||
* Model a K8s pod ConfigMap mount
|
||||
*
|
||||
* See also https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
@ToString(includeNames = true)
|
||||
@EqualsAndHashCode
|
||||
class PodMountConfig {
|
||||
|
||||
String mountPath
|
||||
|
||||
String fileName
|
||||
|
||||
String configName
|
||||
|
||||
String configKey
|
||||
|
||||
PodMountConfig( String config, String mount ) {
|
||||
assert config
|
||||
assert mount
|
||||
|
||||
final path = Paths.get(mount)
|
||||
final tokens = config.tokenize('/')
|
||||
configName = tokens[0].trim()
|
||||
configKey = tokens.size()>1 ? tokens[1].trim() : null
|
||||
if( configKey ) {
|
||||
mountPath = path.parent.toString()
|
||||
fileName = path.fileName.toString()
|
||||
}
|
||||
else {
|
||||
mountPath = path.toString()
|
||||
}
|
||||
}
|
||||
|
||||
PodMountConfig( Map entry ) {
|
||||
this(entry.config as String, entry.mountPath as String)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.model
|
||||
|
||||
import java.nio.file.Paths
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import groovy.transform.ToString
|
||||
|
||||
/**
|
||||
* Model a K8s pod CSI ephemeral volume mount
|
||||
*
|
||||
* See also https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/#csi-ephemeral-volumes
|
||||
*
|
||||
* @author Ben Sherman <bentshermann@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
@ToString(includeNames = true)
|
||||
@EqualsAndHashCode
|
||||
class PodMountCsiEphemeral {
|
||||
|
||||
String mountPath
|
||||
|
||||
Map csi
|
||||
|
||||
PodMountCsiEphemeral( Map csi, String mountPath ) {
|
||||
assert csi
|
||||
assert mountPath
|
||||
|
||||
this.csi = csi
|
||||
this.mountPath = mountPath
|
||||
}
|
||||
|
||||
PodMountCsiEphemeral( Map entry ) {
|
||||
this(entry.csi as Map, entry.mountPath as String)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.model
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import groovy.transform.ToString
|
||||
|
||||
/**
|
||||
* Model a K8s pod emptyDir mount
|
||||
*
|
||||
* See also https://kubernetes.io/docs/concepts/storage/volumes/#emptydir
|
||||
*
|
||||
* @author Ben Sherman <bentshermann@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
@ToString(includeNames = true)
|
||||
@EqualsAndHashCode
|
||||
class PodMountEmptyDir {
|
||||
|
||||
String mountPath
|
||||
|
||||
Map emptyDir
|
||||
|
||||
PodMountEmptyDir( Map emptyDir, String mountPath ) {
|
||||
assert mountPath
|
||||
|
||||
this.emptyDir = emptyDir
|
||||
this.mountPath = mountPath
|
||||
}
|
||||
|
||||
PodMountEmptyDir( Map entry ) {
|
||||
this(entry.emptyDir as Map, entry.mountPath as String)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.model
|
||||
|
||||
import java.nio.file.Paths
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import groovy.transform.ToString
|
||||
|
||||
/**
|
||||
* Model a K8s Secret file mount
|
||||
*
|
||||
* https://kubernetes.io/docs/concepts/configuration/secret/
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
@ToString(includeNames = true)
|
||||
@EqualsAndHashCode
|
||||
class PodMountSecret {
|
||||
|
||||
String mountPath
|
||||
|
||||
String fileName
|
||||
|
||||
String secretName
|
||||
|
||||
String secretKey
|
||||
|
||||
PodMountSecret(String secret, String mount) {
|
||||
assert secret
|
||||
assert mount
|
||||
|
||||
final path = Paths.get(mount)
|
||||
final tokens = secret.tokenize('/')
|
||||
secretName = tokens[0].trim()
|
||||
secretKey = tokens.size()>1 ? tokens[1].trim() : null
|
||||
if( secretKey ) {
|
||||
mountPath = path.parent.toString()
|
||||
fileName = path.fileName.toString()
|
||||
}
|
||||
else {
|
||||
mountPath = path.toString()
|
||||
}
|
||||
}
|
||||
|
||||
PodMountSecret(Map entry) {
|
||||
this(entry.secret as String, entry.mountPath as String)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.model
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import groovy.transform.ToString
|
||||
|
||||
/**
|
||||
* Model a Pod nodeSelector spec
|
||||
*
|
||||
* https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
@ToString(includeNames = true)
|
||||
@EqualsAndHashCode(includeFields = true)
|
||||
class PodNodeSelector {
|
||||
|
||||
private Map spec = [:]
|
||||
|
||||
PodNodeSelector(selector) {
|
||||
if( selector instanceof CharSequence )
|
||||
createWithString(selector.toString())
|
||||
|
||||
else if( selector instanceof Map )
|
||||
createWithMap(selector)
|
||||
|
||||
else if( selector != null )
|
||||
throw new IllegalArgumentException("K8s invalid pod nodeSelector value: $selector [${selector.getClass().getName()}]")
|
||||
}
|
||||
|
||||
private createWithMap(Map selection ) {
|
||||
if(selection) {
|
||||
for( Map.Entry entry : selection.entrySet() ) {
|
||||
spec.put(entry.key.toString(), entry.value?.toString())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param selector
|
||||
* A string representing a comma separated list of pairs
|
||||
* e.g. foo=1,bar=2
|
||||
*
|
||||
*/
|
||||
private createWithString( String selector ) {
|
||||
if(!selector) return
|
||||
def entries = selector.tokenize(',')
|
||||
for( String item : entries ) {
|
||||
def pair = item.tokenize('=')
|
||||
spec.put( trim(pair[0]), trim(pair[1]) ?: 'true' )
|
||||
}
|
||||
}
|
||||
|
||||
private String trim(String v) {
|
||||
v?.trim()
|
||||
}
|
||||
|
||||
Map<String,String> toSpec() { spec }
|
||||
|
||||
String toString() {
|
||||
"PodNodeSelector[ ${spec?.toString()} ]"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,323 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.model
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import groovy.transform.PackageScope
|
||||
import groovy.transform.ToString
|
||||
|
||||
/**
|
||||
* Model K8s pod options such as environment variables,
|
||||
* secret and config-maps
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
@ToString(includeNames = true)
|
||||
@EqualsAndHashCode(includeFields = true)
|
||||
class PodOptions {
|
||||
|
||||
private String imagePullPolicy
|
||||
|
||||
private String imagePullSecret
|
||||
|
||||
private Collection<PodEnv> envVars
|
||||
|
||||
private Collection<PodMountConfig> mountConfigMaps
|
||||
|
||||
private Collection<PodMountCsiEphemeral> mountCsiEphemerals
|
||||
|
||||
private Collection<PodMountEmptyDir> mountEmptyDirs
|
||||
|
||||
private Collection<PodMountSecret> mountSecrets
|
||||
|
||||
private Collection<PodVolumeClaim> mountClaims
|
||||
|
||||
private Collection<PodHostMount> mountHostPaths
|
||||
|
||||
private Map<String,String> labels = [:]
|
||||
|
||||
private Map<String,String> annotations = [:]
|
||||
|
||||
private PodNodeSelector nodeSelector
|
||||
|
||||
private Map affinity
|
||||
|
||||
private PodSecurityContext securityContext
|
||||
|
||||
private boolean automountServiceAccountToken
|
||||
|
||||
private String priorityClassName
|
||||
|
||||
private List<Map> tolerations
|
||||
|
||||
private Boolean privileged
|
||||
|
||||
private String schedulerName
|
||||
|
||||
private Integer ttlSecondsAfterFinished
|
||||
|
||||
private String runtimeClassName
|
||||
|
||||
PodOptions( List<Map> options=null ) {
|
||||
int size = options ? options.size() : 0
|
||||
envVars = new HashSet<>(size)
|
||||
mountConfigMaps = new HashSet<>(size)
|
||||
mountCsiEphemerals = new HashSet<>(size)
|
||||
mountEmptyDirs = new HashSet<>(size)
|
||||
mountSecrets = new HashSet<>(size)
|
||||
mountClaims = new HashSet<>(size)
|
||||
mountHostPaths = new HashSet<>(10)
|
||||
automountServiceAccountToken = true
|
||||
tolerations = new ArrayList<Map>(size)
|
||||
init(options)
|
||||
}
|
||||
|
||||
@PackageScope void init(List<Map> options) {
|
||||
if( !options ) return
|
||||
for( Map entry : options ) {
|
||||
create(entry)
|
||||
}
|
||||
}
|
||||
|
||||
@PackageScope void create(Map<String,String> entry) {
|
||||
if( entry.env && entry.value ) {
|
||||
envVars << PodEnv.value(entry.env, entry.value)
|
||||
}
|
||||
else if( entry.env && entry.fieldPath ) {
|
||||
envVars << PodEnv.fieldPath(entry.env, entry.fieldPath)
|
||||
}
|
||||
else if( entry.env && entry.secret ) {
|
||||
envVars << PodEnv.secret(entry.env, entry.secret)
|
||||
}
|
||||
else if( entry.env && entry.config ) {
|
||||
envVars << PodEnv.config(entry.env, entry.config)
|
||||
}
|
||||
else if( entry.mountPath && entry.secret ) {
|
||||
mountSecrets << new PodMountSecret(entry)
|
||||
}
|
||||
else if( entry.mountPath && entry.config ) {
|
||||
mountConfigMaps << new PodMountConfig(entry)
|
||||
}
|
||||
else if( entry.mountPath && entry.csi ) {
|
||||
mountCsiEphemerals << new PodMountCsiEphemeral(entry)
|
||||
}
|
||||
else if( entry.mountPath && entry.emptyDir != null ) {
|
||||
mountEmptyDirs << new PodMountEmptyDir(entry)
|
||||
}
|
||||
else if( entry.mountPath && entry.volumeClaim ) {
|
||||
mountClaims << new PodVolumeClaim(entry)
|
||||
}
|
||||
else if( entry.mountPath && entry.hostPath instanceof CharSequence ) {
|
||||
mountHostPaths << new PodHostMount(entry.hostPath, entry.mountPath)
|
||||
}
|
||||
else if( entry.pullPolicy || entry.imagePullPolicy ) {
|
||||
this.imagePullPolicy = entry.pullPolicy ?: entry.imagePullPolicy as String
|
||||
}
|
||||
else if( entry.imagePullSecret || entry.imagePullSecrets ) {
|
||||
this.imagePullSecret = entry.imagePullSecret ?: entry.imagePullSecrets
|
||||
}
|
||||
else if( entry.label && entry.value ) {
|
||||
this.labels.put(entry.label as String, entry.value as String)
|
||||
}
|
||||
else if( entry.runAsUser != null ) {
|
||||
this.securityContext = new PodSecurityContext(entry.runAsUser)
|
||||
}
|
||||
else if( entry.securityContext instanceof Map ) {
|
||||
this.securityContext = new PodSecurityContext(entry.securityContext as Map)
|
||||
}
|
||||
else if( entry.nodeSelector ) {
|
||||
this.nodeSelector = new PodNodeSelector(entry.nodeSelector)
|
||||
}
|
||||
else if( entry.affinity instanceof Map ) {
|
||||
this.affinity = entry.affinity as Map
|
||||
}
|
||||
else if( entry.annotation && entry.value ) {
|
||||
this.annotations.put(entry.annotation as String, entry.value as String)
|
||||
}
|
||||
else if( entry.automountServiceAccountToken instanceof Boolean ) {
|
||||
this.automountServiceAccountToken = entry.automountServiceAccountToken as Boolean
|
||||
}
|
||||
else if( entry.priorityClassName ) {
|
||||
this.priorityClassName = entry.priorityClassName
|
||||
}
|
||||
else if( entry.toleration instanceof Map ) {
|
||||
tolerations << (entry.toleration as Map)
|
||||
}
|
||||
else if( entry.privileged instanceof Boolean ) {
|
||||
this.privileged = entry.privileged as Boolean
|
||||
}
|
||||
else if( entry.schedulerName ) {
|
||||
this.schedulerName = entry.schedulerName
|
||||
}
|
||||
else if( entry.ttlSecondsAfterFinished instanceof Integer ) {
|
||||
this.ttlSecondsAfterFinished = entry.ttlSecondsAfterFinished as Integer
|
||||
}
|
||||
else if( entry.runtimeClassName ) {
|
||||
this.runtimeClassName = entry.runtimeClassName
|
||||
}
|
||||
else
|
||||
throw new IllegalArgumentException("Unknown pod options: $entry")
|
||||
}
|
||||
|
||||
|
||||
Collection<PodEnv> getEnvVars() { envVars }
|
||||
|
||||
Collection<PodMountConfig> getMountConfigMaps() { mountConfigMaps }
|
||||
|
||||
Collection<PodMountCsiEphemeral> getMountCsiEphemerals() { mountCsiEphemerals }
|
||||
|
||||
Collection<PodMountEmptyDir> getMountEmptyDirs() { mountEmptyDirs }
|
||||
|
||||
Collection<PodMountSecret> getMountSecrets() { mountSecrets }
|
||||
|
||||
Collection<PodHostMount> getMountHostPaths() { mountHostPaths }
|
||||
|
||||
Collection<PodVolumeClaim> getVolumeClaims() { mountClaims }
|
||||
|
||||
Map<String,String> getLabels() { labels }
|
||||
|
||||
Map<String,String> getAnnotations() { annotations }
|
||||
|
||||
PodNodeSelector getNodeSelector() { nodeSelector }
|
||||
|
||||
PodOptions setNodeSelector( PodNodeSelector sel ) {
|
||||
nodeSelector = sel
|
||||
return this
|
||||
}
|
||||
|
||||
Map getAffinity() { affinity }
|
||||
|
||||
PodSecurityContext getSecurityContext() { securityContext }
|
||||
|
||||
PodOptions setSecurityContext( PodSecurityContext ctx ) {
|
||||
this.securityContext = ctx
|
||||
return this
|
||||
}
|
||||
|
||||
String getImagePullSecret() { imagePullSecret }
|
||||
|
||||
PodOptions setImagePullSecret( String secret ) {
|
||||
this.imagePullSecret = secret
|
||||
return this
|
||||
}
|
||||
|
||||
String getImagePullPolicy() { imagePullPolicy }
|
||||
|
||||
PodOptions setImagePullPolicy( String policy ) {
|
||||
this.imagePullPolicy = policy
|
||||
return this
|
||||
}
|
||||
|
||||
boolean getAutomountServiceAccountToken() { automountServiceAccountToken }
|
||||
|
||||
PodOptions setAutomountServiceAccountToken( boolean mount ) {
|
||||
this.automountServiceAccountToken = mount
|
||||
return this
|
||||
}
|
||||
|
||||
String getPriorityClassName() { priorityClassName }
|
||||
|
||||
String getSchedulerName() { schedulerName }
|
||||
|
||||
List<Map> getTolerations() { tolerations }
|
||||
|
||||
Boolean getPrivileged() { privileged }
|
||||
|
||||
Integer getTtlSecondsAfterFinished() { ttlSecondsAfterFinished }
|
||||
|
||||
String getRuntimeClassName() { runtimeClassName }
|
||||
|
||||
PodOptions plus( PodOptions other ) {
|
||||
def result = new PodOptions()
|
||||
|
||||
// env vars
|
||||
result.envVars.addAll(envVars)
|
||||
result.envVars.addAll( other.envVars )
|
||||
|
||||
// config maps
|
||||
result.mountConfigMaps.addAll( mountConfigMaps )
|
||||
result.mountConfigMaps.addAll( other.mountConfigMaps )
|
||||
|
||||
// csi ephemeral volumes
|
||||
result.mountCsiEphemerals.addAll( mountCsiEphemerals )
|
||||
result.mountCsiEphemerals.addAll( other.mountCsiEphemerals )
|
||||
|
||||
// empty dirs
|
||||
result.mountEmptyDirs.addAll( mountEmptyDirs )
|
||||
result.mountEmptyDirs.addAll( other.mountEmptyDirs )
|
||||
|
||||
// host paths
|
||||
result.mountHostPaths.addAll( mountHostPaths )
|
||||
result.mountHostPaths.addAll( other.mountHostPaths )
|
||||
|
||||
// secrets
|
||||
result.mountSecrets.addAll( mountSecrets )
|
||||
result.mountSecrets.addAll( other.mountSecrets )
|
||||
|
||||
// volume claims
|
||||
result.volumeClaims.addAll( volumeClaims )
|
||||
result.volumeClaims.addAll( other.volumeClaims )
|
||||
|
||||
// sec context
|
||||
result.securityContext = other.securityContext ?: this.securityContext
|
||||
|
||||
// node selector
|
||||
result.nodeSelector = other.nodeSelector ?: this.nodeSelector
|
||||
|
||||
// affinity
|
||||
result.affinity = other.affinity ?: this.affinity
|
||||
|
||||
// pull policy
|
||||
result.imagePullPolicy = other.imagePullPolicy ?: this.imagePullPolicy
|
||||
|
||||
// image secret
|
||||
result.imagePullSecret = other.imagePullSecret ?: this.imagePullSecret
|
||||
|
||||
// labels
|
||||
result.labels.putAll(labels)
|
||||
result.labels.putAll(other.labels)
|
||||
|
||||
// annotations
|
||||
result.annotations.putAll(annotations)
|
||||
result.annotations.putAll(other.annotations)
|
||||
|
||||
// automount service account token
|
||||
result.automountServiceAccountToken = other.automountServiceAccountToken & this.automountServiceAccountToken
|
||||
|
||||
// priority class name
|
||||
result.priorityClassName = other.priorityClassName ?: this.priorityClassName
|
||||
|
||||
// tolerations
|
||||
result.tolerations = other.tolerations ?: this.tolerations
|
||||
|
||||
// privileged execution
|
||||
result.privileged = other.privileged!=null ? other.privileged : this.privileged
|
||||
|
||||
// scheduler name
|
||||
result.schedulerName = other.schedulerName ?: this.schedulerName
|
||||
|
||||
// ttl seconds after finished (job)
|
||||
result.ttlSecondsAfterFinished = other.ttlSecondsAfterFinished!=null ? other.ttlSecondsAfterFinished : this.ttlSecondsAfterFinished
|
||||
|
||||
// runtime class name
|
||||
result.runtimeClassName = other.runtimeClassName ?: this.runtimeClassName
|
||||
|
||||
return result
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.model
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import groovy.transform.ToString
|
||||
|
||||
/**
|
||||
* Models K8s pod security context
|
||||
*
|
||||
* See
|
||||
* https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
@ToString(includeNames = true)
|
||||
@EqualsAndHashCode(includeFields = true)
|
||||
class PodSecurityContext {
|
||||
|
||||
private Map spec
|
||||
|
||||
PodSecurityContext(def user) {
|
||||
spec = [runAsUser: user]
|
||||
}
|
||||
|
||||
PodSecurityContext(Map ctx) {
|
||||
assert ctx
|
||||
spec = ctx
|
||||
}
|
||||
|
||||
Map toSpec() { spec }
|
||||
|
||||
String toString() {
|
||||
"PodSecurityContext[ ${spec?.toString()} ]"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,814 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.model
|
||||
|
||||
import java.nio.file.Path
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
import groovy.transform.CompileDynamic
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.PackageScope
|
||||
import nextflow.executor.res.AcceleratorResource
|
||||
import nextflow.util.MemoryUnit
|
||||
import groovy.util.logging.Slf4j
|
||||
|
||||
/**
|
||||
* Object build for a K8s pod specification
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
@Slf4j
|
||||
class PodSpecBuilder {
|
||||
|
||||
static enum MetaType { LABEL, ANNOTATION }
|
||||
|
||||
static enum SegmentType {
|
||||
PREFIX (253),
|
||||
NAME (63),
|
||||
VALUE (63)
|
||||
|
||||
private final int maxSize;
|
||||
SegmentType(int maxSize) {
|
||||
this.maxSize = maxSize;
|
||||
}
|
||||
}
|
||||
|
||||
static @PackageScope AtomicInteger VOLUMES = new AtomicInteger()
|
||||
|
||||
String podName
|
||||
|
||||
String imageName
|
||||
|
||||
String imagePullPolicy
|
||||
|
||||
String imagePullSecret
|
||||
|
||||
List<String> command = []
|
||||
|
||||
List<String> args = new ArrayList<>()
|
||||
|
||||
Map<String,String> labels = [:]
|
||||
|
||||
Map<String,String> annotations = [:]
|
||||
|
||||
String namespace
|
||||
|
||||
String restart
|
||||
|
||||
List<PodEnv> envVars = []
|
||||
|
||||
String workDir
|
||||
|
||||
Integer cpus
|
||||
|
||||
boolean cpuLimits
|
||||
|
||||
String memory
|
||||
|
||||
String disk
|
||||
|
||||
String serviceAccount
|
||||
|
||||
boolean automountServiceAccountToken = true
|
||||
|
||||
AcceleratorResource accelerator
|
||||
|
||||
Collection<PodMountConfig> configMaps = []
|
||||
|
||||
Collection<PodMountCsiEphemeral> csiEphemerals = []
|
||||
|
||||
Collection<PodMountEmptyDir> emptyDirs = []
|
||||
|
||||
Collection<PodMountSecret> secrets = []
|
||||
|
||||
Collection<PodHostMount> hostMounts = []
|
||||
|
||||
Collection<PodVolumeClaim> volumeClaims = []
|
||||
|
||||
PodSecurityContext securityContext
|
||||
|
||||
PodNodeSelector nodeSelector
|
||||
|
||||
Map affinity
|
||||
|
||||
String priorityClassName
|
||||
|
||||
List<Map> tolerations = []
|
||||
|
||||
boolean privileged
|
||||
|
||||
int activeDeadlineSeconds
|
||||
|
||||
Map<String,List<String>> capabilities
|
||||
|
||||
List<String> devices
|
||||
|
||||
Map<String,?> resourcesLimits
|
||||
|
||||
String schedulerName
|
||||
|
||||
Integer ttlSecondsAfterFinished
|
||||
|
||||
String runtimeClassName
|
||||
|
||||
String nodeName
|
||||
|
||||
Integer port = null
|
||||
|
||||
/**
|
||||
* @return A sequential volume unique identifier
|
||||
*/
|
||||
static protected String nextVolName() {
|
||||
"vol-${VOLUMES.incrementAndGet()}".toString()
|
||||
}
|
||||
|
||||
PodSpecBuilder withPodName(String name) {
|
||||
this.podName = name
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withImageName(String name) {
|
||||
this.imageName = name
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withImagePullPolicy(String policy) {
|
||||
this.imagePullPolicy = policy
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withWorkDir( String path ) {
|
||||
this.workDir = path
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withWorkDir(Path path ) {
|
||||
this.workDir = path.toString()
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withNamespace(String name) {
|
||||
this.namespace = name
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withServiceAccount(String name) {
|
||||
this.serviceAccount = name
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withCommand( cmd ) {
|
||||
if( cmd==null ) return this
|
||||
assert cmd instanceof List || cmd instanceof CharSequence, "Missing or invalid K8s command parameter: $cmd"
|
||||
this.command = cmd instanceof List ? cmd as List<String> : ['/bin/bash','-c', cmd.toString()]
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withArgs( args ) {
|
||||
if( args==null ) return this
|
||||
assert args instanceof List || args instanceof CharSequence, "Missing or invalid K8s args parameter: $args"
|
||||
this.args = args instanceof List ? args as List<String> : ['/bin/bash','-c', args.toString()]
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withCpus( Integer cpus ) {
|
||||
this.cpus = cpus
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withCpuLimits(boolean cpuLimits) {
|
||||
this.cpuLimits = cpuLimits
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withMemory(String mem) {
|
||||
this.memory = mem
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withMemory(MemoryUnit mem) {
|
||||
this.memory = "${mem.mega}Mi".toString()
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withDisk(String disk) {
|
||||
this.disk = disk
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withDisk(MemoryUnit disk) {
|
||||
this.disk = "${disk.mega}Mi".toString()
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withAccelerator(AcceleratorResource acc) {
|
||||
this.accelerator = acc
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withLabel( String name, String value ) {
|
||||
this.labels.put(name, value)
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withLabels(Map labels) {
|
||||
this.labels.putAll(labels)
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withAnnotation( String name, String value ) {
|
||||
this.annotations.put(name, value)
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withAnnotations(Map annotations) {
|
||||
this.annotations.putAll(annotations)
|
||||
return this
|
||||
}
|
||||
|
||||
|
||||
PodSpecBuilder withEnv( PodEnv env ) {
|
||||
envVars.add(env)
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withEnv( Collection envs ) {
|
||||
envVars.addAll(envs)
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withVolumeClaim( PodVolumeClaim claim ) {
|
||||
volumeClaims.add(claim)
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withVolumeClaims( Collection<PodVolumeClaim> claims ) {
|
||||
volumeClaims.addAll(claims)
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withConfigMaps( Collection<PodMountConfig> configMaps ) {
|
||||
this.configMaps.addAll(configMaps)
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withConfigMap( PodMountConfig configMap ) {
|
||||
this.configMaps.add(configMap)
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withCsiEphemerals( Collection<PodMountCsiEphemeral> csiEphemerals ) {
|
||||
this.csiEphemerals.addAll(csiEphemerals)
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withCsiEphemeral( PodMountCsiEphemeral csiEphemeral ) {
|
||||
this.csiEphemerals.add(csiEphemeral)
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withEmptyDirs( Collection<PodMountEmptyDir> emptyDirs ) {
|
||||
this.emptyDirs.addAll(emptyDirs)
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withEmptyDir( PodMountEmptyDir emptyDir ) {
|
||||
this.emptyDirs.add(emptyDir)
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withSecrets( Collection<PodMountSecret> secrets ) {
|
||||
this.secrets.addAll(secrets)
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withSecret( PodMountSecret secret ) {
|
||||
this.secrets.add(secret)
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withHostMounts( Collection<PodHostMount> mounts ) {
|
||||
this.hostMounts.addAll(mounts)
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withHostMount( String host, String mount ) {
|
||||
this.hostMounts.add( new PodHostMount(host, mount))
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withPrivileged(boolean value) {
|
||||
this.privileged = value
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withCapabilities(Map<String,List<String>> cap) {
|
||||
this.capabilities = cap
|
||||
for( String it : cap.keySet() ) {
|
||||
if( it !in ['add','drop']) throw new IllegalArgumentException("K8s capability action can be either 'add' or 'drop' - offending value '$it'")
|
||||
}
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withActiveDeadline(int seconds) {
|
||||
this.activeDeadlineSeconds = seconds
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withResourcesLimits(Map<String,?> limits) {
|
||||
this.resourcesLimits = limits
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withPodOptions(PodOptions opts) {
|
||||
// -- pull policy
|
||||
if( opts.imagePullPolicy )
|
||||
imagePullPolicy = opts.imagePullPolicy
|
||||
if( opts.imagePullSecret )
|
||||
imagePullSecret = opts.imagePullSecret
|
||||
// -- env vars
|
||||
if( opts.getEnvVars() )
|
||||
envVars.addAll( opts.getEnvVars() )
|
||||
// -- configMaps
|
||||
if( opts.getMountConfigMaps() )
|
||||
configMaps.addAll( opts.getMountConfigMaps() )
|
||||
// -- csi ephemeral volumes
|
||||
if( opts.getMountCsiEphemerals() )
|
||||
csiEphemerals.addAll( opts.getMountCsiEphemerals() )
|
||||
// -- emptyDirs
|
||||
if( opts.getMountEmptyDirs() )
|
||||
emptyDirs.addAll( opts.getMountEmptyDirs() )
|
||||
// -- host paths
|
||||
if( opts.getMountHostPaths() )
|
||||
hostMounts.addAll( opts.getMountHostPaths() )
|
||||
// -- secrets
|
||||
if( opts.getMountSecrets() )
|
||||
secrets.addAll( opts.getMountSecrets() )
|
||||
// -- volume claims
|
||||
if( opts.getVolumeClaims() )
|
||||
volumeClaims.addAll( opts.getVolumeClaims() )
|
||||
// -- labels
|
||||
if( opts.labels ) {
|
||||
def keys = opts.labels.keySet()
|
||||
if( 'app' in keys ) throw new IllegalArgumentException("Invalid pod label -- `app` is a reserved label")
|
||||
if( 'runName' in keys ) throw new IllegalArgumentException("Invalid pod label -- `runName` is a reserved label")
|
||||
labels.putAll( opts.labels )
|
||||
}
|
||||
// - annotations
|
||||
if( opts.annotations ) {
|
||||
annotations.putAll( opts.annotations )
|
||||
}
|
||||
// -- security context
|
||||
if( opts.securityContext )
|
||||
securityContext = opts.securityContext
|
||||
// -- node selector
|
||||
if( opts.nodeSelector )
|
||||
nodeSelector = opts.nodeSelector
|
||||
// -- affinity
|
||||
if( opts.affinity )
|
||||
affinity = opts.affinity
|
||||
// -- automount service account token
|
||||
automountServiceAccountToken = opts.automountServiceAccountToken
|
||||
// -- priority class name
|
||||
priorityClassName = opts.priorityClassName
|
||||
// -- tolerations
|
||||
if( opts.tolerations )
|
||||
tolerations.addAll(opts.tolerations)
|
||||
// -- privileged
|
||||
privileged = opts.privileged
|
||||
// -- scheduler name
|
||||
schedulerName = opts.schedulerName
|
||||
// -- ttl seconds after finished (job)
|
||||
if( opts.ttlSecondsAfterFinished != null )
|
||||
ttlSecondsAfterFinished = opts.ttlSecondsAfterFinished
|
||||
// runtime class name
|
||||
if( opts.runtimeClassName != null )
|
||||
runtimeClassName = opts.runtimeClassName
|
||||
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withNodeName(String value) {
|
||||
this.nodeName = value
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withPort(int value) {
|
||||
this.port = new Integer(value)
|
||||
return this
|
||||
}
|
||||
|
||||
@PackageScope List<Map> createPullSecret() {
|
||||
def result = new ArrayList(1)
|
||||
def entry = new LinkedHashMap(1)
|
||||
entry.name = imagePullSecret
|
||||
result.add(entry)
|
||||
return result
|
||||
}
|
||||
|
||||
Map build() {
|
||||
assert this.podName, 'Missing K8s podName parameter'
|
||||
assert this.imageName, 'Missing K8s imageName parameter'
|
||||
assert this.command || this.args, 'Missing K8s command parameter'
|
||||
|
||||
final restart = this.restart ?: 'Never'
|
||||
|
||||
final metadata = new LinkedHashMap<String,Object>()
|
||||
metadata.name = podName
|
||||
metadata.namespace = namespace ?: 'default'
|
||||
|
||||
final labels = this.labels ?: [:]
|
||||
final env = []
|
||||
for( PodEnv entry : this.envVars ) {
|
||||
env.add(entry.toSpec())
|
||||
}
|
||||
|
||||
final container = [ name: this.podName, image: this.imageName ]
|
||||
if( this.command )
|
||||
container.command = this.command
|
||||
if( this.args )
|
||||
container.args = args
|
||||
|
||||
if( this.workDir )
|
||||
container.put('workingDir', workDir)
|
||||
|
||||
if( imagePullPolicy )
|
||||
container.imagePullPolicy = imagePullPolicy
|
||||
|
||||
if( this.port ) {
|
||||
List<Map> ports = []
|
||||
ports << [containerPort: this.port.intValue()]
|
||||
container.ports = ports
|
||||
}
|
||||
|
||||
final secContext = new LinkedHashMap(10)
|
||||
if( privileged ) {
|
||||
// note: privileged flag needs to be defined in the *container* securityContext
|
||||
// not the 'spec' securityContext (see below)
|
||||
secContext.privileged =true
|
||||
}
|
||||
if( capabilities ) {
|
||||
secContext.capabilities = capabilities
|
||||
}
|
||||
if( secContext ) {
|
||||
container.securityContext = secContext
|
||||
}
|
||||
|
||||
final spec = [
|
||||
restartPolicy: restart,
|
||||
containers: [ container ],
|
||||
]
|
||||
|
||||
if( nodeSelector )
|
||||
spec.nodeSelector = nodeSelector.toSpec()
|
||||
|
||||
if( schedulerName )
|
||||
spec.schedulerName = schedulerName
|
||||
|
||||
if( affinity )
|
||||
spec.affinity = affinity
|
||||
|
||||
if( this.serviceAccount )
|
||||
spec.serviceAccountName = this.serviceAccount
|
||||
|
||||
if( ! this.automountServiceAccountToken )
|
||||
spec.automountServiceAccountToken = false
|
||||
|
||||
if( securityContext )
|
||||
spec.securityContext = securityContext.toSpec()
|
||||
|
||||
if( imagePullSecret )
|
||||
spec.imagePullSecrets = createPullSecret()
|
||||
|
||||
if( priorityClassName )
|
||||
spec.priorityClassName = priorityClassName
|
||||
|
||||
// tolerations
|
||||
if( this.tolerations )
|
||||
spec.tolerations = this.tolerations
|
||||
|
||||
// add labels
|
||||
if( labels )
|
||||
metadata.labels = sanitize(labels, MetaType.LABEL)
|
||||
|
||||
if( annotations )
|
||||
metadata.annotations = sanitize(annotations, MetaType.ANNOTATION)
|
||||
|
||||
// time directive
|
||||
if ( activeDeadlineSeconds > 0)
|
||||
spec.activeDeadlineSeconds = activeDeadlineSeconds
|
||||
|
||||
if ( runtimeClassName )
|
||||
spec.runtimeClassName = runtimeClassName
|
||||
|
||||
if ( nodeName )
|
||||
spec.nodeName = nodeName
|
||||
|
||||
final pod = [
|
||||
apiVersion: 'v1',
|
||||
kind: 'Pod',
|
||||
metadata: metadata,
|
||||
spec: spec
|
||||
]
|
||||
|
||||
// add environment
|
||||
if( env )
|
||||
container.env = env
|
||||
|
||||
// add resources
|
||||
if( this.cpus ) {
|
||||
container.resources = addCpuResources(this.cpus, container.resources as Map)
|
||||
}
|
||||
|
||||
if( this.memory ) {
|
||||
container.resources = addMemoryResources(this.memory, container.resources as Map)
|
||||
}
|
||||
|
||||
if( this.accelerator ) {
|
||||
container.resources = addAcceleratorResources(this.accelerator, container.resources as Map)
|
||||
}
|
||||
|
||||
if( this.disk ) {
|
||||
container.resources = addDiskResources(this.disk, container.resources as Map)
|
||||
}
|
||||
|
||||
if( this.resourcesLimits ) {
|
||||
container.resources = addResourcesLimits(this.resourcesLimits, container.resources as Map)
|
||||
}
|
||||
|
||||
// add storage definitions ie. volumes and mounts
|
||||
final List<Map> mounts = []
|
||||
final List<Map> volumes = []
|
||||
final namesMap = [:]
|
||||
|
||||
// creates a volume name for each unique claim name
|
||||
for( String claimName : volumeClaims.collect { it.claimName }.unique() ) {
|
||||
final volName = nextVolName()
|
||||
namesMap[claimName] = volName
|
||||
volumes << [name: volName, persistentVolumeClaim: [claimName: claimName]]
|
||||
}
|
||||
|
||||
// -- persistent volume claims
|
||||
for( PodVolumeClaim entry : volumeClaims ) {
|
||||
//check if we already have a volume for the pvc
|
||||
final name = namesMap.get(entry.claimName)
|
||||
final claim = [name: name, mountPath: entry.mountPath ]
|
||||
if( entry.subPath )
|
||||
claim.subPath = entry.subPath
|
||||
if( entry.readOnly )
|
||||
claim.readOnly = entry.readOnly
|
||||
mounts << claim
|
||||
}
|
||||
|
||||
// -- configMap volumes
|
||||
for( PodMountConfig entry : configMaps ) {
|
||||
final name = nextVolName()
|
||||
configMapToSpec(name, entry, mounts, volumes)
|
||||
}
|
||||
|
||||
// -- csi ephemeral volumes
|
||||
for( PodMountCsiEphemeral entry : csiEphemerals ) {
|
||||
final name = nextVolName()
|
||||
mounts << [name: name, mountPath: entry.mountPath, readOnly: entry.csi.readOnly ?: false]
|
||||
volumes << [name: name, csi: entry.csi]
|
||||
}
|
||||
|
||||
// -- emptyDir volumes
|
||||
for( PodMountEmptyDir entry : emptyDirs ) {
|
||||
final name = nextVolName()
|
||||
mounts << [name: name, mountPath: entry.mountPath]
|
||||
volumes << [name: name, emptyDir: entry.emptyDir]
|
||||
}
|
||||
|
||||
// -- secret volumes
|
||||
for( PodMountSecret entry : secrets ) {
|
||||
final name = nextVolName()
|
||||
secretToSpec(name, entry, mounts, volumes)
|
||||
}
|
||||
|
||||
// -- host path volumes
|
||||
for( PodHostMount entry : hostMounts ) {
|
||||
final name = nextVolName()
|
||||
mounts << [name: name, mountPath: entry.mountPath]
|
||||
volumes << [name: name, hostPath: [path: entry.hostPath]]
|
||||
}
|
||||
|
||||
|
||||
if( volumes )
|
||||
spec.volumes = volumes
|
||||
if( mounts )
|
||||
container.volumeMounts = mounts
|
||||
|
||||
return pod
|
||||
}
|
||||
|
||||
Map buildAsJob() {
|
||||
final pod = build()
|
||||
final spec = [
|
||||
backoffLimit: 0,
|
||||
template: [
|
||||
metadata: pod.metadata,
|
||||
spec: pod.spec
|
||||
]
|
||||
]
|
||||
|
||||
if( ttlSecondsAfterFinished != null )
|
||||
spec.ttlSecondsAfterFinished = ttlSecondsAfterFinished
|
||||
|
||||
return [
|
||||
apiVersion: 'batch/v1',
|
||||
kind: 'Job',
|
||||
metadata: pod.metadata,
|
||||
spec: spec
|
||||
]
|
||||
}
|
||||
|
||||
@PackageScope
|
||||
Map addResourcesLimits(Map limits, Map result) {
|
||||
if( result == null )
|
||||
result = new LinkedHashMap(2)
|
||||
|
||||
final limits0 = result.limits as Map ?: new LinkedHashMap(10)
|
||||
limits0.putAll( limits )
|
||||
result.limits = limits0
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
@PackageScope
|
||||
Map addCpuResources(Integer cpus, Map res) {
|
||||
if( res == null )
|
||||
res = new LinkedHashMap(2)
|
||||
|
||||
final requests0 = res.requests as Map ?: new LinkedHashMap<>(10)
|
||||
requests0.cpu = cpus
|
||||
res.requests = requests0
|
||||
|
||||
if( cpuLimits ) {
|
||||
final limits0 = res.limits as Map ?: new LinkedHashMap(10)
|
||||
limits0.cpu = cpus
|
||||
res.limits = limits0
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
@PackageScope
|
||||
Map addMemoryResources(String memory, Map res) {
|
||||
if( res == null )
|
||||
res = new LinkedHashMap(2)
|
||||
|
||||
final req = res.requests as Map ?: new LinkedHashMap(10)
|
||||
req.memory = memory
|
||||
res.requests = req
|
||||
|
||||
final lim = res.limits as Map ?: new LinkedHashMap(10)
|
||||
lim.memory = memory
|
||||
res.limits = lim
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
@PackageScope
|
||||
Map addDiskResources(String diskRequest, Map res) {
|
||||
if( res == null )
|
||||
res = new LinkedHashMap(2)
|
||||
|
||||
final req = res.requests as Map ?: new LinkedHashMap(10)
|
||||
req.'ephemeral-storage' = diskRequest
|
||||
res.requests = req
|
||||
|
||||
final lim = res.limits as Map ?: new LinkedHashMap(10)
|
||||
lim.'ephemeral-storage' = diskRequest
|
||||
res.limits = lim
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
@PackageScope
|
||||
String getAcceleratorType(AcceleratorResource accelerator) {
|
||||
|
||||
def type = accelerator.type ?: 'nvidia.com'
|
||||
|
||||
if ( type.contains('/') )
|
||||
// Assume the user has fully specified the resource type.
|
||||
return type
|
||||
|
||||
// Assume we're using GPU and update as necessary.
|
||||
if( !type.contains('.') ) type += '.com'
|
||||
type += '/gpu'
|
||||
|
||||
return type
|
||||
}
|
||||
|
||||
|
||||
@PackageScope
|
||||
Map addAcceleratorResources(AcceleratorResource accelerator, Map res) {
|
||||
|
||||
if( res == null )
|
||||
res = new LinkedHashMap(2)
|
||||
|
||||
def type = getAcceleratorType(accelerator)
|
||||
|
||||
if( accelerator.request ) {
|
||||
final req = res.requests as Map ?: new LinkedHashMap<>(2)
|
||||
req.put(type, accelerator.request)
|
||||
res.requests = req
|
||||
}
|
||||
if( accelerator.limit ) {
|
||||
final lim = res.limits as Map ?: new LinkedHashMap<>(2)
|
||||
lim.put(type, accelerator.limit)
|
||||
res.limits = lim
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
@PackageScope
|
||||
@CompileDynamic
|
||||
static void secretToSpec(String volName, PodMountSecret entry, List mounts, List volumes ) {
|
||||
assert entry
|
||||
|
||||
final secret = [secretName: entry.secretName]
|
||||
if( entry.secretKey ) {
|
||||
secret.items = [ [key: entry.secretKey, path: entry.fileName ] ]
|
||||
}
|
||||
|
||||
mounts << [name: volName, mountPath: entry.mountPath]
|
||||
volumes << [name: volName, secret: secret ]
|
||||
}
|
||||
|
||||
@PackageScope
|
||||
@CompileDynamic
|
||||
static void configMapToSpec(String volName, PodMountConfig entry, List<Map> mounts, List<Map> volumes ) {
|
||||
assert entry
|
||||
|
||||
final config = [name: entry.configName]
|
||||
if( entry.configKey ) {
|
||||
config.items = [ [key: entry.configKey, path: entry.fileName ] ]
|
||||
}
|
||||
|
||||
mounts << [name: volName, mountPath: entry.mountPath]
|
||||
volumes << [name: volName, configMap: config ]
|
||||
}
|
||||
|
||||
protected Map sanitize(Map map, MetaType kind) {
|
||||
final result = new HashMap(map.size())
|
||||
for( Map.Entry entry : map ) {
|
||||
final key = sanitizeKey(entry.key as String, kind)
|
||||
final value = (kind == MetaType.LABEL)
|
||||
? sanitizeValue(entry.value, kind, SegmentType.VALUE)
|
||||
: entry.value
|
||||
|
||||
result.put(key, value)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
protected String sanitizeKey(String value, MetaType kind) {
|
||||
final parts = value.tokenize('/')
|
||||
|
||||
if (parts.size() == 2) {
|
||||
return "${sanitizeValue(parts[0], kind, SegmentType.PREFIX)}/${sanitizeValue(parts[1], kind, SegmentType.NAME)}"
|
||||
}
|
||||
if( parts.size() == 1 ) {
|
||||
return sanitizeValue(parts[0], kind, SegmentType.NAME)
|
||||
}
|
||||
else {
|
||||
throw new IllegalArgumentException("Invalid key in pod ${kind.toString().toLowerCase()} -- Key can only contain exactly one '/' character")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Sanitize a string value to contain only alphanumeric characters, '-', '_' or '.',
|
||||
* and to start and end with an alphanumeric character.
|
||||
*/
|
||||
protected String sanitizeValue(value, MetaType kind, SegmentType segment) {
|
||||
def str = String.valueOf(value)
|
||||
if( str.length() > segment.maxSize ) {
|
||||
log.debug "K8s $kind $segment exceeds allowed size: $segment.maxSize -- offending str=$str"
|
||||
str = str.substring(0,segment.maxSize)
|
||||
}
|
||||
str = str.replaceAll(/[^a-zA-Z0-9\.\_\-]+/, '_')
|
||||
str = str.replaceAll(/^[^a-zA-Z0-9]+/, '')
|
||||
str = str.replaceAll(/[^a-zA-Z0-9]+$/, '')
|
||||
return str
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.model
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import groovy.transform.ToString
|
||||
|
||||
/**
|
||||
* Model a K8s pod persistent volume claim mount
|
||||
*
|
||||
* See https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
@ToString(includeNames = true)
|
||||
@EqualsAndHashCode
|
||||
class PodVolumeClaim {
|
||||
|
||||
String claimName
|
||||
|
||||
String mountPath
|
||||
|
||||
String subPath
|
||||
|
||||
boolean readOnly
|
||||
|
||||
PodVolumeClaim(String name, String mount, String subPath=null, boolean readOnly=false) {
|
||||
assert name
|
||||
assert mount
|
||||
this.claimName = name
|
||||
this.mountPath = sanitize(mount)
|
||||
this.subPath = subPath
|
||||
this.readOnly = readOnly
|
||||
validate(mountPath)
|
||||
}
|
||||
|
||||
PodVolumeClaim(Map entry) {
|
||||
assert entry.volumeClaim
|
||||
assert entry.mountPath
|
||||
this.claimName = entry.volumeClaim
|
||||
this.mountPath = sanitize(entry.mountPath)
|
||||
this.subPath = entry.subPath
|
||||
this.readOnly = entry.readOnly ?: false
|
||||
validate(mountPath)
|
||||
}
|
||||
|
||||
private static validate(String path) {
|
||||
if( !path.startsWith('/') )
|
||||
throw new IllegalArgumentException("K8s volume claim path must be an absolute path: $path")
|
||||
}
|
||||
|
||||
static private String sanitize(path) {
|
||||
if( !path ) return null
|
||||
def result = path.toString().trim()
|
||||
while( result.endsWith('/') && result.size()>1 )
|
||||
result = result.substring(0,result.size()-1)
|
||||
return result
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.model
|
||||
|
||||
/**
|
||||
* Model the resource type to be used to run nextflow tasks
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
enum ResourceType {
|
||||
Pod, Job;
|
||||
|
||||
String lower() {
|
||||
return this.name().toLowerCase()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,384 @@
|
||||
package recreationaltech.plugin.strategies
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import recreationaltech.plugin.K8sDVFSClient
|
||||
import recreationaltech.plugin.K8sRuntimeEstimator
|
||||
import recreationaltech.plugin.K8sSchedulingDecision
|
||||
import recreationaltech.plugin.K8sSchedulingRequest
|
||||
import recreationaltech.plugin.K8sSchedulingStrategy
|
||||
import recreationaltech.plugin.K8sTaskHandler
|
||||
import recreationaltech.plugin.K8sTaskScheduler
|
||||
import recreationaltech.plugin.client.K8sClient
|
||||
import nextflow.processor.TaskRun
|
||||
import nextflow.util.Duration
|
||||
|
||||
/**
|
||||
* Implements a scheduling strategy utilizing dvfs to reduce the energy consumption
|
||||
* of workflow execution, while attempting to maintain the same makespan.
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class K8sDVFSSchedulingStrategy implements K8sSchedulingStrategy {
|
||||
|
||||
/** Used for passing K8sExecutor.getClient.
|
||||
* We cannot pass the client directly, becaue it can be refreshed
|
||||
* during workflow execution.
|
||||
*/
|
||||
public interface K8sClientGetter {
|
||||
K8sClient getClient()
|
||||
}
|
||||
|
||||
private static long getTaskMemoryRequirment(TaskRun task) {
|
||||
return task.config.getMemory() ? task.config.getMemory().bytes : 64 * 1024 * 1024
|
||||
}
|
||||
|
||||
private static int getTaskCPURequirement(TaskRun task) {
|
||||
return task.config.hasCpus() ? task.config.getCpus() * 1000 : 1000
|
||||
}
|
||||
|
||||
@Slf4j
|
||||
private class WorkerNode {
|
||||
private class AssignedTask {
|
||||
TaskRun task
|
||||
long frequency
|
||||
|
||||
AssignedTask(TaskRun t, long f) {
|
||||
this.task = t
|
||||
this.frequency = f
|
||||
}
|
||||
}
|
||||
|
||||
String name
|
||||
|
||||
long maxFrequency
|
||||
long minFrequency
|
||||
long currentFrequency
|
||||
|
||||
ArrayList<AssignedTask> tasks
|
||||
|
||||
WorkerNode(String name, long maxF, long minF, long curF, K8sClientGetter clientGetter) {
|
||||
this.name = name
|
||||
this.maxFrequency = maxF
|
||||
this.minFrequency = minF
|
||||
this.currentFrequency = curF
|
||||
this.tasks = new ArrayList<>()
|
||||
}
|
||||
|
||||
/* Return the number of available (unoccupied) bytes */
|
||||
long getAvailableMemory() {
|
||||
def k8sClient = clientGetter.getClient()
|
||||
Long available = k8sClient.getNodeMemoryAvailableBytes(this.name)
|
||||
if (available == null) {
|
||||
log.warn "[K8s] failed to retrieve available memory for node ${name}"
|
||||
// Fallback: try capacity - allocated
|
||||
Long capacity = k8sClient.getNodeMemoryCapacityBytes(this.name)
|
||||
Long allocated = k8sClient.getNodeMemoryAllocatedBytes(this.name)
|
||||
if (capacity != null && allocated != null) {
|
||||
return capacity - allocated
|
||||
}
|
||||
return 0
|
||||
}
|
||||
return available
|
||||
}
|
||||
|
||||
/* Return the number of available (unoccupied) cpu cores */
|
||||
long getAvailableCPUs() {
|
||||
def k8sClient = clientGetter.getClient()
|
||||
Long available = k8sClient.getNodeCpuAvailableMillis(this.name)
|
||||
if (available == null) {
|
||||
log.warn "[K8s] failed to retrieve available CPU for node ${name}"
|
||||
// Fallback: try capacity - allocated
|
||||
Long capacity = k8sClient.getNodeCpuCapacityMillis(this.name)
|
||||
Long allocated = k8sClient.getNodeCpuAllocatedMillis(this.name)
|
||||
if (capacity != null && allocated != null) {
|
||||
return capacity - allocated
|
||||
}
|
||||
return 0
|
||||
}
|
||||
return available
|
||||
}
|
||||
|
||||
/* Return the total amount of installed memory */
|
||||
long getMemoryAmount() {
|
||||
def k8sClient = clientGetter.getClient()
|
||||
Long capacity = k8sClient.getNodeMemoryCapacityBytes(this.name)
|
||||
return capacity != null ? capacity.longValue() : 0
|
||||
}
|
||||
|
||||
/* Return the total number of installed cpu cores */
|
||||
long getCPUCount() {
|
||||
def k8sClient = clientGetter.getClient()
|
||||
Long capacity = k8sClient.getNodeCpuCapacityMillis(this.name)
|
||||
return capacity != null ? capacity.longValue(): 0
|
||||
}
|
||||
|
||||
// Sets the frequency to the max. requested frequency of all currently running tasks.
|
||||
private void updateFrequency(K8sDVFSClient dvfsClient) {
|
||||
if (tasks.size() == 0)
|
||||
return
|
||||
long max = Long.MIN_VALUE
|
||||
for (AssignedTask t : tasks) {
|
||||
max = Long.max(t.frequency, max)
|
||||
}
|
||||
log.info "[K8s] node ${name} running at ${max} Hz / ${maxFrequency} Hz ${((double)max/(double)maxFrequency) * 100.0}%"
|
||||
dvfsClient.setNodeFrequency(name, (int)max)
|
||||
this.currentFrequency = max
|
||||
}
|
||||
|
||||
void assignTask(TaskRun task, long frequency, K8sDVFSClient dvfsClient) {
|
||||
log.info "[K8s] node ${name}: task ${task.name} assigned with ${frequency}/${maxFrequency}"
|
||||
this.tasks.add(new AssignedTask(task, frequency))
|
||||
updateFrequency(dvfsClient)
|
||||
}
|
||||
|
||||
void taskFinished(TaskRun task, K8sDVFSClient dvfsClient) {
|
||||
log.info "[K8s] node ${name}: task ${task.name} finished"
|
||||
this.tasks.removeIf {it.task == task}
|
||||
updateFrequency(dvfsClient)
|
||||
}
|
||||
}
|
||||
|
||||
class SchedulingRequestComparator implements Comparator<K8sSchedulingRequest> {
|
||||
K8sRuntimeEstimator runtimeEstimator
|
||||
long currentTime
|
||||
double epsilon
|
||||
|
||||
@Override
|
||||
int compare(K8sSchedulingRequest o1, K8sSchedulingRequest o2) {
|
||||
// First, check if one of the tasks is (estimated to be) on the critical path
|
||||
double t1 = runtimeEstimator.estimate(o1.handler)
|
||||
double t2 = runtimeEstimator.estimate(o2.handler)
|
||||
|
||||
if (t1 > t2 + epsilon)
|
||||
return -1
|
||||
else if (t2 > t1 + epsilon)
|
||||
return 1
|
||||
|
||||
// Both are not on the critical path. Sort based on the time they spent in the queue
|
||||
long w1 = currentTime - o1.submitTimeMillis
|
||||
long w2 = currentTime - o2.submitTimeMillis
|
||||
if (w1 > w2)
|
||||
return -1
|
||||
else if (w2 > w1)
|
||||
return 1
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
private K8sRuntimeEstimator runtimeEstimator
|
||||
private K8sDVFSClient dvfsClient
|
||||
|
||||
private ArrayList<WorkerNode> nodes
|
||||
private HashMap<String, WorkerNode> taskToNode
|
||||
private long globalMaxFrequency
|
||||
private long globalMinFrequency
|
||||
|
||||
private K8sClientGetter clientGetter
|
||||
|
||||
private double comparisonEpsilonMillis
|
||||
|
||||
private double[] topRuntimes
|
||||
private double averageRuntime
|
||||
private double finishedTaskCount
|
||||
|
||||
boolean fullSpeedMode
|
||||
|
||||
K8sDVFSSchedulingStrategy(K8sRuntimeEstimator runtimeEstimator,
|
||||
K8sDVFSClient dvfsClient,
|
||||
K8sClientGetter clientGetter,
|
||||
Duration runtimeComparisonEpsilon,
|
||||
int topRuntimeCount) {
|
||||
this.runtimeEstimator = runtimeEstimator
|
||||
this.dvfsClient = dvfsClient
|
||||
this.nodes = new ArrayList<>()
|
||||
this.taskToNode = new HashMap<>();
|
||||
this.clientGetter = clientGetter
|
||||
this.comparisonEpsilonMillis = (double)runtimeComparisonEpsilon.toMillis()
|
||||
this.topRuntimes = new double[topRuntimeCount]
|
||||
for (int i = 0; i < topRuntimeCount; i++) {
|
||||
this.topRuntimes[i] = 0.0
|
||||
}
|
||||
this.averageRuntime = 0.0
|
||||
this.finishedTaskCount = 0.0
|
||||
}
|
||||
|
||||
private boolean isInTopRuntimes(double rt) {
|
||||
for (int i = 0; i < topRuntimes.size(); i++) {
|
||||
if (rt >= topRuntimes[i])
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
private void updateTopRuntimes(double rt) {
|
||||
for (int i = 0; i < topRuntimes.size(); i++) {
|
||||
if (rt > topRuntimes[i]) {
|
||||
/* Move all one down */
|
||||
for (int j = topRuntimes.size() - 1; j > i; j--) {
|
||||
topRuntimes[j] = topRuntimes[j - 1];
|
||||
}
|
||||
topRuntimes[i] = rt
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
K8sSchedulingDecision schedule(K8sTaskScheduler scheduler, List<K8sSchedulingRequest> queue) {
|
||||
if (nodes.isEmpty()) {
|
||||
if (!initNodes(scheduler))
|
||||
return null
|
||||
}
|
||||
|
||||
/* Step 1: Sort by task priority. We will attempt to schedule tasks "in order", so that the
|
||||
* highest priority tasks are assigned to nodes as soon as possible.
|
||||
*
|
||||
* Priority is based on a) the tasks estimated runtime and b) the wait time of the task.
|
||||
*/
|
||||
SchedulingRequestComparator comparator = new SchedulingRequestComparator()
|
||||
comparator.runtimeEstimator = runtimeEstimator
|
||||
comparator.epsilon = comparisonEpsilonMillis
|
||||
comparator.currentTime = System.currentTimeMillis()
|
||||
queue.sort(comparator)
|
||||
|
||||
/* Step 2: For each task attempt to schedule it onto a node */
|
||||
for (K8sSchedulingRequest req : queue) {
|
||||
/* Step 2.1: Determine if the task is on the critical path.
|
||||
* If yes, we just schedule it at max frequency on the node with the highest available
|
||||
* frequency. If not, we determine a frequency (see below).
|
||||
*/
|
||||
final double taskEstimation = runtimeEstimator.estimate(req.handler)
|
||||
final boolean isCriticalPath = isInTopRuntimes(taskEstimation)
|
||||
long frequency = globalMaxFrequency
|
||||
if (!isCriticalPath && !fullSpeedMode) {
|
||||
/* Set frequency so that we expect the runtime to be close to the mean runtime. */
|
||||
frequency = (long)Math.floor((taskEstimation * globalMaxFrequency) / averageRuntime)
|
||||
frequency = Math.max(frequency, globalMinFrequency)
|
||||
frequency = Math.min(frequency, globalMaxFrequency)
|
||||
}
|
||||
|
||||
/* Step 2.2: Filter nodes based on task requirements */
|
||||
ArrayList<WorkerNode> suitableNodes = filterNodes(req.task)
|
||||
if (suitableNodes.size() == 0) {
|
||||
if (!anyNode(req.task)) {
|
||||
log.error "[K8s] unable to schedule task ${req.task} - no node satisfies resource requirements ${getTaskMemoryRequirment(req.task)} bytes ${getTaskCPURequirement(req.task)} cpus"
|
||||
return null
|
||||
}
|
||||
/* No node can currently execute this task, but it should be possible in the future */
|
||||
/* log.info "[K8s] ${req.task} can not be scheduled: ${getTaskMemoryRequirment(req.task)} bytes ${getTaskCPURequirement(req.task)} CPUs"
|
||||
for (WorkerNode n : this.nodes) {
|
||||
log.info "[K8s] node ${n.name} - ${n.availableMemory}, ${n.availableCPUs}"
|
||||
} */
|
||||
continue
|
||||
}
|
||||
|
||||
/* Step 2.3: Assign to node based on "best fit" - current node frequency is closest to determined frequency */
|
||||
long minDist = Math.abs(suitableNodes[0].currentFrequency - frequency)
|
||||
WorkerNode closest = suitableNodes[0]
|
||||
for (WorkerNode node : suitableNodes) {
|
||||
long dist = Math.abs(node.currentFrequency - frequency)
|
||||
if (dist < minDist) {
|
||||
closest = node
|
||||
minDist = dist
|
||||
}
|
||||
}
|
||||
|
||||
closest.assignTask(req.task, frequency, dvfsClient)
|
||||
taskToNode.put(req.task.hash.toString(), closest)
|
||||
log.info "[K8s] DVFS: Assigned task ${req.task} to node ${closest.name} - ${taskToNode.size()} assigned tasks"
|
||||
return new K8sSchedulingDecision(req, closest.name)
|
||||
}
|
||||
|
||||
log.info "[K8s] unable to schedule any task. The queue contains ${queue.size()} tasks."
|
||||
return null
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean scheduleImmediately(K8sTaskScheduler scheduler, List<K8sSchedulingRequest> queue) {
|
||||
if (nodes.isEmpty()) {
|
||||
if (!initNodes(scheduler))
|
||||
return false
|
||||
}
|
||||
/* We want to schedule immediately if there are unoccupied nodes */
|
||||
boolean doIt = queue != null && queue.size() > 0 && taskToNode.size() < nodes.size()
|
||||
log.info "[K8s] scheduleImmediately: ${queue.size()} tasks in queue, ${taskToNode.size()} tasks running on ${nodes.size()} nodes: ${doIt}"
|
||||
return doIt
|
||||
}
|
||||
|
||||
@Override
|
||||
synchronized void taskFinished(K8sTaskHandler task) {
|
||||
/* TODO: This just uses elapsed wall-clock time, regardless of the frequency used to execute the task.
|
||||
* This will skew the average towards longer runtimes, which is undesirable, because it will lead to more
|
||||
* tasks classified as "critical path".
|
||||
* A simple (rough) solution could be to keep track of the tasks "relative" frequency and just scale the
|
||||
* elapsed time based on that.
|
||||
*/
|
||||
double runtime = (double)(task.getCompleteTimeMillis() - task.getStartTimeMillis())
|
||||
averageRuntime = (runtime + finishedTaskCount * averageRuntime) / (finishedTaskCount + 1.0)
|
||||
finishedTaskCount += 1.0
|
||||
updateTopRuntimes(runtime)
|
||||
|
||||
/* Free resources allocated by this task */
|
||||
WorkerNode node = taskToNode.get(task.task.hash.toString())
|
||||
if (node != null) {
|
||||
node.taskFinished(task.task, dvfsClient)
|
||||
taskToNode.remove(task.task.hash.toString())
|
||||
} else {
|
||||
log.warn "[K8s] no node recorded for task ${task.toString()}"
|
||||
}
|
||||
|
||||
log.info "[K8s] task ${task.toString()} finished - ${taskToNode.size()} tasks running"
|
||||
if (node != null) {
|
||||
log.info "[K8s] task ran on node ${node.name} - ${node.availableMemory} bytes ${node.availableCPUs}"
|
||||
}
|
||||
}
|
||||
|
||||
private synchronized boolean initNodes(K8sTaskScheduler scheduler) {
|
||||
this.globalMaxFrequency = Long.MAX_VALUE
|
||||
final nodes = scheduler.getNodes()
|
||||
for (String node : nodes) {
|
||||
final cur = dvfsClient.getNodeCurrentFrequency(node)
|
||||
final min = dvfsClient.getNodeMinFrequency(node)
|
||||
final max = dvfsClient.getNodeMaxFrequency(node)
|
||||
final cpus = dvfsClient.getCPUCount(node)
|
||||
final mem = dvfsClient.getMemoryAmount(node)
|
||||
|
||||
if (cur.empty || min.empty || max.empty || cpus.empty || mem.empty) {
|
||||
log.error "[K8s] failed to query node $node information"
|
||||
continue
|
||||
}
|
||||
globalMaxFrequency = Long.min(globalMaxFrequency, max.asLong)
|
||||
globalMinFrequency = Long.max(globalMinFrequency, min.asLong)
|
||||
|
||||
log.info "[K8s] node ${node}: ${cpus.asLong} CPUs, ${mem.asLong} bytes RAM ${min.asLong} Hz - ${max.asLong} Hz current ${cur.asLong}"
|
||||
this.nodes.add(new WorkerNode(node, max.asLong, min.asLong, cur.asLong, clientGetter))
|
||||
}
|
||||
return !this.nodes.isEmpty()
|
||||
}
|
||||
|
||||
/* Returns a list of nodes that fulfill the tasks resource requirements
|
||||
*/
|
||||
private ArrayList<WorkerNode> filterNodes(TaskRun task) {
|
||||
final long reqBytes = getTaskMemoryRequirment(task)
|
||||
final int reqCPUs = getTaskCPURequirement(task)
|
||||
ArrayList<WorkerNode> suitableNodes = new ArrayList<>()
|
||||
for (WorkerNode n : nodes) {
|
||||
if (n.availableMemory >= reqBytes && n.availableCPUs >= reqCPUs) {
|
||||
log.info "[K8s] task ${task.name}: ${reqBytes} bytes ${reqCPUs} cpus: node ${n.name} has ${n.availableMemory} bytes, ${n.availableCPUs} cpus"
|
||||
suitableNodes.add(n)
|
||||
}
|
||||
}
|
||||
return suitableNodes
|
||||
}
|
||||
|
||||
private boolean anyNode(TaskRun task) {
|
||||
final long reqBytes = getTaskMemoryRequirment(task)
|
||||
final int reqCPUs = getTaskCPURequirement(task)
|
||||
for (WorkerNode n : nodes) {
|
||||
if (n.memoryAmount >= reqBytes && n.CPUCount >= reqCPUs)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package recreationaltech.plugin.strategies
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import recreationaltech.plugin.K8sSchedulingDecision
|
||||
import recreationaltech.plugin.K8sSchedulingRequest
|
||||
import recreationaltech.plugin.K8sSchedulingStrategy
|
||||
import recreationaltech.plugin.K8sTaskHandler
|
||||
import recreationaltech.plugin.K8sTaskScheduler
|
||||
|
||||
@CompileStatic
|
||||
class K8sHashSchedulingStrategy implements K8sSchedulingStrategy {
|
||||
|
||||
@Override
|
||||
K8sSchedulingDecision schedule(K8sTaskScheduler scheduler, List<K8sSchedulingRequest> queue) {
|
||||
if (!queue)
|
||||
return null
|
||||
|
||||
final freeNodes = scheduler.nodes
|
||||
if (freeNodes.size() == 0)
|
||||
return null
|
||||
|
||||
final request = queue[0]
|
||||
final index = Math.floorMod(request.task.hash.asInt(), freeNodes.size())
|
||||
return new K8sSchedulingDecision(request, freeNodes[index])
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean scheduleImmediately(K8sTaskScheduler scheduler, List<K8sSchedulingRequest> queue) {
|
||||
return false
|
||||
}
|
||||
|
||||
@Override
|
||||
void taskFinished(K8sTaskHandler task) { /* nop */ }
|
||||
}
|
||||
@@ -0,0 +1,497 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin
|
||||
|
||||
import nextflow.BuildInfo
|
||||
import nextflow.SysEnv
|
||||
import recreationaltech.plugin.client.ClientConfig
|
||||
import recreationaltech.plugin.model.PodEnv
|
||||
import recreationaltech.plugin.model.PodSecurityContext
|
||||
import recreationaltech.plugin.model.PodVolumeClaim
|
||||
import nextflow.util.Duration
|
||||
import spock.lang.Specification
|
||||
import spock.lang.Unroll
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class K8sConfigTest extends Specification {
|
||||
|
||||
def 'should create config object' () {
|
||||
|
||||
when:
|
||||
def cfg = new K8sConfig()
|
||||
then:
|
||||
cfg.getNamespace() == null
|
||||
cfg.getServiceAccount() == null
|
||||
!cfg.getDebug().getYaml()
|
||||
|
||||
when:
|
||||
cfg = new K8sConfig( namespace:'foo', serviceAccount: 'bar', debug: [yaml: true] )
|
||||
then:
|
||||
cfg.getNamespace() == 'foo'
|
||||
cfg.getServiceAccount() == 'bar'
|
||||
cfg.getDebug().getYaml()
|
||||
cfg.debug.yaml
|
||||
|
||||
}
|
||||
|
||||
def 'should set cleanup' () {
|
||||
given:
|
||||
K8sConfig cfg
|
||||
|
||||
when:
|
||||
cfg = new K8sConfig()
|
||||
then: 'it should return true when missing value'
|
||||
cfg.getCleanup()
|
||||
|
||||
when:
|
||||
cfg = new K8sConfig()
|
||||
then: 'it should return false specified as default'
|
||||
!cfg.getCleanup(false)
|
||||
|
||||
when:
|
||||
cfg = new K8sConfig(cleanup:false)
|
||||
then: 'it should return false'
|
||||
!cfg.getCleanup()
|
||||
|
||||
when:
|
||||
cfg = new K8sConfig(cleanup:true)
|
||||
then: 'it should return true'
|
||||
cfg.getCleanup()
|
||||
|
||||
when:
|
||||
cfg = new K8sConfig(cleanup:true)
|
||||
then: 'the default value should be ignored'
|
||||
cfg.getCleanup(false)
|
||||
}
|
||||
|
||||
def 'should create config with storage claims' () {
|
||||
|
||||
when:
|
||||
def cfg = new K8sConfig(storageClaimName: 'pvc-1')
|
||||
then:
|
||||
cfg.getStorageClaimName() == 'pvc-1'
|
||||
cfg.getStorageMountPath() == '/workspace'
|
||||
cfg.getPodOptions().getVolumeClaims() == [ new PodVolumeClaim('pvc-1', '/workspace') ] as Set
|
||||
|
||||
when:
|
||||
cfg = new K8sConfig([
|
||||
storageClaimName: 'pvc-2',
|
||||
storageMountPath: '/data',
|
||||
pod: [ [volumeClaim:'foo', mountPath: '/here'],
|
||||
[volumeClaim: 'bar', mountPath: '/there']] ])
|
||||
then:
|
||||
cfg.getStorageClaimName() == 'pvc-2'
|
||||
cfg.getStorageMountPath() == '/data'
|
||||
cfg.getPodOptions().getVolumeClaims() == [
|
||||
new PodVolumeClaim('pvc-2', '/data'),
|
||||
new PodVolumeClaim('foo', '/here'),
|
||||
new PodVolumeClaim('bar', '/there')
|
||||
] as Set
|
||||
|
||||
|
||||
when:
|
||||
cfg = new K8sConfig(storageClaimName: 'pvc-3', storageMountPath: '/some/path', storageSubPath: '/bar')
|
||||
then:
|
||||
cfg.getStorageClaimName() == 'pvc-3'
|
||||
cfg.getStorageMountPath() == '/some/path'
|
||||
cfg.getStorageSubPath() == '/bar'
|
||||
cfg.getPodOptions().getVolumeClaims() == [ new PodVolumeClaim('pvc-3', '/some/path', '/bar') ] as Set
|
||||
|
||||
}
|
||||
|
||||
def 'should set device plugin' () {
|
||||
when:
|
||||
def cfg = new K8sConfig([:])
|
||||
then:
|
||||
cfg.fuseDevicePlugin() == ['nextflow.io/fuse':1]
|
||||
|
||||
when:
|
||||
cfg = new K8sConfig([fuseDevicePlugin:['foo/fuse':10]])
|
||||
then:
|
||||
cfg.fuseDevicePlugin() == ['foo/fuse':10]
|
||||
}
|
||||
|
||||
def 'should create client config' () {
|
||||
|
||||
given:
|
||||
def CONFIG = [namespace: 'this', serviceAccount: 'that', client: [server: 'http://foo']]
|
||||
|
||||
when:
|
||||
def config = new K8sConfig(CONFIG)
|
||||
def client = config.getClient()
|
||||
then:
|
||||
client.server == 'http://foo'
|
||||
client.namespace == 'this'
|
||||
client.serviceAccount == 'that'
|
||||
client.httpConnectTimeout == null // testing default null
|
||||
client.httpReadTimeout == null // testing default null
|
||||
client.retryConfig.maxAttempts == 4
|
||||
|
||||
}
|
||||
|
||||
def 'should create client config with http request timeouts' () {
|
||||
|
||||
given:
|
||||
def CONFIG = [
|
||||
namespace: 'this',
|
||||
serviceAccount: 'that',
|
||||
client: [server: 'http://foo'],
|
||||
httpReadTimeout: '20s',
|
||||
httpConnectTimeout: '25s' ]
|
||||
|
||||
when:
|
||||
def config = new K8sConfig(CONFIG)
|
||||
def client = config.getClient()
|
||||
then:
|
||||
client.server == 'http://foo'
|
||||
client.namespace == 'this'
|
||||
client.serviceAccount == 'that'
|
||||
client.httpConnectTimeout == Duration.of('25s')
|
||||
client.httpReadTimeout == Duration.of('20s')
|
||||
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should create client config with discovery' () {
|
||||
|
||||
given:
|
||||
def CONFIG = [context: CONTEXT, namespace: NAMESPACE, serviceAccount: SERVICE_ACCOUNT]
|
||||
K8sConfig config = Spy(K8sConfig, constructorArgs: [ CONFIG ])
|
||||
|
||||
when:
|
||||
def client = config.getClient()
|
||||
then:
|
||||
1 * config.clientDiscovery(CONTEXT, NAMESPACE, SERVICE_ACCOUNT) >> new ClientConfig(namespace: NAMESPACE, server: SERVER)
|
||||
and:
|
||||
client.server == SERVER
|
||||
client.namespace == NAMESPACE ?: 'default'
|
||||
client.serviceAccount == SERVICE_ACCOUNT ?: 'default'
|
||||
|
||||
where:
|
||||
CONTEXT | SERVER | NAMESPACE | SERVICE_ACCOUNT
|
||||
'foo' | 'host.com'| null | null
|
||||
'bar' | 'this.com'| 'ns1' | 'sa2'
|
||||
|
||||
}
|
||||
|
||||
def 'should get nextflow image name' () {
|
||||
|
||||
when:
|
||||
def cfg = new K8sConfig()
|
||||
then:
|
||||
cfg.getNextflowImageName() == "nextflow/nextflow:${BuildInfo.version}"
|
||||
|
||||
}
|
||||
|
||||
def 'should get autoMountHostPaths' () {
|
||||
|
||||
when:
|
||||
def cfg = new K8sConfig()
|
||||
then:
|
||||
!cfg.getAutoMountHostPaths()
|
||||
|
||||
when:
|
||||
cfg = new K8sConfig(autoMountHostPaths: true)
|
||||
then:
|
||||
cfg.getAutoMountHostPaths()
|
||||
|
||||
when:
|
||||
cfg = new K8sConfig(autoMountHostPaths: false)
|
||||
then:
|
||||
!cfg.getAutoMountHostPaths()
|
||||
}
|
||||
|
||||
|
||||
def 'should get podOptions' () {
|
||||
|
||||
when:
|
||||
def cfg = new K8sConfig()
|
||||
def opts = cfg.getPodOptions()
|
||||
then:
|
||||
opts.envVars == [] as Set
|
||||
opts.mountSecrets == [] as Set
|
||||
opts.mountConfigMaps == [] as Set
|
||||
opts.volumeClaims == [] as Set
|
||||
|
||||
|
||||
when:
|
||||
opts = new K8sConfig(pod: [ [pullPolicy: 'Always'], [env: 'HELLO', value: 'WORLD'] ]).getPodOptions()
|
||||
then:
|
||||
opts.getImagePullPolicy() == 'Always'
|
||||
opts.getEnvVars() == [ PodEnv.value('HELLO','WORLD') ] as Set
|
||||
}
|
||||
|
||||
def 'should return user name' () {
|
||||
|
||||
when:
|
||||
def cfg = new K8sConfig()
|
||||
then:
|
||||
cfg.getUserName() == System.properties.get('user.name')
|
||||
|
||||
when:
|
||||
cfg = new K8sConfig(userName: 'foo')
|
||||
then:
|
||||
cfg.getUserName() == 'foo'
|
||||
}
|
||||
|
||||
def 'should return user dir' () {
|
||||
when:
|
||||
def cfg = new K8sConfig()
|
||||
then:
|
||||
cfg.getLaunchDir() == '/workspace/' + System.properties.get('user.name')
|
||||
|
||||
when:
|
||||
cfg = new K8sConfig(storageMountPath: '/this/path', userName: 'foo')
|
||||
then:
|
||||
cfg.getLaunchDir() == '/this/path/foo'
|
||||
|
||||
when:
|
||||
cfg = new K8sConfig(storageMountPath: '/this/path', userName: 'foo', launchDir: '/my/path')
|
||||
then:
|
||||
cfg.getLaunchDir() == '/my/path'
|
||||
|
||||
}
|
||||
|
||||
def 'should return work dir' () {
|
||||
when:
|
||||
def cfg = new K8sConfig()
|
||||
then:
|
||||
cfg.getWorkDir() == "/workspace/${System.properties.get('user.name')}/work"
|
||||
|
||||
when:
|
||||
cfg = new K8sConfig(launchDir: '/my/dir')
|
||||
then:
|
||||
cfg.getWorkDir() == "/my/dir/work"
|
||||
|
||||
when:
|
||||
cfg = new K8sConfig(launchDir: '/my/dir', workDir: '/the/wor/dir')
|
||||
then:
|
||||
cfg.getWorkDir() == "/the/wor/dir"
|
||||
}
|
||||
|
||||
def 'should return project dir' () {
|
||||
when:
|
||||
def cfg = new K8sConfig()
|
||||
then:
|
||||
cfg.getProjectDir() == '/workspace/projects'
|
||||
|
||||
when:
|
||||
cfg = new K8sConfig(storageMountPath: '/foo')
|
||||
then:
|
||||
cfg.getProjectDir() == '/foo/projects'
|
||||
|
||||
when:
|
||||
cfg = new K8sConfig(storageMountPath: '/foo', projectDir: '/my/project/dir')
|
||||
then:
|
||||
cfg.getProjectDir() == '/my/project/dir'
|
||||
}
|
||||
|
||||
def 'should return storage dir' () {
|
||||
|
||||
when:
|
||||
def cfg = new K8sConfig()
|
||||
then:
|
||||
cfg.getStorageMountPath() == '/workspace'
|
||||
|
||||
when:
|
||||
cfg = new K8sConfig(storageMountPath: '/mnt/there')
|
||||
then:
|
||||
cfg.getStorageMountPath() == '/mnt/there'
|
||||
|
||||
}
|
||||
|
||||
def 'should return compute resource type' () {
|
||||
|
||||
when:
|
||||
def cfg = new K8sConfig()
|
||||
then:
|
||||
!cfg.useJobResource()
|
||||
|
||||
when:
|
||||
cfg = new K8sConfig(computeResourceType: 'Job')
|
||||
then:
|
||||
cfg.useJobResource()
|
||||
|
||||
}
|
||||
|
||||
def 'should return storage claim name' () {
|
||||
when:
|
||||
def cfg = new K8sConfig()
|
||||
then:
|
||||
cfg.getStorageClaimName() == null
|
||||
|
||||
when:
|
||||
cfg = new K8sConfig(storageClaimName: 'xxx')
|
||||
then:
|
||||
cfg.getStorageClaimName() == 'xxx'
|
||||
}
|
||||
|
||||
def 'should create k8s config with one volume claim' () {
|
||||
|
||||
when:
|
||||
def cfg = new K8sConfig( pod: [runAsUser: 1000] )
|
||||
then:
|
||||
cfg.getPodOptions().getSecurityContext() == new PodSecurityContext(1000)
|
||||
cfg.getPodOptions().getVolumeClaims().size() == 0
|
||||
|
||||
when:
|
||||
cfg = new K8sConfig( pod: [volumeClaim: 'nf-0001', mountPath: '/workspace'] )
|
||||
then:
|
||||
cfg.getPodOptions().getSecurityContext() == null
|
||||
cfg.getPodOptions().getVolumeClaims() == [new PodVolumeClaim('nf-0001', '/workspace')] as Set
|
||||
|
||||
|
||||
when:
|
||||
cfg = new K8sConfig( pod: [
|
||||
[runAsUser: 1000],
|
||||
[volumeClaim: 'nf-0001', mountPath: '/workspace'],
|
||||
[volumeClaim: 'nf-0002', mountPath: '/data', subPath: '/home']
|
||||
])
|
||||
then:
|
||||
cfg.getPodOptions().getSecurityContext() == new PodSecurityContext(1000)
|
||||
cfg.getPodOptions().getVolumeClaims() == [
|
||||
new PodVolumeClaim('nf-0001', '/workspace'),
|
||||
new PodVolumeClaim('nf-0002', '/data', '/home')
|
||||
] as Set
|
||||
|
||||
}
|
||||
|
||||
|
||||
def 'should set the sec context'( ) {
|
||||
|
||||
given:
|
||||
def ctx = [runAsUser: 500, fsGroup: 200, allowPrivilegeEscalation: true, seLinuxOptions: [level: "s0:c123,c456"]]
|
||||
|
||||
when:
|
||||
def cfg = new K8sConfig( runAsUser: 500 )
|
||||
then:
|
||||
cfg.getPodOptions().getSecurityContext() == new PodSecurityContext(500)
|
||||
|
||||
when:
|
||||
cfg = new K8sConfig( securityContext: ctx )
|
||||
then:
|
||||
cfg.getPodOptions().getSecurityContext() == new PodSecurityContext(ctx)
|
||||
|
||||
}
|
||||
|
||||
def 'should set env and sec context' () {
|
||||
given:
|
||||
def ctx = [
|
||||
[env: 'FUSION_BUCKETS', value: 's3://nextflow-ci'],
|
||||
[securityContext: [privileged: true]]]
|
||||
|
||||
when:
|
||||
def cfg = new K8sConfig( pod: ctx )
|
||||
then:
|
||||
cfg.getPodOptions().getEnvVars().first() == PodEnv.value('FUSION_BUCKETS', 's3://nextflow-ci')
|
||||
cfg.getPodOptions().getSecurityContext().toSpec() == [privileged:true]
|
||||
|
||||
}
|
||||
|
||||
def 'should set the image pull policy' () {
|
||||
when:
|
||||
def cfg = new K8sConfig( pullPolicy: 'always' )
|
||||
then:
|
||||
cfg.getPodOptions().getImagePullPolicy() == 'always'
|
||||
}
|
||||
|
||||
def 'should set preserve entrypoint setting'( ) {
|
||||
|
||||
when:
|
||||
def cfg = new K8sConfig([:])
|
||||
then:
|
||||
!cfg.entrypointOverride()
|
||||
|
||||
when:
|
||||
SysEnv.push(NXF_CONTAINER_ENTRYPOINT_OVERRIDE: 'true')
|
||||
cfg = new K8sConfig()
|
||||
def result = cfg.entrypointOverride()
|
||||
SysEnv.pop()
|
||||
then:
|
||||
result
|
||||
|
||||
}
|
||||
|
||||
def 'should set debug.yaml' () {
|
||||
when:
|
||||
def cfg = new K8sConfig( debug: [yaml: true] )
|
||||
then:
|
||||
cfg.getDebug().getYaml()
|
||||
|
||||
when:
|
||||
cfg = new K8sConfig( debug: [yaml: false] )
|
||||
then:
|
||||
!cfg.getDebug().getYaml()
|
||||
|
||||
when:
|
||||
cfg = new K8sConfig( debug: null )
|
||||
then:
|
||||
!cfg.getDebug().getYaml()
|
||||
|
||||
when:
|
||||
cfg = new K8sConfig( debug: [:] )
|
||||
then:
|
||||
!cfg.getDebug().getYaml()
|
||||
}
|
||||
|
||||
def 'should set fetchNodeName' () {
|
||||
when:
|
||||
def cfg = new K8sConfig( fetchNodeName: true )
|
||||
then:
|
||||
cfg.fetchNodeName() == true
|
||||
|
||||
when:
|
||||
cfg = new K8sConfig( fetchNodeName: false )
|
||||
then:
|
||||
cfg.fetchNodeName() == false
|
||||
|
||||
when:
|
||||
cfg = new K8sConfig()
|
||||
then:
|
||||
cfg.fetchNodeName() == false
|
||||
}
|
||||
|
||||
def 'should set clientRefreshInterval' () {
|
||||
when:
|
||||
def cfg = new K8sConfig()
|
||||
then:
|
||||
cfg.clientRefreshInterval == Duration.of('50m')
|
||||
|
||||
when:
|
||||
cfg = new K8sConfig(clientRefreshInterval: '30m')
|
||||
then:
|
||||
cfg.clientRefreshInterval == Duration.of('30m')
|
||||
|
||||
when:
|
||||
cfg = new K8sConfig(clientRefreshInterval: '1h')
|
||||
then:
|
||||
cfg.clientRefreshInterval == Duration.of('1h')
|
||||
}
|
||||
|
||||
def 'should have nodeInit image' () {
|
||||
when:
|
||||
def cfg = new K8sConfig(
|
||||
nodeInit: [
|
||||
image: 'some-image:0'
|
||||
]
|
||||
)
|
||||
then:
|
||||
cfg.nodeInit.image == 'some-image:0'
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,674 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin
|
||||
|
||||
import java.nio.file.Files
|
||||
|
||||
import nextflow.cli.CliOptions
|
||||
import nextflow.cli.CmdKubeRun
|
||||
import nextflow.cli.Launcher
|
||||
import recreationaltech.plugin.client.ClientConfig
|
||||
import recreationaltech.plugin.client.K8sClient
|
||||
import recreationaltech.plugin.model.PodMountConfig
|
||||
import recreationaltech.plugin.model.PodOptions
|
||||
import recreationaltech.plugin.model.PodSpecBuilder
|
||||
import recreationaltech.plugin.model.PodVolumeClaim
|
||||
import spock.lang.Specification
|
||||
import spock.lang.Unroll
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class K8sDriverLauncherTest extends Specification {
|
||||
|
||||
def setup() {
|
||||
PodSpecBuilder.VOLUMES.set(0)
|
||||
}
|
||||
|
||||
def 'should execute run' () {
|
||||
given:
|
||||
def NAME = 'nxf-foo'
|
||||
def NF_CONFIG = [process:[executor:'k8s']]
|
||||
def K8S_CONFIG = Mock(K8sConfig)
|
||||
def K8S_CLIENT = Mock(K8sClient)
|
||||
def driver = Spy(K8sDriverLauncher)
|
||||
|
||||
when:
|
||||
driver.run(NAME, ['a','b','c'])
|
||||
then:
|
||||
1 * driver.makeConfig(NAME) >> NF_CONFIG
|
||||
1 * driver.makeK8sConfig(NF_CONFIG) >> K8S_CONFIG
|
||||
1 * driver.makeK8sClient(K8S_CONFIG) >> K8S_CLIENT
|
||||
1 * K8S_CONFIG.checkStorageAndPaths(K8S_CLIENT)
|
||||
1 * driver.createK8sConfigMap() >> null
|
||||
1 * driver.createK8sLauncherPod() >> null
|
||||
1 * driver.waitPodStart() >> null
|
||||
1 * driver.printK8sPodOutput() >> null
|
||||
|
||||
driver.pipelineName == NAME
|
||||
driver.interactive == false
|
||||
driver.config == NF_CONFIG
|
||||
driver.k8sConfig == K8S_CONFIG
|
||||
driver.k8sClient == K8S_CLIENT
|
||||
}
|
||||
|
||||
def 'should make k8s config' () {
|
||||
|
||||
given:
|
||||
K8sConfig k8sConfig
|
||||
K8sDriverLauncher driver = Spy(K8sDriverLauncher)
|
||||
|
||||
when:
|
||||
k8sConfig = driver.makeK8sConfig([:])
|
||||
then:
|
||||
k8sConfig != null
|
||||
|
||||
when:
|
||||
k8sConfig = driver.makeK8sConfig(k8s: [storageClaimName: 'foo', storageMountPath: '/mnt'])
|
||||
then:
|
||||
k8sConfig.getStorageClaimName() == 'foo'
|
||||
k8sConfig.getStorageMountPath() == '/mnt'
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Unroll
|
||||
def 'should get cmd cli' () {
|
||||
|
||||
given:
|
||||
def l = new K8sDriverLauncher(cmd: cmd, pipelineName: 'foo')
|
||||
|
||||
when:
|
||||
cmd.launcher = new Launcher(options: new CliOptions())
|
||||
then:
|
||||
l.getLaunchCli() == expected
|
||||
|
||||
where:
|
||||
cmd | expected
|
||||
new CmdKubeRun() | 'nextflow run foo'
|
||||
new CmdKubeRun(cacheable: false) | 'nextflow run foo -cache false'
|
||||
new CmdKubeRun(resume: true) | 'nextflow run foo -resume true'
|
||||
new CmdKubeRun(poolSize: 10) | 'nextflow run foo -ps 10'
|
||||
new CmdKubeRun(pollInterval: 5) | 'nextflow run foo -pi 5'
|
||||
new CmdKubeRun(queueSize: 9) | 'nextflow run foo -qs 9'
|
||||
new CmdKubeRun(revision: 'xyz') | 'nextflow run foo -r xyz'
|
||||
new CmdKubeRun(latest: true) | 'nextflow run foo -latest true'
|
||||
new CmdKubeRun(withTrace: true) | 'nextflow run foo -with-trace true'
|
||||
new CmdKubeRun(withTimeline: true) | 'nextflow run foo -with-timeline true'
|
||||
new CmdKubeRun(withDag: true) | 'nextflow run foo -with-dag true'
|
||||
new CmdKubeRun(dumpHashes: true) | 'nextflow run foo -dump-hashes true'
|
||||
new CmdKubeRun(dumpChannels: 'lala') | 'nextflow run foo -dump-channels lala'
|
||||
new CmdKubeRun(env: [XX:'hello', YY: 'world']) | 'nextflow run foo -e.XX hello -e.YY world'
|
||||
new CmdKubeRun(process: [mem: '100',cpus:'2']) | 'nextflow run foo -process.mem 100 -process.cpus 2'
|
||||
new CmdKubeRun(params: [alpha:'x', beta:'y']) | 'nextflow run foo --alpha x --beta y'
|
||||
new CmdKubeRun(params: [alpha: '/path/*.txt']) | 'nextflow run foo --alpha /path/\\*.txt'
|
||||
new CmdKubeRun(entryName: 'lala') | 'nextflow run foo -entry lala'
|
||||
}
|
||||
|
||||
def 'should set the run name' () {
|
||||
given:
|
||||
def cmd = new CmdKubeRun()
|
||||
cmd.launcher = new Launcher(options: new CliOptions())
|
||||
|
||||
when:
|
||||
def l = new K8sDriverLauncher(cmd: cmd, pipelineName: 'foo', runName: 'bar')
|
||||
then:
|
||||
l.getLaunchCli() == 'nextflow run foo -name bar'
|
||||
}
|
||||
|
||||
|
||||
|
||||
def 'should create launcher spec pod' () {
|
||||
|
||||
given:
|
||||
def pod = Mock(PodOptions)
|
||||
pod.getVolumeClaims() >> [ new PodVolumeClaim('pvc-1', '/mnt/path/data') ]
|
||||
pod.getMountConfigMaps() >> [ new PodMountConfig('cfg-2', '/mnt/path/cfg') ]
|
||||
pod.automountServiceAccountToken = false
|
||||
|
||||
def k8s = Mock(K8sConfig)
|
||||
k8s.getNextflowImageName() >> 'the-image'
|
||||
k8s.getLaunchDir() >> '/the/user/dir'
|
||||
k8s.getWorkDir() >> '/the/work/dir'
|
||||
k8s.getProjectDir() >> '/the/project/dir'
|
||||
k8s.getPodOptions() >> pod
|
||||
|
||||
and:
|
||||
def driver = Spy(K8sDriverLauncher)
|
||||
driver.@runName = 'foo-boo'
|
||||
driver.@k8sClient = new K8sClient(new ClientConfig(namespace: 'foo', serviceAccount: 'bar'))
|
||||
driver.@k8sConfig = k8s
|
||||
|
||||
when:
|
||||
def spec = driver.makeLauncherSpec()
|
||||
then:
|
||||
driver.getLaunchCli() >> 'nextflow run foo'
|
||||
|
||||
spec == [
|
||||
apiVersion: 'v1',
|
||||
kind: 'Pod',
|
||||
metadata: [name:'foo-boo', namespace:'foo', labels:[app:'nextflow', runName:'foo-boo']],
|
||||
spec: [
|
||||
restartPolicy: 'Never',
|
||||
containers: [
|
||||
[
|
||||
name: 'foo-boo',
|
||||
image: 'the-image',
|
||||
command: ['/bin/bash', '-c', "source /etc/nextflow/init.sh; nextflow run foo; if [ -x /etc/nextflow/node-init-cleanup.sh ]; then /etc/nextflow/node-init-cleanup.sh || true; fi; exit \$status"],
|
||||
env: [
|
||||
[name:'NXF_WORK', value:'/the/work/dir'],
|
||||
[name:'NXF_ASSETS', value:'/the/project/dir'],
|
||||
[name:'NXF_EXECUTOR', value:'k8s'],
|
||||
[name:'NXF_ANSI_LOG', value: 'false']
|
||||
],
|
||||
volumeMounts: [
|
||||
[name:'vol-1', mountPath:'/mnt/path/data'],
|
||||
[name:'vol-2', mountPath:'/mnt/path/cfg']
|
||||
]
|
||||
]
|
||||
],
|
||||
serviceAccountName: 'bar',
|
||||
automountServiceAccountToken: false,
|
||||
volumes: [
|
||||
[name:'vol-1', persistentVolumeClaim:[claimName:'pvc-1']],
|
||||
[name:'vol-2', configMap:[name:'cfg-2']]
|
||||
]
|
||||
]
|
||||
]
|
||||
|
||||
}
|
||||
|
||||
def 'should create launcher spec job' () {
|
||||
|
||||
given:
|
||||
def pod = Mock(PodOptions)
|
||||
pod.getVolumeClaims() >> [ new PodVolumeClaim('pvc-1', '/mnt/path/data') ]
|
||||
pod.getMountConfigMaps() >> [ new PodMountConfig('cfg-2', '/mnt/path/cfg') ]
|
||||
|
||||
def k8s = Mock(K8sConfig)
|
||||
k8s.getNextflowImageName() >> 'the-image'
|
||||
k8s.getLaunchDir() >> '/the/user/dir'
|
||||
k8s.getWorkDir() >> '/the/work/dir'
|
||||
k8s.getProjectDir() >> '/the/project/dir'
|
||||
k8s.getPodOptions() >> pod
|
||||
k8s.useJobResource() >> true
|
||||
|
||||
and:
|
||||
def driver = Spy(K8sDriverLauncher)
|
||||
driver.@runName = 'foo-boo'
|
||||
driver.@k8sClient = new K8sClient(new ClientConfig(namespace: 'foo', serviceAccount: 'bar'))
|
||||
driver.@k8sConfig = k8s
|
||||
|
||||
and:
|
||||
def metadata = [name: 'foo-boo', namespace: 'foo', labels: [app: 'nextflow', runName: 'foo-boo']]
|
||||
|
||||
when:
|
||||
def spec = driver.makeLauncherSpec()
|
||||
then:
|
||||
driver.getLaunchCli() >> 'nextflow run foo'
|
||||
|
||||
spec == [
|
||||
apiVersion: 'batch/v1',
|
||||
kind: 'Job',
|
||||
metadata: metadata,
|
||||
spec: [
|
||||
backoffLimit: 0,
|
||||
template: [
|
||||
metadata: metadata,
|
||||
spec: [
|
||||
restartPolicy: 'Never',
|
||||
containers: [
|
||||
[
|
||||
name: 'foo-boo',
|
||||
image: 'the-image',
|
||||
command: ['/bin/bash', '-c', "source /etc/nextflow/init.sh; nextflow run foo; if [ -x /etc/nextflow/node-init-cleanup.sh ]; then /etc/nextflow/node-init-cleanup.sh || true; fi; exit \$status"],
|
||||
env: [
|
||||
[name:'NXF_WORK', value:'/the/work/dir'],
|
||||
[name:'NXF_ASSETS', value:'/the/project/dir'],
|
||||
[name:'NXF_EXECUTOR', value:'k8s'],
|
||||
[name:'NXF_ANSI_LOG', value: 'false']
|
||||
],
|
||||
volumeMounts: [
|
||||
[name:'vol-1', mountPath:'/mnt/path/data'],
|
||||
[name:'vol-2', mountPath:'/mnt/path/cfg']
|
||||
]
|
||||
]
|
||||
],
|
||||
serviceAccountName: 'bar',
|
||||
automountServiceAccountToken: false,
|
||||
volumes: [
|
||||
[name:'vol-1', persistentVolumeClaim:[claimName:'pvc-1']],
|
||||
[name:'vol-2', configMap:[name:'cfg-2']]
|
||||
]
|
||||
]
|
||||
]
|
||||
]
|
||||
]
|
||||
}
|
||||
|
||||
def 'should use user provided pod image' () {
|
||||
|
||||
given:
|
||||
def pod = Mock(PodOptions)
|
||||
pod.getVolumeClaims() >> [ new PodVolumeClaim('pvc-1', '/mnt/path/data') ]
|
||||
pod.getMountConfigMaps() >> [ new PodMountConfig('cfg-2', '/mnt/path/cfg') ]
|
||||
|
||||
def k8s = Mock(K8sConfig)
|
||||
k8s.getLaunchDir() >> '/the/user/dir'
|
||||
k8s.getWorkDir() >> '/the/work/dir'
|
||||
k8s.getProjectDir() >> '/the/project/dir'
|
||||
k8s.getPodOptions() >> pod
|
||||
|
||||
and:
|
||||
def driver = Spy(K8sDriverLauncher)
|
||||
driver.@runName = 'foo-boo'
|
||||
driver.@k8sClient = new K8sClient(new ClientConfig(namespace: 'foo', serviceAccount: 'bar'))
|
||||
driver.@k8sConfig = k8s
|
||||
driver.@headImage = 'foo/bar'
|
||||
|
||||
when:
|
||||
def result = driver.makeLauncherSpec()
|
||||
then:
|
||||
driver.getLaunchCli() >> 'nextflow run foo'
|
||||
and:
|
||||
result.spec.containers[0].image == 'foo/bar'
|
||||
|
||||
}
|
||||
|
||||
def 'should use user provided head-cpu and head-memory request' () {
|
||||
given:
|
||||
def pod = Mock(PodOptions)
|
||||
pod.getVolumeClaims() >> [ new PodVolumeClaim('pvc-1', '/mnt/path/data') ]
|
||||
pod.getMountConfigMaps() >> [ new PodMountConfig('cfg-2', '/mnt/path/cfg') ]
|
||||
|
||||
def k8s = Mock(K8sConfig)
|
||||
k8s.getNextflowImageName() >> 'the-image'
|
||||
k8s.getLaunchDir() >> '/the/user/dir'
|
||||
k8s.getWorkDir() >> '/the/work/dir'
|
||||
k8s.getProjectDir() >> '/the/project/dir'
|
||||
k8s.getPodOptions() >> pod
|
||||
|
||||
and:
|
||||
def driver = Spy(K8sDriverLauncher)
|
||||
driver.@runName = 'foo-boo'
|
||||
driver.@k8sClient = new K8sClient(new ClientConfig(namespace: 'foo', serviceAccount: 'bar'))
|
||||
driver.@k8sConfig = k8s
|
||||
driver.@headCpus = 2
|
||||
driver.@headMemory = '200Mi'
|
||||
|
||||
when:
|
||||
def result = driver.makeLauncherSpec()
|
||||
then:
|
||||
driver.getLaunchCli() >> 'nextflow run foo'
|
||||
and:
|
||||
result.spec.containers[0].resources == [
|
||||
requests: [cpu: 2, memory: '200Mi'],
|
||||
limits: [memory: '200Mi']
|
||||
]
|
||||
}
|
||||
|
||||
def 'should use user provided head-cpu and head-memory limits' () {
|
||||
given:
|
||||
def pod = Mock(PodOptions)
|
||||
pod.getVolumeClaims() >> [ new PodVolumeClaim('pvc-1', '/mnt/path/data') ]
|
||||
pod.getMountConfigMaps() >> [ new PodMountConfig('cfg-2', '/mnt/path/cfg') ]
|
||||
|
||||
def k8s = Mock(K8sConfig)
|
||||
k8s.getNextflowImageName() >> 'the-image'
|
||||
k8s.getLaunchDir() >> '/the/user/dir'
|
||||
k8s.getWorkDir() >> '/the/work/dir'
|
||||
k8s.getProjectDir() >> '/the/project/dir'
|
||||
k8s.getPodOptions() >> pod
|
||||
k8s.cpuLimitsEnabled() >> true
|
||||
|
||||
and:
|
||||
def driver = Spy(K8sDriverLauncher)
|
||||
driver.@runName = 'foo-boo'
|
||||
driver.@k8sClient = new K8sClient(new ClientConfig(namespace: 'foo', serviceAccount: 'bar'))
|
||||
driver.@k8sConfig = k8s
|
||||
driver.@headCpus = 2
|
||||
driver.@headMemory = '200Mi'
|
||||
|
||||
when:
|
||||
def result = driver.makeLauncherSpec()
|
||||
then:
|
||||
driver.getLaunchCli() >> 'nextflow run foo'
|
||||
and:
|
||||
result.spec.containers[0].resources == [
|
||||
requests: [cpu: 2, memory: '200Mi'],
|
||||
limits: [cpu: 2, memory: '200Mi']
|
||||
]
|
||||
}
|
||||
|
||||
def 'should create config map' () {
|
||||
|
||||
given:
|
||||
def folder = Files.createTempDirectory('foo')
|
||||
|
||||
def params = folder.resolve('params.json')
|
||||
params.text = 'bla-bla'
|
||||
def driver = Spy(K8sDriverLauncher)
|
||||
def NXF_CONFIG = [foo: 'bar'].toConfigObject()
|
||||
|
||||
def SCM_FILE = folder.resolve('scm')
|
||||
SCM_FILE.text = "hello = 'world'\n"
|
||||
|
||||
|
||||
def EXPECTED = [:]
|
||||
EXPECTED['init.sh'] == ''
|
||||
|
||||
def POD_OPTIONS = new PodOptions()
|
||||
|
||||
def K8S_CONFIG = Mock(K8sConfig)
|
||||
K8S_CONFIG.getLaunchDir() >> '/launch/dir'
|
||||
K8S_CONFIG.getPodOptions() >> POD_OPTIONS
|
||||
|
||||
when:
|
||||
driver.@config = NXF_CONFIG
|
||||
driver.@k8sConfig = K8S_CONFIG
|
||||
driver.@cmd = new CmdKubeRun(paramsFile: params.toString())
|
||||
|
||||
driver.createK8sConfigMap()
|
||||
then:
|
||||
1 * driver.getScmFile() >> SCM_FILE
|
||||
1 * driver.makeConfigMapName(_ as Map) >> 'nf-config-123'
|
||||
1 * driver.tryCreateConfigMap('nf-config-123', _ as Map) >> { name, cfg ->
|
||||
assert cfg.'init.sh' == "mkdir -p '/launch/dir'; if [ -d '/launch/dir' ]; then cd '/launch/dir'; else echo 'Cannot create directory: /launch/dir'; exit 1; fi; [ -f /etc/nextflow/scm ] && ln -s /etc/nextflow/scm \$NXF_HOME/scm; [ -f /etc/nextflow/nextflow.config ] && cp /etc/nextflow/nextflow.config \$PWD/nextflow.config; "
|
||||
assert cfg.'nextflow.config' == "foo = 'bar'\n"
|
||||
assert cfg.'scm' == "hello = 'world'\n"
|
||||
assert cfg.'params.json' == 'bla-bla'
|
||||
return null
|
||||
}
|
||||
|
||||
POD_OPTIONS.getMountConfigMaps() == [ new PodMountConfig('nf-config-123', '/etc/nextflow') ] as Set
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
def 'should create config map with pre-script' () {
|
||||
|
||||
given:
|
||||
def folder = Files.createTempDirectory('foo')
|
||||
|
||||
def params = folder.resolve('params.json')
|
||||
params.text = 'bla-bla'
|
||||
def driver = Spy(K8sDriverLauncher)
|
||||
driver.@headPreScript = '/bin/foo.sh'
|
||||
def NXF_CONFIG = [foo: 'bar'].toConfigObject()
|
||||
|
||||
def SCM_FILE = folder.resolve('scm')
|
||||
SCM_FILE.text = "hello = 'world'\n"
|
||||
|
||||
|
||||
def EXPECTED = [:]
|
||||
EXPECTED['init.sh'] == ''
|
||||
|
||||
def POD_OPTIONS = new PodOptions()
|
||||
|
||||
def K8S_CONFIG = Mock(K8sConfig)
|
||||
K8S_CONFIG.getLaunchDir() >> '/launch/dir'
|
||||
K8S_CONFIG.getPodOptions() >> POD_OPTIONS
|
||||
|
||||
when:
|
||||
driver.@config = NXF_CONFIG
|
||||
driver.@k8sConfig = K8S_CONFIG
|
||||
driver.@cmd = new CmdKubeRun(paramsFile: params.toString())
|
||||
|
||||
driver.createK8sConfigMap()
|
||||
then:
|
||||
1 * driver.getScmFile() >> SCM_FILE
|
||||
1 * driver.makeConfigMapName(_ as Map) >> 'nf-config-123'
|
||||
1 * driver.tryCreateConfigMap('nf-config-123', _ as Map) >> { name, cfg ->
|
||||
assert cfg.'init.sh' == "mkdir -p '/launch/dir'; if [ -d '/launch/dir' ]; then cd '/launch/dir'; else echo 'Cannot create directory: /launch/dir'; exit 1; fi; [ -f /etc/nextflow/scm ] && ln -s /etc/nextflow/scm \$NXF_HOME/scm; [ -f /etc/nextflow/nextflow.config ] && cp /etc/nextflow/nextflow.config \$PWD/nextflow.config; [ -f '/bin/foo.sh' ] && '/bin/foo.sh'; "
|
||||
assert cfg.'nextflow.config' == "foo = 'bar'\n"
|
||||
assert cfg.'scm' == "hello = 'world'\n"
|
||||
assert cfg.'params.json' == 'bla-bla'
|
||||
return null
|
||||
}
|
||||
|
||||
POD_OPTIONS.getMountConfigMaps() == [ new PodMountConfig('nf-config-123', '/etc/nextflow') ] as Set
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
|
||||
def 'should make config' () {
|
||||
given:
|
||||
Map config
|
||||
def driver = Spy(K8sDriverLauncher)
|
||||
def NAME = 'somePipelineName'
|
||||
def CFG_EMPTY = new ConfigObject()
|
||||
def CFG_WITH_MOUNTS = new ConfigObject()
|
||||
CFG_WITH_MOUNTS.k8s.storageClaimName = 'pvc'
|
||||
CFG_WITH_MOUNTS.k8s.storageMountPath = '/foo'
|
||||
|
||||
when:
|
||||
driver.@cmd = new CmdKubeRun()
|
||||
config = driver.makeConfig(NAME).toMap()
|
||||
then:
|
||||
1 * driver.loadConfig(NAME) >> CFG_EMPTY
|
||||
config.process.executor == 'k8s'
|
||||
config.k8s.pod == null
|
||||
config.k8s.storageMountPath == null
|
||||
config.k8s.storageClaimName == null
|
||||
|
||||
when:
|
||||
driver.@cmd = new CmdKubeRun()
|
||||
config = driver.makeConfig(NAME).toMap()
|
||||
then:
|
||||
1 * driver.loadConfig(NAME) >> CFG_WITH_MOUNTS
|
||||
config.process.executor == 'k8s'
|
||||
config.k8s.storageClaimName == 'pvc'
|
||||
config.k8s.storageMountPath == '/foo'
|
||||
and:
|
||||
new K8sConfig(config.k8s).getStorageClaimName() == 'pvc'
|
||||
new K8sConfig(config.k8s).getStorageMountPath() == '/foo'
|
||||
new K8sConfig(config.k8s).getPodOptions() == new PodOptions([ [volumeClaim:'pvc', mountPath: '/foo'] ])
|
||||
|
||||
when:
|
||||
driver.@cmd = new CmdKubeRun(volMounts: ['pvc-1:/this','pvc-2:/that'] )
|
||||
config = driver.makeConfig(NAME).toMap()
|
||||
then:
|
||||
1 * driver.loadConfig(NAME) >> CFG_EMPTY
|
||||
config.process.executor == 'k8s'
|
||||
config.k8s.storageClaimName == 'pvc-1'
|
||||
config.k8s.storageMountPath == '/this'
|
||||
config.k8s.pod == [ [volumeClaim: 'pvc-2', mountPath: '/that'] ]
|
||||
and:
|
||||
new K8sConfig(config.k8s).getStorageClaimName() == 'pvc-1'
|
||||
new K8sConfig(config.k8s).getStorageMountPath() == '/this'
|
||||
new K8sConfig(config.k8s).getPodOptions() == new PodOptions([
|
||||
[volumeClaim:'pvc-1', mountPath: '/this'],
|
||||
[volumeClaim:'pvc-2', mountPath: '/that']
|
||||
])
|
||||
|
||||
|
||||
when:
|
||||
driver.@cmd = new CmdKubeRun(volMounts: ['xyz:/this'] )
|
||||
config = driver.makeConfig(NAME).toMap()
|
||||
then:
|
||||
1 * driver.loadConfig(NAME) >> CFG_WITH_MOUNTS
|
||||
config.process.executor == 'k8s'
|
||||
config.k8s.storageClaimName == 'xyz'
|
||||
config.k8s.storageMountPath == '/this'
|
||||
config.k8s.pod == null
|
||||
and:
|
||||
new K8sConfig(config.k8s).getStorageClaimName() == 'xyz'
|
||||
new K8sConfig(config.k8s).getStorageMountPath() == '/this'
|
||||
new K8sConfig(config.k8s).getPodOptions() == new PodOptions([
|
||||
[volumeClaim:'xyz', mountPath: '/this']
|
||||
])
|
||||
|
||||
|
||||
when:
|
||||
driver.@cmd = new CmdKubeRun(volMounts: ['xyz', 'bar:/mnt/bar'] )
|
||||
config = driver.makeConfig(NAME).toMap()
|
||||
then:
|
||||
1 * driver.loadConfig(NAME) >> CFG_WITH_MOUNTS
|
||||
config.process.executor == 'k8s'
|
||||
config.k8s.storageClaimName == 'xyz'
|
||||
config.k8s.storageMountPath == null
|
||||
config.k8s.pod == [ [volumeClaim: 'bar', mountPath: '/mnt/bar'] ]
|
||||
and:
|
||||
new K8sConfig(config.k8s).getStorageClaimName() == 'xyz'
|
||||
new K8sConfig(config.k8s).getStorageMountPath() == '/workspace'
|
||||
new K8sConfig(config.k8s).getPodOptions() == new PodOptions([
|
||||
[volumeClaim:'xyz', mountPath: '/workspace'],
|
||||
[volumeClaim:'bar', mountPath: '/mnt/bar']
|
||||
])
|
||||
|
||||
}
|
||||
|
||||
def 'should add the plugin into the config' () {
|
||||
given:
|
||||
def cmd = new CmdKubeRun()
|
||||
cmd.launcher = new Launcher(options: new CliOptions())
|
||||
|
||||
when:
|
||||
def l = new K8sDriverLauncher(cmd: cmd, plugins: 'nf-cws@1.0.0', runName: 'bar')
|
||||
then:
|
||||
l.makeConfig( "/bar").get('plugins') == [ 'nf-cws@1.0.0' ]
|
||||
}
|
||||
|
||||
def 'should make config - deprecated' () {
|
||||
|
||||
given:
|
||||
Map config
|
||||
def driver = Spy(K8sDriverLauncher)
|
||||
def NAME = 'somePipelineName'
|
||||
def CFG_EMPTY = new ConfigObject()
|
||||
def CFG_WITH_MOUNTS = new ConfigObject()
|
||||
CFG_WITH_MOUNTS.k8s.volumeClaims = [ pvc: [mountPath:'/foo'] ]
|
||||
|
||||
when:
|
||||
driver.@cmd = new CmdKubeRun()
|
||||
config = driver.makeConfig(NAME).toMap()
|
||||
then:
|
||||
1 * driver.loadConfig(NAME) >> CFG_EMPTY
|
||||
config.process.executor == 'k8s'
|
||||
|
||||
when:
|
||||
driver.@cmd = new CmdKubeRun()
|
||||
config = driver.makeConfig(NAME).toMap()
|
||||
then:
|
||||
1 * driver.loadConfig(NAME) >> CFG_WITH_MOUNTS
|
||||
config.process.executor == 'k8s'
|
||||
config.k8s.storageClaimName == 'pvc'
|
||||
config.k8s.storageMountPath == '/foo'
|
||||
|
||||
when:
|
||||
driver.@cmd = new CmdKubeRun(volMounts: ['pvc-1:/this','pvc-2:/that'] )
|
||||
config = driver.makeConfig(NAME).toMap()
|
||||
then:
|
||||
1 * driver.loadConfig(NAME) >> CFG_EMPTY
|
||||
config.process.executor == 'k8s'
|
||||
config.k8s.storageClaimName == 'pvc-1'
|
||||
config.k8s.storageMountPath == '/this'
|
||||
config.k8s.pod == [ [volumeClaim: 'pvc-2', mountPath: '/that'] ]
|
||||
|
||||
|
||||
when:
|
||||
driver.@cmd = new CmdKubeRun(volMounts: ['xyz:/this'] )
|
||||
config = driver.makeConfig(NAME).toMap()
|
||||
then:
|
||||
1 * driver.loadConfig(NAME) >> CFG_WITH_MOUNTS
|
||||
config.process.executor == 'k8s'
|
||||
config.k8s.storageClaimName == 'xyz'
|
||||
config.k8s.storageMountPath == '/this'
|
||||
config.k8s.pod == null
|
||||
and:
|
||||
new K8sConfig(config.k8s).getStorageClaimName() == 'xyz'
|
||||
new K8sConfig(config.k8s).getStorageMountPath() == '/this'
|
||||
new K8sConfig(config.k8s).getPodOptions() == new PodOptions([
|
||||
[volumeClaim:'xyz', mountPath: '/this']
|
||||
])
|
||||
|
||||
}
|
||||
|
||||
def 'should return pod exit status' () {
|
||||
given:
|
||||
def POD_NAME = 'pod-x'
|
||||
def client = Mock(K8sClient)
|
||||
def driver = Spy(K8sDriverLauncher)
|
||||
driver.@k8sClient = client
|
||||
driver.@runName = POD_NAME
|
||||
driver.@k8sConfig = Mock(K8sConfig)
|
||||
|
||||
when:
|
||||
def status = driver.waitPodTermination()
|
||||
then:
|
||||
1 * client.podState(POD_NAME) >> [terminated: [exitCode: 99]]
|
||||
1 * driver.k8sConfig.useJobResource() >> [:]
|
||||
then:
|
||||
status == 99
|
||||
|
||||
when:
|
||||
status = driver.waitPodTermination()
|
||||
then:
|
||||
1 * client.podState(POD_NAME) >> [:]
|
||||
then:
|
||||
1 * client.podState(POD_NAME) >> [terminated: [exitCode: 99]]
|
||||
then:
|
||||
status == 99
|
||||
|
||||
when:
|
||||
status = driver.waitPodTermination()
|
||||
then:
|
||||
1 * client.podState(POD_NAME) >> [:]
|
||||
1 * driver.isWaitTimedOut(_) >> true
|
||||
then:
|
||||
status == 127
|
||||
}
|
||||
|
||||
|
||||
def 'should delete configMap' () {
|
||||
given:
|
||||
def POD_NAME = 'pod-x'
|
||||
def config = Mock(K8sConfig)
|
||||
def driver = Spy(K8sDriverLauncher)
|
||||
driver.@k8sConfig = config
|
||||
driver.@runName = POD_NAME
|
||||
driver.@initDeployer = new K8sNodeInitDeployer(driver.k8sClient, config)
|
||||
|
||||
when:
|
||||
driver.shutdown()
|
||||
then:
|
||||
1 * driver.waitPodTermination() >> 0
|
||||
then:
|
||||
1 * config.getCleanup(true) >> true
|
||||
1 * driver.deleteConfigMap() >> null
|
||||
|
||||
when:
|
||||
driver.shutdown()
|
||||
then:
|
||||
1 * driver.waitPodTermination() >> 1
|
||||
then:
|
||||
1 * config.getCleanup(false) >> true
|
||||
1 * driver.deleteConfigMap() >> null
|
||||
|
||||
when:
|
||||
driver.shutdown()
|
||||
then:
|
||||
1 * driver.waitPodTermination() >> 1
|
||||
then:
|
||||
1 * config.getCleanup(false) >> false
|
||||
0 * driver.deleteConfigMap() >> null
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin
|
||||
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
import com.google.common.cache.CacheBuilder
|
||||
import recreationaltech.plugin.client.ClientConfig
|
||||
import recreationaltech.plugin.client.K8sClient
|
||||
import spock.lang.Specification
|
||||
|
||||
/**
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class K8sExecutorTest extends Specification {
|
||||
|
||||
def 'should cache k8s client and refresh after expiration' () {
|
||||
given:
|
||||
def CONFIG = new K8sConfig(
|
||||
client: [server: 'http://k8s-server'],
|
||||
namespace: 'test-ns',
|
||||
serviceAccount: 'test-sa',
|
||||
clientRefreshInterval: '100ms'
|
||||
)
|
||||
and:
|
||||
def executor = Spy(K8sExecutor)
|
||||
executor.getK8sConfig() >> CONFIG
|
||||
// use a short-lived cache for the test
|
||||
executor.@clientCache = CacheBuilder.newBuilder()
|
||||
.expireAfterWrite(100, TimeUnit.MILLISECONDS)
|
||||
.build()
|
||||
|
||||
when: 'first call to getClient'
|
||||
def client1 = executor.getClient()
|
||||
then: 'a new K8sClient is created'
|
||||
client1 instanceof K8sClient
|
||||
client1.config.server == 'http://k8s-server'
|
||||
|
||||
when: 'second call within cache interval'
|
||||
def client2 = executor.getClient()
|
||||
then: 'returns the same cached instance'
|
||||
client2.is(client1)
|
||||
|
||||
when: 'call after cache expiration'
|
||||
sleep(150)
|
||||
def client3 = executor.getClient()
|
||||
then: 'a new K8sClient instance is created'
|
||||
client3 instanceof K8sClient
|
||||
!client3.is(client1)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
package recreationaltech.plugin
|
||||
|
||||
import recreationaltech.plugin.client.K8sClient
|
||||
import recreationaltech.plugin.model.PodSpecBuilder
|
||||
import spock.lang.Specification
|
||||
|
||||
class K8sNodeInitDeployerTest extends Specification {
|
||||
|
||||
def setup() {
|
||||
PodSpecBuilder.VOLUMES.set(0)
|
||||
}
|
||||
|
||||
def 'should not deploy pods when node init is disabled' () {
|
||||
given:
|
||||
def client = Mock(K8sClient)
|
||||
def config = new K8sConfig(nodeInit: [enabled: false])
|
||||
def deployer = new K8sNodeInitDeployer(client, config)
|
||||
|
||||
when:
|
||||
deployer.deploy()
|
||||
|
||||
then:
|
||||
0 * client.nodeList()
|
||||
0 * client.podCreate(_)
|
||||
}
|
||||
|
||||
def 'should deploy one init pod for each node' () {
|
||||
given:
|
||||
def client = Mock(K8sClient)
|
||||
def config = new K8sConfig(nodeInit: [
|
||||
enabled: true,
|
||||
image: 'ubuntu:latest',
|
||||
command: ['/bin/bash', '-c', 'echo init']
|
||||
])
|
||||
def deployer = new K8sNodeInitDeployer(client, config)
|
||||
|
||||
when:
|
||||
deployer.deploy()
|
||||
|
||||
then:
|
||||
1 * client.nodeList() >> [
|
||||
items: [
|
||||
[metadata: [name: 'node-a']],
|
||||
[metadata: [name: 'node-b']]
|
||||
]
|
||||
]
|
||||
|
||||
then:
|
||||
1 * client.podCreate({ Map spec ->
|
||||
spec.kind == 'Pod'
|
||||
spec.metadata.name == 'nf-init-node-a'
|
||||
spec.metadata.namespace == 'default'
|
||||
spec.spec.nodeName == 'node-a'
|
||||
spec.spec.restartPolicy == 'Never'
|
||||
|
||||
def container = spec.spec.containers[0]
|
||||
container.name == 'nf-init-node-a'
|
||||
container.image == 'ubuntu:latest'
|
||||
container.command == ['/bin/bash', '-c', 'echo init']
|
||||
container.securityContext.privileged == true
|
||||
|
||||
spec.spec.volumes*.hostPath*.path as Set == ['/sys', '/dev', '/lib/modules'] as Set
|
||||
container.volumeMounts*.mountPath as Set == ['/sys', '/dev', '/lib/modules'] as Set
|
||||
})
|
||||
|
||||
then:
|
||||
1 * client.podCreate({ Map spec ->
|
||||
spec.kind == 'Pod'
|
||||
spec.metadata.name == 'nf-init-node-b'
|
||||
spec.spec.nodeName == 'node-b'
|
||||
|
||||
def container = spec.spec.containers[0]
|
||||
container.name == 'nf-init-node-b'
|
||||
container.image == 'ubuntu:latest'
|
||||
container.command == ['/bin/bash', '-c', 'echo init']
|
||||
container.securityContext.privileged == true
|
||||
})
|
||||
|
||||
0 * client._
|
||||
}
|
||||
|
||||
def 'should lowercase and truncate generated pod names' () {
|
||||
given:
|
||||
def client = Mock(K8sClient)
|
||||
def config = new K8sConfig(nodeInit: [
|
||||
enabled: true,
|
||||
image: 'ubuntu:latest',
|
||||
command: ['true']
|
||||
])
|
||||
def deployer = new K8sNodeInitDeployer(client, config)
|
||||
|
||||
when:
|
||||
deployer.deploy()
|
||||
|
||||
then:
|
||||
1 * client.nodeList() >> [
|
||||
items: [
|
||||
[metadata: [name: 'NODE-WITH-A-VERY-LONG-NAME-ABCDEFGHIJKLMNOPQRSTUVWXYZ-0123']]
|
||||
]
|
||||
]
|
||||
|
||||
then:
|
||||
1 * client.podCreate({ Map spec ->
|
||||
spec.metadata.name == 'nf-init-node-with-a-very-long-name-abcdefghijklmnopqrstuvwxyz-0'
|
||||
spec.metadata.name.size() == 63
|
||||
spec.metadata.name == spec.metadata.name.toLowerCase()
|
||||
spec.spec.nodeName == 'NODE-WITH-A-VERY-LONG-NAME-ABCDEFGHIJKLMNOPQRSTUVWXYZ-0123'
|
||||
})
|
||||
|
||||
then:
|
||||
0 * client._
|
||||
}
|
||||
|
||||
def 'should not cleanup pods when node init is disabled' () {
|
||||
given:
|
||||
def client = Mock(K8sClient)
|
||||
def config = new K8sConfig(nodeInit: [enabled: false, cleanup: true])
|
||||
def deployer = new K8sNodeInitDeployer(client, config)
|
||||
|
||||
when:
|
||||
deployer.cleanup()
|
||||
|
||||
then:
|
||||
0 * client.nodeList()
|
||||
0 * client.podDelete(_)
|
||||
}
|
||||
|
||||
def 'should not cleanup pods when cleanup is disabled' () {
|
||||
given:
|
||||
def client = Mock(K8sClient)
|
||||
def config = new K8sConfig(nodeInit: [enabled: true, cleanup: false])
|
||||
def deployer = new K8sNodeInitDeployer(client, config)
|
||||
|
||||
when:
|
||||
deployer.cleanup()
|
||||
|
||||
then:
|
||||
0 * client.nodeList()
|
||||
0 * client.podDelete(_)
|
||||
}
|
||||
|
||||
def 'should cleanup one init pod for each node' () {
|
||||
given:
|
||||
def client = Mock(K8sClient)
|
||||
def config = new K8sConfig(nodeInit: [enabled: true, cleanup: true])
|
||||
def deployer = new K8sNodeInitDeployer(client, config)
|
||||
|
||||
when:
|
||||
deployer.cleanup()
|
||||
|
||||
then:
|
||||
1 * client.nodeList() >> [
|
||||
items: [
|
||||
[metadata: [name: 'node-a']],
|
||||
[metadata: [name: 'NODE-B']]
|
||||
]
|
||||
]
|
||||
|
||||
then:
|
||||
1 * client.podDelete('nf-init-node-a')
|
||||
1 * client.podDelete('nf-init-node-b')
|
||||
0 * client._
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package recreationaltech.plugin
|
||||
|
||||
import spock.lang.Specification
|
||||
|
||||
class K8sRuntimeEstimatorTest extends Specification {
|
||||
|
||||
def 'does extract a function' () {
|
||||
given:
|
||||
def observations = [a: [new Tuple2(0, 0), new Tuple2(100, 100)]]
|
||||
def estimator = new K8sRuntimeEstimator(observations)
|
||||
|
||||
when:
|
||||
def estimation = estimator.estimate("a", 1234)
|
||||
|
||||
then:
|
||||
estimation != Double.POSITIVE_INFINITY
|
||||
estimation == 1234.0
|
||||
}
|
||||
|
||||
def 'can extract a function from single measurement' () {
|
||||
given:
|
||||
def observations = [a: [new Tuple2(100, 100)]]
|
||||
def estimator = new K8sRuntimeEstimator(observations)
|
||||
|
||||
when:
|
||||
def estimation = estimator.estimate("a", 1234)
|
||||
|
||||
then:
|
||||
estimation == 100.0 /* Single measurement implies constant runtime */
|
||||
}
|
||||
|
||||
def 'unknown function returns positive infinity' () {
|
||||
given:
|
||||
def observations = [] as HashMap<String, ArrayList>
|
||||
def estimator = new K8sRuntimeEstimator(observations)
|
||||
|
||||
when:
|
||||
def estimation = estimator.estimate("b", 1234)
|
||||
|
||||
then:
|
||||
estimation == Double.POSITIVE_INFINITY
|
||||
}
|
||||
|
||||
def 'extracts function from multiple measurements' () {
|
||||
given:
|
||||
def observations = [a: [new Tuple2(10, 15), new Tuple2(20, 34), new Tuple2(30, 46)]]
|
||||
def estimator = new K8sRuntimeEstimator(observations)
|
||||
|
||||
when:
|
||||
def estimation = estimator.estimate("a", 40)
|
||||
|
||||
then:
|
||||
estimation != Double.POSITIVE_INFINITY
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package recreationaltech.plugin
|
||||
|
||||
import nextflow.processor.TaskRun
|
||||
import spock.lang.Specification
|
||||
|
||||
class K8sSchedulingRequestTest extends Specification {
|
||||
|
||||
def 'should create scheduling request from task handler' () {
|
||||
given:
|
||||
def task = Mock(TaskRun)
|
||||
def handler = Spy(K8sTaskHandler)
|
||||
handler.task = task
|
||||
|
||||
when:
|
||||
def request = new K8sSchedulingRequest(handler)
|
||||
|
||||
then:
|
||||
request.handler == handler
|
||||
request.task == task
|
||||
request.submitTimeMillis > 0
|
||||
request.submitTimeMillis <= System.currentTimeMillis()
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,62 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin
|
||||
|
||||
import java.nio.file.Files
|
||||
|
||||
import nextflow.Session
|
||||
import nextflow.executor.Executor
|
||||
import nextflow.processor.TaskConfig
|
||||
import nextflow.processor.TaskProcessor
|
||||
import nextflow.processor.TaskRun
|
||||
import spock.lang.Specification
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class K8sWrapperBuilderTest extends Specification {
|
||||
|
||||
def 'should render launcher script' () {
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
and:
|
||||
def sess = Mock(Session)
|
||||
def exec = Mock(Executor)
|
||||
def proc = Mock(TaskProcessor) { getSession() >> sess; getExecutor() >> exec }
|
||||
def config = new TaskConfig()
|
||||
def task = Mock(TaskRun) {
|
||||
getName() >> 'foo'
|
||||
getConfig() >> config
|
||||
getProcessor() >> proc
|
||||
getWorkDir() >> folder
|
||||
getInputFilesMap() >> [:]
|
||||
getOutputFilesNames() >> []
|
||||
}
|
||||
|
||||
and:
|
||||
def builder = Spy(new K8sWrapperBuilder(task)) { getSecretsEnv() >> null; fixOwnership() >> false }
|
||||
|
||||
when:
|
||||
def binding = builder.makeBinding()
|
||||
|
||||
then:
|
||||
binding.header_script == "NXF_CHDIR=${folder}"
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.client
|
||||
|
||||
import java.nio.file.Files
|
||||
|
||||
import spock.lang.Specification
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class ClientConfigTest extends Specification {
|
||||
|
||||
def 'should stringify a config' () {
|
||||
|
||||
when:
|
||||
final CERT = 'LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNTRENDQVRDZ0F3SUJBZ0lJRlFsM1l2Y2k1TWN3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB4T0RBeE1UTXlNREEyTlRaYUZ3MHhPVEF4TVRNeU1EQTJOVFphTURNeApGREFTQmdOVkJBb1RDMFJ2WTJ0bGNpQkpibU11TVJzd0dRWURWUVFERXhKa2IyTnJaWEl0Wm05eUxXUmxjMnQwCmIzQXdnWjh3RFFZSktvWklodmNOQVFFQkJRQURnWTBBTUlHSkFvR0JBUEtYT0ZsV2t2THIzb29ETGNFOElyME0KTzNBMHZqQlVvUzZ0bUdBbFRYYTd0QWQwM3BTMXNJNit0WVRwVlU2YXR6ZU9vU0VrOWhmaWxBdVNYdG1hSHZCUAp1czFEcG1LZEZRMWI3OFRkSnQ4OGV3c3BRajFxYUwvQldHeitMUzUrRHUrNUJuUGtmZlhDS1UxQTdUc2tZamJyClhxeDhlN2FWZURWTmFjZXc0Z0RqQWdNQkFBR2pBakFBTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFBMXVtVlAKR29EZTVCRXJrb21qWXdITXhiTTd4UStibTYrUDE1T0pINUo0UGNQeU11d25ocC9ORVp1NnpsTTZSUUo3SUNKQgpHWTRBMnFKVmJsWUkwQkJzRkF1TXMreTAyazdVVVVoK0NRYVd0SXhBcFNmbkQ4dUVXQ0g5VE1ZNGdLbTZjTDhVCk1OVVl1RnpUQ2hmTS96RjdUMXVaZWxJYXNrYXFaWSt3a3hxa3YyRUQxQ2F5MDUxSXRWRXZVbDIvSVZyVHdrT20KZ25nL3Q4L2RkeDhpOUkzTFJrMTlTaERKdXlQZ1NrTTZRSWlSd09mRHk4V0ZFaURpd0hBS0ErSEZhTGhOOFJTMwpieDUvdEhEN01id0FpdnorNTU4YUFEQjNEd1ZpekthM2d5Wm4yUzRjUGFqZnNwODFqRkNIQS9QekdQdTU2MzJwCkxRN0gyRW1aYmJuUHFYTFgKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo='.decodeBase64()
|
||||
final KEY = 'LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlDWGdJQkFBS0JnUUR5bHpoWlZwTHk2OTZLQXkzQlBDSzlERHR3Tkw0d1ZLRXVyWmhnSlUxMnU3UUhkTjZVCnRiQ092cldFNlZWT21yYzNqcUVoSlBZWDRwUUxrbDdabWg3d1Q3ck5RNlppblJVTlcrL0UzU2JmUEhzTEtVSTkKYW1pL3dWaHMvaTB1Zmc3dnVRWno1SDMxd2lsTlFPMDdKR0kyNjE2c2ZIdTJsWGcxVFduSHNPSUE0d0lEQVFBQgpBb0dBYWRUOCtVU2lvU1d6bFVRanZ1eHNQMHRKMXY2N2hqdzFnVGFzaGkxZjZRK2tUNmgxdml5eGxPU3dMZ2JaCmQ0eFpwL3dxWVZwTm5rZnp6RVNUNnB5cEo5WTEwdHY1cFpSWG9HbG1NT2tIZSswUW45N0c5ZDRzL2JCV3lmYXYKRzhRTC9tZFN6Vy85YUdrSkpiNWU0VDlsSURvRDNFVDgwYUFWbzl2V0NPVUxsdWtDUVFEK0hINU5ucVBuSTdnTApWOUJKZzlRRVBwUTVYa2traW8rejZ2YkRHQU5rR1VPV1dmRURKUHE2Q2JBb1dqeWh1Qy9KS1dYRWs4Rkt0M1Y2CkhVNllYeVpGQWtFQTlHVE9XOFM4KzVNNHE5R3lNeURxN1ZkVHA3M2daeSsvNjVQam5hNlpDUnhTZklxL2xKUVoKY2F6MkhGYVRzRFdLbkdhWGNxTmdBVXNEODNyWTlzM3hCd0pCQUt5Vjc1YUtPMm0rRWI3cWVsV2p5bmpEZytwZQp4akNpUnkxOFZQSjJPYjlmaFU3MWNVS2dlQVdvbE5NalRuREw1dkNxUkNzNTZ4cnk5VC9sN2I2QlNUMENRUURnCjRoV2xDZTdnQzhOZEQzTkxhdUhpRGJZenB4dmp0Mk9Ca2E4ai9ISmptTVVxUnI0dEtPNFUxUlFPVlhoRzc2MmgKWnlHNjRpeklZOCs1N3ZQUWZ3Wm5Ba0VBdW9RWW1lUi90UWhIakhRNFlhZGRHbkNBQ2hZZ29ObEFzSGhGTElxVQo1ZTZaMXN2Q3VKU285TDVVRCtrclFUYWlGU01pRHZwZlJyVE1ZKzZ5Q0tTajd3PT0KLS0tLS1FTkQgUlNBIFBSSVZBVEUgS0VZLS0tLS0K'.decodeBase64()
|
||||
|
||||
def config = new ClientConfig()
|
||||
config.clientCert = CERT
|
||||
config.clientKey = KEY
|
||||
config.sslCert = CERT
|
||||
println config.toString()
|
||||
|
||||
then:
|
||||
noExceptionThrown()
|
||||
|
||||
}
|
||||
|
||||
def 'should create a client config from a map' () {
|
||||
|
||||
given:
|
||||
def MAP = [
|
||||
server:'foo.com',
|
||||
token: 'blah-blah',
|
||||
namespace: 'my-namespace',
|
||||
verifySsl: true,
|
||||
sslCert: 'fizzbuzz'.bytes.encodeBase64().toString(),
|
||||
clientCert: 'hello'.bytes.encodeBase64().toString(),
|
||||
clientKey: 'world'.bytes.encodeBase64().toString() ]
|
||||
|
||||
when:
|
||||
def result = ClientConfig.fromNextflowConfig(MAP, null, null)
|
||||
|
||||
then:
|
||||
result.server == 'foo.com'
|
||||
result.token == 'blah-blah'
|
||||
result.namespace == 'my-namespace'
|
||||
result.serviceAccount == 'default'
|
||||
result.verifySsl
|
||||
result.clientCert == 'hello'.bytes
|
||||
result.clientKey == 'world'.bytes
|
||||
result.sslCert == 'fizzbuzz'.bytes
|
||||
|
||||
when:
|
||||
result = ClientConfig.fromNextflowConfig(MAP, 'ns1', 'sa2')
|
||||
then:
|
||||
result.server == 'foo.com'
|
||||
result.token == 'blah-blah'
|
||||
result.namespace == 'ns1'
|
||||
result.serviceAccount == 'sa2'
|
||||
result.verifySsl
|
||||
result.clientCert == 'hello'.bytes
|
||||
result.clientKey == 'world'.bytes
|
||||
result.sslCert == 'fizzbuzz'.bytes
|
||||
}
|
||||
|
||||
def 'should create a client config from a map with files' () {
|
||||
|
||||
given:
|
||||
def folder = Files.createTempDirectory('test')
|
||||
def file1 = folder.resolve('file1')
|
||||
def file2 = folder.resolve('file2')
|
||||
def file3 = folder.resolve('file3')
|
||||
file1.text = 'fizzbuzz'.bytes.encodeBase64().toString()
|
||||
file2.text = 'hello'.bytes.encodeBase64().toString()
|
||||
file3.text = 'world'.bytes.encodeBase64().toString()
|
||||
|
||||
def MAP = [
|
||||
server:'foo.com',
|
||||
token: 'blah-blah',
|
||||
namespace: 'my-namespace',
|
||||
verifySsl: false,
|
||||
sslCertFile: file1,
|
||||
clientCertFile: file2,
|
||||
clientKeyFile: file3 ]
|
||||
|
||||
when:
|
||||
def result = ClientConfig.fromNextflowConfig(MAP, null, null)
|
||||
|
||||
then:
|
||||
result.server == 'foo.com'
|
||||
result.token == 'blah-blah'
|
||||
result.namespace == 'my-namespace'
|
||||
result.serviceAccount == 'default'
|
||||
!result.verifySsl
|
||||
result.sslCert == file1.text.bytes
|
||||
result.clientCert == file2.text.bytes
|
||||
result.clientKey == file3.text.bytes
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,437 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.client
|
||||
import javax.net.ssl.KeyManager
|
||||
import java.nio.file.Files
|
||||
|
||||
import spock.lang.Specification
|
||||
import test.TestHelper
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class ConfigDiscoveryTest extends Specification {
|
||||
|
||||
def 'should read config from file' () {
|
||||
|
||||
given:
|
||||
final CERT_DATA = "d29ybGQgaGVsbG8="
|
||||
final CLIENT_CERT = "aGVsbG8gd29ybGQ="
|
||||
final CLIENT_KEY = "Y2lhbyBtaWFv"
|
||||
|
||||
def CONFIG = TestHelper.createInMemTempFile('config')
|
||||
CONFIG.text = """
|
||||
apiVersion: v1
|
||||
clusters:
|
||||
- cluster:
|
||||
insecure-skip-tls-verify: true
|
||||
server: https://localhost:6443
|
||||
certificate-authority-data: $CERT_DATA
|
||||
name: docker-for-desktop-cluster
|
||||
contexts:
|
||||
- context:
|
||||
cluster: docker-for-desktop-cluster
|
||||
user: docker-for-desktop
|
||||
name: docker-for-desktop
|
||||
current-context: docker-for-desktop
|
||||
kind: Config
|
||||
preferences: {}
|
||||
users:
|
||||
- name: docker-for-desktop
|
||||
user:
|
||||
client-certificate-data: $CLIENT_CERT
|
||||
client-key-data: $CLIENT_KEY
|
||||
"""
|
||||
.stripIndent()
|
||||
|
||||
def discovery = Spy(ConfigDiscovery)
|
||||
def KEY_MANAGERS = [] as KeyManager[]
|
||||
|
||||
when:
|
||||
def config = discovery.fromKubeConfig(CONFIG, null, null, null)
|
||||
then:
|
||||
0 * discovery.discoverAuthToken(_, 'default',null) >> 'secret-token'
|
||||
1 * discovery.createKeyManagers(CLIENT_CERT.decodeBase64(), CLIENT_KEY.decodeBase64()) >> KEY_MANAGERS
|
||||
config.server == 'https://localhost:6443'
|
||||
config.token == null
|
||||
config.namespace == 'default'
|
||||
config.serviceAccount == 'default'
|
||||
config.clientCert == CLIENT_CERT.decodeBase64()
|
||||
config.clientKey == CLIENT_KEY.decodeBase64()
|
||||
config.sslCert == CERT_DATA.decodeBase64()
|
||||
config.keyManagers.is( KEY_MANAGERS )
|
||||
!config.verifySsl
|
||||
!config.isFromCluster
|
||||
|
||||
}
|
||||
|
||||
def 'should read config from file with provided namespace' () {
|
||||
|
||||
given:
|
||||
final CERT_DATA = "d29ybGQgaGVsbG8="
|
||||
final CLIENT_CERT = "aGVsbG8gd29ybGQ="
|
||||
final CLIENT_KEY = "Y2lhbyBtaWFv"
|
||||
|
||||
def CONFIG = TestHelper.createInMemTempFile('config')
|
||||
CONFIG.text = """
|
||||
apiVersion: v1
|
||||
clusters:
|
||||
- cluster:
|
||||
insecure-skip-tls-verify: true
|
||||
server: https://localhost:6443
|
||||
certificate-authority-data: $CERT_DATA
|
||||
name: docker-for-desktop-cluster
|
||||
contexts:
|
||||
- context:
|
||||
cluster: docker-for-desktop-cluster
|
||||
user: docker-for-desktop
|
||||
name: docker-for-desktop
|
||||
current-context: docker-for-desktop
|
||||
kind: Config
|
||||
preferences: {}
|
||||
users:
|
||||
- name: docker-for-desktop
|
||||
user:
|
||||
client-certificate-data: $CLIENT_CERT
|
||||
client-key-data: $CLIENT_KEY
|
||||
"""
|
||||
.stripIndent()
|
||||
|
||||
def discovery = Spy(ConfigDiscovery)
|
||||
def KEY_MANAGERS = [] as KeyManager[]
|
||||
|
||||
when:
|
||||
def config = discovery.fromKubeConfig(CONFIG, 'docker-for-desktop', 'ns1', 'sa2')
|
||||
then:
|
||||
0 * discovery.discoverAuthToken('docker-for-desktop','ns1','sa2') >> 'secret-token'
|
||||
1 * discovery.createKeyManagers(CLIENT_CERT.decodeBase64(), CLIENT_KEY.decodeBase64()) >> KEY_MANAGERS
|
||||
config.server == 'https://localhost:6443'
|
||||
config.token == null
|
||||
config.namespace == 'ns1'
|
||||
config.serviceAccount == 'sa2'
|
||||
config.clientCert == CLIENT_CERT.decodeBase64()
|
||||
config.clientKey == CLIENT_KEY.decodeBase64()
|
||||
config.sslCert == CERT_DATA.decodeBase64()
|
||||
config.keyManagers.is( KEY_MANAGERS )
|
||||
!config.verifySsl
|
||||
!config.isFromCluster
|
||||
|
||||
}
|
||||
|
||||
def 'should read config from file with cert files' () {
|
||||
|
||||
given:
|
||||
def folder = Files.createTempDirectory(null)
|
||||
def CA_FILE = folder.resolve('ca'); CA_FILE.text = 'ca-content'
|
||||
def CLIENT_CERT_FILE = folder.resolve('client-cert'); CLIENT_CERT_FILE.text = 'client-cert-content'
|
||||
def CLIENT_KEY_FILE = folder.resolve('client-key'); CLIENT_KEY_FILE.text = 'client-key-content'
|
||||
def CONFIG = folder.resolve('config')
|
||||
def KEY_MANAGERS = [] as KeyManager[]
|
||||
|
||||
CONFIG.text = """
|
||||
apiVersion: v1
|
||||
clusters:
|
||||
- cluster:
|
||||
insecure-skip-tls-verify: true
|
||||
server: https://localhost:6443
|
||||
certificate-authority: $CA_FILE
|
||||
name: docker-for-desktop-cluster
|
||||
contexts:
|
||||
- context:
|
||||
cluster: docker-for-desktop-cluster
|
||||
user: docker-for-desktop
|
||||
name: docker-for-desktop
|
||||
current-context: docker-for-desktop
|
||||
kind: Config
|
||||
preferences: {}
|
||||
users:
|
||||
- name: docker-for-desktop
|
||||
user:
|
||||
client-certificate: $CLIENT_CERT_FILE
|
||||
client-key: $CLIENT_KEY_FILE
|
||||
"""
|
||||
.stripIndent()
|
||||
|
||||
def discovery = Spy(ConfigDiscovery)
|
||||
|
||||
when:
|
||||
def config = discovery.fromKubeConfig(CONFIG, null, null, null)
|
||||
then:
|
||||
1 * discovery.createKeyManagers( CLIENT_CERT_FILE.bytes, CLIENT_KEY_FILE.bytes ) >> KEY_MANAGERS
|
||||
config.server == 'https://localhost:6443'
|
||||
config.token == null
|
||||
config.namespace == 'default'
|
||||
config.serviceAccount == 'default'
|
||||
config.clientCert == CLIENT_CERT_FILE.bytes
|
||||
config.clientKey == CLIENT_KEY_FILE.bytes
|
||||
config.sslCert == CA_FILE.bytes
|
||||
config.keyManagers.is( KEY_MANAGERS )
|
||||
!config.verifySsl
|
||||
!config.isFromCluster
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
def 'should read config and use token' () {
|
||||
|
||||
given:
|
||||
def folder = Files.createTempDirectory(null)
|
||||
def CONFIG = folder.resolve('config')
|
||||
|
||||
CONFIG.text = """
|
||||
apiVersion: v1
|
||||
clusters:
|
||||
- cluster:
|
||||
insecure-skip-tls-verify: true
|
||||
server: https://localhost:6443
|
||||
name: docker-for-desktop-cluster
|
||||
contexts:
|
||||
- context:
|
||||
cluster: docker-for-desktop-cluster
|
||||
user: docker-for-desktop
|
||||
name: docker-for-desktop
|
||||
current-context: docker-for-desktop
|
||||
kind: Config
|
||||
preferences: {}
|
||||
users:
|
||||
- name: docker-for-desktop
|
||||
user:
|
||||
token: 90s090s98s7f8s
|
||||
"""
|
||||
.stripIndent()
|
||||
|
||||
def discovery = Spy(ConfigDiscovery)
|
||||
|
||||
when:
|
||||
def config = discovery.fromKubeConfig(CONFIG, null, null, null)
|
||||
then:
|
||||
0 * discovery.discoverAuthToken(_,_,_) >> 'secret-token'
|
||||
0 * discovery.createKeyManagers( _, _ ) >> null
|
||||
config.server == 'https://localhost:6443'
|
||||
config.token == '90s090s98s7f8s'
|
||||
config.namespace == 'default'
|
||||
config.serviceAccount == 'default'
|
||||
!config.verifySsl
|
||||
!config.isFromCluster
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
def 'should read config and discover token' () {
|
||||
|
||||
given:
|
||||
def folder = Files.createTempDirectory(null)
|
||||
def CONFIG = folder.resolve('config')
|
||||
|
||||
CONFIG.text = """
|
||||
apiVersion: v1
|
||||
clusters:
|
||||
- cluster:
|
||||
insecure-skip-tls-verify: true
|
||||
server: https://localhost:6443
|
||||
name: docker-for-desktop-cluster
|
||||
contexts:
|
||||
- context:
|
||||
cluster: docker-for-desktop-cluster
|
||||
user: docker-for-desktop
|
||||
name: docker-for-desktop
|
||||
current-context: docker-for-desktop
|
||||
kind: Config
|
||||
preferences: {}
|
||||
users:
|
||||
- name: docker-for-desktop
|
||||
user:
|
||||
foo: bar
|
||||
"""
|
||||
.stripIndent()
|
||||
|
||||
def discovery = Spy(ConfigDiscovery)
|
||||
|
||||
when:
|
||||
def config = discovery.fromKubeConfig(CONFIG, null, null, null)
|
||||
then:
|
||||
1 * discovery.discoverAuthToken(_, _, _) >> 'secret-token'
|
||||
0 * discovery.createKeyManagers( _, _ ) >> null
|
||||
config.server == 'https://localhost:6443'
|
||||
config.token == 'secret-token'
|
||||
config.namespace == 'default'
|
||||
config.serviceAccount == 'default'
|
||||
!config.verifySsl
|
||||
!config.isFromCluster
|
||||
|
||||
cleanup:
|
||||
folder?.deleteDir()
|
||||
}
|
||||
|
||||
def 'should read config from given context' () {
|
||||
|
||||
given:
|
||||
def folder = Files.createTempDirectory(null)
|
||||
folder.resolve('fake-cert-file').text = 'fake-cert-content'
|
||||
folder.resolve('fake-key-file').text = 'fake-key-content'
|
||||
folder.resolve('fake-ca-file').text = 'fake-ca-content'
|
||||
|
||||
def CONFIG = folder.resolve('config')
|
||||
CONFIG.text = '''
|
||||
apiVersion: v1
|
||||
clusters:
|
||||
- cluster:
|
||||
certificate-authority: fake-ca-file
|
||||
server: https://1.2.3.4
|
||||
name: development
|
||||
- cluster:
|
||||
insecure-skip-tls-verify: true
|
||||
server: https://5.6.7.8
|
||||
name: scratch
|
||||
contexts:
|
||||
- context:
|
||||
cluster: development
|
||||
namespace: frontend
|
||||
user: developer
|
||||
name: dev-frontend
|
||||
- context:
|
||||
cluster: development
|
||||
namespace: storage
|
||||
user: developer
|
||||
name: dev-storage
|
||||
- context:
|
||||
cluster: scratch
|
||||
namespace: default
|
||||
user: experimenter
|
||||
name: exp-scratch
|
||||
current-context: ""
|
||||
kind: Config
|
||||
preferences: {}
|
||||
users:
|
||||
- name: developer
|
||||
user:
|
||||
client-certificate: fake-cert-file
|
||||
- name: experimenter
|
||||
user:
|
||||
password: some-password
|
||||
username: exp
|
||||
'''.stripIndent()
|
||||
|
||||
when:
|
||||
def cfg1 = new ConfigDiscovery().fromKubeConfig(CONFIG, 'dev-frontend', null, null)
|
||||
then:
|
||||
cfg1.server == 'https://1.2.3.4'
|
||||
cfg1.sslCert == 'fake-ca-content'.bytes
|
||||
cfg1.isVerifySsl()
|
||||
cfg1.namespace == 'frontend'
|
||||
cfg1.serviceAccount == 'default'
|
||||
cfg1.clientCert == 'fake-cert-content'.bytes
|
||||
|
||||
when:
|
||||
def cfg2 = new ConfigDiscovery().fromKubeConfig(CONFIG, 'dev-storage', null, null)
|
||||
then:
|
||||
cfg2.server == 'https://1.2.3.4'
|
||||
cfg2.sslCert == 'fake-ca-content'.bytes
|
||||
cfg2.isVerifySsl()
|
||||
cfg2.namespace == 'storage'
|
||||
cfg2.serviceAccount == 'default'
|
||||
cfg2.clientCert == 'fake-cert-content'.bytes
|
||||
|
||||
when:
|
||||
def cfg3 = new ConfigDiscovery().fromKubeConfig(CONFIG, 'exp-scratch', null, null)
|
||||
then:
|
||||
cfg3.server == 'https://5.6.7.8'
|
||||
cfg3.sslCert == null
|
||||
!cfg3.isVerifySsl()
|
||||
cfg3.namespace == 'default'
|
||||
cfg3.serviceAccount == 'default'
|
||||
|
||||
when:
|
||||
new ConfigDiscovery().fromKubeConfig(CONFIG, 'foo', null, null)
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
|
||||
true
|
||||
cleanup:
|
||||
folder.deleteDir()
|
||||
}
|
||||
|
||||
def 'should load from cluster env' () {
|
||||
given:
|
||||
def CERT_FILE = TestHelper.createInMemTempFile('ca'); CERT_FILE.text = 'ca-content'
|
||||
def TOKEN_FILE = TestHelper.createInMemTempFile('token'); TOKEN_FILE.text = 'my-token'
|
||||
def NAMESPACE_FILE = TestHelper.createInMemTempFile('namespace'); NAMESPACE_FILE.text = 'foo-namespace'
|
||||
|
||||
def discovery = Spy(ConfigDiscovery)
|
||||
|
||||
when:
|
||||
def env = [ KUBERNETES_SERVICE_HOST: 'foo.com', KUBERNETES_SERVICE_PORT: '4343' ]
|
||||
def config = discovery.fromCluster(env, null, null)
|
||||
then:
|
||||
1 * discovery.path('/var/run/secrets/kubernetes.io/serviceaccount/ca.crt') >> CERT_FILE
|
||||
1 * discovery.path('/var/run/secrets/kubernetes.io/serviceaccount/token') >> TOKEN_FILE
|
||||
1 * discovery.path('/var/run/secrets/kubernetes.io/serviceaccount/namespace') >> NAMESPACE_FILE
|
||||
0 * discovery.createKeyManagers(_,_) >> null
|
||||
and:
|
||||
config.server == 'foo.com:4343'
|
||||
config.namespace == 'foo-namespace'
|
||||
config.token == 'my-token'
|
||||
config.sslCert == CERT_FILE.text.bytes
|
||||
config.isFromCluster
|
||||
|
||||
when:
|
||||
env = [ KUBERNETES_SERVICE_HOST: 'https://host.com' ]
|
||||
config = discovery.fromCluster(env, 'my-namespace', null)
|
||||
then:
|
||||
1 * discovery.path('/var/run/secrets/kubernetes.io/serviceaccount/ca.crt') >> CERT_FILE
|
||||
1 * discovery.path('/var/run/secrets/kubernetes.io/serviceaccount/token') >> TOKEN_FILE
|
||||
1 * discovery.path('/var/run/secrets/kubernetes.io/serviceaccount/namespace') >> NAMESPACE_FILE
|
||||
and:
|
||||
config.server == 'https://host.com'
|
||||
config.namespace == 'my-namespace'
|
||||
}
|
||||
|
||||
def 'should create key managers' () {
|
||||
given:
|
||||
final CERT = 'LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNTRENDQVRDZ0F3SUJBZ0lJRlFsM1l2Y2k1TWN3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB4T0RBeE1UTXlNREEyTlRaYUZ3MHhPVEF4TVRNeU1EQTJOVFphTURNeApGREFTQmdOVkJBb1RDMFJ2WTJ0bGNpQkpibU11TVJzd0dRWURWUVFERXhKa2IyTnJaWEl0Wm05eUxXUmxjMnQwCmIzQXdnWjh3RFFZSktvWklodmNOQVFFQkJRQURnWTBBTUlHSkFvR0JBUEtYT0ZsV2t2THIzb29ETGNFOElyME0KTzNBMHZqQlVvUzZ0bUdBbFRYYTd0QWQwM3BTMXNJNit0WVRwVlU2YXR6ZU9vU0VrOWhmaWxBdVNYdG1hSHZCUAp1czFEcG1LZEZRMWI3OFRkSnQ4OGV3c3BRajFxYUwvQldHeitMUzUrRHUrNUJuUGtmZlhDS1UxQTdUc2tZamJyClhxeDhlN2FWZURWTmFjZXc0Z0RqQWdNQkFBR2pBakFBTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFBMXVtVlAKR29EZTVCRXJrb21qWXdITXhiTTd4UStibTYrUDE1T0pINUo0UGNQeU11d25ocC9ORVp1NnpsTTZSUUo3SUNKQgpHWTRBMnFKVmJsWUkwQkJzRkF1TXMreTAyazdVVVVoK0NRYVd0SXhBcFNmbkQ4dUVXQ0g5VE1ZNGdLbTZjTDhVCk1OVVl1RnpUQ2hmTS96RjdUMXVaZWxJYXNrYXFaWSt3a3hxa3YyRUQxQ2F5MDUxSXRWRXZVbDIvSVZyVHdrT20KZ25nL3Q4L2RkeDhpOUkzTFJrMTlTaERKdXlQZ1NrTTZRSWlSd09mRHk4V0ZFaURpd0hBS0ErSEZhTGhOOFJTMwpieDUvdEhEN01id0FpdnorNTU4YUFEQjNEd1ZpekthM2d5Wm4yUzRjUGFqZnNwODFqRkNIQS9QekdQdTU2MzJwCkxRN0gyRW1aYmJuUHFYTFgKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo='.decodeBase64()
|
||||
final KEY = 'LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlDWGdJQkFBS0JnUUR5bHpoWlZwTHk2OTZLQXkzQlBDSzlERHR3Tkw0d1ZLRXVyWmhnSlUxMnU3UUhkTjZVCnRiQ092cldFNlZWT21yYzNqcUVoSlBZWDRwUUxrbDdabWg3d1Q3ck5RNlppblJVTlcrL0UzU2JmUEhzTEtVSTkKYW1pL3dWaHMvaTB1Zmc3dnVRWno1SDMxd2lsTlFPMDdKR0kyNjE2c2ZIdTJsWGcxVFduSHNPSUE0d0lEQVFBQgpBb0dBYWRUOCtVU2lvU1d6bFVRanZ1eHNQMHRKMXY2N2hqdzFnVGFzaGkxZjZRK2tUNmgxdml5eGxPU3dMZ2JaCmQ0eFpwL3dxWVZwTm5rZnp6RVNUNnB5cEo5WTEwdHY1cFpSWG9HbG1NT2tIZSswUW45N0c5ZDRzL2JCV3lmYXYKRzhRTC9tZFN6Vy85YUdrSkpiNWU0VDlsSURvRDNFVDgwYUFWbzl2V0NPVUxsdWtDUVFEK0hINU5ucVBuSTdnTApWOUJKZzlRRVBwUTVYa2traW8rejZ2YkRHQU5rR1VPV1dmRURKUHE2Q2JBb1dqeWh1Qy9KS1dYRWs4Rkt0M1Y2CkhVNllYeVpGQWtFQTlHVE9XOFM4KzVNNHE5R3lNeURxN1ZkVHA3M2daeSsvNjVQam5hNlpDUnhTZklxL2xKUVoKY2F6MkhGYVRzRFdLbkdhWGNxTmdBVXNEODNyWTlzM3hCd0pCQUt5Vjc1YUtPMm0rRWI3cWVsV2p5bmpEZytwZQp4akNpUnkxOFZQSjJPYjlmaFU3MWNVS2dlQVdvbE5NalRuREw1dkNxUkNzNTZ4cnk5VC9sN2I2QlNUMENRUURnCjRoV2xDZTdnQzhOZEQzTkxhdUhpRGJZenB4dmp0Mk9Ca2E4ai9ISmptTVVxUnI0dEtPNFUxUlFPVlhoRzc2MmgKWnlHNjRpeklZOCs1N3ZQUWZ3Wm5Ba0VBdW9RWW1lUi90UWhIakhRNFlhZGRHbkNBQ2hZZ29ObEFzSGhGTElxVQo1ZTZaMXN2Q3VKU285TDVVRCtrclFUYWlGU01pRHZwZlJyVE1ZKzZ5Q0tTajd3PT0KLS0tLS1FTkQgUlNBIFBSSVZBVEUgS0VZLS0tLS0K'.decodeBase64()
|
||||
final discovery = new ConfigDiscovery()
|
||||
when:
|
||||
def managers = discovery.createKeyManagers(CERT, KEY)
|
||||
then:
|
||||
managers.size()==1
|
||||
}
|
||||
|
||||
def 'should create key managers from an EC client key' () {
|
||||
given:
|
||||
final CERT = 'LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNTRENDQVRDZ0F3SUJBZ0lJRlFsM1l2Y2k1TWN3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB4T0RBeE1UTXlNREEyTlRaYUZ3MHhPVEF4TVRNeU1EQTJOVFphTURNeApGREFTQmdOVkJBb1RDMFJ2WTJ0bGNpQkpibU11TVJzd0dRWURWUVFERXhKa2IyTnJaWEl0Wm05eUxXUmxjMnQwCmIzQXdnWjh3RFFZSktvWklodmNOQVFFQkJRQURnWTBBTUlHSkFvR0JBUEtYT0ZsV2t2THIzb29ETGNFOElyME0KTzNBMHZqQlVvUzZ0bUdBbFRYYTd0QWQwM3BTMXNJNit0WVRwVlU2YXR6ZU9vU0VrOWhmaWxBdVNYdG1hSHZCUAp1czFEcG1LZEZRMWI3OFRkSnQ4OGV3c3BRajFxYUwvQldHeitMUzUrRHUrNUJuUGtmZlhDS1UxQTdUc2tZamJyClhxeDhlN2FWZURWTmFjZXc0Z0RqQWdNQkFBR2pBakFBTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFBMXVtVlAKR29EZTVCRXJrb21qWXdITXhiTTd4UStibTYrUDE1T0pINUo0UGNQeU11d25ocC9ORVp1NnpsTTZSUUo3SUNKQgpHWTRBMnFKVmJsWUkwQkJzRkF1TXMreTAyazdVVVVoK0NRYVd0SXhBcFNmbkQ4dUVXQ0g5VE1ZNGdLbTZjTDhVCk1OVVl1RnpUQ2hmTS96RjdUMXVaZWxJYXNrYXFaWSt3a3hxa3YyRUQxQ2F5MDUxSXRWRXZVbDIvSVZyVHdrT20KZ25nL3Q4L2RkeDhpOUkzTFJrMTlTaERKdXlQZ1NrTTZRSWlSd09mRHk4V0ZFaURpd0hBS0ErSEZhTGhOOFJTMwpieDUvdEhEN01id0FpdnorNTU4YUFEQjNEd1ZpekthM2d5Wm4yUzRjUGFqZnNwODFqRkNIQS9QekdQdTU2MzJwCkxRN0gyRW1aYmJuUHFYTFgKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo='.decodeBase64()
|
||||
final KEY = 'LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JR0hBZ0VBTUJNR0J5cUdTTTQ5QWdFR0NDcUdTTTQ5QXdFSEJHMHdhd0lCQVFRZ21aZFZ3NmJRU0w1T1l5RjQKbzJ4V0hUQ05BSW1hRTkycGd2dGMzK2Z2UDVxaFJBTkNBQVJSd0RpUVptTUNqcWxvbFBzRTdiZjgwWjhrZkRXTworS2U4NUdVSll2MlBubWVxbDhkYjdwcmFlMHFPQUJaaXR2Mmh2SmJFeFdsUFR0MS9CYTNMK1B5NAotLS0tLUVORCBQUklWQVRFIEtFWS0tLS0tCg=='.decodeBase64()
|
||||
final discovery = new ConfigDiscovery()
|
||||
when:
|
||||
def managers = discovery.createKeyManagers(CERT, KEY)
|
||||
then:
|
||||
managers.size()==1
|
||||
}
|
||||
|
||||
def 'should create key managers from an EC-encrypted client key' () {
|
||||
given:
|
||||
final CERT = 'LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrVENDQVRlZ0F3SUJBZ0lJSGw1Zmx0UmRTdDB3Q2dZSUtvWkl6ajBFQXdJd0l6RWhNQjhHQTFVRUF3d1kKYXpOekxXTnNhV1Z1ZEMxallVQXhOekUwTnpRM09UVTNNQjRYRFRJME1EVXdNekUwTlRJek4xb1hEVEkxTURVdwpNekUwTlRJek4xb3dNREVYTUJVR0ExVUVDaE1PYzNsemRHVnRPbTFoYzNSbGNuTXhGVEFUQmdOVkJBTVRESE41CmMzUmxiVHBoWkcxcGJqQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJQN1Q5RHVvUlllLzBlUkwKUmNHV2RoYnl2Q3BucXlsSVIyaUwxdGkwc1hVdEpZZjUrVXhIOWFBMjdzY2FSYW1qbjdnTTFrKzZNaVk5cm15OApyRmdoWm1xalNEQkdNQTRHQTFVZER3RUIvd1FFQXdJRm9EQVRCZ05WSFNVRUREQUtCZ2dyQmdFRkJRY0RBakFmCkJnTlZIU01FR0RBV2dCU0NhdXFoQVEvWEdoaFRtaFBoY21vRVdOeWluakFLQmdncWhrak9QUVFEQWdOSUFEQkYKQWlCZzRaNmlWeFV3Mk5uMHBQTG02VlovUGttQnVuTDEwZG50dEg3UVdIcklCd0loQU0vTDhVMGxQN0IyeFEyZwpsZjlhNHNhbzJ1bE5ONnQvQ0dibzlxTlo1QzZHCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0KLS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJkekNDQVIyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWpNU0V3SHdZRFZRUUREQmhyTTNNdFkyeHAKWlc1MExXTmhRREUzTVRRM05EYzVOVGN3SGhjTk1qUXdOVEF6TVRRMU1qTTNXaGNOTXpRd05UQXhNVFExTWpNMwpXakFqTVNFd0h3WURWUVFEREJock0zTXRZMnhwWlc1MExXTmhRREUzTVRRM05EYzVOVGN3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFRZHFYVHdIQS9mVjRKZGdYa2FubXB1OVE0QStwUGRGaXZGdytiUmVhdEYKUXVOUTBKWndIbzlaa2ltb2lEUU5qb2h0TWdHckdtTVlsTTZuaXM4ZVFvM3RvMEl3UURBT0JnTlZIUThCQWY4RQpCQU1DQXFRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVWdtcnFvUUVQMXhvWVU1b1Q0WEpxCkJGamNvcDR3Q2dZSUtvWkl6ajBFQXdJRFNBQXdSUUloQUlvb2ZmNzdvb1VYS2hmNVo3aVRzdExhOTVwU2VaRmUKRHZjMXdFQXVEa3NTQWlBNzJQajJxNnpBclhpYkpUa0s2RTBHTEtVODdhTHhHc3BmS29uVVJnalI2Zz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K'.decodeBase64()
|
||||
final KEY = 'LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUNvQTNvRHkzN3NXdmszM3JGRGtRdlZ1Wkh1cCt1Uk40V3RqbUlPR1c4cHBvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFL3RQME82aEZoNy9SNUV0RndaWjJGdks4S21lcktVaEhhSXZXMkxTeGRTMGxoL241VEVmMQpvRGJ1eHhwRnFhT2Z1QXpXVDdveUpqMnViTHlzV0NGbWFnPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo='.decodeBase64()
|
||||
final discovery = new ConfigDiscovery()
|
||||
when:
|
||||
def managers = discovery.createKeyManagers(CERT, KEY)
|
||||
then:
|
||||
managers.size()==1
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,66 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.client
|
||||
|
||||
import spock.lang.Specification
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class K8sResponseExceptionTest extends Specification {
|
||||
|
||||
def 'should create response from valid json' () {
|
||||
|
||||
when:
|
||||
def resp = new K8sResponseException(
|
||||
'Request /this/that failed',
|
||||
new K8sResponseJson('{"foo":"one","bar":"two"}'))
|
||||
then:
|
||||
resp.getMessage() == '''
|
||||
Request /this/that failed
|
||||
|
||||
{
|
||||
"foo": "one",
|
||||
"bar": "two"
|
||||
}
|
||||
'''.stripIndent().leftTrim()
|
||||
}
|
||||
|
||||
|
||||
def 'should create response from error message' () {
|
||||
|
||||
when:
|
||||
def resp = new K8sResponseException(
|
||||
'Request /this/that failed',
|
||||
new K8sResponseJson('Oops.. it crashed badly'))
|
||||
then:
|
||||
resp.getMessage() == 'Request /this/that failed -- Oops.. it crashed badly'
|
||||
}
|
||||
|
||||
def 'should contain the response object passed to it' () {
|
||||
|
||||
given:
|
||||
def resp_json = new K8sResponseJson('{"error": "out of cheese error"}')
|
||||
|
||||
when:
|
||||
def resp = new K8sResponseException("Error occurred",resp_json)
|
||||
|
||||
then:
|
||||
resp.response == resp_json
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.client
|
||||
|
||||
import spock.lang.Specification
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class K8sResponseJsonTest extends Specification {
|
||||
|
||||
def 'should create a response from a map' () {
|
||||
|
||||
given:
|
||||
def MAP = [foo: 'one', bar: 'two']
|
||||
|
||||
when:
|
||||
def resp = new K8sResponseJson(MAP)
|
||||
then:
|
||||
resp.foo == 'one'
|
||||
resp.bar == 'two'
|
||||
resp.toString() == '''
|
||||
{
|
||||
"foo": "one",
|
||||
"bar": "two"
|
||||
}
|
||||
'''.stripIndent().trim()
|
||||
|
||||
}
|
||||
|
||||
def 'should create a response from a json string' () {
|
||||
|
||||
when:
|
||||
def resp = new K8sResponseJson('{"foo":"one","bar":"two"}')
|
||||
then:
|
||||
resp.foo == 'one'
|
||||
resp.bar == 'two'
|
||||
resp.toString() == '''
|
||||
{
|
||||
"foo": "one",
|
||||
"bar": "two"
|
||||
}
|
||||
'''.stripIndent().trim()
|
||||
|
||||
}
|
||||
|
||||
def 'should create a response from an error message' () {
|
||||
when:
|
||||
def resp = new K8sResponseJson('Ooops .. this crashed')
|
||||
then:
|
||||
resp.toString() == 'Ooops .. this crashed'
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.model
|
||||
|
||||
import spock.lang.Specification
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class PodEnvTest extends Specification {
|
||||
|
||||
def 'should return env spec' () {
|
||||
expect:
|
||||
PodEnv.value('ALPHA', 'aaa').toSpec() == [name:'ALPHA', value:'aaa']
|
||||
}
|
||||
|
||||
def 'should create env fieldPath spec' () {
|
||||
expect:
|
||||
PodEnv.fieldPath('ALPHA', 'aaa').toSpec() == [
|
||||
name:'ALPHA',
|
||||
valueFrom: [fieldRef:[fieldPath: 'aaa']]
|
||||
]
|
||||
}
|
||||
|
||||
def 'should create env secret spec' () {
|
||||
expect:
|
||||
PodEnv.secret('ALPHA', 'data/key-1').toSpec() == [
|
||||
name: 'ALPHA',
|
||||
valueFrom: [secretKeyRef:[name:'data', key:'key-1']]
|
||||
]
|
||||
|
||||
PodEnv.secret('ALPHA', 'data').toSpec() == [
|
||||
name: 'ALPHA',
|
||||
valueFrom: [secretKeyRef:[name:'data', key:'ALPHA']]
|
||||
]
|
||||
|
||||
}
|
||||
|
||||
def 'should create env config spec' () {
|
||||
expect:
|
||||
PodEnv.config('ALPHA', 'data/key-1').toSpec() == [
|
||||
name: 'ALPHA',
|
||||
valueFrom: [configMapKeyRef:[name:'data', key:'key-1']]
|
||||
]
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.model
|
||||
|
||||
import spock.lang.Specification
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class PodMountConfigTest extends Specification {
|
||||
|
||||
def 'should create mount for configmap' () {
|
||||
|
||||
when:
|
||||
def opt = new PodMountConfig(mountPath: '/etc/some/name', config: 'here' )
|
||||
then:
|
||||
opt.mountPath == '/etc/some/name'
|
||||
opt.fileName == null
|
||||
opt.configName == 'here'
|
||||
opt.configKey == null
|
||||
|
||||
when:
|
||||
opt = new PodMountConfig(mountPath: '/etc/some/name', config: 'here/there.txt' )
|
||||
then:
|
||||
opt.mountPath == '/etc/some'
|
||||
opt.fileName == 'name'
|
||||
opt.configName == 'here'
|
||||
opt.configKey == 'there.txt'
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.model
|
||||
|
||||
import spock.lang.Specification
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class PodMountSecretTest extends Specification {
|
||||
|
||||
def 'should create mount for configmap' () {
|
||||
|
||||
when:
|
||||
def opt = new PodMountSecret(mountPath: '/etc/some/name', secret: 'here' )
|
||||
then:
|
||||
opt.mountPath == '/etc/some/name'
|
||||
opt.fileName == null
|
||||
opt.secretName == 'here'
|
||||
opt.secretKey == null
|
||||
|
||||
when:
|
||||
opt = new PodMountSecret(mountPath: '/etc/some/name', secret: 'here/there.txt' )
|
||||
then:
|
||||
opt.mountPath == '/etc/some'
|
||||
opt.fileName == 'name'
|
||||
opt.secretName == 'here'
|
||||
opt.secretKey == 'there.txt'
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.model
|
||||
|
||||
import spock.lang.Specification
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class PodNodeSelectorTest extends Specification {
|
||||
|
||||
def 'should create node selector' () {
|
||||
|
||||
expect:
|
||||
new PodNodeSelector(selector).toSpec() == spec
|
||||
|
||||
where:
|
||||
selector | spec
|
||||
'' | [:]
|
||||
'foo=1' | [foo:'1']
|
||||
'x=a,y=2,z=9' | [x:'a',y:'2',z:'9']
|
||||
'x= a , y=2 , z =9' | [x:'a',y:'2',z:'9']
|
||||
'gpu,intel' | [gpu:'true',intel: 'true']
|
||||
[foo:1, bar: 'two'] | [foo:'1', bar:'two']
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,568 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.model
|
||||
|
||||
import spock.lang.Specification
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class PodOptionsTest extends Specification {
|
||||
|
||||
|
||||
def 'should create empty options' () {
|
||||
|
||||
when:
|
||||
def options = new PodOptions(null)
|
||||
then:
|
||||
options.getEnvVars() == [] as Set
|
||||
options.getMountConfigMaps() == [] as Set
|
||||
options.getMountCsiEphemerals() == [] as Set
|
||||
options.getMountEmptyDirs() == [] as Set
|
||||
options.getMountSecrets() == [] as Set
|
||||
options.getAutomountServiceAccountToken() == true
|
||||
}
|
||||
|
||||
def 'should set pullPolicy' () {
|
||||
|
||||
when:
|
||||
def options = new PodOptions()
|
||||
then:
|
||||
options.getImagePullPolicy() == null
|
||||
|
||||
when:
|
||||
options = new PodOptions([ [pullPolicy:'Always'] ])
|
||||
then:
|
||||
options.getImagePullPolicy() == 'Always'
|
||||
|
||||
when:
|
||||
options = new PodOptions([ [imagePullPolicy:'latest'] ])
|
||||
then:
|
||||
options.getImagePullPolicy() == 'latest'
|
||||
}
|
||||
|
||||
def 'should set imagePullSecret' () {
|
||||
|
||||
when:
|
||||
def options = new PodOptions()
|
||||
then:
|
||||
options.imagePullSecret == null
|
||||
|
||||
when:
|
||||
options = new PodOptions([ [imagePullSecret:'foo'] ])
|
||||
then:
|
||||
options.imagePullSecret == 'foo'
|
||||
|
||||
when:
|
||||
options = new PodOptions([ [imagePullSecrets:'bar'] ])
|
||||
then:
|
||||
options.imagePullSecret == 'bar'
|
||||
}
|
||||
|
||||
def 'should return config mounts' () {
|
||||
|
||||
given:
|
||||
def options = [
|
||||
[mountPath: '/this/path1.txt', config: 'name/key1'],
|
||||
[mountPath: '/this/path2.txt', config: 'name/key2'],
|
||||
[mountPath: '/this/path2.txt', config: 'name/key2'], // <-- identical entry are ignored
|
||||
[mountPath: '/this/path2.txt', secret: 'name/secret'],
|
||||
[env: 'FOO', config: '/name/foo']
|
||||
]
|
||||
|
||||
when:
|
||||
def configs = new PodOptions(options).getMountConfigMaps()
|
||||
then:
|
||||
configs.size() == 2
|
||||
configs == [
|
||||
new PodMountConfig(mountPath: '/this/path1.txt', config: 'name/key1'),
|
||||
new PodMountConfig(mountPath: '/this/path2.txt', config: 'name/key2')
|
||||
] as Set
|
||||
}
|
||||
|
||||
def 'should return csi ephemeral mounts' () {
|
||||
|
||||
given:
|
||||
def options = [
|
||||
[
|
||||
mountPath: '/data',
|
||||
csi: [
|
||||
driver: 'inline.storage.kubernetes.io',
|
||||
volumeAttributes: [foo: 'bar']
|
||||
]
|
||||
]
|
||||
]
|
||||
|
||||
when:
|
||||
def csiEphemerals = new PodOptions(options).getMountCsiEphemerals()
|
||||
then:
|
||||
csiEphemerals == [
|
||||
new PodMountCsiEphemeral(mountPath: '/data', csi: options[0].csi)
|
||||
] as Set
|
||||
}
|
||||
|
||||
def 'should return emptyDir mounts' () {
|
||||
|
||||
given:
|
||||
def options = [
|
||||
[mountPath: '/scratch1', emptyDir: [medium: 'Memory']],
|
||||
[mountPath: '/scratch2', emptyDir: [medium: 'Disk']]
|
||||
]
|
||||
|
||||
when:
|
||||
def emptyDirs = new PodOptions(options).getMountEmptyDirs()
|
||||
then:
|
||||
emptyDirs.size() == 2
|
||||
emptyDirs == [
|
||||
new PodMountEmptyDir(options[0]),
|
||||
new PodMountEmptyDir(options[1]) ] as Set
|
||||
}
|
||||
|
||||
def 'should return secret mounts' () {
|
||||
|
||||
given:
|
||||
def options = [
|
||||
[mountPath: '/this/path1.txt', config: 'name/key1'],
|
||||
[mountPath: '/this/alpha.txt', secret: 'name/secret1'],
|
||||
[mountPath: '/this/beta.txt', secret: 'name/secret2'],
|
||||
[mountPath: '/this/beta.txt', secret: 'name/secret2'],
|
||||
[env: 'FOO', config: '/name/foo']
|
||||
]
|
||||
|
||||
when:
|
||||
def secrets = new PodOptions(options).getMountSecrets()
|
||||
then:
|
||||
secrets.size() == 2
|
||||
secrets == [
|
||||
new PodMountSecret(mountPath: '/this/alpha.txt', secret: 'name/secret1'),
|
||||
new PodMountSecret(mountPath: '/this/beta.txt', secret: 'name/secret2'),
|
||||
] as Set
|
||||
}
|
||||
|
||||
|
||||
def 'should return env definitions' () {
|
||||
|
||||
given:
|
||||
def options = [
|
||||
[mountPath: '/this/path1.txt', config: 'name/key1'],
|
||||
[mountPath: '/this/alpha.txt', secret: 'name/secret1'],
|
||||
[mountPath: '/this/beta.txt', secret: 'name/secret2'],
|
||||
[env: 'FOO', config: '/name/foo'],
|
||||
[env: 'FOO', config: '/name/foo'],
|
||||
[env: 'BAR', config: '/name/BAR'],
|
||||
[env: 'ALPHA', value: 'aaa'],
|
||||
[env: 'ALPHA', value: 'aaa'],
|
||||
[env: 'BETA', value: 'bbb'],
|
||||
[env: 'PASSWORD', secret:'name/key'],
|
||||
[env: 'PASSWORD', secret:'name/key'],
|
||||
]
|
||||
|
||||
when:
|
||||
def env = new PodOptions(options).getEnvVars()
|
||||
then:
|
||||
env.size() == 5
|
||||
env == [
|
||||
PodEnv.config('FOO', '/name/foo'),
|
||||
PodEnv.config('BAR', '/name/BAR'),
|
||||
PodEnv.value('ALPHA', 'aaa'),
|
||||
PodEnv.value('BETA', 'bbb'),
|
||||
PodEnv.secret('PASSWORD', 'name/key'),
|
||||
] as Set
|
||||
|
||||
}
|
||||
|
||||
def 'should create persistent volume claims' () {
|
||||
given:
|
||||
def options = [
|
||||
[volumeClaim:'pvc1', mountPath: '/this/path'],
|
||||
[volumeClaim:'pvc2', mountPath: '/that/path'],
|
||||
[volumeClaim:'pvc3', mountPath: '/some/data', subPath: '/foo']
|
||||
]
|
||||
|
||||
when:
|
||||
def claims = new PodOptions(options).getVolumeClaims()
|
||||
then:
|
||||
claims.size() == 3
|
||||
claims == [
|
||||
new PodVolumeClaim('pvc1', '/this/path'),
|
||||
new PodVolumeClaim('pvc2', '/that/path'),
|
||||
new PodVolumeClaim('pvc3', '/some/data', '/foo')
|
||||
] as Set
|
||||
|
||||
}
|
||||
|
||||
def 'should create host path' () {
|
||||
given:
|
||||
def options = [
|
||||
[hostPath: '/host/one', mountPath: '/pod/1'],
|
||||
[hostPath: '/host/two', mountPath: '/pod/2']
|
||||
]
|
||||
when:
|
||||
def mounts = new PodOptions(options).getMountHostPaths()
|
||||
|
||||
then:
|
||||
mounts == [
|
||||
new PodHostMount('/host/one', '/pod/1'),
|
||||
new PodHostMount('/host/two', '/pod/2')
|
||||
] as Set
|
||||
|
||||
}
|
||||
|
||||
def 'should not create env' () {
|
||||
when:
|
||||
new PodOptions([ [env:'FOO'] ])
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
|
||||
when:
|
||||
new PodOptions([ [secret:'FOO'] ])
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
|
||||
when:
|
||||
new PodOptions([ [config:'FOO'] ])
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
|
||||
when:
|
||||
new PodOptions([ [volumeClaim:'FOO'] ])
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
}
|
||||
|
||||
def 'should merge podOptions' () {
|
||||
|
||||
given:
|
||||
def list1 = [
|
||||
[env: 'HELLO', value: 'WORLD'],
|
||||
[config: 'data/key', mountPath: '/data/file.txt'],
|
||||
[secret: 'secret/key', mountPath: '/etc/secret'],
|
||||
[volumeClaim: 'pvc', mountPath: '/mnt/claim'],
|
||||
[runAsUser: 500]
|
||||
]
|
||||
|
||||
def list2 = [
|
||||
[env: 'ALPHA', value: 'GAMMA'],
|
||||
[config: 'bar/key', mountPath: '/b/bb'],
|
||||
[secret: 'foo/key', mountPath: '/a/aa'],
|
||||
[volumeClaim: 'cvp', mountPath: '/c/cc'],
|
||||
|
||||
[env: 'DELTA', value: 'LAMBDA'],
|
||||
[config: 'y', mountPath: '/y'],
|
||||
[secret: 'x', mountPath: '/x'],
|
||||
[volumeClaim: 'z', mountPath: '/z'],
|
||||
]
|
||||
|
||||
def list3 = [
|
||||
[env: 'HELLO', value: 'WORLD'],
|
||||
[config: 'data/key', mountPath: '/data/file.txt'],
|
||||
[secret: 'secret/key', mountPath: '/etc/secret'],
|
||||
[volumeClaim: 'pvc', mountPath: '/mnt/claim'],
|
||||
|
||||
[env: 'DELTA', value: 'LAMBDA'],
|
||||
[config: 'y', mountPath: '/y'],
|
||||
[secret: 'x', mountPath: '/x'],
|
||||
[volumeClaim: 'z', mountPath: '/z'],
|
||||
|
||||
[csi: [driver: 'inline.storage.kubernetes.io'], mountPath: '/data'],
|
||||
[emptyDir: [:], mountPath: '/scratch1'],
|
||||
[securityContext: [runAsUser: 1000, fsGroup: 200, allowPrivilegeEscalation: true]],
|
||||
[nodeSelector: 'foo=X, bar=Y'],
|
||||
[automountServiceAccountToken: false],
|
||||
[priorityClassName: 'high-priority']
|
||||
]
|
||||
|
||||
PodOptions opts
|
||||
|
||||
when:
|
||||
opts = new PodOptions() + new PodOptions()
|
||||
then:
|
||||
opts == new PodOptions()
|
||||
|
||||
when:
|
||||
opts = new PodOptions(list1) + new PodOptions()
|
||||
then:
|
||||
opts == new PodOptions(list1)
|
||||
opts.securityContext.toSpec() == [runAsUser:500]
|
||||
|
||||
when:
|
||||
opts = new PodOptions() + new PodOptions(list1)
|
||||
then:
|
||||
opts == new PodOptions(list1)
|
||||
opts.securityContext.toSpec() == [runAsUser:500]
|
||||
|
||||
when:
|
||||
opts = new PodOptions(list1) + new PodOptions(list1)
|
||||
then:
|
||||
opts == new PodOptions(list1)
|
||||
opts.securityContext.toSpec() == [runAsUser:500]
|
||||
|
||||
when:
|
||||
opts = new PodOptions(list1) + new PodOptions(list2)
|
||||
then:
|
||||
opts == new PodOptions(list1 + list2)
|
||||
opts.securityContext.toSpec() == [runAsUser:500]
|
||||
|
||||
when:
|
||||
opts = new PodOptions(list1) + new PodOptions(list3)
|
||||
then:
|
||||
opts.getEnvVars() == [
|
||||
PodEnv.value('HELLO','WORLD'),
|
||||
PodEnv.value('DELTA','LAMBDA')
|
||||
] as Set
|
||||
|
||||
opts.getMountConfigMaps() == [
|
||||
new PodMountConfig('data/key', '/data/file.txt'),
|
||||
new PodMountConfig('y', '/y'),
|
||||
] as Set
|
||||
|
||||
opts.getMountCsiEphemerals() == [
|
||||
new PodMountCsiEphemeral([driver: 'inline.storage.kubernetes.io'], '/data')
|
||||
] as Set
|
||||
|
||||
opts.getMountEmptyDirs() == [
|
||||
new PodMountEmptyDir([:], '/scratch1'),
|
||||
] as Set
|
||||
|
||||
opts.getMountSecrets() == [
|
||||
new PodMountSecret('secret/key', '/etc/secret'),
|
||||
new PodMountSecret('x', '/x')
|
||||
] as Set
|
||||
|
||||
opts.getVolumeClaims() == [
|
||||
new PodVolumeClaim('pvc','/mnt/claim'),
|
||||
new PodVolumeClaim('z','/z'),
|
||||
] as Set
|
||||
|
||||
opts.securityContext.toSpec() == [runAsUser: 1000, fsGroup: 200, allowPrivilegeEscalation: true]
|
||||
|
||||
opts.nodeSelector.toSpec() == [foo: 'X', bar: "Y"]
|
||||
|
||||
opts.getAutomountServiceAccountToken() == false
|
||||
|
||||
opts.getPriorityClassName() == 'high-priority'
|
||||
}
|
||||
|
||||
def 'should copy image pull policy' (){
|
||||
given:
|
||||
def data = [
|
||||
[imagePullPolicy : 'FOO']
|
||||
]
|
||||
|
||||
when:
|
||||
def opts = new PodOptions() + new PodOptions(data)
|
||||
then:
|
||||
opts.imagePullPolicy == 'FOO'
|
||||
|
||||
when:
|
||||
opts = new PodOptions(data) + new PodOptions()
|
||||
then:
|
||||
opts.imagePullPolicy == 'FOO'
|
||||
}
|
||||
|
||||
def 'should copy image pull secret' (){
|
||||
given:
|
||||
def data = [
|
||||
[imagePullSecret : 'BAR']
|
||||
]
|
||||
|
||||
when:
|
||||
def opts = new PodOptions() + new PodOptions(data)
|
||||
then:
|
||||
opts.imagePullSecret == 'BAR'
|
||||
|
||||
when:
|
||||
opts = new PodOptions(data) + new PodOptions()
|
||||
then:
|
||||
opts.imagePullSecret == 'BAR'
|
||||
}
|
||||
|
||||
def 'should copy pod labels' (){
|
||||
given:
|
||||
def data = [
|
||||
[label: "LABEL", value: 'VALUE']
|
||||
]
|
||||
|
||||
when:
|
||||
def opts = new PodOptions() + new PodOptions(data)
|
||||
then:
|
||||
opts.labels == ["LABEL": "VALUE"]
|
||||
|
||||
when:
|
||||
opts = new PodOptions(data) + new PodOptions()
|
||||
then:
|
||||
opts.labels == ["LABEL": "VALUE"]
|
||||
|
||||
when:
|
||||
opts = new PodOptions([[label:"FOO", value:'one']]) + new PodOptions([[label:"BAR", value:'two']])
|
||||
then:
|
||||
opts.labels == [FOO: 'one', BAR: 'two']
|
||||
}
|
||||
|
||||
def 'should copy host paths' (){
|
||||
given:
|
||||
def data = [
|
||||
[hostPath: "/foo", mountPath: '/one']
|
||||
]
|
||||
|
||||
when:
|
||||
def opts = new PodOptions() + new PodOptions(data)
|
||||
then:
|
||||
opts.getMountHostPaths() == [new PodHostMount('/foo', '/one')] as Set
|
||||
|
||||
when:
|
||||
opts = new PodOptions(data) + new PodOptions()
|
||||
then:
|
||||
opts.getMountHostPaths() == [new PodHostMount('/foo', '/one')] as Set
|
||||
|
||||
when:
|
||||
opts = new PodOptions([[hostPath:"/foo", mountPath: '/one']]) + new PodOptions([[hostPath:"/bar", mountPath: '/two']])
|
||||
then:
|
||||
opts.getMountHostPaths() == [
|
||||
new PodHostMount('/foo','/one'),
|
||||
new PodHostMount('/bar','/two')
|
||||
] as Set
|
||||
}
|
||||
|
||||
def 'should create pod labels' () {
|
||||
|
||||
given:
|
||||
def options = [
|
||||
[label: 'ALPHA', value: 'aaa'],
|
||||
[label: 'DELTA', value: 'bbb'],
|
||||
[label: 'DELTA', value: 'ddd']
|
||||
]
|
||||
|
||||
when:
|
||||
def opts = new PodOptions(options)
|
||||
then:
|
||||
opts.labels.size() == 2
|
||||
opts.labels == [ALPHA: 'aaa', DELTA: 'ddd']
|
||||
|
||||
}
|
||||
|
||||
def 'should copy pod annotations' (){
|
||||
given:
|
||||
def data = [
|
||||
[annotation: "ANNOTATION", value: 'VALUE']
|
||||
]
|
||||
|
||||
when:
|
||||
def opts = new PodOptions() + new PodOptions(data)
|
||||
then:
|
||||
opts.annotations == ["ANNOTATION": "VALUE"]
|
||||
|
||||
when:
|
||||
opts = new PodOptions(data) + new PodOptions()
|
||||
then:
|
||||
opts.annotations == ["ANNOTATION": "VALUE"]
|
||||
|
||||
when:
|
||||
opts = new PodOptions([[annotation:"FOO", value:'one']]) + new PodOptions([[annotation:"BAR", value:'two']])
|
||||
then:
|
||||
opts.annotations == [FOO: 'one', BAR: 'two']
|
||||
}
|
||||
|
||||
def 'should create pod annotations' () {
|
||||
|
||||
given:
|
||||
def options = [
|
||||
[annotation: 'ALPHA', value: 'aaa'],
|
||||
[annotation: 'DELTA', value: 'bbb'],
|
||||
[annotation: 'DELTA', value: 'ddd']
|
||||
]
|
||||
|
||||
when:
|
||||
def opts = new PodOptions(options)
|
||||
then:
|
||||
opts.annotations.size() == 2
|
||||
opts.annotations == [ALPHA: 'aaa', DELTA: 'ddd']
|
||||
|
||||
}
|
||||
|
||||
def 'should create user security context' () {
|
||||
when:
|
||||
def opts = new PodOptions([ [runAsUser: 1000] ])
|
||||
then:
|
||||
opts.getSecurityContext() == new PodSecurityContext(1000)
|
||||
|
||||
when:
|
||||
opts = new PodOptions([ [runAsUser: 'foo'] ])
|
||||
then:
|
||||
opts.getSecurityContext() == new PodSecurityContext('foo')
|
||||
|
||||
when:
|
||||
opts = new PodOptions([ [runAsUser: 'foo'] ])
|
||||
then:
|
||||
opts.getSecurityContext() != new PodSecurityContext('bar')
|
||||
|
||||
when:
|
||||
def ctx = [runAsUser: 500, fsGroup: 200, allowPrivilegeEscalation: true, seLinuxOptions: [level: "s0:c123,c456"]]
|
||||
def expected = new PodSecurityContext(ctx)
|
||||
opts = new PodOptions([ [securityContext: ctx] ])
|
||||
then:
|
||||
opts.getSecurityContext() == expected
|
||||
opts.getSecurityContext().toSpec() == ctx
|
||||
}
|
||||
|
||||
def 'should create pod node selector' () {
|
||||
when:
|
||||
def opts = new PodOptions([ [nodeSelector: 'foo=1, bar=true, baz=Z'] ])
|
||||
then:
|
||||
opts.nodeSelector.toSpec() == [foo: '1', bar: 'true', baz: 'Z']
|
||||
|
||||
}
|
||||
|
||||
def 'should set pod automount service token' () {
|
||||
when:
|
||||
def opts = new PodOptions([[automountServiceAccountToken: false]])
|
||||
then:
|
||||
opts.getAutomountServiceAccountToken() == false
|
||||
}
|
||||
|
||||
def 'should set pod priority class name' () {
|
||||
when:
|
||||
def opts = new PodOptions([[priorityClassName: 'high-priority']])
|
||||
then:
|
||||
opts.getPriorityClassName() == 'high-priority'
|
||||
}
|
||||
|
||||
def 'should set pod privileged' () {
|
||||
when:
|
||||
def opts = new PodOptions([:])
|
||||
then:
|
||||
!opts.getPrivileged()
|
||||
|
||||
when:
|
||||
opts = new PodOptions([[privileged: true]])
|
||||
then:
|
||||
opts.getPrivileged()
|
||||
}
|
||||
|
||||
def 'should set pod schedulerName' () {
|
||||
when:
|
||||
def opts = new PodOptions()
|
||||
then:
|
||||
opts.getSchedulerName() == null
|
||||
|
||||
when:
|
||||
opts = new PodOptions([ [schedulerName:'my-scheduler'] ])
|
||||
then:
|
||||
opts.getSchedulerName() == 'my-scheduler'
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,914 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.model
|
||||
|
||||
import nextflow.executor.res.AcceleratorResource
|
||||
import nextflow.util.MemoryUnit
|
||||
import spock.lang.Specification
|
||||
import spock.lang.Unroll
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class PodSpecBuilderTest extends Specification {
|
||||
|
||||
def setup() {
|
||||
PodSpecBuilder.VOLUMES.set(0)
|
||||
}
|
||||
|
||||
|
||||
def 'should create pod spec' () {
|
||||
|
||||
when:
|
||||
def spec = new PodSpecBuilder()
|
||||
.withPodName('foo')
|
||||
.withImageName('busybox')
|
||||
.withWorkDir('/some/work/dir')
|
||||
.withCommand(['echo', 'hello'])
|
||||
.build()
|
||||
|
||||
then:
|
||||
spec == [
|
||||
apiVersion: 'v1',
|
||||
kind: 'Pod',
|
||||
metadata: [name:'foo', namespace:'default'],
|
||||
spec: [
|
||||
restartPolicy:'Never',
|
||||
containers:[[
|
||||
name:'foo',
|
||||
image:'busybox',
|
||||
command:['echo', 'hello'],
|
||||
workingDir:'/some/work/dir'
|
||||
]]
|
||||
]
|
||||
]
|
||||
|
||||
}
|
||||
|
||||
def 'should create pod spec with args' () {
|
||||
|
||||
when:
|
||||
def pod = new PodSpecBuilder()
|
||||
.withPodName('foo')
|
||||
.withImageName('busybox')
|
||||
.withArgs(['echo', 'hello'])
|
||||
.build()
|
||||
|
||||
then:
|
||||
pod.spec.containers[0].args == ['echo', 'hello']
|
||||
|
||||
}
|
||||
|
||||
def 'should create pod spec with args string' () {
|
||||
|
||||
when:
|
||||
def pod = new PodSpecBuilder()
|
||||
.withPodName('foo')
|
||||
.withImageName('busybox')
|
||||
.withArgs('echo foo')
|
||||
.build()
|
||||
|
||||
then:
|
||||
pod.spec.containers[0].args == ['/bin/bash', '-c', 'echo foo']
|
||||
|
||||
}
|
||||
|
||||
def 'should create pod spec with privileged' () {
|
||||
|
||||
when:
|
||||
def pod = new PodSpecBuilder()
|
||||
.withPodName('foo')
|
||||
.withImageName('busybox')
|
||||
.withCommand('echo foo')
|
||||
.withPrivileged(true)
|
||||
.build()
|
||||
|
||||
then:
|
||||
pod.spec.containers[0].securityContext == [privileged: true]
|
||||
|
||||
}
|
||||
|
||||
def 'should create pod with resources limits' () {
|
||||
when:
|
||||
def pod1 = new PodSpecBuilder()
|
||||
.withPodName('foo')
|
||||
.withImageName('busybox')
|
||||
.withCommand('echo foo')
|
||||
.withResourcesLimits('nextflow.io/fuse': 1)
|
||||
.build()
|
||||
|
||||
then:
|
||||
pod1.spec.containers[0].resources == [limits:['nextflow.io/fuse':1]]
|
||||
|
||||
|
||||
when:
|
||||
def pod2 = new PodSpecBuilder()
|
||||
.withPodName('foo')
|
||||
.withImageName('busybox')
|
||||
.withCommand('echo foo')
|
||||
.withCpus(8)
|
||||
.withCpuLimits(true)
|
||||
.withMemory(MemoryUnit.of('10GB'))
|
||||
.withResourcesLimits('nextflow.io/fuse': 1)
|
||||
.build()
|
||||
|
||||
then:
|
||||
pod2.spec.containers[0].resources == [
|
||||
requests: ['cpu':8, 'memory':'10240Mi'],
|
||||
limits: ['cpu':8, 'memory':'10240Mi', 'nextflow.io/fuse':1] ]
|
||||
}
|
||||
|
||||
def 'should set namespace, labels and annotations' () {
|
||||
|
||||
when:
|
||||
def pod = new PodSpecBuilder()
|
||||
.withPodName('foo')
|
||||
.withImageName('busybox')
|
||||
.withCommand(['sh', '-c', 'echo hello'])
|
||||
.withNamespace('xyz')
|
||||
.withLabel('app','myApp')
|
||||
.withLabel('runName','something')
|
||||
.withLabel('version','3.6.1')
|
||||
.withAnnotation("anno1", "value1")
|
||||
.withAnnotations([anno2: "value2", anno3: "value3"])
|
||||
.build()
|
||||
|
||||
then:
|
||||
pod.metadata.namespace == 'xyz'
|
||||
pod.metadata.labels == [
|
||||
app: 'myApp',
|
||||
runName: 'something',
|
||||
version: '3.6.1'
|
||||
]
|
||||
pod.metadata.annotations == [
|
||||
anno1: "value1",
|
||||
anno2: "value2",
|
||||
anno3: "value3"
|
||||
]
|
||||
}
|
||||
|
||||
def 'should truncate labels longer than 63 chars' () {
|
||||
|
||||
when:
|
||||
def pod = new PodSpecBuilder()
|
||||
.withPodName('foo')
|
||||
.withImageName('busybox')
|
||||
.withCommand(['sh', '-c', 'echo hello'])
|
||||
.withLabel('app','myApp')
|
||||
.withLabel('runName','something')
|
||||
.withLabel('tag','somethingreallylonggggggggggggggggggggggggggggggggggggggggggendEXTRABIT')
|
||||
.withLabels([tag2: 'somethingreallylonggggggggggggggggggggggggggggggggggggggggggendEXTRABIT', tag3: 'somethingreallylonggggggggggggggggggggggggggggggggggggggggggendEXTRABIT'])
|
||||
.build()
|
||||
|
||||
then:
|
||||
pod.metadata.labels == [
|
||||
app: 'myApp',
|
||||
runName: 'something',
|
||||
tag: 'somethingreallylonggggggggggggggggggggggggggggggggggggggggggend',
|
||||
tag2: 'somethingreallylonggggggggggggggggggggggggggggggggggggggggggend',
|
||||
tag3: 'somethingreallylonggggggggggggggggggggggggggggggggggggggggggend'
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
def 'should set resources and env' () {
|
||||
|
||||
when:
|
||||
def pod = new PodSpecBuilder()
|
||||
.withPodName('foo')
|
||||
.withImageName('busybox')
|
||||
.withCommand('echo hello')
|
||||
.withEnv(PodEnv.value('ALPHA','hello'))
|
||||
.withEnv(PodEnv.value('DELTA', 'world'))
|
||||
.withCpus(8)
|
||||
.withAccelerator( new AcceleratorResource(request: 5, limit:10, type: 'foo.org') )
|
||||
.withMemory('100Gi')
|
||||
.withDisk('10Gi')
|
||||
.build()
|
||||
|
||||
then:
|
||||
pod.spec.containers[0].env == [
|
||||
[name:'ALPHA', value:'hello'],
|
||||
[name:'DELTA', value:'world']
|
||||
]
|
||||
pod.spec.containers[0].resources == [
|
||||
requests: ['foo.org/gpu':5, cpu:8, memory:'100Gi', 'ephemeral-storage':'10Gi'],
|
||||
limits: ['foo.org/gpu':10, memory:'100Gi', 'ephemeral-storage':'10Gi']
|
||||
]
|
||||
}
|
||||
|
||||
def 'should get storage spec for volume claims' () {
|
||||
|
||||
when:
|
||||
def pod = new PodSpecBuilder()
|
||||
.withPodName('foo')
|
||||
.withImageName('busybox')
|
||||
.withCommand(['echo'])
|
||||
.withVolumeClaim(new PodVolumeClaim('first','/work'))
|
||||
.withVolumeClaim(new PodVolumeClaim('second', '/data', '/foo'))
|
||||
.withVolumeClaim(new PodVolumeClaim('third', '/things', null, true))
|
||||
.build()
|
||||
then:
|
||||
pod.spec.containers[0].volumeMounts == [
|
||||
[name:'vol-1', mountPath:'/work'],
|
||||
[name:'vol-2', mountPath:'/data', subPath: '/foo'],
|
||||
[name:'vol-3', mountPath:'/things', readOnly: true]
|
||||
]
|
||||
pod.spec.volumes == [
|
||||
[name:'vol-1', persistentVolumeClaim:[claimName:'first']],
|
||||
[name:'vol-2', persistentVolumeClaim:[claimName:'second']],
|
||||
[name:'vol-3', persistentVolumeClaim:[claimName:'third']]
|
||||
]
|
||||
|
||||
}
|
||||
|
||||
def 'should only define one volume per persistentVolumeClaim' () {
|
||||
|
||||
when:
|
||||
def pod = new PodSpecBuilder()
|
||||
.withPodName('foo')
|
||||
.withImageName('busybox')
|
||||
.withCommand(['echo'])
|
||||
.withVolumeClaim(new PodVolumeClaim('first','/work'))
|
||||
.withVolumeClaim(new PodVolumeClaim('first','/work2', '/bar'))
|
||||
.withVolumeClaim(new PodVolumeClaim('second', '/data', '/foo'))
|
||||
.withVolumeClaim(new PodVolumeClaim('second', '/data2', '/fooz'))
|
||||
.build()
|
||||
then:
|
||||
pod.spec.containers[0].volumeMounts == [
|
||||
[name:'vol-1', mountPath:'/work'],
|
||||
[name:'vol-1', mountPath:'/work2', subPath: '/bar'],
|
||||
[name:'vol-2', mountPath:'/data', subPath: '/foo'],
|
||||
[name:'vol-2', mountPath:'/data2', subPath: '/fooz']
|
||||
]
|
||||
pod.spec.volumes == [
|
||||
[name:'vol-1', persistentVolumeClaim:[claimName:'first']],
|
||||
[name:'vol-2', persistentVolumeClaim:[claimName:'second']]
|
||||
]
|
||||
|
||||
}
|
||||
|
||||
def 'should get config map mounts' () {
|
||||
|
||||
when:
|
||||
def pod = new PodSpecBuilder()
|
||||
.withPodName('foo')
|
||||
.withImageName('busybox')
|
||||
.withCommand(['echo'])
|
||||
.withConfigMap(new PodMountConfig(config: 'cfg1', mountPath: '/etc/config'))
|
||||
.withConfigMap(new PodMountConfig(config: 'data2', mountPath: '/data/path'))
|
||||
.build()
|
||||
then:
|
||||
pod.spec.containers[0].volumeMounts == [
|
||||
[name:'vol-1', mountPath:'/etc/config'],
|
||||
[name:'vol-2', mountPath:'/data/path']
|
||||
]
|
||||
pod.spec.volumes == [
|
||||
[name:'vol-1', configMap:[name:'cfg1']],
|
||||
[name:'vol-2', configMap:[name:'data2']]
|
||||
]
|
||||
|
||||
}
|
||||
|
||||
def 'should get csi ephemeral mounts' () {
|
||||
|
||||
when:
|
||||
def pod = new PodSpecBuilder()
|
||||
.withPodName('foo')
|
||||
.withImageName('busybox')
|
||||
.withCommand(['echo'])
|
||||
.withCsiEphemeral(new PodMountCsiEphemeral(csi: [driver: 'inline.storage.kubernetes.io', readOnly: true], mountPath: '/data'))
|
||||
.build()
|
||||
then:
|
||||
pod.spec.containers[0].volumeMounts == [
|
||||
[name: 'vol-1', mountPath: '/data', readOnly: true]
|
||||
]
|
||||
pod.spec.volumes == [
|
||||
[name: 'vol-1', csi: [driver: 'inline.storage.kubernetes.io', readOnly: true]]
|
||||
]
|
||||
}
|
||||
|
||||
def 'should get empty dir mounts' () {
|
||||
|
||||
when:
|
||||
def pod = new PodSpecBuilder()
|
||||
.withPodName('foo')
|
||||
.withImageName('busybox')
|
||||
.withCommand(['echo'])
|
||||
.withEmptyDir(new PodMountEmptyDir(mountPath: '/scratch1', emptyDir: [medium: 'Disk']))
|
||||
.withEmptyDir(new PodMountEmptyDir(mountPath: '/scratch2', emptyDir: [medium: 'Memory']))
|
||||
.build()
|
||||
then:
|
||||
pod.spec.containers[0].volumeMounts == [
|
||||
[name: 'vol-1', mountPath: '/scratch1'],
|
||||
[name: 'vol-2', mountPath: '/scratch2']
|
||||
]
|
||||
pod.spec.volumes == [
|
||||
[name: 'vol-1', emptyDir: [medium: 'Disk']],
|
||||
[name: 'vol-2', emptyDir: [medium: 'Memory']]
|
||||
]
|
||||
}
|
||||
|
||||
def 'should consume env secrets' () {
|
||||
|
||||
when:
|
||||
def pod = new PodSpecBuilder()
|
||||
.withPodName('foo')
|
||||
.withImageName('busybox')
|
||||
.withCommand(['echo'])
|
||||
.withEnv( PodEnv.value('FOO','abc'))
|
||||
.withEnv( PodEnv.secret('VAR_X', 'delta/bar'))
|
||||
.withEnv( PodEnv.secret('VAR_Y', 'gamma'))
|
||||
.build()
|
||||
|
||||
then:
|
||||
pod.spec.containers[0].env == [
|
||||
[name: 'FOO', value: 'abc'],
|
||||
[name: 'VAR_X', valueFrom: [secretKeyRef: [name:'delta', key:'bar']]],
|
||||
[name: 'VAR_Y', valueFrom: [secretKeyRef: [name:'gamma', key:'VAR_Y']]]
|
||||
]
|
||||
}
|
||||
|
||||
def 'should consume env configMap' () {
|
||||
|
||||
when:
|
||||
def pod = new PodSpecBuilder()
|
||||
.withPodName('foo')
|
||||
.withImageName('busybox')
|
||||
.withCommand(['echo'])
|
||||
.withEnv( PodEnv.value('FOO','abc'))
|
||||
.withEnv( PodEnv.config('VAR_X', 'data'))
|
||||
.withEnv( PodEnv.config('VAR_Y', 'omega/bar-2'))
|
||||
.build()
|
||||
|
||||
then:
|
||||
pod.spec.containers[0].env == [
|
||||
[name: 'FOO', value: 'abc'],
|
||||
[name: 'VAR_X', valueFrom: [configMapKeyRef: [name:'data', key:'VAR_X']]],
|
||||
[name: 'VAR_Y', valueFrom: [configMapKeyRef: [name:'omega', key:'bar-2']]]
|
||||
]
|
||||
}
|
||||
|
||||
def 'should consume file secrets' () {
|
||||
|
||||
when:
|
||||
def pod = new PodSpecBuilder()
|
||||
.withPodName('foo')
|
||||
.withImageName('busybox')
|
||||
.withCommand(['echo'])
|
||||
.withSecret(new PodMountSecret(secret: 'alpha', mountPath: '/this/and/that'))
|
||||
.withSecret(new PodMountSecret(secret: 'delta/foo', mountPath: '/etc/mnt/bar.txt'))
|
||||
.build()
|
||||
|
||||
then:
|
||||
pod.spec.containers[0].volumeMounts == [
|
||||
[name:'vol-1', mountPath:'/this/and/that'],
|
||||
[name:'vol-2', mountPath:'/etc/mnt']
|
||||
]
|
||||
pod.spec.volumes == [
|
||||
[name:'vol-1', secret:[secretName: 'alpha']],
|
||||
[name:'vol-2', secret:[
|
||||
secretName: 'delta',
|
||||
items: [
|
||||
[ key: 'foo', path:'bar.txt' ]
|
||||
]
|
||||
]]
|
||||
]
|
||||
}
|
||||
|
||||
def 'should get host path mounts' () {
|
||||
|
||||
when:
|
||||
def pod = new PodSpecBuilder()
|
||||
.withPodName('foo')
|
||||
.withImageName('busybox')
|
||||
.withCommand(['echo'])
|
||||
.withHostMount('/tmp','/scratch')
|
||||
.withHostMount('/host/data','/mnt/container')
|
||||
.build()
|
||||
|
||||
then:
|
||||
pod.spec.containers[0].volumeMounts == [
|
||||
[name:'vol-1', mountPath:'/scratch'],
|
||||
[name:'vol-2', mountPath:'/mnt/container']
|
||||
]
|
||||
pod.spec.volumes == [
|
||||
[name:'vol-1', hostPath: [path:'/tmp']],
|
||||
[name:'vol-2', hostPath: [path:'/host/data']]
|
||||
]
|
||||
|
||||
}
|
||||
|
||||
|
||||
def 'should return secret file volume and mounts' () {
|
||||
|
||||
given:
|
||||
List mounts
|
||||
List volumes
|
||||
def builder = new PodSpecBuilder()
|
||||
|
||||
when:
|
||||
def secret1 = new PodMountSecret(secret:'foo', mountPath: '/etc/conf')
|
||||
builder.secretToSpec( 'vol1', secret1, mounts=[], volumes=[] )
|
||||
|
||||
then:
|
||||
mounts == [
|
||||
[ name: 'vol1', mountPath: '/etc/conf']
|
||||
]
|
||||
|
||||
volumes == [
|
||||
[ name: 'vol1', secret: [secretName: 'foo']]
|
||||
]
|
||||
|
||||
|
||||
when:
|
||||
def secret2 = new PodMountSecret(secret:'bar/hello.txt', mountPath: '/etc/conf/world.txt')
|
||||
builder.secretToSpec( 'vol2', secret2, mounts=[], volumes=[] )
|
||||
|
||||
then:
|
||||
mounts == [
|
||||
[ name: 'vol2', mountPath: '/etc/conf']
|
||||
]
|
||||
|
||||
volumes == [
|
||||
[ name: 'vol2', secret: [
|
||||
secretName: 'bar',
|
||||
items: [ [key: 'hello.txt', path:'world.txt'] ]
|
||||
]]
|
||||
]
|
||||
|
||||
}
|
||||
|
||||
def 'should return configmap file volume and mounts' () {
|
||||
|
||||
given:
|
||||
List mounts
|
||||
List volumes
|
||||
def builder = new PodSpecBuilder()
|
||||
|
||||
when:
|
||||
def config1 = new PodMountConfig(config:'foo', mountPath: '/etc/conf')
|
||||
builder.configMapToSpec( 'vol1', config1, mounts=[], volumes=[] )
|
||||
|
||||
then:
|
||||
mounts == [
|
||||
[ name: 'vol1', mountPath: '/etc/conf']
|
||||
]
|
||||
|
||||
volumes == [
|
||||
[ name: 'vol1', configMap: [name: 'foo']]
|
||||
]
|
||||
|
||||
|
||||
when:
|
||||
def config2 = new PodMountConfig(config:'bar/hello.txt', mountPath: '/etc/conf/world.txt')
|
||||
builder.configMapToSpec( 'vol2', config2, mounts=[], volumes=[] )
|
||||
|
||||
then:
|
||||
mounts == [
|
||||
[ name: 'vol2', mountPath: '/etc/conf']
|
||||
]
|
||||
|
||||
volumes == [
|
||||
[ name: 'vol2', configMap: [
|
||||
name: 'bar',
|
||||
items: [ [key: 'hello.txt', path:'world.txt'] ]
|
||||
]]
|
||||
]
|
||||
|
||||
}
|
||||
|
||||
|
||||
def 'should create pod spec with pod options' () {
|
||||
|
||||
given:
|
||||
def affinity = [
|
||||
nodeAffinity: [
|
||||
requiredDuringSchedulingIgnoredDuringExecution: [
|
||||
nodeSelectorTerms: [
|
||||
[key: 'foo', operator: 'In', values: ['bar', 'baz']]
|
||||
]
|
||||
]
|
||||
]
|
||||
]
|
||||
def tolerations = [[
|
||||
key: 'example-key',
|
||||
operator: 'Exists',
|
||||
effect: 'NoSchedule'
|
||||
]]
|
||||
def opts = Mock(PodOptions)
|
||||
and:
|
||||
def builder = new PodSpecBuilder()
|
||||
.withPodName('foo')
|
||||
.withImageName('busybox')
|
||||
.withCommand(['echo'])
|
||||
.withLabel('runName', 'crazy_john')
|
||||
.withAnnotation('evict', 'false')
|
||||
|
||||
when:
|
||||
def pod = builder.withPodOptions(opts).build()
|
||||
then:
|
||||
_ * opts.getAffinity() >> affinity
|
||||
_ * opts.getAnnotations() >> [OMEGA:'zzz', SIGMA:'www']
|
||||
_ * opts.getAutomountServiceAccountToken() >> false
|
||||
2 * opts.getEnvVars() >> [ PodEnv.value('HELLO','WORLD') ]
|
||||
2 * opts.getImagePullPolicy() >> 'always'
|
||||
2 * opts.getImagePullSecret() >> 'myPullSecret'
|
||||
_ * opts.getLabels() >> [ALPHA: 'xxx', GAMMA: 'yyy']
|
||||
2 * opts.getVolumeClaims() >> [ new PodVolumeClaim('pvc1', '/work') ]
|
||||
2 * opts.getMountConfigMaps() >> [ new PodMountConfig('data', '/home/user') ]
|
||||
2 * opts.getMountSecrets() >> [ new PodMountSecret('blah', '/etc/secret.txt') ]
|
||||
_ * opts.getNodeSelector() >> new PodNodeSelector(gpu:true, queue: 'fast')
|
||||
_ * opts.getPriorityClassName() >> 'high-priority'
|
||||
_ * opts.getSecurityContext() >> new PodSecurityContext(1000)
|
||||
_ * opts.getTolerations() >> tolerations
|
||||
and:
|
||||
pod.metadata == [
|
||||
name:'foo',
|
||||
namespace:'default',
|
||||
labels:[runName:'crazy_john', ALPHA:'xxx', GAMMA:'yyy'],
|
||||
annotations: [evict: 'false', OMEGA:'zzz', SIGMA:'www']
|
||||
]
|
||||
and:
|
||||
pod.spec.affinity == affinity
|
||||
pod.spec.automountServiceAccountToken == false
|
||||
pod.spec.imagePullSecrets == [[ name: 'myPullSecret' ]]
|
||||
pod.spec.nodeSelector == [gpu: 'true', queue: 'fast']
|
||||
pod.spec.priorityClassName == 'high-priority'
|
||||
pod.spec.securityContext == [ runAsUser: 1000 ]
|
||||
pod.spec.tolerations == tolerations
|
||||
pod.spec.containers[0].imagePullPolicy == 'always'
|
||||
pod.spec.containers[0].env == [[name:'HELLO', value:'WORLD']]
|
||||
pod.spec.containers[0].volumeMounts == [
|
||||
[name:'vol-1', mountPath:'/work'],
|
||||
[name:'vol-2', mountPath:'/home/user'],
|
||||
[name:'vol-3', mountPath:'/etc/secret.txt']
|
||||
]
|
||||
and:
|
||||
pod.spec.volumes == [
|
||||
[name:'vol-1', persistentVolumeClaim:[claimName:'pvc1']],
|
||||
[name:'vol-2', configMap:[name:'data']],
|
||||
[name:'vol-3', secret:[secretName:'blah']]
|
||||
]
|
||||
|
||||
}
|
||||
|
||||
def 'should create pod spec with activeDeadlineSeconds' () {
|
||||
|
||||
when:
|
||||
def pod = new PodSpecBuilder()
|
||||
.withPodName('foo')
|
||||
.withImageName('busybox')
|
||||
.withCommand(['echo', 'hello'])
|
||||
.withActiveDeadline(100)
|
||||
.build()
|
||||
|
||||
then:
|
||||
pod.spec.activeDeadlineSeconds == 100
|
||||
|
||||
}
|
||||
|
||||
def 'should create pod spec with schedulerName' () {
|
||||
|
||||
when:
|
||||
def pod = new PodSpecBuilder()
|
||||
.withPodName('foo')
|
||||
.withImageName('busybox')
|
||||
.withCommand(['echo', 'hello'])
|
||||
.withPodOptions(new PodOptions(schedulerName: 'my-scheduler'))
|
||||
.build()
|
||||
|
||||
then:
|
||||
pod.spec.schedulerName == 'my-scheduler'
|
||||
|
||||
}
|
||||
|
||||
def 'should create image pull request map' () {
|
||||
given:
|
||||
def builder = new PodSpecBuilder(imagePullSecret: 'MySecret')
|
||||
when:
|
||||
def result = builder.createPullSecret()
|
||||
then:
|
||||
result.size() == 1
|
||||
result.get(0).name == 'MySecret'
|
||||
}
|
||||
|
||||
|
||||
def 'should return the resources map' () {
|
||||
|
||||
given:
|
||||
def builder = new PodSpecBuilder()
|
||||
|
||||
when:
|
||||
def res = builder.addAcceleratorResources(new AcceleratorResource(request:2, limit: 5), null)
|
||||
then:
|
||||
res.requests == ['nvidia.com/gpu': 2]
|
||||
res.limits == ['nvidia.com/gpu': 5]
|
||||
|
||||
when:
|
||||
res = builder.addAcceleratorResources(new AcceleratorResource(limit: 5, type:'foo'), null)
|
||||
then:
|
||||
res.requests == ['foo.com/gpu': 5]
|
||||
res.limits == ['foo.com/gpu': 5]
|
||||
|
||||
when:
|
||||
res = builder.addAcceleratorResources(new AcceleratorResource(request: 5, type:'foo.org'), null)
|
||||
then:
|
||||
res.requests == ['foo.org/gpu': 5]
|
||||
res.limits == null
|
||||
|
||||
when:
|
||||
res = builder.addAcceleratorResources(new AcceleratorResource(request: 5, type: 'foo.org'), [requests: [cpu: 2]])
|
||||
then:
|
||||
res.requests == [cpu: 2, 'foo.org/gpu': 5]
|
||||
res.limits == null
|
||||
|
||||
when:
|
||||
res = builder.addAcceleratorResources(new AcceleratorResource(request: 5, limit: 10, type: 'foo.org'), [requests: [cpu: 2]])
|
||||
then:
|
||||
res.requests == [cpu: 2, 'foo.org/gpu': 5]
|
||||
res.limits == ['foo.org/gpu': 10]
|
||||
|
||||
when:
|
||||
res = builder.addAcceleratorResources(new AcceleratorResource(request: 5, type:'example.com/fpga'), null)
|
||||
then:
|
||||
res.requests == ['example.com/fpga': 5]
|
||||
res.limits == null
|
||||
|
||||
when:
|
||||
res = builder.addAcceleratorResources(new AcceleratorResource(request: 5, limit: 10, type: 'example.com/fpga'), [requests: [cpu: 2]])
|
||||
then:
|
||||
res.requests == [cpu: 2, 'example.com/fpga': 5]
|
||||
res.limits == ['example.com/fpga': 10]
|
||||
}
|
||||
|
||||
def 'should add resources limits' () {
|
||||
given:
|
||||
def builder = new PodSpecBuilder()
|
||||
Map resources
|
||||
|
||||
when:
|
||||
resources = builder.addResourcesLimits(['foo':1], null)
|
||||
then:
|
||||
resources == [limits:[foo:1]]
|
||||
|
||||
when:
|
||||
resources = builder.addResourcesLimits(['foo':1], [requests: ['x':1], limits: ['y': 2]])
|
||||
then:
|
||||
resources == [requests:[x:1], limits:[y:2, foo:1]]
|
||||
}
|
||||
|
||||
|
||||
@Unroll
|
||||
def 'should sanitize k8s label value: #label' () {
|
||||
given:
|
||||
def builder = new PodSpecBuilder()
|
||||
|
||||
expect:
|
||||
builder.sanitizeValue(label, PodSpecBuilder.MetaType.LABEL, PodSpecBuilder.SegmentType.VALUE) == str
|
||||
|
||||
where:
|
||||
label | str
|
||||
null | 'null'
|
||||
'hello' | 'hello'
|
||||
'hello world' | 'hello_world'
|
||||
'hello world' | 'hello_world'
|
||||
'hello.world' | 'hello.world'
|
||||
'hello-world' | 'hello-world'
|
||||
'hello_world' | 'hello_world'
|
||||
'hello_world-' | 'hello_world'
|
||||
'hello_world_' | 'hello_world'
|
||||
'hello_world.' | 'hello_world'
|
||||
'hello_123' | 'hello_123'
|
||||
'HELLO 123' | 'HELLO_123'
|
||||
'123hello' | '123hello'
|
||||
'x2345678901234567890123456789012345678901234567890123456789012345' | 'x23456789012345678901234567890123456789012345678901234567890123'
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should sanitize k8s label key: #label_key' () {
|
||||
given:
|
||||
def builder = new PodSpecBuilder()
|
||||
|
||||
expect:
|
||||
builder.sanitizeKey(label_key, PodSpecBuilder.MetaType.LABEL) == str
|
||||
|
||||
where:
|
||||
label_key | str
|
||||
'foo' | 'foo'
|
||||
'key 1' | 'key_1'
|
||||
'foo.bar/key 2' | 'foo.bar/key_2'
|
||||
'foo.bar/' | 'foo.bar'
|
||||
'/foo.bar' | 'foo.bar'
|
||||
'x2345678901234567890123456789012345678901234567890123456789012345' | 'x23456789012345678901234567890123456789012345678901234567890123'
|
||||
'x23456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345/key 2' | 'x234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123/key_2'
|
||||
'foo.bar/x2345678901234567890123456789012345678901234567890123456789012345' | 'foo.bar/x23456789012345678901234567890123456789012345678901234567890123'
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should report error if sanitizing k8s label with more than one slash character: #label_key' () {
|
||||
given:
|
||||
def builder = new PodSpecBuilder()
|
||||
|
||||
when:
|
||||
builder.sanitizeKey(label_key, PodSpecBuilder.MetaType.LABEL)
|
||||
|
||||
then:
|
||||
def error = thrown(expectedException)
|
||||
error.message == expectedMessage
|
||||
|
||||
where:
|
||||
label_key | expectedException | expectedMessage
|
||||
'foo.bar/key 2/key 3' | IllegalArgumentException | "Invalid key in pod label -- Key can only contain exactly one '/' character"
|
||||
'foo.bar/foo/bar/bar' | IllegalArgumentException | "Invalid key in pod label -- Key can only contain exactly one '/' character"
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should sanitize k8s label map' () {
|
||||
given:
|
||||
def builder = new PodSpecBuilder()
|
||||
|
||||
expect:
|
||||
builder.sanitize(KEY_VALUE, PodSpecBuilder.MetaType.LABEL) == EXPECTED
|
||||
|
||||
where:
|
||||
KEY_VALUE | EXPECTED
|
||||
[foo:'bar'] | [foo:'bar']
|
||||
['key 1':'value 2'] | [key_1:'value_2']
|
||||
['foo.bar/key 2':'value 3'] | ['foo.bar/key_2':'value_3']
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should sanitize k8s annotation key' () {
|
||||
given:
|
||||
def builder = new PodSpecBuilder()
|
||||
|
||||
expect:
|
||||
builder.sanitize(KEY_VALUE, PodSpecBuilder.MetaType.ANNOTATION) == EXPECTED
|
||||
|
||||
where:
|
||||
KEY_VALUE | EXPECTED
|
||||
[foo:'bar'] | [foo:'bar']
|
||||
['key 1':'value 2'] | [key_1:'value 2']
|
||||
['foo.bar/key 2':'value 3'] | ['foo.bar/key_2':'value 3']
|
||||
['x2345678901234567890123456789012345678901234567890123456789012345':'value 5'] | ['x23456789012345678901234567890123456789012345678901234567890123':'value 5']
|
||||
['x23456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345/key 4':'value 6'] | ['x234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123/key_4':'value 6']
|
||||
['foo.bar/x2345678901234567890123456789012345678901234567890123456789012345':'value 7'] | ['foo.bar/x23456789012345678901234567890123456789012345678901234567890123':'value 7']
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should report error if sanitizing k8s annotation key with more than one slash character: #annotation_key' () {
|
||||
given:
|
||||
def builder = new PodSpecBuilder()
|
||||
|
||||
when:
|
||||
builder.sanitizeKey(annotation_key, PodSpecBuilder.MetaType.ANNOTATION)
|
||||
|
||||
then:
|
||||
def error = thrown(expectedException)
|
||||
error.message == expectedMessage
|
||||
|
||||
where:
|
||||
annotation_key | expectedException | expectedMessage
|
||||
'foo.bar/key 2/key 3' | IllegalArgumentException | "Invalid key in pod annotation -- Key can only contain exactly one '/' character"
|
||||
'foo.bar/foo/bar/bar' | IllegalArgumentException | "Invalid key in pod annotation -- Key can only contain exactly one '/' character"
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should not sanitize k8s annotation value' () {
|
||||
given:
|
||||
def builder = new PodSpecBuilder()
|
||||
|
||||
expect:
|
||||
builder.sanitize(ANNOTATION, PodSpecBuilder.MetaType.ANNOTATION) == EXPECTED
|
||||
|
||||
where:
|
||||
ANNOTATION | EXPECTED
|
||||
['foo':'value 1'] | ['foo':'value 1']
|
||||
['foo':'foo.bar / *'] | ['foo':'foo.bar / *']
|
||||
['foo':'value 2 \n value 3'] | ['foo':'value 2 \n value 3']
|
||||
['foo':'value 3'] | ['foo':'value 3']
|
||||
['foo':'x2345678901234567890123456789012345678901234567890123456789012345'] | ['foo':'x2345678901234567890123456789012345678901234567890123456789012345']
|
||||
}
|
||||
|
||||
def 'should create job spec' () {
|
||||
|
||||
when:
|
||||
def spec = new PodSpecBuilder()
|
||||
.withPodName('foo')
|
||||
.withImageName('busybox')
|
||||
.withCommand(['echo', 'hello'])
|
||||
.buildAsJob()
|
||||
|
||||
then:
|
||||
spec == [
|
||||
apiVersion: 'batch/v1',
|
||||
kind: 'Job',
|
||||
metadata: [name: 'foo', namespace: 'default'],
|
||||
spec: [
|
||||
backoffLimit: 0,
|
||||
template: [
|
||||
metadata: [name: 'foo', namespace: 'default'],
|
||||
spec: [
|
||||
restartPolicy: 'Never',
|
||||
containers: [[
|
||||
name: 'foo',
|
||||
image: 'busybox',
|
||||
command: ['echo', 'hello'],
|
||||
]]
|
||||
]
|
||||
]
|
||||
]
|
||||
]
|
||||
}
|
||||
|
||||
def 'should create job spec with labels and annotations' () {
|
||||
|
||||
when:
|
||||
def job = new PodSpecBuilder()
|
||||
.withPodName('foo')
|
||||
.withImageName('busybox')
|
||||
.withCommand(['echo', 'hello'])
|
||||
.withLabel('app','someApp')
|
||||
.withLabel('runName','someName')
|
||||
.withLabel('version','3.8.1')
|
||||
.withAnnotation('anno1', 'val1')
|
||||
.withAnnotations([anno2: 'val2', anno3: 'val3'])
|
||||
.buildAsJob()
|
||||
|
||||
def metadata = [
|
||||
name: 'foo',
|
||||
namespace: 'default',
|
||||
labels: [
|
||||
app: 'someApp',
|
||||
runName: 'someName',
|
||||
version: '3.8.1'
|
||||
],
|
||||
annotations: [
|
||||
anno1: 'val1',
|
||||
anno2: 'val2',
|
||||
anno3: 'val3'
|
||||
]
|
||||
]
|
||||
|
||||
then:
|
||||
job.metadata == metadata
|
||||
job.spec.template.metadata == metadata
|
||||
}
|
||||
|
||||
def 'should create job spec with ttl seconds' () {
|
||||
when:
|
||||
def job = new PodSpecBuilder()
|
||||
.withPodName('foo')
|
||||
.withImageName('busybox')
|
||||
.withCommand(['echo', 'hello'])
|
||||
.buildAsJob()
|
||||
then:
|
||||
!job.spec.ttlSecondsAfterFinished
|
||||
|
||||
when:
|
||||
job = new PodSpecBuilder()
|
||||
.withPodName('foo')
|
||||
.withImageName('busybox')
|
||||
.withCommand(['echo', 'hello'])
|
||||
.withPodOptions( new PodOptions(ttlSecondsAfterFinished: 60) )
|
||||
.buildAsJob()
|
||||
then:
|
||||
job.spec.ttlSecondsAfterFinished == 60
|
||||
}
|
||||
|
||||
def 'should create pod spec with runtimeClassName' () {
|
||||
when:
|
||||
def pod = new PodSpecBuilder()
|
||||
.withPodName('foo')
|
||||
.withImageName('busybox')
|
||||
.withCommand(['echo', 'hello'])
|
||||
.build()
|
||||
then:
|
||||
!pod.spec.runtimeClassName
|
||||
|
||||
when:
|
||||
pod = new PodSpecBuilder()
|
||||
.withPodName('foo')
|
||||
.withImageName('busybox')
|
||||
.withCommand(['echo', 'hello'])
|
||||
.withPodOptions(new PodOptions(runtimeClassName: 'val1'))
|
||||
.build()
|
||||
then:
|
||||
pod.spec.runtimeClassName == 'val1'
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package recreationaltech.plugin.model
|
||||
|
||||
import spock.lang.Specification
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class PodVolumeClaimTest extends Specification {
|
||||
|
||||
|
||||
def 'should create a pod volume claim' () {
|
||||
when:
|
||||
def vol1 = new PodVolumeClaim('foo', '/bar')
|
||||
|
||||
then:
|
||||
vol1.claimName == 'foo'
|
||||
vol1.mountPath == '/bar'
|
||||
vol1.readOnly == false
|
||||
|
||||
when:
|
||||
def vol2 = new PodVolumeClaim(volumeClaim: 'alpha', mountPath: '/gamma')
|
||||
then:
|
||||
vol2.claimName == 'alpha'
|
||||
vol2.mountPath == '/gamma'
|
||||
vol2.readOnly == false
|
||||
|
||||
when:
|
||||
def vol3 = new PodVolumeClaim('aaa', '/bbb', null, true)
|
||||
|
||||
then:
|
||||
vol3.claimName == 'aaa'
|
||||
vol3.mountPath == '/bbb'
|
||||
vol3.readOnly == true
|
||||
|
||||
when:
|
||||
def vol4 = new PodVolumeClaim(volumeClaim: 'ccc', mountPath: '/ddd', readOnly: true)
|
||||
|
||||
then:
|
||||
vol4.claimName == 'ccc'
|
||||
vol4.mountPath == '/ddd'
|
||||
vol4.readOnly == true
|
||||
|
||||
}
|
||||
|
||||
def 'should sanitize paths' () {
|
||||
|
||||
expect :
|
||||
new PodVolumeClaim('foo','/data/work//').mountPath == '/data/work'
|
||||
new PodVolumeClaim('foo','//').mountPath == '/'
|
||||
new PodVolumeClaim('foo','/data').mountPath == '/data'
|
||||
|
||||
when:
|
||||
new PodVolumeClaim('foo','data')
|
||||
then:
|
||||
thrown(IllegalArgumentException)
|
||||
}
|
||||
|
||||
}
|
||||
4
mise.toml
Normal file
4
mise.toml
Normal file
@@ -0,0 +1,4 @@
|
||||
[tools]
|
||||
go='1.25.6'
|
||||
java = "21.0.2"
|
||||
groovy = "4.0.28"
|
||||
@@ -1,12 +1,37 @@
|
||||
FROM eclipse-temurin:17-jre
|
||||
FROM eclipse-temurin:17-jdk AS build
|
||||
|
||||
WORKDIR /opt/nextflow
|
||||
|
||||
COPY . /opt/nextflow
|
||||
|
||||
RUN chmod +x /opt/nextflow/launch.sh /opt/nextflow/nextflow || true
|
||||
RUN chmod +x ./gradlew
|
||||
|
||||
ENV PATH="/opt/nextflow:${PATH}"
|
||||
RUN ./gradlew clean assemble pack
|
||||
|
||||
RUN mkdir -p /tmp/nf-unpack /opt/nf-dist \
|
||||
&& cd /tmp/nf-unpack \
|
||||
&& jar xf /opt/nextflow/modules/nextflow/build/distributions/nextflow-26.04.0.zip \
|
||||
&& cp -a nextflow-*/* /opt/nf-dist/ \
|
||||
&& chmod +x /opt/nf-dist/bin/nextflow
|
||||
|
||||
RUN mkdir -p /tmp/nxf-home/plugins/nf-k8s-1.5.2 \
|
||||
&& cd /tmp/nxf-home/plugins/nf-k8s-1.5.2 \
|
||||
&& jar xf /opt/nextflow/plugins/nf-k8s/build/distributions/nf-k8s-1.5.2.zip
|
||||
|
||||
RUN mkdir -p /tmp/nxf-home/plugins/nf-amazon-3.9.0 \
|
||||
&& cd /tmp/nxf-home/plugins/nf-amazon-3.9.0 \
|
||||
&& jar xf /opt/nextflow/plugins/nf-amazon/build/distributions/nf-amazon-3.9.0.zip
|
||||
|
||||
FROM eclipse-temurin:17-jre
|
||||
|
||||
WORKDIR /opt/nextflow
|
||||
|
||||
COPY --from=build /opt/nf-dist /opt/nextflow
|
||||
COPY --from=build /tmp/nxf-home /opt/nextflow/.nextflow
|
||||
|
||||
ENV PATH="/opt/nextflow/bin:${PATH}"
|
||||
ENV NXF_HOME="/opt/nextflow/.nextflow"
|
||||
ENV NXF_PLUGINS_DIR="/opt/nextflow/.nextflow/plugins"
|
||||
ENV JAVA_OPTS="--add-opens java.base/java.nio.file.spi=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.util.concurrent=ALL-UNNAMED"
|
||||
|
||||
ENTRYPOINT ["/opt/nextflow/launch.sh"]
|
||||
ENTRYPOINT ["/opt/nextflow/bin/nextflow"]
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
build=0
|
||||
version=26.04.0
|
||||
timestamp=1778418181930
|
||||
commitId=ad0d91a
|
||||
timestamp=1784045783078
|
||||
commitId=7b9188a
|
||||
|
||||
@@ -39,6 +39,9 @@ import nextflow.k8s.model.PodSecurityContext
|
||||
import nextflow.k8s.model.PodVolumeClaim
|
||||
import nextflow.k8s.model.ResourceType
|
||||
import nextflow.util.Duration
|
||||
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
/**
|
||||
* Model Kubernetes specific settings defined in the nextflow
|
||||
* configuration file
|
||||
@@ -221,6 +224,54 @@ class K8sConfig implements ConfigScope {
|
||||
""")
|
||||
final String nextflowImage
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The run interval of the kubernetes scheduler
|
||||
""")
|
||||
final Duration schedulerInterval
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Enables task runtime recording
|
||||
""")
|
||||
final boolean recordTaskRuntimes
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The runtime recording file
|
||||
""")
|
||||
final String runtimeRecordPath
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Selects a scheduling strategy
|
||||
""")
|
||||
final String schedulingStrategy
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Selects a runtime estimator
|
||||
""")
|
||||
final String runtimeEstimator
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Noise magnitude used for the noise runtime estimator
|
||||
""")
|
||||
final Duration noiseRuntimeEstimatorNoiseMagnitude
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Max. amount of time two runtime estimates can differ to be considered equal.
|
||||
""")
|
||||
final Duration runtimeComparisonEpsilon
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Number of saved top runtimes used to classify a task as critical.
|
||||
""")
|
||||
final int dvfsSchedulingNumTopRuntimes
|
||||
|
||||
/* required by extension point -- do not remove */
|
||||
K8sConfig() {
|
||||
this(Collections.emptyMap())
|
||||
@@ -251,10 +302,19 @@ class K8sConfig implements ConfigScope {
|
||||
storageSubPath = opts.storageSubPath
|
||||
userName = opts.userName
|
||||
nextflowImage = opts.nextflowImage ?: "nextflow/nextflow:${BuildInfo.version}"
|
||||
schedulerInterval = opts.schedulerInterval as Duration ?: new Duration(10, TimeUnit.SECONDS)
|
||||
recordTaskRuntimes = opts.recordTaskRuntimes as boolean ?: false
|
||||
|
||||
launchDir = opts.launchDir ?: "${storageMountPath}/${getUserName()}"
|
||||
projectDir = opts.projectDir ?: "${storageMountPath}/projects"
|
||||
workDir = opts.workDir ?: "${launchDir}/work"
|
||||
runtimeRecordPath = opts.runtimeRecordPath as String ?: "${workDir}/runtimes.csv"
|
||||
|
||||
schedulingStrategy = opts.schedulingStrategy as String ?: "Hash"
|
||||
runtimeEstimator = opts.runtimeEstimator as String ?: "LinearFit"
|
||||
noiseRuntimeEstimatorNoiseMagnitude = opts.noiseRuntimeEstimatorNoiseMagnitude as Duration ?: new Duration(30, TimeUnit.SECONDS)
|
||||
runtimeComparisonEpsilon = opts.runtimeComparisonEpsilon as Duration ?: new Duration(10, TimeUnit.SECONDS)
|
||||
dvfsSchedulingNumTopRuntimes = opts.dvfsSchedulingTopRuntimes as int ?: 3
|
||||
|
||||
// -- shortcut to pod image pull-policy
|
||||
if( imagePullPolicy )
|
||||
@@ -272,7 +332,6 @@ class K8sConfig implements ConfigScope {
|
||||
else if( securityContext )
|
||||
pod.securityContext = new PodSecurityContext(securityContext)
|
||||
|
||||
log.info("Hello sailor")
|
||||
nodeInit = new K8sNodeInitConfig(opts.nodeInit as Map ?: Collections.emptyMap())
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
package nextflow.k8s
|
||||
|
||||
import groovy.util.logging.Slf4j
|
||||
|
||||
import java.net.http.HttpClient
|
||||
import java.net.http.HttpRequest
|
||||
import java.net.http.HttpResponse
|
||||
import java.nio.channels.UnresolvedAddressException
|
||||
|
||||
/**
|
||||
* The DVFS client uses the deployed DVFS agents to control the operating frequency of the worker nodes.
|
||||
*/
|
||||
@Slf4j
|
||||
class K8sDVFSClient {
|
||||
private HttpClient httpClient
|
||||
private Map<String, String> ipTable
|
||||
|
||||
K8sDVFSClient(String[] nodes, String[] ips) {
|
||||
assert(nodes.length == ips.length)
|
||||
this.httpClient = HttpClient.newBuilder().build()
|
||||
this.ipTable = new HashMap<>()
|
||||
for (int i = 0; i < nodes.length; i++) {
|
||||
this.ipTable.put(nodes[i], ips[i])
|
||||
}
|
||||
}
|
||||
|
||||
OptionalLong getNodeCurrentFrequency(String node) {
|
||||
log.debug("Getting current frequency of node ${node}")
|
||||
return getNodeFrequency(node, "current")
|
||||
}
|
||||
|
||||
OptionalLong getNodeMaxFrequency(String node) {
|
||||
log.debug("Getting max frequency of node ${node}")
|
||||
return getNodeFrequency(node, "max")
|
||||
}
|
||||
|
||||
OptionalLong getNodeMinFrequency(String node) {
|
||||
log.debug("Getting min frequency of node ${node}")
|
||||
return getNodeFrequency(node, "min")
|
||||
}
|
||||
|
||||
private OptionalLong getNodeFrequency(String node, String endpoint) {
|
||||
return getLong(HttpRequest.newBuilder()
|
||||
.uri(new URI("http://${agentAddress(node)}/cpu/frequency/${endpoint}"))
|
||||
.GET()
|
||||
.build())
|
||||
}
|
||||
|
||||
boolean setNodeFrequency(String node, int frequency) {
|
||||
String body = String.valueOf(frequency)
|
||||
|
||||
HttpRequest request = HttpRequest.newBuilder()
|
||||
.uri(new URI("http://${agentAddress(node)}/cpu/frequency/current"))
|
||||
.PUT(HttpRequest.BodyPublishers.ofString(body))
|
||||
.build()
|
||||
HttpResponse<Void> response = httpClient.send(request, HttpResponse.BodyHandlers.discarding())
|
||||
if (response.statusCode() != 200) {
|
||||
log.error("Request PUT ${request.uri().toString()} returned ${response.statusCode()}")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
OptionalLong getCPUCount(String node) {
|
||||
return getLong(HttpRequest.newBuilder()
|
||||
.uri(new URI("http://${agentAddress(node)}/cpu/count"))
|
||||
.GET()
|
||||
.build())
|
||||
}
|
||||
|
||||
OptionalLong getMemoryAmount(String node) {
|
||||
return getLong(HttpRequest.newBuilder()
|
||||
.uri(new URI("http://${agentAddress(node)}/mem/amount"))
|
||||
.GET()
|
||||
.build())
|
||||
}
|
||||
|
||||
private OptionalLong getLong(HttpRequest request) {
|
||||
try {
|
||||
HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString())
|
||||
if (response.statusCode() != 200) {
|
||||
log.error("Request GET ${request.uri().toString()} returned ${response.statusCode()}")
|
||||
return OptionalLong.empty()
|
||||
}
|
||||
long parsed = Long.parseLong(response.body())
|
||||
return OptionalLong.of(parsed)
|
||||
} catch (NumberFormatException ex) {
|
||||
log.error("Unexpected response ${response.body()} - ${ex.message}")
|
||||
return OptionalLong.empty()
|
||||
} catch (ConnectException ex) {
|
||||
log.error("Request failed ${request.uri().toString()}: ${ex.message}")
|
||||
return OptionalLong.empty()
|
||||
} catch (UnresolvedAddressException ex) {
|
||||
log.error("Failed to resolve address ${request.uri().toString()}: ${ex.message}")
|
||||
return OptionalLong.empty()
|
||||
}
|
||||
}
|
||||
|
||||
private String agentAddress(String node) {
|
||||
return ipTable.get(node) + ":8080"
|
||||
}
|
||||
}
|
||||
@@ -151,7 +151,7 @@ class K8sDriverLauncher {
|
||||
this.k8sConfig = makeK8sConfig(config.toMap())
|
||||
this.k8sClient = makeK8sClient(k8sConfig)
|
||||
this.k8sConfig.checkStorageAndPaths(k8sClient)
|
||||
this.initDeployer = new K8sNodeInitDeployer(k8sClient, k8sConfig, runName)
|
||||
this.initDeployer = new K8sNodeInitDeployer(k8sClient, k8sConfig)
|
||||
|
||||
createK8sConfigMap()
|
||||
|
||||
|
||||
@@ -16,6 +16,8 @@
|
||||
|
||||
package nextflow.k8s
|
||||
|
||||
import groovy.transform.CompileDynamic
|
||||
import nextflow.k8s.strategies.K8sDVFSSchedulingStrategy
|
||||
import nextflow.k8s.strategies.K8sHashSchedulingStrategy
|
||||
|
||||
import java.util.concurrent.TimeUnit
|
||||
@@ -52,19 +54,22 @@ class K8sExecutor extends Executor implements ExtensionPoint {
|
||||
*/
|
||||
private Cache<String, K8sClient> clientCache
|
||||
|
||||
private K8sTaskScheduler taskScheduler
|
||||
private K8sTaskScheduler taskScheduler
|
||||
private Thread schedulerThread
|
||||
|
||||
K8sRuntimeRecorder runtimeRecorder
|
||||
K8sRuntimeEstimator runtimeEstimator
|
||||
|
||||
/**
|
||||
* @return The Kubernetes HTTP client. Delegates to a Guava cache that refreshes
|
||||
* the client (including the service account token) when the configured interval expires.
|
||||
*/
|
||||
protected K8sClient getClient() {
|
||||
K8sClient getClient() {
|
||||
clientCache.get('client', () -> new K8sClient(k8sConfig.getClient()))
|
||||
}
|
||||
|
||||
protected K8sTaskScheduler getTaskScheduler() {
|
||||
if( taskScheduler == null )
|
||||
taskScheduler = new K8sTaskScheduler(getClient(), new K8sHashSchedulingStrategy())
|
||||
assert taskScheduler != null
|
||||
return taskScheduler
|
||||
}
|
||||
|
||||
@@ -88,7 +93,62 @@ class K8sExecutor extends Executor implements ExtensionPoint {
|
||||
.expireAfterWrite(refreshInterval.toMillis(), TimeUnit.MILLISECONDS)
|
||||
.build()
|
||||
final client = getClient()
|
||||
|
||||
log.debug "[K8s] config=$k8sConfig; API client config=$client.config"
|
||||
|
||||
this.runtimeRecorder = new K8sRuntimeRecorder(k8sConfig.recordTaskRuntimes, k8sConfig.runtimeRecordPath)
|
||||
|
||||
if (k8sConfig.runtimeEstimator == "LinearFit") {
|
||||
this.runtimeEstimator = new K8sLinearFitRuntimeEstimator(k8sConfig.runtimeRecordPath)
|
||||
} else if (k8sConfig.runtimeEstimator == "Noise") {
|
||||
this.runtimeEstimator = new K8sNoiseRuntimeEstimator(k8sConfig.runtimeEstimator, k8sConfig.noiseRuntimeEstimatorNoiseMagnitude)
|
||||
} else {
|
||||
log.error "[K8s] invalid runtime estimator ${k8sConfig.runtimeEstimator} - falling back on \"LinearFit\""
|
||||
this.runtimeEstimator = new K8sLinearFitRuntimeEstimator(k8sConfig.runtimeRecordPath)
|
||||
}
|
||||
|
||||
String[] nodes = getNodeList()
|
||||
|
||||
K8sSchedulingStrategy strategy = null
|
||||
if (k8sConfig.schedulingStrategy == "Hash") {
|
||||
strategy = new K8sHashSchedulingStrategy()
|
||||
} else if (k8sConfig.schedulingStrategy == "DVFS" || k8sConfig.schedulingStrategy == "DVFS-SPEED") {
|
||||
String[] ips = new String[nodes.length]
|
||||
for (int i = 0; i < nodes.length; i++) {
|
||||
ips[i] = client.getPodIpAddress(K8sNodeInitDeployer.buildPodName(nodes[i]))
|
||||
log.info "[K8s] node ${nodes[i]} -> ${ips[i]}"
|
||||
}
|
||||
strategy = new K8sDVFSSchedulingStrategy(this.runtimeEstimator,
|
||||
new K8sDVFSClient(nodes, ips),
|
||||
() -> getClient(),
|
||||
k8sConfig.runtimeComparisonEpsilon,
|
||||
k8sConfig.dvfsSchedulingNumTopRuntimes)
|
||||
strategy.fullSpeedMode = k8sConfig.schedulingStrategy == "DVFS-SPEED"
|
||||
} else {
|
||||
log.error "[K8s] invalid scheduling strategy $k8sConfig.schedulingStrategy, falling back on \"Hash\""
|
||||
strategy = new K8sHashSchedulingStrategy()
|
||||
}
|
||||
|
||||
this.taskScheduler = new K8sTaskScheduler(nodes, strategy, k8sConfig.schedulerInterval)
|
||||
this.schedulerThread = new Thread(this.taskScheduler)
|
||||
this.schedulerThread.start()
|
||||
}
|
||||
|
||||
@CompileDynamic
|
||||
private String[] getNodeList() {
|
||||
final resp = getClient().nodeList()
|
||||
ArrayList<String> nodes = new ArrayList<String>()
|
||||
for ( Map item : resp.items ) {
|
||||
nodes.add(item.metadata.name as String)
|
||||
}
|
||||
return nodes.toArray()
|
||||
}
|
||||
|
||||
@Override
|
||||
void shutdown() {
|
||||
this.runtimeRecorder.write()
|
||||
this.taskScheduler.stop()
|
||||
this.schedulerThread.join()
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
package nextflow.k8s
|
||||
|
||||
import groovy.util.logging.Slf4j
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
|
||||
/**
|
||||
* Provides runtime estimates for tasks based on task-name and input file size
|
||||
*/
|
||||
@Slf4j
|
||||
class K8sLinearFitRuntimeEstimator extends K8sRuntimeEstimator {
|
||||
|
||||
class Function {
|
||||
private double m
|
||||
private double n
|
||||
|
||||
Function(double m, double n) {
|
||||
this.m = m
|
||||
this.n = n
|
||||
}
|
||||
|
||||
double estimate(long x) {
|
||||
return m * (double)x + n
|
||||
}
|
||||
}
|
||||
|
||||
HashMap<String, Function> estimators;
|
||||
|
||||
/**
|
||||
* Initialize the estimator with data recorded by K8sRuntimeRecorder
|
||||
* @param dataFilePath
|
||||
*/
|
||||
K8sLinearFitRuntimeEstimator(String dataFilePath) {
|
||||
def data = parseDataFile(dataFilePath)
|
||||
fit(data)
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes the runtime estimator with statically known data.
|
||||
* @param data Map from task name to list of recordings, where each recording is a tuple (input-size, runtime-in-ms)
|
||||
*/
|
||||
K8sLinearFitRuntimeEstimator(HashMap<String, ArrayList<Tuple2<Long, Long>>> data) {
|
||||
fit(data)
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns an estimation of the task runtime in milliseconds
|
||||
* @param taskName the name of the task
|
||||
* @param inputSize the total input size in bytes
|
||||
* @return estimated task runtime in milliseconds OR infinity if the task is unknown.
|
||||
*/
|
||||
double estimate(String taskName, long inputSize) {
|
||||
Function estimator = estimators.get(taskName)
|
||||
if (estimator == null) {
|
||||
//log.warn "[K8s] Unable to estimate take ${taskName}: unknown task"
|
||||
return Double.POSITIVE_INFINITY
|
||||
}
|
||||
return estimator.estimate(inputSize)
|
||||
}
|
||||
|
||||
private void fit(HashMap<String, ArrayList<Tuple2<Long, Long>>> data) {
|
||||
estimators = new HashMap<>()
|
||||
for (Map.Entry<String, ArrayList<Tuple2<Long, Long>>> t : data) {
|
||||
Function f = fit(t.value)
|
||||
estimators.put(t.key, f)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Uses linear regression to fit a linear function (y = m * x + n) to the provided observations
|
||||
* @param observations list of tuples (input size, runtime in ms)
|
||||
* @return linear function fitted to the input
|
||||
*/
|
||||
private Function fit(ArrayList<Tuple2<Long, Long>> observations) throws IllegalArgumentException {
|
||||
int n = observations.size()
|
||||
if (n > 1) {
|
||||
double sumX = 0, sumY = 0, sumXY = 0, sumX2 = 0
|
||||
for (Tuple2<Long, Long> o : observations) {
|
||||
double x = (double) o.get(0)
|
||||
double y = (double) o.get(1)
|
||||
sumX += x
|
||||
sumY += y
|
||||
sumXY += x * y
|
||||
sumX2 += x * x
|
||||
}
|
||||
double m = (n * sumXY - sumX * sumY) / (n * sumX2 - sumX * sumX)
|
||||
return new Function(
|
||||
m,
|
||||
(sumY - m * sumX) / n
|
||||
)
|
||||
} else if (n == 1) {
|
||||
// Special case: We only have 1 measurement. We will just assume that the runtime is constant,
|
||||
// because in our observed data, it is.
|
||||
return new Function(0.0, (double) observations[0].get(1))
|
||||
}
|
||||
throw new IllegalArgumentException("requires at least 1 observation")
|
||||
}
|
||||
}
|
||||
@@ -9,12 +9,10 @@ import nextflow.k8s.model.PodSpecBuilder
|
||||
class K8sNodeInitDeployer {
|
||||
private K8sClient client
|
||||
private K8sConfig config
|
||||
private String runName
|
||||
|
||||
K8sNodeInitDeployer(K8sClient client, K8sConfig config, String runName) {
|
||||
K8sNodeInitDeployer(K8sClient client, K8sConfig config) {
|
||||
this.client = client
|
||||
this.config = config
|
||||
this.runName = runName
|
||||
}
|
||||
|
||||
void deploy() {
|
||||
@@ -110,16 +108,16 @@ exit 0
|
||||
private List<String> getNodes() {
|
||||
final resp = client.nodeList()
|
||||
ArrayList<String> nodes = new ArrayList<String>()
|
||||
for ( Map item : resp.items ) {
|
||||
for ( Map item: resp.items ) {
|
||||
nodes.add(item.metadata.name as String)
|
||||
}
|
||||
return nodes
|
||||
}
|
||||
|
||||
private String buildPodName(String nodeName) {
|
||||
public static String buildPodName(String nodeName) {
|
||||
// TODO: Remove anything that is not lowercase alpha-numeric or dash
|
||||
String sanitizedNodeName = nodeName.toLowerCase()
|
||||
String name = "nf-init-${runName}-${sanitizedNodeName}"
|
||||
String name = "nf-init-${sanitizedNodeName}"
|
||||
if ( name.length() > 63 )
|
||||
name = name.substring(0, 63)
|
||||
return name
|
||||
@@ -138,6 +136,7 @@ exit 0
|
||||
.withPrivileged(true)
|
||||
.withHostMounts(mounts)
|
||||
.withPodName(buildPodName(nodeName))
|
||||
.withPort(8080)
|
||||
.build()
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
package nextflow.k8s
|
||||
|
||||
import nextflow.util.Duration
|
||||
|
||||
class K8sNoiseRuntimeEstimator extends K8sRuntimeEstimator {
|
||||
// Magnitude of the noise we add
|
||||
private long noiseMag
|
||||
|
||||
private HashMap<String, ArrayList<Tuple2<Long, Long>>> data;
|
||||
private synchronized Random rnd;
|
||||
|
||||
/**
|
||||
* Initialize the estimator with data recorded by K8sRuntimeRecorder
|
||||
* @param dataFilePath
|
||||
* @param noiseMagnitude magnitude of the noise added to recordings
|
||||
*/
|
||||
K8sNoiseRuntimeEstimator(String dataFilePath, Duration noiseMagnitude) {
|
||||
this.data = parseDataFile(dataFilePath)
|
||||
this.noiseMag = noiseMagnitude.toMillis()
|
||||
this.rnd = new Random()
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes the runtime estimator with statically known data.
|
||||
* @param data Map from task name to list of recordings, where each recording is a tuple (input-size, runtime-in-ms)
|
||||
*/
|
||||
K8sNoiseRuntimeEstimator(HashMap<String, ArrayList<Tuple2<Long, Long>>> data, Duration noiseMagnitude) {
|
||||
this.data = data;
|
||||
this.noiseMag = noiseMagnitude.toMillis()
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns an estimation of the task runtime in milliseconds
|
||||
* @param taskName the name of the task
|
||||
* @param inputSize the total input size in bytes
|
||||
* @return estimated task runtime in milliseconds OR infinity if the task is unknown.
|
||||
*/
|
||||
@Override
|
||||
double estimate(String taskName, long inputSize) {
|
||||
ArrayList recordings = data.get(taskName)
|
||||
for (Tuple2<Long, Long> recording : recordings) {
|
||||
if (recording.get(0).longValue() == inputSize) {
|
||||
double noise = rnd.nextDouble(-1.0, 1.0) * (double)noiseMag
|
||||
long ms = recording.get(1).longValue()
|
||||
return (double)ms + noise
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package nextflow.k8s
|
||||
|
||||
import groovy.util.logging.Slf4j
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
|
||||
/**
|
||||
* Provides runtime estimates for tasks based on task-name and input file size
|
||||
*/
|
||||
@Slf4j
|
||||
abstract class K8sRuntimeEstimator {
|
||||
/**
|
||||
* Returns an estimation of the task runtime in milliseconds
|
||||
* @param handler the task handler
|
||||
* @return estimated task runtime in milliseconds OR infinity if the task is unknown.
|
||||
*/
|
||||
double estimate(K8sTaskHandler handler) {
|
||||
long x = getTaskHandlerInputSize(handler)
|
||||
return estimate(handler.task.processor.name, x)
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns an estimation of the task runtime in milliseconds
|
||||
* @param taskName the name of the task
|
||||
* @param inputSize the total input size in bytes
|
||||
* @return estimated task runtime in milliseconds OR infinity if the task is unknown.
|
||||
*/
|
||||
abstract double estimate(String taskName, long inputSize);
|
||||
|
||||
protected long getTaskHandlerInputSize(K8sTaskHandler handler) {
|
||||
long inputSizeSum = 0
|
||||
// File input
|
||||
def inputFiles = handler.task.getInputFilesMap()
|
||||
for (Map.Entry<String, Path> f : inputFiles) {
|
||||
try {
|
||||
inputSizeSum += Files.size(f.value)
|
||||
} catch (IOException ex) {
|
||||
log.error "[K8s] failed to get size of input file ${f.value} of task ${task.task.name}: ${ex.message}"
|
||||
}
|
||||
}
|
||||
// Non file input
|
||||
def inputs = handler.task.getInputs()
|
||||
for (Map.Entry i : inputs) {
|
||||
inputSizeSum += i.value.toString().length()
|
||||
}
|
||||
|
||||
return inputSizeSum
|
||||
}
|
||||
|
||||
/// @brief Parses the runtime recording data file
|
||||
/// @return Map from task name to list of recordings, where each recording is a tuple (input-size, runtime-in-ms)
|
||||
protected HashMap<String, ArrayList<Tuple2<Long, Long>>> parseDataFile(String dataFilePath) {
|
||||
HashMap<String, ArrayList<Tuple2<Long, Long>>> data = new HashMap<>();
|
||||
try {
|
||||
BufferedReader reader = new BufferedReader(new FileReader(dataFilePath))
|
||||
String line = reader.readLine()
|
||||
while (line != null) {
|
||||
// <task-name>,<input-size>,<runtime-ms>
|
||||
String[] elems = line.split(",")
|
||||
if (elems.length != 3) {
|
||||
log.warn "[K8s] ${dataFilePath}: Unexpected line ${line}"
|
||||
continue
|
||||
}
|
||||
|
||||
ArrayList<Tuple2<Long, Long>> taskData = data.get(elems[0])
|
||||
if (taskData == null) {
|
||||
taskData = new ArrayList<>()
|
||||
data.put(elems[0], taskData)
|
||||
}
|
||||
try {
|
||||
taskData.add(new Tuple2(Long.parseLong(elems[1]), Long.parseLong(elems[2])))
|
||||
} catch (NumberFormatException ex) {
|
||||
log.error "[K8s] ${dataFilePath} invalid data: ${ex.message}"
|
||||
}
|
||||
line = reader.readLine()
|
||||
}
|
||||
reader.close()
|
||||
} catch (IOException ex) {
|
||||
log.error "[K8s] Failed to load ${dataFilePath}: ${ex.message}"
|
||||
}
|
||||
return data
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package nextflow.k8s
|
||||
|
||||
record K8sRuntimeRecord(
|
||||
String taskName,
|
||||
long inputSize,
|
||||
long runtimeMillis
|
||||
) {}
|
||||
@@ -0,0 +1,67 @@
|
||||
package nextflow.k8s
|
||||
|
||||
import groovy.util.logging.Slf4j
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
|
||||
@Slf4j
|
||||
class K8sRuntimeRecorder {
|
||||
private final boolean enabled
|
||||
private final String recordPath
|
||||
|
||||
private ArrayList<K8sRuntimeRecord> records
|
||||
|
||||
K8sRuntimeRecorder(boolean enabled, String recordPath) {
|
||||
this.enabled = enabled
|
||||
this.recordPath = recordPath
|
||||
this.records = new ArrayList<>()
|
||||
}
|
||||
|
||||
/**
|
||||
* Records the runtime of a task.
|
||||
* This should be called for a finished task, which has its
|
||||
* start and end timestamps set.
|
||||
* @param task
|
||||
*/
|
||||
synchronized void record(K8sTaskHandler task) {
|
||||
if (!enabled)
|
||||
return;
|
||||
long runtimeMilis = task.completeTimeMillis - task.startTimeMillis
|
||||
long inputSizeSum = 0
|
||||
|
||||
// Input files
|
||||
def inputFiles = task.task.getInputFilesMap()
|
||||
for (Map.Entry<String, Path> f : inputFiles) {
|
||||
try {
|
||||
inputSizeSum += Files.size(f.value)
|
||||
} catch (IOException ex) {
|
||||
log.error "[K8s] failed to get size of input file ${f.value} of task ${task.task.name}: ${ex.message}"
|
||||
}
|
||||
}
|
||||
|
||||
// Non file input
|
||||
def inputs = task.task.getInputs()
|
||||
for (Map.Entry i : inputs) {
|
||||
inputSizeSum += i.value.toString().length()
|
||||
}
|
||||
|
||||
log.info "[K8s] task ${task.task.processor.name} - input ${inputSizeSum} bytes - ran ${runtimeMilis} ms (${task.startTimeMillis} -> ${task.completeTimeMillis})"
|
||||
records.add(new K8sRuntimeRecord(task.task.processor.name, inputSizeSum, runtimeMilis))
|
||||
}
|
||||
|
||||
void write() {
|
||||
if (!enabled)
|
||||
return;
|
||||
try {
|
||||
FileWriter out = new FileWriter(recordPath)
|
||||
for (K8sRuntimeRecord record : records) {
|
||||
out.write("${record.taskName},${record.inputSize},${record.runtimeMillis}\n")
|
||||
}
|
||||
out.close()
|
||||
log.info "[K8s] written runtime recording to ${recordPath}"
|
||||
} catch (IOException err) {
|
||||
log.error "[K8s] failed to write runtime recording ${recordPath}: ${err.message}"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -14,4 +14,8 @@ class K8sSchedulingRequest {
|
||||
this.task = handler.task
|
||||
this.submitTimeMillis = System.currentTimeMillis()
|
||||
}
|
||||
|
||||
String getTaskName() {
|
||||
return task.processor.name
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,9 +5,23 @@ interface K8sSchedulingStrategy {
|
||||
* Selects the next task to run from the given task queue.
|
||||
* If the scheduler should wait, returns {@code null} instead
|
||||
*
|
||||
* @param scheduler the calling scheduler object
|
||||
* @param queue Pending scheduling requests
|
||||
* @param nodes Available Kubernetes node names
|
||||
* @return A launch decision, or {@code null} when no task should be launched now
|
||||
*/
|
||||
K8sSchedulingDecision schedule(List<K8sSchedulingRequest> queue, List<String> nodes)
|
||||
K8sSchedulingDecision schedule(K8sTaskScheduler scheduler, List<K8sSchedulingRequest> queue)
|
||||
|
||||
/**
|
||||
* Decides if the scheduler should immediately invoke the @ref schedule method of the strategy
|
||||
* @param scheduler the calling scheduler object
|
||||
* @param queue pending scheduling requests
|
||||
* @return {@code true} if @ref schedule should immediately be called
|
||||
*/
|
||||
boolean scheduleImmediately(K8sTaskScheduler scheduler, List<K8sSchedulingRequest> queue)
|
||||
|
||||
/**
|
||||
* Called when a task has finished execution.
|
||||
* @param task the task
|
||||
*/
|
||||
void taskFinished(K8sTaskHandler task)
|
||||
}
|
||||
|
||||
@@ -334,6 +334,7 @@ class K8sTaskHandler extends TaskHandler implements FusionAwareTask {
|
||||
void submit() {
|
||||
builder = createBashWrapper(task)
|
||||
builder.build()
|
||||
log.info "[K8s] submitting task ${this.task.name}"
|
||||
executor.taskScheduler.submit(this)
|
||||
}
|
||||
|
||||
@@ -470,6 +471,12 @@ class K8sTaskHandler extends TaskHandler implements FusionAwareTask {
|
||||
deleteJobIfSuccessful(task)
|
||||
updateTimestamps(state.terminated as Map)
|
||||
determineNode()
|
||||
|
||||
// Signal the scheduler that this task has finished running
|
||||
if (executor != null) {
|
||||
executor.taskScheduler.taskFinished(this)
|
||||
executor.runtimeRecorder.record(this)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
|
||||
@@ -1,60 +1,96 @@
|
||||
package nextflow.k8s
|
||||
|
||||
import groovy.transform.CompileDynamic
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.k8s.client.K8sClient
|
||||
import nextflow.util.Duration
|
||||
import java.util.concurrent.LinkedBlockingQueue
|
||||
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class K8sTaskScheduler {
|
||||
private final K8sClient client
|
||||
class K8sTaskScheduler implements Runnable {
|
||||
private final Duration interval
|
||||
private final K8sSchedulingStrategy strategy
|
||||
private final LinkedBlockingQueue<K8sSchedulingRequest> queue = new LinkedBlockingQueue<>()
|
||||
private List<String> cachedNodes
|
||||
private String[] nodes
|
||||
|
||||
K8sTaskScheduler(K8sClient client, K8sSchedulingStrategy strategy) {
|
||||
this.client = client
|
||||
private synchronized boolean shouldStop
|
||||
|
||||
K8sTaskScheduler(String[] nodes, K8sSchedulingStrategy strategy, Duration interval) {
|
||||
this.interval = interval
|
||||
this.strategy = strategy
|
||||
this.nodes = nodes
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a task to the queue of outstanding tasks
|
||||
* @param handler
|
||||
*/
|
||||
void submit(K8sTaskHandler handler) {
|
||||
log.debug "[K8s] received queued task ${handler.task.name}"
|
||||
log.info "[K8s] received queued task ${handler.task.name}"
|
||||
queue.add(new K8sSchedulingRequest(handler))
|
||||
|
||||
drain()
|
||||
final pending = new ArrayList<K8sSchedulingRequest>(queue)
|
||||
if (strategy.scheduleImmediately(this, pending))
|
||||
schedule()
|
||||
}
|
||||
|
||||
protected synchronized void drain() {
|
||||
while( true ) {
|
||||
final pending = new ArrayList<K8sSchedulingRequest>(queue)
|
||||
final decision = strategy.schedule(pending, getNodes())
|
||||
/**
|
||||
* Notify the scheduler that a task has finished execution
|
||||
* @param handler
|
||||
*/
|
||||
void taskFinished(K8sTaskHandler handler) {
|
||||
strategy.taskFinished(handler)
|
||||
|
||||
if ( !decision )
|
||||
return
|
||||
/* If we have pending tasks, now would be a good time to schedule a new one.
|
||||
* Because resources were freed right now */
|
||||
if (queue.size() > 0)
|
||||
schedule()
|
||||
}
|
||||
|
||||
if ( !queue.remove(decision.request) )
|
||||
continue
|
||||
protected synchronized void schedule() {
|
||||
final pending = new ArrayList<K8sSchedulingRequest>(queue)
|
||||
final decision = strategy.schedule(this, pending)
|
||||
|
||||
log.debug "[K8s] launching queued task ${decision.request.task.name} on node: ${decision.nodeName}"
|
||||
decision.request.handler.submitNow(decision.nodeName)
|
||||
if (!decision)
|
||||
return
|
||||
|
||||
if (!queue.remove(decision.request)) {
|
||||
log.warn "[K8s] failed to remove selected task from queue ${decision.request.task.name}"
|
||||
return
|
||||
}
|
||||
|
||||
log.info "[K8s] launching queued task ${decision.request.task.name} on node: ${decision.nodeName}"
|
||||
decision.request.handler.submitNow(decision.nodeName)
|
||||
}
|
||||
|
||||
protected List<String> getNodes() {
|
||||
if ( cachedNodes == null )
|
||||
cachedNodes = fetchNodes()
|
||||
return cachedNodes
|
||||
/* Scheduling Strategy Interface */
|
||||
List<String> getNodes() {
|
||||
return nodes.toList()
|
||||
}
|
||||
|
||||
@CompileDynamic
|
||||
private List<String> fetchNodes() {
|
||||
final resp = client.nodeList()
|
||||
ArrayList<String> nodes = new ArrayList<String>()
|
||||
for ( Map item : resp.items ) {
|
||||
nodes.add(item.metadata.name as String)
|
||||
/**
|
||||
* Run is the scheduler threads main function
|
||||
* */
|
||||
void run() {
|
||||
this.shouldStop = false
|
||||
final interval = this.interval
|
||||
if (interval.toMillis() == 0L) {
|
||||
log.info("[K8s] scheduler loop disabled (interval is 0)")
|
||||
return
|
||||
}
|
||||
return nodes
|
||||
log.info("[K8s] launched scheduler loop (${interval.toString()} interval)")
|
||||
while (!shouldStop) {
|
||||
sleep(interval.toMillis())
|
||||
schedule()
|
||||
}
|
||||
log.info("[K8s] terminated scheduler loop")
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop terminates the scheduler thread
|
||||
*/
|
||||
void stop() {
|
||||
log.info("[K8s] stopping scheduler loop")
|
||||
this.shouldStop = true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -332,6 +332,12 @@ class K8sClient {
|
||||
return new K8sResponseJson(resp.text)
|
||||
}
|
||||
|
||||
String getPodIpAddress(String podName) {
|
||||
assert podName
|
||||
final K8sResponseJson resp = podStatus0(podName)
|
||||
(resp?.status as Map)?.podIP as String
|
||||
}
|
||||
|
||||
/*
|
||||
* https://v1-8.docs.kubernetes.io/docs/api-reference/v1.8/#read-status-69
|
||||
*/
|
||||
@@ -534,6 +540,446 @@ class K8sClient {
|
||||
new K8sResponseJson(resp.text)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a specific node by name
|
||||
* @param name The node name
|
||||
* @return Response object containing the node details.
|
||||
*/
|
||||
K8sResponseJson nodeDescribe(String name) {
|
||||
assert name
|
||||
final action = "/api/v1/nodes/$name"
|
||||
final resp = get(action)
|
||||
trace('GET', action, resp.text)
|
||||
new K8sResponseJson(resp.text)
|
||||
}
|
||||
|
||||
/**
|
||||
* Query the CPU capacity of a node
|
||||
* @param nodeName The name of the node
|
||||
* @return The CPU capacity in cores (as a String representing the quantity, e.g., "4", "2.5")
|
||||
*/
|
||||
String getNodeCpuCapacity(String nodeName) {
|
||||
assert nodeName
|
||||
final resp = nodeDescribe(nodeName)
|
||||
final status = resp.status as Map
|
||||
final capacity = status?.capacity as Map
|
||||
capacity?.cpu as String
|
||||
}
|
||||
|
||||
/**
|
||||
* Query the currently used CPU of a node
|
||||
* Note: This requires the metrics server to be installed in the cluster.
|
||||
* Falls back to the allocated CPU from the node's allocated resources if metrics are not available.
|
||||
* @param nodeName The name of the node
|
||||
* @return The used CPU in cores (as a String representing the quantity, e.g., "1", "0.5")
|
||||
*/
|
||||
String getNodeCpuUsed(String nodeName) {
|
||||
assert nodeName
|
||||
|
||||
// First, try to get metrics from the metrics server
|
||||
try {
|
||||
final action = "/apis/metrics.k8s.io/v1beta1/nodes/$nodeName"
|
||||
final resp = get(action)
|
||||
trace('GET', action, resp.text)
|
||||
final metrics = new K8sResponseJson(resp.text)
|
||||
final usage = metrics.usage as Map
|
||||
return usage?.cpu as String
|
||||
}
|
||||
catch (Exception e) {
|
||||
// Fall back to allocated CPU from node status
|
||||
log.debug("Metrics server not available or error fetching metrics for node $nodeName, falling back to allocated resources: ${e.message}")
|
||||
final nodeResp = nodeDescribe(nodeName)
|
||||
final status = nodeResp.status as Map
|
||||
final allocatable = status?.allocatable as Map
|
||||
final allocated = status?.allocated as Map
|
||||
|
||||
// Calculate used as allocatable minus available (if we had that info)
|
||||
// For now, return the allocated CPU if available
|
||||
if (allocated?.cpu) {
|
||||
return allocated.cpu as String
|
||||
}
|
||||
|
||||
// If we can't get used metrics, return null
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Query the memory capacity of a node
|
||||
* @param nodeName The name of the node
|
||||
* @return The memory capacity in bytes (as a String representing the quantity, e.g., "16Gi", "8192Mi")
|
||||
*/
|
||||
String getNodeMemoryCapacity(String nodeName) {
|
||||
assert nodeName
|
||||
final resp = nodeDescribe(nodeName)
|
||||
final status = resp.status as Map
|
||||
final capacity = status?.capacity as Map
|
||||
capacity?.memory as String
|
||||
}
|
||||
|
||||
/**
|
||||
* Query the currently used memory of a node
|
||||
* Note: This requires the metrics server to be installed in the cluster.
|
||||
* Falls back to the allocated memory from the node's allocated resources if metrics are not available.
|
||||
* @param nodeName The name of the node
|
||||
* @return The used memory in bytes (as a String representing the quantity, e.g., "4Gi", "2048Mi")
|
||||
*/
|
||||
String getNodeMemoryUsed(String nodeName) {
|
||||
assert nodeName
|
||||
|
||||
// First, try to get metrics from the metrics server
|
||||
try {
|
||||
final action = "/apis/metrics.k8s.io/v1beta1/nodes/$nodeName"
|
||||
final resp = get(action)
|
||||
trace('GET', action, resp.text)
|
||||
final metrics = new K8sResponseJson(resp.text)
|
||||
final usage = metrics.usage as Map
|
||||
return usage?.memory as String
|
||||
}
|
||||
catch (Exception e) {
|
||||
// Fall back to allocated memory from node status
|
||||
log.debug("Metrics server not available or error fetching metrics for node $nodeName, falling back to allocated resources: ${e.message}")
|
||||
final nodeResp = nodeDescribe(nodeName)
|
||||
final status = nodeResp.status as Map
|
||||
final allocated = status?.allocated as Map
|
||||
|
||||
// Return the allocated memory if available
|
||||
if (allocated?.memory) {
|
||||
return allocated.memory as String
|
||||
}
|
||||
|
||||
// If we can't get used metrics, return null
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a Kubernetes quantity string (e.g., "16Gi", "8192Mi", "1024Ki", "1500m") into a long value.
|
||||
* For memory: supports binary (Ki, Mi, Gi, Ti, Pi, Ei) and the value is returned in bytes.
|
||||
* For CPU: supports millicores (e.g., "1500m" = 1.5 cores = 1500 millicores) and cores (e.g., "2" = 2000 millicores).
|
||||
*
|
||||
* @param quantity The Kubernetes quantity string (e.g., "16Gi", "1500m", "2")
|
||||
* @param forCpu If true, parses as CPU (millicores); if false, parses as memory (bytes)
|
||||
* @return The value as a Long, or null if unparseable
|
||||
*/
|
||||
static Long parseK8sQuantity(String quantity, boolean forCpu=false) {
|
||||
if (!quantity) return null
|
||||
|
||||
// Reject quantities with spaces - Kubernetes quantity strings don't have spaces
|
||||
if (quantity.contains(' ')) {
|
||||
return null
|
||||
}
|
||||
|
||||
// Extract numeric value and suffix separately
|
||||
// Kubernetes quantities: https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/#quantity
|
||||
def suffix = null
|
||||
def numericStr = quantity
|
||||
|
||||
// Try to extract suffix - check for the longest suffixes first
|
||||
// Binary: KiB, MiB, GiB, TiB, PiB, EiB, Ki, Mi, Gi, Ti, Pi, Ei
|
||||
// Decimal: KB, MB, GB, TB, PB, EB, K, M, G, T, P, E
|
||||
// CPU: m (millicores), n (nanocores)
|
||||
def suffixes = ['kib', 'mib', 'gib', 'tib', 'pib', 'eib',
|
||||
'ki', 'mi', 'gi', 'ti', 'pi', 'ei',
|
||||
'kb', 'mb', 'gb', 'tb', 'pb', 'eb',
|
||||
'k', 'm', 'g', 't', 'p', 'e']
|
||||
|
||||
// For CPU, we only care about 'm' (millicores), 'u' (microcores) and 'n' (nanocores) suffixes
|
||||
if (forCpu) {
|
||||
suffixes = ['m', 'u', 'n']
|
||||
}
|
||||
|
||||
for (def s : suffixes) {
|
||||
if (quantity.toLowerCase().endsWith(s)) {
|
||||
suffix = s
|
||||
numericStr = quantity.substring(0, quantity.length() - s.length())
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// If we couldn't extract a suffix, try to parse the whole thing as a number
|
||||
// Validate that numericStr only contains valid numeric characters
|
||||
if (numericStr && !numericStr.matches(/^[+-]?[0-9]*(\.[0-9]*)?$/)) {
|
||||
return null
|
||||
}
|
||||
|
||||
def numericValue
|
||||
try {
|
||||
numericValue = numericStr.toDouble()
|
||||
} catch (Exception e) {
|
||||
return null
|
||||
}
|
||||
|
||||
if (forCpu) {
|
||||
// CPU: handle millicores (m), microcores (u) and nanocores (n) suffixes
|
||||
if (suffix == 'm') {
|
||||
// Already in millicores
|
||||
return (long)(numericValue)
|
||||
} else if (suffix == 'n') {
|
||||
// Nanocores: convert to millicores (1 millicore = 1,000,000 nanocores)
|
||||
return (long)(numericValue / 1000000.0)
|
||||
} else if (suffix == 'u') {
|
||||
// Microcores: convert to millicores (1 millicore = 1,000 microcores)
|
||||
return (long)(numericValue / 1000)
|
||||
} else {
|
||||
// Default is cores, convert to millicores
|
||||
return (long)(numericValue * 1000)
|
||||
}
|
||||
} else {
|
||||
// Memory: convert to bytes
|
||||
def multiplier = 1L
|
||||
|
||||
if (suffix == null || suffix == '') {
|
||||
// No suffix - assume bytes
|
||||
return (long)numericValue
|
||||
}
|
||||
|
||||
// Binary suffixes (Ki, Mi, Gi, Ti, Pi, Ei)
|
||||
switch (suffix) {
|
||||
case 'ki': case 'kib':
|
||||
multiplier = 1024L
|
||||
break
|
||||
case 'mi': case 'mib':
|
||||
multiplier = 1024L * 1024L
|
||||
break
|
||||
case 'gi': case 'gib':
|
||||
multiplier = 1024L * 1024L * 1024L
|
||||
break
|
||||
case 'ti': case 'tib':
|
||||
multiplier = 1024L * 1024L * 1024L * 1024L
|
||||
break
|
||||
case 'pi': case 'pib':
|
||||
multiplier = 1024L * 1024L * 1024L * 1024L * 1024L
|
||||
break
|
||||
case 'ei': case 'eib':
|
||||
multiplier = 1024L * 1024L * 1024L * 1024L * 1024L * 1024L
|
||||
break
|
||||
// Decimal suffixes (K, M, G, T, P, E) - Kubernetes uses binary by default
|
||||
// but we support decimal for completeness
|
||||
case 'k': case 'kb':
|
||||
multiplier = 1000L
|
||||
break
|
||||
case 'm': case 'mb':
|
||||
multiplier = 1000L * 1000L
|
||||
break
|
||||
case 'g': case 'gb':
|
||||
multiplier = 1000L * 1000L * 1000L
|
||||
break
|
||||
case 't': case 'tb':
|
||||
multiplier = 1000L * 1000L * 1000L * 1000L
|
||||
break
|
||||
case 'p': case 'pb':
|
||||
multiplier = 1000L * 1000L * 1000L * 1000L * 1000L
|
||||
break
|
||||
case 'e': case 'eb':
|
||||
multiplier = 1000L * 1000L * 1000L * 1000L * 1000L * 1000L
|
||||
break
|
||||
}
|
||||
|
||||
return (long)(numericValue * multiplier)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Query the CPU capacity of a node in millicores (1 core = 1000 millicores)
|
||||
* @param nodeName The name of the node
|
||||
* @return The CPU capacity in millicores as a Long, or null if unparseable
|
||||
*/
|
||||
Long getNodeCpuCapacityMillis(String nodeName) {
|
||||
assert nodeName
|
||||
final capacity = getNodeCpuCapacity(nodeName)
|
||||
if (!capacity) {
|
||||
log.debug("Node $nodeName CPU capacity not available")
|
||||
return null
|
||||
}
|
||||
final result = parseK8sQuantity(capacity, true)
|
||||
if (result == null) {
|
||||
log.warn("Unable to parse CPU capacity for node $nodeName: '$capacity'")
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Query the currently used CPU of a node in millicores (1 core = 1000 millicores)
|
||||
* @param nodeName The name of the node
|
||||
* @return The used CPU in millicores as a Long, or null if unparseable
|
||||
*/
|
||||
Long getNodeCpuUsedMillis(String nodeName) {
|
||||
assert nodeName
|
||||
final used = getNodeCpuUsed(nodeName)
|
||||
if (!used) {
|
||||
log.debug("Node $nodeName CPU usage not available (metrics server not installed or node has no allocated resources)")
|
||||
return null
|
||||
}
|
||||
final result = parseK8sQuantity(used, true)
|
||||
if (result == null) {
|
||||
log.warn("Unable to parse CPU usage for node $nodeName: '$used'")
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Query the memory capacity of a node in bytes
|
||||
* @param nodeName The name of the node
|
||||
* @return The memory capacity in bytes as a Long, or null if unparseable
|
||||
*/
|
||||
Long getNodeMemoryCapacityBytes(String nodeName) {
|
||||
assert nodeName
|
||||
final capacity = getNodeMemoryCapacity(nodeName)
|
||||
if (!capacity) {
|
||||
log.debug("Node $nodeName memory capacity not available")
|
||||
return null
|
||||
}
|
||||
final result = parseK8sQuantity(capacity, false)
|
||||
if (result == null) {
|
||||
log.warn("Unable to parse memory capacity for node $nodeName: '$capacity'")
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Query the currently used memory of a node in bytes
|
||||
* @param nodeName The name of the node
|
||||
* @return The used memory in bytes as a Long, or null if unparseable
|
||||
*/
|
||||
Long getNodeMemoryUsedBytes(String nodeName) {
|
||||
assert nodeName
|
||||
final used = getNodeMemoryUsed(nodeName)
|
||||
if (!used) {
|
||||
log.debug("Node $nodeName memory usage not available (metrics server not installed or node has no allocated resources)")
|
||||
return null
|
||||
}
|
||||
final result = parseK8sQuantity(used, false)
|
||||
if (result == null) {
|
||||
log.warn("Unable to parse memory usage for node $nodeName: '$used'")
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Query the total CPU allocated to pods on a node (sum of pod requests)
|
||||
* @param nodeName The name of the node
|
||||
* @return The allocated CPU in millicores as a Long, or null if unparseable
|
||||
*/
|
||||
Long getNodeCpuAllocatedMillis(String nodeName) {
|
||||
assert nodeName
|
||||
try {
|
||||
// List all pods on the node and sum their CPU requests
|
||||
final action = "/api/v1/pods?fieldSelector=spec.nodeName=$nodeName"
|
||||
final resp = get(action)
|
||||
trace('GET', action, resp.text)
|
||||
final podList = new K8sResponseJson(resp.text)
|
||||
final items = podList.items as List<Map>
|
||||
|
||||
long totalAllocated = 0L
|
||||
if (items) {
|
||||
for (Map pod : items) {
|
||||
final spec = pod.spec as Map
|
||||
if (spec) {
|
||||
final containers = spec.containers as List<Map>
|
||||
if (containers) {
|
||||
for (Map container : containers) {
|
||||
final resources = container.resources as Map
|
||||
final requests = resources?.requests as Map
|
||||
if (requests?.cpu) {
|
||||
final cpuStr = requests.cpu as String
|
||||
final cpuMillis = parseK8sQuantity(cpuStr, true)
|
||||
if (cpuMillis != null) {
|
||||
totalAllocated += cpuMillis
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return totalAllocated
|
||||
}
|
||||
catch (Exception e) {
|
||||
log.warn("Failed to get allocated CPU for node $nodeName: ${e.message}")
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Query the available CPU on a node (allocatable - allocated)
|
||||
* For scheduling purposes, we need allocated (requests), not usage (from metrics server)
|
||||
* @param nodeName The name of the node
|
||||
* @return The available CPU in millicores as a Long, or null if unparseable
|
||||
*/
|
||||
Long getNodeCpuAvailableMillis(String nodeName) {
|
||||
assert nodeName
|
||||
final capacity = getNodeCpuCapacityMillis(nodeName)
|
||||
final allocated = getNodeCpuAllocatedMillis(nodeName)
|
||||
|
||||
if (capacity == null || allocated == null) {
|
||||
return null
|
||||
}
|
||||
return capacity - allocated
|
||||
}
|
||||
|
||||
/**
|
||||
* Query the total memory allocated to pods on a node (sum of pod requests)
|
||||
* @param nodeName The name of the node
|
||||
* @return The allocated memory in bytes as a Long, or null if unparseable
|
||||
*/
|
||||
Long getNodeMemoryAllocatedBytes(String nodeName) {
|
||||
assert nodeName
|
||||
try {
|
||||
// List all pods on the node and sum their memory requests
|
||||
final action = "/api/v1/pods?fieldSelector=spec.nodeName=$nodeName"
|
||||
final resp = get(action)
|
||||
trace('GET', action, resp.text)
|
||||
final podList = new K8sResponseJson(resp.text)
|
||||
final items = podList.items as List<Map>
|
||||
|
||||
long totalAllocated = 0L
|
||||
if (items) {
|
||||
for (Map pod : items) {
|
||||
final spec = pod.spec as Map
|
||||
if (spec) {
|
||||
final containers = spec.containers as List<Map>
|
||||
if (containers) {
|
||||
for (Map container : containers) {
|
||||
final resources = container.resources as Map
|
||||
final requests = resources?.requests as Map
|
||||
if (requests?.memory) {
|
||||
final memStr = requests.memory as String
|
||||
final memBytes = parseK8sQuantity(memStr, false)
|
||||
if (memBytes != null) {
|
||||
totalAllocated += memBytes
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return totalAllocated
|
||||
}
|
||||
catch (Exception e) {
|
||||
log.warn("Failed to get allocated memory for node $nodeName: ${e.message}")
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Query the available memory on a node (allocatable - allocated)
|
||||
* For scheduling purposes, we need allocated (requests), not usage (from metrics server)
|
||||
* @param nodeName The name of the node
|
||||
* @return The available memory in bytes as a Long, or null if unparseable
|
||||
*/
|
||||
Long getNodeMemoryAvailableBytes(String nodeName) {
|
||||
assert nodeName
|
||||
final capacity = getNodeMemoryCapacityBytes(nodeName)
|
||||
final allocated = getNodeMemoryAllocatedBytes(nodeName)
|
||||
|
||||
if (capacity == null || allocated == null) {
|
||||
return null
|
||||
}
|
||||
return capacity - allocated
|
||||
}
|
||||
|
||||
protected void checkInvalidWaitingState( Map waiting, K8sResponseJson resp ) {
|
||||
if( waiting.reason == 'ErrImagePull' || waiting.reason == 'ImagePullBackOff') {
|
||||
def message = "K8s pod image cannot be pulled"
|
||||
|
||||
@@ -128,6 +128,8 @@ class PodSpecBuilder {
|
||||
|
||||
String nodeName
|
||||
|
||||
Integer port = null
|
||||
|
||||
/**
|
||||
* @return A sequential volume unique identifier
|
||||
*/
|
||||
@@ -406,6 +408,11 @@ class PodSpecBuilder {
|
||||
return this
|
||||
}
|
||||
|
||||
PodSpecBuilder withPort(int value) {
|
||||
this.port = new Integer(value)
|
||||
return this
|
||||
}
|
||||
|
||||
@PackageScope List<Map> createPullSecret() {
|
||||
def result = new ArrayList(1)
|
||||
def entry = new LinkedHashMap(1)
|
||||
@@ -443,6 +450,12 @@ class PodSpecBuilder {
|
||||
if( imagePullPolicy )
|
||||
container.imagePullPolicy = imagePullPolicy
|
||||
|
||||
if( this.port ) {
|
||||
List<Map> ports = []
|
||||
ports << [containerPort: this.port.intValue()]
|
||||
container.ports = ports
|
||||
}
|
||||
|
||||
final secContext = new LinkedHashMap(10)
|
||||
if( privileged ) {
|
||||
// note: privileged flag needs to be defined in the *container* securityContext
|
||||
|
||||
@@ -0,0 +1,384 @@
|
||||
package nextflow.k8s.strategies
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.k8s.K8sDVFSClient
|
||||
import nextflow.k8s.K8sRuntimeEstimator
|
||||
import nextflow.k8s.K8sSchedulingDecision
|
||||
import nextflow.k8s.K8sSchedulingRequest
|
||||
import nextflow.k8s.K8sSchedulingStrategy
|
||||
import nextflow.k8s.K8sTaskHandler
|
||||
import nextflow.k8s.K8sTaskScheduler
|
||||
import nextflow.k8s.client.K8sClient
|
||||
import nextflow.processor.TaskRun
|
||||
import nextflow.util.Duration
|
||||
|
||||
/**
|
||||
* Implements a scheduling strategy utilizing dvfs to reduce the energy consumption
|
||||
* of workflow execution, while attempting to maintain the same makespan.
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class K8sDVFSSchedulingStrategy implements K8sSchedulingStrategy {
|
||||
|
||||
/** Used for passing K8sExecutor.getClient.
|
||||
* We cannot pass the client directly, becaue it can be refreshed
|
||||
* during workflow execution.
|
||||
*/
|
||||
public interface K8sClientGetter {
|
||||
K8sClient getClient()
|
||||
}
|
||||
|
||||
private static long getTaskMemoryRequirment(TaskRun task) {
|
||||
return task.config.getMemory() ? task.config.getMemory().bytes : 64 * 1024 * 1024
|
||||
}
|
||||
|
||||
private static int getTaskCPURequirement(TaskRun task) {
|
||||
return task.config.hasCpus() ? task.config.getCpus() * 1000 : 1000
|
||||
}
|
||||
|
||||
@Slf4j
|
||||
private class WorkerNode {
|
||||
private class AssignedTask {
|
||||
TaskRun task
|
||||
long frequency
|
||||
|
||||
AssignedTask(TaskRun t, long f) {
|
||||
this.task = t
|
||||
this.frequency = f
|
||||
}
|
||||
}
|
||||
|
||||
String name
|
||||
|
||||
long maxFrequency
|
||||
long minFrequency
|
||||
long currentFrequency
|
||||
|
||||
ArrayList<AssignedTask> tasks
|
||||
|
||||
WorkerNode(String name, long maxF, long minF, long curF, K8sClientGetter clientGetter) {
|
||||
this.name = name
|
||||
this.maxFrequency = maxF
|
||||
this.minFrequency = minF
|
||||
this.currentFrequency = curF
|
||||
this.tasks = new ArrayList<>()
|
||||
}
|
||||
|
||||
/* Return the number of available (unoccupied) bytes */
|
||||
long getAvailableMemory() {
|
||||
def k8sClient = clientGetter.getClient()
|
||||
Long available = k8sClient.getNodeMemoryAvailableBytes(this.name)
|
||||
if (available == null) {
|
||||
log.warn "[K8s] failed to retrieve available memory for node ${name}"
|
||||
// Fallback: try capacity - allocated
|
||||
Long capacity = k8sClient.getNodeMemoryCapacityBytes(this.name)
|
||||
Long allocated = k8sClient.getNodeMemoryAllocatedBytes(this.name)
|
||||
if (capacity != null && allocated != null) {
|
||||
return capacity - allocated
|
||||
}
|
||||
return 0
|
||||
}
|
||||
return available
|
||||
}
|
||||
|
||||
/* Return the number of available (unoccupied) cpu cores */
|
||||
long getAvailableCPUs() {
|
||||
def k8sClient = clientGetter.getClient()
|
||||
Long available = k8sClient.getNodeCpuAvailableMillis(this.name)
|
||||
if (available == null) {
|
||||
log.warn "[K8s] failed to retrieve available CPU for node ${name}"
|
||||
// Fallback: try capacity - allocated
|
||||
Long capacity = k8sClient.getNodeCpuCapacityMillis(this.name)
|
||||
Long allocated = k8sClient.getNodeCpuAllocatedMillis(this.name)
|
||||
if (capacity != null && allocated != null) {
|
||||
return capacity - allocated
|
||||
}
|
||||
return 0
|
||||
}
|
||||
return available
|
||||
}
|
||||
|
||||
/* Return the total amount of installed memory */
|
||||
long getMemoryAmount() {
|
||||
def k8sClient = clientGetter.getClient()
|
||||
Long capacity = k8sClient.getNodeMemoryCapacityBytes(this.name)
|
||||
return capacity != null ? capacity.longValue() : 0
|
||||
}
|
||||
|
||||
/* Return the total number of installed cpu cores */
|
||||
long getCPUCount() {
|
||||
def k8sClient = clientGetter.getClient()
|
||||
Long capacity = k8sClient.getNodeCpuCapacityMillis(this.name)
|
||||
return capacity != null ? capacity.longValue(): 0
|
||||
}
|
||||
|
||||
// Sets the frequency to the max. requested frequency of all currently running tasks.
|
||||
private void updateFrequency(K8sDVFSClient dvfsClient) {
|
||||
if (tasks.size() == 0)
|
||||
return
|
||||
long max = Long.MIN_VALUE
|
||||
for (AssignedTask t : tasks) {
|
||||
max = Long.max(t.frequency, max)
|
||||
}
|
||||
log.info "[K8s] node ${name} running at ${max} Hz / ${maxFrequency} Hz ${((double)max/(double)maxFrequency) * 100.0}%"
|
||||
dvfsClient.setNodeFrequency(name, (int)max)
|
||||
this.currentFrequency = max
|
||||
}
|
||||
|
||||
void assignTask(TaskRun task, long frequency, K8sDVFSClient dvfsClient) {
|
||||
log.info "[K8s] node ${name}: task ${task.name} assigned with ${frequency}/${maxFrequency}"
|
||||
this.tasks.add(new AssignedTask(task, frequency))
|
||||
updateFrequency(dvfsClient)
|
||||
}
|
||||
|
||||
void taskFinished(TaskRun task, K8sDVFSClient dvfsClient) {
|
||||
log.info "[K8s] node ${name}: task ${task.name} finished"
|
||||
this.tasks.removeIf {it.task == task}
|
||||
updateFrequency(dvfsClient)
|
||||
}
|
||||
}
|
||||
|
||||
class SchedulingRequestComparator implements Comparator<K8sSchedulingRequest> {
|
||||
K8sRuntimeEstimator runtimeEstimator
|
||||
long currentTime
|
||||
double epsilon
|
||||
|
||||
@Override
|
||||
int compare(K8sSchedulingRequest o1, K8sSchedulingRequest o2) {
|
||||
// First, check if one of the tasks is (estimated to be) on the critical path
|
||||
double t1 = runtimeEstimator.estimate(o1.handler)
|
||||
double t2 = runtimeEstimator.estimate(o2.handler)
|
||||
|
||||
if (t1 > t2 + epsilon)
|
||||
return -1
|
||||
else if (t2 > t1 + epsilon)
|
||||
return 1
|
||||
|
||||
// Both are not on the critical path. Sort based on the time they spent in the queue
|
||||
long w1 = currentTime - o1.submitTimeMillis
|
||||
long w2 = currentTime - o2.submitTimeMillis
|
||||
if (w1 > w2)
|
||||
return -1
|
||||
else if (w2 > w1)
|
||||
return 1
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
private K8sRuntimeEstimator runtimeEstimator
|
||||
private K8sDVFSClient dvfsClient
|
||||
|
||||
private ArrayList<WorkerNode> nodes
|
||||
private HashMap<String, WorkerNode> taskToNode
|
||||
private long globalMaxFrequency
|
||||
private long globalMinFrequency
|
||||
|
||||
private K8sClientGetter clientGetter
|
||||
|
||||
private double comparisonEpsilonMillis
|
||||
|
||||
private double[] topRuntimes
|
||||
private double averageRuntime
|
||||
private double finishedTaskCount
|
||||
|
||||
boolean fullSpeedMode
|
||||
|
||||
K8sDVFSSchedulingStrategy(K8sRuntimeEstimator runtimeEstimator,
|
||||
K8sDVFSClient dvfsClient,
|
||||
K8sClientGetter clientGetter,
|
||||
Duration runtimeComparisonEpsilon,
|
||||
int topRuntimeCount) {
|
||||
this.runtimeEstimator = runtimeEstimator
|
||||
this.dvfsClient = dvfsClient
|
||||
this.nodes = new ArrayList<>()
|
||||
this.taskToNode = new HashMap<>();
|
||||
this.clientGetter = clientGetter
|
||||
this.comparisonEpsilonMillis = (double)runtimeComparisonEpsilon.toMillis()
|
||||
this.topRuntimes = new double[topRuntimeCount]
|
||||
for (int i = 0; i < topRuntimeCount; i++) {
|
||||
this.topRuntimes[i] = 0.0
|
||||
}
|
||||
this.averageRuntime = 0.0
|
||||
this.finishedTaskCount = 0.0
|
||||
}
|
||||
|
||||
private boolean isInTopRuntimes(double rt) {
|
||||
for (int i = 0; i < topRuntimes.size(); i++) {
|
||||
if (rt >= topRuntimes[i])
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
private void updateTopRuntimes(double rt) {
|
||||
for (int i = 0; i < topRuntimes.size(); i++) {
|
||||
if (rt > topRuntimes[i]) {
|
||||
/* Move all one down */
|
||||
for (int j = topRuntimes.size() - 1; j > i; j--) {
|
||||
topRuntimes[j] = topRuntimes[j - 1];
|
||||
}
|
||||
topRuntimes[i] = rt
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
K8sSchedulingDecision schedule(K8sTaskScheduler scheduler, List<K8sSchedulingRequest> queue) {
|
||||
if (nodes.isEmpty()) {
|
||||
if (!initNodes(scheduler))
|
||||
return null
|
||||
}
|
||||
|
||||
/* Step 1: Sort by task priority. We will attempt to schedule tasks "in order", so that the
|
||||
* highest priority tasks are assigned to nodes as soon as possible.
|
||||
*
|
||||
* Priority is based on a) the tasks estimated runtime and b) the wait time of the task.
|
||||
*/
|
||||
SchedulingRequestComparator comparator = new SchedulingRequestComparator()
|
||||
comparator.runtimeEstimator = runtimeEstimator
|
||||
comparator.epsilon = comparisonEpsilonMillis
|
||||
comparator.currentTime = System.currentTimeMillis()
|
||||
queue.sort(comparator)
|
||||
|
||||
/* Step 2: For each task attempt to schedule it onto a node */
|
||||
for (K8sSchedulingRequest req : queue) {
|
||||
/* Step 2.1: Determine if the task is on the critical path.
|
||||
* If yes, we just schedule it at max frequency on the node with the highest available
|
||||
* frequency. If not, we determine a frequency (see below).
|
||||
*/
|
||||
final double taskEstimation = runtimeEstimator.estimate(req.handler)
|
||||
final boolean isCriticalPath = isInTopRuntimes(taskEstimation)
|
||||
long frequency = globalMaxFrequency
|
||||
if (!isCriticalPath && !fullSpeedMode) {
|
||||
/* Set frequency so that we expect the runtime to be close to the mean runtime. */
|
||||
frequency = (long)Math.floor((taskEstimation * globalMaxFrequency) / averageRuntime)
|
||||
frequency = Math.max(frequency, globalMinFrequency)
|
||||
frequency = Math.min(frequency, globalMaxFrequency)
|
||||
}
|
||||
|
||||
/* Step 2.2: Filter nodes based on task requirements */
|
||||
ArrayList<WorkerNode> suitableNodes = filterNodes(req.task)
|
||||
if (suitableNodes.size() == 0) {
|
||||
if (!anyNode(req.task)) {
|
||||
log.error "[K8s] unable to schedule task ${req.task} - no node satisfies resource requirements ${getTaskMemoryRequirment(req.task)} bytes ${getTaskCPURequirement(req.task)} cpus"
|
||||
return null
|
||||
}
|
||||
/* No node can currently execute this task, but it should be possible in the future */
|
||||
/* log.info "[K8s] ${req.task} can not be scheduled: ${getTaskMemoryRequirment(req.task)} bytes ${getTaskCPURequirement(req.task)} CPUs"
|
||||
for (WorkerNode n : this.nodes) {
|
||||
log.info "[K8s] node ${n.name} - ${n.availableMemory}, ${n.availableCPUs}"
|
||||
} */
|
||||
continue
|
||||
}
|
||||
|
||||
/* Step 2.3: Assign to node based on "best fit" - current node frequency is closest to determined frequency */
|
||||
long minDist = Math.abs(suitableNodes[0].currentFrequency - frequency)
|
||||
WorkerNode closest = suitableNodes[0]
|
||||
for (WorkerNode node : suitableNodes) {
|
||||
long dist = Math.abs(node.currentFrequency - frequency)
|
||||
if (dist < minDist) {
|
||||
closest = node
|
||||
minDist = dist
|
||||
}
|
||||
}
|
||||
|
||||
closest.assignTask(req.task, frequency, dvfsClient)
|
||||
taskToNode.put(req.task.hash.toString(), closest)
|
||||
log.info "[K8s] DVFS: Assigned task ${req.task} to node ${closest.name} - ${taskToNode.size()} assigned tasks"
|
||||
return new K8sSchedulingDecision(req, closest.name)
|
||||
}
|
||||
|
||||
log.info "[K8s] unable to schedule any task. The queue contains ${queue.size()} tasks."
|
||||
return null
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean scheduleImmediately(K8sTaskScheduler scheduler, List<K8sSchedulingRequest> queue) {
|
||||
if (nodes.isEmpty()) {
|
||||
if (!initNodes(scheduler))
|
||||
return false
|
||||
}
|
||||
/* We want to schedule immediately if there are unoccupied nodes */
|
||||
boolean doIt = queue != null && queue.size() > 0 && taskToNode.size() < nodes.size()
|
||||
log.info "[K8s] scheduleImmediately: ${queue.size()} tasks in queue, ${taskToNode.size()} tasks running on ${nodes.size()} nodes: ${doIt}"
|
||||
return doIt
|
||||
}
|
||||
|
||||
@Override
|
||||
synchronized void taskFinished(K8sTaskHandler task) {
|
||||
/* TODO: This just uses elapsed wall-clock time, regardless of the frequency used to execute the task.
|
||||
* This will skew the average towards longer runtimes, which is undesirable, because it will lead to more
|
||||
* tasks classified as "critical path".
|
||||
* A simple (rough) solution could be to keep track of the tasks "relative" frequency and just scale the
|
||||
* elapsed time based on that.
|
||||
*/
|
||||
double runtime = (double)(task.getCompleteTimeMillis() - task.getStartTimeMillis())
|
||||
averageRuntime = (runtime + finishedTaskCount * averageRuntime) / (finishedTaskCount + 1.0)
|
||||
finishedTaskCount += 1.0
|
||||
updateTopRuntimes(runtime)
|
||||
|
||||
/* Free resources allocated by this task */
|
||||
WorkerNode node = taskToNode.get(task.task.hash.toString())
|
||||
if (node != null) {
|
||||
node.taskFinished(task.task, dvfsClient)
|
||||
taskToNode.remove(task.task.hash.toString())
|
||||
} else {
|
||||
log.warn "[K8s] no node recorded for task ${task.toString()}"
|
||||
}
|
||||
|
||||
log.info "[K8s] task ${task.toString()} finished - ${taskToNode.size()} tasks running"
|
||||
if (node != null) {
|
||||
log.info "[K8s] task ran on node ${node.name} - ${node.availableMemory} bytes ${node.availableCPUs}"
|
||||
}
|
||||
}
|
||||
|
||||
private synchronized boolean initNodes(K8sTaskScheduler scheduler) {
|
||||
this.globalMaxFrequency = Long.MAX_VALUE
|
||||
final nodes = scheduler.getNodes()
|
||||
for (String node : nodes) {
|
||||
final cur = dvfsClient.getNodeCurrentFrequency(node)
|
||||
final min = dvfsClient.getNodeMinFrequency(node)
|
||||
final max = dvfsClient.getNodeMaxFrequency(node)
|
||||
final cpus = dvfsClient.getCPUCount(node)
|
||||
final mem = dvfsClient.getMemoryAmount(node)
|
||||
|
||||
if (cur.empty || min.empty || max.empty || cpus.empty || mem.empty) {
|
||||
log.error "[K8s] failed to query node $node information"
|
||||
continue
|
||||
}
|
||||
globalMaxFrequency = Long.min(globalMaxFrequency, max.asLong)
|
||||
globalMinFrequency = Long.max(globalMinFrequency, min.asLong)
|
||||
|
||||
log.info "[K8s] node ${node}: ${cpus.asLong} CPUs, ${mem.asLong} bytes RAM ${min.asLong} Hz - ${max.asLong} Hz current ${cur.asLong}"
|
||||
this.nodes.add(new WorkerNode(node, max.asLong, min.asLong, cur.asLong, clientGetter))
|
||||
}
|
||||
return !this.nodes.isEmpty()
|
||||
}
|
||||
|
||||
/* Returns a list of nodes that fulfill the tasks resource requirements
|
||||
*/
|
||||
private ArrayList<WorkerNode> filterNodes(TaskRun task) {
|
||||
final long reqBytes = getTaskMemoryRequirment(task)
|
||||
final int reqCPUs = getTaskCPURequirement(task)
|
||||
ArrayList<WorkerNode> suitableNodes = new ArrayList<>()
|
||||
for (WorkerNode n : nodes) {
|
||||
if (n.availableMemory >= reqBytes && n.availableCPUs >= reqCPUs) {
|
||||
log.info "[K8s] task ${task.name}: ${reqBytes} bytes ${reqCPUs} cpus: node ${n.name} has ${n.availableMemory} bytes, ${n.availableCPUs} cpus"
|
||||
suitableNodes.add(n)
|
||||
}
|
||||
}
|
||||
return suitableNodes
|
||||
}
|
||||
|
||||
private boolean anyNode(TaskRun task) {
|
||||
final long reqBytes = getTaskMemoryRequirment(task)
|
||||
final int reqCPUs = getTaskCPURequirement(task)
|
||||
for (WorkerNode n : nodes) {
|
||||
if (n.memoryAmount >= reqBytes && n.CPUCount >= reqCPUs)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -4,16 +4,31 @@ import groovy.transform.CompileStatic
|
||||
import nextflow.k8s.K8sSchedulingDecision
|
||||
import nextflow.k8s.K8sSchedulingRequest
|
||||
import nextflow.k8s.K8sSchedulingStrategy
|
||||
import nextflow.k8s.K8sTaskHandler
|
||||
import nextflow.k8s.K8sTaskScheduler
|
||||
|
||||
@CompileStatic
|
||||
class K8sHashSchedulingStrategy implements K8sSchedulingStrategy {
|
||||
|
||||
@Override
|
||||
K8sSchedulingDecision schedule(List<K8sSchedulingRequest> queue, List<String> nodes) {
|
||||
if ( !queue || !nodes )
|
||||
K8sSchedulingDecision schedule(K8sTaskScheduler scheduler, List<K8sSchedulingRequest> queue) {
|
||||
if (!queue)
|
||||
return null
|
||||
|
||||
final freeNodes = scheduler.nodes
|
||||
if (freeNodes.size() == 0)
|
||||
return null
|
||||
|
||||
final request = queue[0]
|
||||
final index = Math.floorMod(request.task.hash.asInt(), nodes.size())
|
||||
return new K8sSchedulingDecision(request, nodes[index])
|
||||
final index = Math.floorMod(request.task.hash.asInt(), freeNodes.size())
|
||||
return new K8sSchedulingDecision(request, freeNodes[index])
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean scheduleImmediately(K8sTaskScheduler scheduler, List<K8sSchedulingRequest> queue) {
|
||||
return false
|
||||
}
|
||||
|
||||
@Override
|
||||
void taskFinished(K8sTaskHandler task) { /* nop */ }
|
||||
}
|
||||
|
||||
@@ -644,7 +644,7 @@ class K8sDriverLauncherTest extends Specification {
|
||||
def driver = Spy(K8sDriverLauncher)
|
||||
driver.@k8sConfig = config
|
||||
driver.@runName = POD_NAME
|
||||
driver.@initDeployer = new K8sNodeInitDeployer(driver.k8sClient, config, driver.runName)
|
||||
driver.@initDeployer = new K8sNodeInitDeployer(driver.k8sClient, config)
|
||||
|
||||
when:
|
||||
driver.shutdown()
|
||||
|
||||
@@ -14,7 +14,7 @@ class K8sNodeInitDeployerTest extends Specification {
|
||||
given:
|
||||
def client = Mock(K8sClient)
|
||||
def config = new K8sConfig(nodeInit: [enabled: false])
|
||||
def deployer = new K8sNodeInitDeployer(client, config, 'run-foo')
|
||||
def deployer = new K8sNodeInitDeployer(client, config)
|
||||
|
||||
when:
|
||||
deployer.deploy()
|
||||
@@ -32,7 +32,7 @@ class K8sNodeInitDeployerTest extends Specification {
|
||||
image: 'ubuntu:latest',
|
||||
command: ['/bin/bash', '-c', 'echo init']
|
||||
])
|
||||
def deployer = new K8sNodeInitDeployer(client, config, 'run-foo')
|
||||
def deployer = new K8sNodeInitDeployer(client, config)
|
||||
|
||||
when:
|
||||
deployer.deploy()
|
||||
@@ -48,13 +48,13 @@ class K8sNodeInitDeployerTest extends Specification {
|
||||
then:
|
||||
1 * client.podCreate({ Map spec ->
|
||||
spec.kind == 'Pod'
|
||||
spec.metadata.name == 'nf-init-run-foo-node-a'
|
||||
spec.metadata.name == 'nf-init-node-a'
|
||||
spec.metadata.namespace == 'default'
|
||||
spec.spec.nodeName == 'node-a'
|
||||
spec.spec.restartPolicy == 'Never'
|
||||
|
||||
def container = spec.spec.containers[0]
|
||||
container.name == 'nf-init-run-foo-node-a'
|
||||
container.name == 'nf-init-node-a'
|
||||
container.image == 'ubuntu:latest'
|
||||
container.command == ['/bin/bash', '-c', 'echo init']
|
||||
container.securityContext.privileged == true
|
||||
@@ -66,11 +66,11 @@ class K8sNodeInitDeployerTest extends Specification {
|
||||
then:
|
||||
1 * client.podCreate({ Map spec ->
|
||||
spec.kind == 'Pod'
|
||||
spec.metadata.name == 'nf-init-run-foo-node-b'
|
||||
spec.metadata.name == 'nf-init-node-b'
|
||||
spec.spec.nodeName == 'node-b'
|
||||
|
||||
def container = spec.spec.containers[0]
|
||||
container.name == 'nf-init-run-foo-node-b'
|
||||
container.name == 'nf-init-node-b'
|
||||
container.image == 'ubuntu:latest'
|
||||
container.command == ['/bin/bash', '-c', 'echo init']
|
||||
container.securityContext.privileged == true
|
||||
@@ -82,13 +82,12 @@ class K8sNodeInitDeployerTest extends Specification {
|
||||
def 'should lowercase and truncate generated pod names' () {
|
||||
given:
|
||||
def client = Mock(K8sClient)
|
||||
def runName = 'run-name-with-a-very-long-identifier'
|
||||
def config = new K8sConfig(nodeInit: [
|
||||
enabled: true,
|
||||
image: 'ubuntu:latest',
|
||||
command: ['true']
|
||||
])
|
||||
def deployer = new K8sNodeInitDeployer(client, config, runName)
|
||||
def deployer = new K8sNodeInitDeployer(client, config)
|
||||
|
||||
when:
|
||||
deployer.deploy()
|
||||
@@ -96,16 +95,16 @@ class K8sNodeInitDeployerTest extends Specification {
|
||||
then:
|
||||
1 * client.nodeList() >> [
|
||||
items: [
|
||||
[metadata: [name: 'NODE-WITH-A-VERY-LONG-NAME-ABCDEFGHIJKLMNOPQRSTUVWXYZ']]
|
||||
[metadata: [name: 'NODE-WITH-A-VERY-LONG-NAME-ABCDEFGHIJKLMNOPQRSTUVWXYZ-0123']]
|
||||
]
|
||||
]
|
||||
|
||||
then:
|
||||
1 * client.podCreate({ Map spec ->
|
||||
spec.metadata.name == 'nf-init-run-name-with-a-very-long-identifier-node-with-a-very-l'
|
||||
spec.metadata.name == 'nf-init-node-with-a-very-long-name-abcdefghijklmnopqrstuvwxyz-0'
|
||||
spec.metadata.name.size() == 63
|
||||
spec.metadata.name == spec.metadata.name.toLowerCase()
|
||||
spec.spec.nodeName == 'NODE-WITH-A-VERY-LONG-NAME-ABCDEFGHIJKLMNOPQRSTUVWXYZ'
|
||||
spec.spec.nodeName == 'NODE-WITH-A-VERY-LONG-NAME-ABCDEFGHIJKLMNOPQRSTUVWXYZ-0123'
|
||||
})
|
||||
|
||||
then:
|
||||
@@ -116,7 +115,7 @@ class K8sNodeInitDeployerTest extends Specification {
|
||||
given:
|
||||
def client = Mock(K8sClient)
|
||||
def config = new K8sConfig(nodeInit: [enabled: false, cleanup: true])
|
||||
def deployer = new K8sNodeInitDeployer(client, config, 'run-foo')
|
||||
def deployer = new K8sNodeInitDeployer(client, config)
|
||||
|
||||
when:
|
||||
deployer.cleanup()
|
||||
@@ -130,7 +129,7 @@ class K8sNodeInitDeployerTest extends Specification {
|
||||
given:
|
||||
def client = Mock(K8sClient)
|
||||
def config = new K8sConfig(nodeInit: [enabled: true, cleanup: false])
|
||||
def deployer = new K8sNodeInitDeployer(client, config, 'run-foo')
|
||||
def deployer = new K8sNodeInitDeployer(client, config)
|
||||
|
||||
when:
|
||||
deployer.cleanup()
|
||||
@@ -144,7 +143,7 @@ class K8sNodeInitDeployerTest extends Specification {
|
||||
given:
|
||||
def client = Mock(K8sClient)
|
||||
def config = new K8sConfig(nodeInit: [enabled: true, cleanup: true])
|
||||
def deployer = new K8sNodeInitDeployer(client, config, 'run-foo')
|
||||
def deployer = new K8sNodeInitDeployer(client, config)
|
||||
|
||||
when:
|
||||
deployer.cleanup()
|
||||
@@ -158,8 +157,8 @@ class K8sNodeInitDeployerTest extends Specification {
|
||||
]
|
||||
|
||||
then:
|
||||
1 * client.podDelete('nf-init-run-foo-node-a')
|
||||
1 * client.podDelete('nf-init-run-foo-node-b')
|
||||
1 * client.podDelete('nf-init-node-a')
|
||||
1 * client.podDelete('nf-init-node-b')
|
||||
0 * client._
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package nextflow.k8s
|
||||
|
||||
import spock.lang.Specification
|
||||
|
||||
class K8sRuntimeEstimatorTest extends Specification {
|
||||
|
||||
def 'does extract a function' () {
|
||||
given:
|
||||
def observations = [a: [new Tuple2(0, 0), new Tuple2(100, 100)]]
|
||||
def estimator = new K8sRuntimeEstimator(observations)
|
||||
|
||||
when:
|
||||
def estimation = estimator.estimate("a", 1234)
|
||||
|
||||
then:
|
||||
estimation != Double.POSITIVE_INFINITY
|
||||
estimation == 1234.0
|
||||
}
|
||||
|
||||
def 'can extract a function from single measurement' () {
|
||||
given:
|
||||
def observations = [a: [new Tuple2(100, 100)]]
|
||||
def estimator = new K8sRuntimeEstimator(observations)
|
||||
|
||||
when:
|
||||
def estimation = estimator.estimate("a", 1234)
|
||||
|
||||
then:
|
||||
estimation == 100.0 /* Single measurement implies constant runtime */
|
||||
}
|
||||
|
||||
def 'unknown function returns positive infinity' () {
|
||||
given:
|
||||
def observations = [] as HashMap<String, ArrayList>
|
||||
def estimator = new K8sRuntimeEstimator(observations)
|
||||
|
||||
when:
|
||||
def estimation = estimator.estimate("b", 1234)
|
||||
|
||||
then:
|
||||
estimation == Double.POSITIVE_INFINITY
|
||||
}
|
||||
|
||||
def 'extracts function from multiple measurements' () {
|
||||
given:
|
||||
def observations = [a: [new Tuple2(10, 15), new Tuple2(20, 34), new Tuple2(30, 46)]]
|
||||
def estimator = new K8sRuntimeEstimator(observations)
|
||||
|
||||
when:
|
||||
def estimation = estimator.estimate("a", 40)
|
||||
|
||||
then:
|
||||
estimation != Double.POSITIVE_INFINITY
|
||||
}
|
||||
}
|
||||
@@ -1,146 +0,0 @@
|
||||
package nextflow.k8s
|
||||
|
||||
import nextflow.k8s.client.K8sClient
|
||||
import nextflow.processor.TaskRun
|
||||
import spock.lang.Specification
|
||||
|
||||
class K8sTaskSchedulerTest extends Specification {
|
||||
|
||||
def 'should enqueue task handler without immediately submitting it' () {
|
||||
given:
|
||||
def client = Mock(K8sClient)
|
||||
def strategy = Mock(K8sSchedulingStrategy)
|
||||
def scheduler = new K8sTaskScheduler(client, strategy)
|
||||
|
||||
def task = Mock(TaskRun)
|
||||
def handler = Spy(K8sTaskHandler)
|
||||
handler.task = task
|
||||
|
||||
when:
|
||||
scheduler.submit(handler)
|
||||
|
||||
then:
|
||||
0 * strategy._
|
||||
0 * client._
|
||||
0 * handler.submitNow(_)
|
||||
|
||||
and:
|
||||
scheduler.@queue.size() == 1
|
||||
}
|
||||
|
||||
def 'should drain queued task and submit it to selected node' () {
|
||||
given:
|
||||
def client = Mock(K8sClient)
|
||||
def strategy = Mock(K8sSchedulingStrategy)
|
||||
def scheduler = new K8sTaskScheduler(client, strategy)
|
||||
|
||||
def task = Mock(TaskRun)
|
||||
def handler = Spy(K8sTaskHandler)
|
||||
handler.task = task
|
||||
|
||||
when:
|
||||
scheduler.submit(handler)
|
||||
scheduler.drain()
|
||||
|
||||
then:
|
||||
1 * client.nodeList() >> [
|
||||
items: [
|
||||
[metadata: [name: 'node-a']],
|
||||
[metadata: [name: 'node-b']]
|
||||
]
|
||||
]
|
||||
|
||||
then:
|
||||
1 * strategy.schedule({ List<K8sSchedulingRequest> queue ->
|
||||
queue.size() == 1 && queue[0].handler == handler && queue[0].task == task
|
||||
}, ['node-a', 'node-b']) >> { List<K8sSchedulingRequest> queue, List<String> nodes ->
|
||||
new K8sSchedulingDecision(queue[0], 'node-b')
|
||||
}
|
||||
|
||||
then:
|
||||
1 * handler.submitNow('node-b') >> {}
|
||||
|
||||
then:
|
||||
1 * strategy.schedule([], ['node-a', 'node-b']) >> null
|
||||
|
||||
and:
|
||||
scheduler.@queue.size() == 0
|
||||
}
|
||||
|
||||
def 'should keep task queued when strategy returns no decision' () {
|
||||
given:
|
||||
def client = Mock(K8sClient)
|
||||
def strategy = Mock(K8sSchedulingStrategy)
|
||||
def scheduler = new K8sTaskScheduler(client, strategy)
|
||||
|
||||
def task = Mock(TaskRun)
|
||||
def handler = Spy(K8sTaskHandler)
|
||||
handler.task = task
|
||||
|
||||
when:
|
||||
scheduler.submit(handler)
|
||||
scheduler.drain()
|
||||
|
||||
then:
|
||||
1 * client.nodeList() >> [
|
||||
items: [
|
||||
[metadata: [name: 'node-a']]
|
||||
]
|
||||
]
|
||||
|
||||
then:
|
||||
1 * strategy.schedule({ List<K8sSchedulingRequest> queue ->
|
||||
queue.size() == 1 && queue[0].handler == handler
|
||||
}, ['node-a']) >> null
|
||||
|
||||
and:
|
||||
0 * handler.submitNow(_)
|
||||
scheduler.@queue.size() == 1
|
||||
}
|
||||
|
||||
def 'should continue draining when selected request was already removed' () {
|
||||
given:
|
||||
def client = Mock(K8sClient)
|
||||
def strategy = Mock(K8sSchedulingStrategy)
|
||||
def scheduler = new K8sTaskScheduler(client, strategy)
|
||||
|
||||
def task1 = Mock(TaskRun)
|
||||
def handler1 = Spy(K8sTaskHandler)
|
||||
handler1.task = task1
|
||||
def staleRequest = new K8sSchedulingRequest(handler1)
|
||||
|
||||
def task2 = Mock(TaskRun)
|
||||
def handler2 = Spy(K8sTaskHandler)
|
||||
handler2.task = task2
|
||||
|
||||
when:
|
||||
scheduler.submit(handler2)
|
||||
scheduler.drain()
|
||||
|
||||
then:
|
||||
1 * client.nodeList() >> [
|
||||
items: [
|
||||
[metadata: [name: 'node-a']]
|
||||
]
|
||||
]
|
||||
|
||||
then:
|
||||
1 * strategy.schedule(_, ['node-a']) >> new K8sSchedulingDecision(staleRequest, 'node-a')
|
||||
|
||||
then:
|
||||
1 * strategy.schedule({ List<K8sSchedulingRequest> queue ->
|
||||
queue.size() == 1 && queue[0].handler == handler2
|
||||
}, ['node-a']) >> { List<K8sSchedulingRequest> queue, List<String> nodes ->
|
||||
new K8sSchedulingDecision(queue[0], 'node-a')
|
||||
}
|
||||
|
||||
then:
|
||||
1 * handler2.submitNow('node-a') >> {}
|
||||
|
||||
then:
|
||||
1 * strategy.schedule([], ['node-a']) >> null
|
||||
|
||||
and:
|
||||
scheduler.@queue.size() == 0
|
||||
}
|
||||
}
|
||||
@@ -1102,4 +1102,529 @@ class K8sClientTest extends Specification {
|
||||
result.terminated.exitCode == null
|
||||
result.terminated.exitcode == null
|
||||
}
|
||||
|
||||
def 'should describe a node' () {
|
||||
given:
|
||||
def JSON = '''
|
||||
{
|
||||
"kind": "Node",
|
||||
"apiVersion": "v1",
|
||||
"metadata": {
|
||||
"name": "test-node",
|
||||
"namespace": "default"
|
||||
},
|
||||
"status": {
|
||||
"capacity": {
|
||||
"cpu": "4",
|
||||
"memory": "16Gi"
|
||||
},
|
||||
"allocatable": {
|
||||
"cpu": "3.5",
|
||||
"memory": "14Gi"
|
||||
}
|
||||
}
|
||||
}
|
||||
'''
|
||||
|
||||
def client = Spy(K8sClient)
|
||||
final NODE_NAME = 'test-node'
|
||||
def RESP = Mock(K8sResponseApi)
|
||||
RESP.getText() >> JSON
|
||||
|
||||
when:
|
||||
def result = client.nodeDescribe(NODE_NAME)
|
||||
then:
|
||||
1 * client.get('/api/v1/nodes/test-node') >> RESP
|
||||
result.kind == 'Node'
|
||||
result.metadata.name == 'test-node'
|
||||
}
|
||||
|
||||
def 'should get node CPU capacity' () {
|
||||
given:
|
||||
def JSON = '''
|
||||
{
|
||||
"kind": "Node",
|
||||
"apiVersion": "v1",
|
||||
"metadata": {
|
||||
"name": "test-node"
|
||||
},
|
||||
"status": {
|
||||
"capacity": {
|
||||
"cpu": "4",
|
||||
"memory": "16Gi"
|
||||
}
|
||||
}
|
||||
}
|
||||
'''
|
||||
|
||||
def client = Spy(K8sClient)
|
||||
final NODE_NAME = 'test-node'
|
||||
def RESP = Mock(K8sResponseApi)
|
||||
RESP.getText() >> JSON
|
||||
|
||||
when:
|
||||
def result = client.getNodeCpuCapacity(NODE_NAME)
|
||||
then:
|
||||
1 * client.nodeDescribe(NODE_NAME) >> new K8sResponseJson(JSON)
|
||||
result == '4'
|
||||
}
|
||||
|
||||
def 'should get node memory capacity' () {
|
||||
given:
|
||||
def JSON = '''
|
||||
{
|
||||
"kind": "Node",
|
||||
"apiVersion": "v1",
|
||||
"metadata": {
|
||||
"name": "test-node"
|
||||
},
|
||||
"status": {
|
||||
"capacity": {
|
||||
"cpu": "4",
|
||||
"memory": "16Gi"
|
||||
}
|
||||
}
|
||||
}
|
||||
'''
|
||||
|
||||
def client = Spy(K8sClient)
|
||||
final NODE_NAME = 'test-node'
|
||||
|
||||
when:
|
||||
def result = client.getNodeMemoryCapacity(NODE_NAME)
|
||||
then:
|
||||
1 * client.nodeDescribe(NODE_NAME) >> new K8sResponseJson(JSON)
|
||||
result == '16Gi'
|
||||
}
|
||||
|
||||
def 'should get node CPU used from metrics server' () {
|
||||
given:
|
||||
def NODE_JSON = '''
|
||||
{
|
||||
"kind": "Node",
|
||||
"apiVersion": "v1",
|
||||
"metadata": {
|
||||
"name": "test-node"
|
||||
},
|
||||
"status": {
|
||||
"capacity": {
|
||||
"cpu": "4",
|
||||
"memory": "16Gi"
|
||||
}
|
||||
}
|
||||
}
|
||||
'''
|
||||
def METRICS_JSON = '''
|
||||
{
|
||||
"kind": "NodeMetrics",
|
||||
"apiVersion": "metrics.k8s.io/v1beta1",
|
||||
"metadata": {
|
||||
"name": "test-node"
|
||||
},
|
||||
"usage": {
|
||||
"cpu": "1500m",
|
||||
"memory": "2Gi"
|
||||
}
|
||||
}
|
||||
'''
|
||||
|
||||
def client = Spy(K8sClient)
|
||||
final NODE_NAME = 'test-node'
|
||||
def NODE_RESP = Mock(K8sResponseApi)
|
||||
NODE_RESP.getText() >> NODE_JSON
|
||||
def METRICS_RESP = Mock(K8sResponseApi)
|
||||
METRICS_RESP.getText() >> METRICS_JSON
|
||||
|
||||
when:
|
||||
def result = client.getNodeCpuUsed(NODE_NAME)
|
||||
then:
|
||||
1 * client.get('/apis/metrics.k8s.io/v1beta1/nodes/test-node') >> METRICS_RESP
|
||||
result == '1500m'
|
||||
}
|
||||
|
||||
def 'should get node memory used from metrics server' () {
|
||||
given:
|
||||
def NODE_JSON = '''
|
||||
{
|
||||
"kind": "Node",
|
||||
"apiVersion": "v1",
|
||||
"metadata": {
|
||||
"name": "test-node"
|
||||
},
|
||||
"status": {
|
||||
"capacity": {
|
||||
"cpu": "4",
|
||||
"memory": "16Gi"
|
||||
}
|
||||
}
|
||||
}
|
||||
'''
|
||||
def METRICS_JSON = '''
|
||||
{
|
||||
"kind": "NodeMetrics",
|
||||
"apiVersion": "metrics.k8s.io/v1beta1",
|
||||
"metadata": {
|
||||
"name": "test-node"
|
||||
},
|
||||
"usage": {
|
||||
"cpu": "1500m",
|
||||
"memory": "2Gi"
|
||||
}
|
||||
}
|
||||
'''
|
||||
|
||||
def client = Spy(K8sClient)
|
||||
final NODE_NAME = 'test-node'
|
||||
def METRICS_RESP = Mock(K8sResponseApi)
|
||||
METRICS_RESP.getText() >> METRICS_JSON
|
||||
|
||||
when:
|
||||
def result = client.getNodeMemoryUsed(NODE_NAME)
|
||||
then:
|
||||
1 * client.get('/apis/metrics.k8s.io/v1beta1/nodes/test-node') >> METRICS_RESP
|
||||
result == '2Gi'
|
||||
}
|
||||
|
||||
def 'should fallback when metrics server not available for CPU' () {
|
||||
given:
|
||||
def NODE_JSON = '''
|
||||
{
|
||||
"kind": "Node",
|
||||
"apiVersion": "v1",
|
||||
"metadata": {
|
||||
"name": "test-node"
|
||||
},
|
||||
"status": {
|
||||
"capacity": {
|
||||
"cpu": "4",
|
||||
"memory": "16Gi"
|
||||
},
|
||||
"allocated": {
|
||||
"cpu": "2",
|
||||
"memory": "8Gi"
|
||||
}
|
||||
}
|
||||
}
|
||||
'''
|
||||
|
||||
def client = Spy(K8sClient)
|
||||
final NODE_NAME = 'test-node'
|
||||
def NODE_RESP = Mock(K8sResponseApi)
|
||||
NODE_RESP.getText() >> NODE_JSON
|
||||
|
||||
when:
|
||||
def result = client.getNodeCpuUsed(NODE_NAME)
|
||||
then:
|
||||
// First attempt to metrics server fails
|
||||
1 * client.get('/apis/metrics.k8s.io/v1beta1/nodes/test-node') >> { throw new K8sResponseException("Metrics not available", new ByteArrayInputStream('{}'.bytes)) }
|
||||
// Fallback to node describe - which calls get internally
|
||||
1 * client.get('/api/v1/nodes/test-node') >> NODE_RESP
|
||||
result == '2'
|
||||
}
|
||||
|
||||
def 'should fallback when metrics server not available for memory' () {
|
||||
given:
|
||||
def NODE_JSON = '''
|
||||
{
|
||||
"kind": "Node",
|
||||
"apiVersion": "v1",
|
||||
"metadata": {
|
||||
"name": "test-node"
|
||||
},
|
||||
"status": {
|
||||
"capacity": {
|
||||
"cpu": "4",
|
||||
"memory": "16Gi"
|
||||
},
|
||||
"allocated": {
|
||||
"cpu": "2",
|
||||
"memory": "8Gi"
|
||||
}
|
||||
}
|
||||
}
|
||||
'''
|
||||
|
||||
def client = Spy(K8sClient)
|
||||
final NODE_NAME = 'test-node'
|
||||
def NODE_RESP = Mock(K8sResponseApi)
|
||||
NODE_RESP.getText() >> NODE_JSON
|
||||
|
||||
when:
|
||||
def result = client.getNodeMemoryUsed(NODE_NAME)
|
||||
then:
|
||||
// First attempt to metrics server fails
|
||||
1 * client.get('/apis/metrics.k8s.io/v1beta1/nodes/test-node') >> { throw new K8sResponseException("Metrics not available", new ByteArrayInputStream('{}'.bytes)) }
|
||||
// Fallback to node describe - which calls get internally
|
||||
1 * client.get('/api/v1/nodes/test-node') >> NODE_RESP
|
||||
result == '8Gi'
|
||||
}
|
||||
|
||||
def 'should return null when no allocated resources available' () {
|
||||
given:
|
||||
def NODE_JSON = '''
|
||||
{
|
||||
"kind": "Node",
|
||||
"apiVersion": "v1",
|
||||
"metadata": {
|
||||
"name": "test-node"
|
||||
},
|
||||
"status": {
|
||||
"capacity": {
|
||||
"cpu": "4",
|
||||
"memory": "16Gi"
|
||||
}
|
||||
}
|
||||
}
|
||||
'''
|
||||
|
||||
def client = Spy(K8sClient)
|
||||
final NODE_NAME = 'test-node'
|
||||
def NODE_RESP = Mock(K8sResponseApi)
|
||||
NODE_RESP.getText() >> NODE_JSON
|
||||
|
||||
when:
|
||||
def resultCpu = client.getNodeCpuUsed(NODE_NAME)
|
||||
def resultMem = client.getNodeMemoryUsed(NODE_NAME)
|
||||
then:
|
||||
// First attempt to metrics server fails for CPU
|
||||
1 * client.get('/apis/metrics.k8s.io/v1beta1/nodes/test-node') >> { throw new K8sResponseException("Metrics not available", new ByteArrayInputStream('{}'.bytes)) }
|
||||
// Fallback to node describe for CPU
|
||||
1 * client.get('/api/v1/nodes/test-node') >> NODE_RESP
|
||||
// First attempt to metrics server fails for Memory
|
||||
1 * client.get('/apis/metrics.k8s.io/v1beta1/nodes/test-node') >> { throw new K8sResponseException("Metrics not available", new ByteArrayInputStream('{}'.bytes)) }
|
||||
// Fallback to node describe for Memory
|
||||
1 * client.get('/api/v1/nodes/test-node') >> NODE_RESP
|
||||
resultCpu == null
|
||||
resultMem == null
|
||||
}
|
||||
|
||||
def 'should parse memory quantity strings' () {
|
||||
expect:
|
||||
K8sClient.parseK8sQuantity(null) == null
|
||||
K8sClient.parseK8sQuantity('') == null
|
||||
K8sClient.parseK8sQuantity('1024') == 1024L
|
||||
K8sClient.parseK8sQuantity('1Ki') == 1024L
|
||||
K8sClient.parseK8sQuantity('1KiB') == 1024L
|
||||
K8sClient.parseK8sQuantity('1Mi') == 1024L * 1024L
|
||||
K8sClient.parseK8sQuantity('1MiB') == 1024L * 1024L
|
||||
K8sClient.parseK8sQuantity('1Gi') == 1024L * 1024L * 1024L
|
||||
K8sClient.parseK8sQuantity('1GiB') == 1024L * 1024L * 1024L
|
||||
K8sClient.parseK8sQuantity('16Gi') == 16L * 1024L * 1024L * 1024L
|
||||
K8sClient.parseK8sQuantity('8192Mi') == 8192L * 1024L * 1024L
|
||||
K8sClient.parseK8sQuantity('0.5Gi') == (long)(0.5 * 1024L * 1024L * 1024L)
|
||||
}
|
||||
|
||||
def 'should parse CPU quantity strings' () {
|
||||
expect:
|
||||
K8sClient.parseK8sQuantity(null, true) == null
|
||||
K8sClient.parseK8sQuantity('', true) == null
|
||||
K8sClient.parseK8sQuantity('1', true) == 1000L // 1 core = 1000 millicores
|
||||
K8sClient.parseK8sQuantity('2', true) == 2000L // 2 cores = 2000 millicores
|
||||
K8sClient.parseK8sQuantity('0.5', true) == 500L // 0.5 cores = 500 millicores
|
||||
K8sClient.parseK8sQuantity('1500m', true) == 1500L // 1500 millicores
|
||||
K8sClient.parseK8sQuantity('100m', true) == 100L // 100 millicores
|
||||
// Nanocores: 1 core = 1,000,000,000 nanocores, so we divide by 1,000,000 to get millicores
|
||||
K8sClient.parseK8sQuantity('1000000n', true) == 1L // 1,000,000 nanocores = 1 millicore
|
||||
K8sClient.parseK8sQuantity('1000000000n', true) == 1000L // 1,000,000,000 nanocores = 1000 millicores = 1 core
|
||||
K8sClient.parseK8sQuantity('298332671n', true) == 298L // 298,332,671 nanocores ≈ 298.33 millicores
|
||||
K8sClient.parseK8sQuantity('434126984n', true) == 434L // 434,126,984 nanocores ≈ 434.13 millicores
|
||||
}
|
||||
|
||||
def 'should get node CPU capacity in millicores' () {
|
||||
given:
|
||||
def NODE_JSON = '''
|
||||
{
|
||||
"kind": "Node",
|
||||
"apiVersion": "v1",
|
||||
"metadata": {
|
||||
"name": "test-node"
|
||||
},
|
||||
"status": {
|
||||
"capacity": {
|
||||
"cpu": "4",
|
||||
"memory": "16Gi"
|
||||
}
|
||||
}
|
||||
}
|
||||
'''
|
||||
|
||||
def client = Spy(K8sClient)
|
||||
final NODE_NAME = 'test-node'
|
||||
|
||||
when:
|
||||
def result = client.getNodeCpuCapacityMillis(NODE_NAME)
|
||||
then:
|
||||
1 * client.nodeDescribe(NODE_NAME) >> new K8sResponseJson(NODE_JSON)
|
||||
result == 4000L // 4 cores = 4000 millicores
|
||||
}
|
||||
|
||||
def 'should get node CPU used in millicores' () {
|
||||
given:
|
||||
def METRICS_JSON = '''
|
||||
{
|
||||
"kind": "NodeMetrics",
|
||||
"apiVersion": "metrics.k8s.io/v1beta1",
|
||||
"metadata": {
|
||||
"name": "test-node"
|
||||
},
|
||||
"usage": {
|
||||
"cpu": "1500m",
|
||||
"memory": "2Gi"
|
||||
}
|
||||
}
|
||||
'''
|
||||
|
||||
def client = Spy(K8sClient)
|
||||
final NODE_NAME = 'test-node'
|
||||
def METRICS_RESP = Mock(K8sResponseApi)
|
||||
METRICS_RESP.getText() >> METRICS_JSON
|
||||
|
||||
when:
|
||||
def result = client.getNodeCpuUsedMillis(NODE_NAME)
|
||||
then:
|
||||
1 * client.get('/apis/metrics.k8s.io/v1beta1/nodes/test-node') >> METRICS_RESP
|
||||
result == 1500L // 1500 millicores
|
||||
}
|
||||
|
||||
def 'should get node memory capacity in bytes' () {
|
||||
given:
|
||||
def NODE_JSON = '''
|
||||
{
|
||||
"kind": "Node",
|
||||
"apiVersion": "v1",
|
||||
"metadata": {
|
||||
"name": "test-node"
|
||||
},
|
||||
"status": {
|
||||
"capacity": {
|
||||
"cpu": "4",
|
||||
"memory": "16Gi"
|
||||
}
|
||||
}
|
||||
}
|
||||
'''
|
||||
|
||||
def client = Spy(K8sClient)
|
||||
final NODE_NAME = 'test-node'
|
||||
|
||||
when:
|
||||
def result = client.getNodeMemoryCapacityBytes(NODE_NAME)
|
||||
then:
|
||||
1 * client.nodeDescribe(NODE_NAME) >> new K8sResponseJson(NODE_JSON)
|
||||
result == 16L * 1024L * 1024L * 1024L // 16 GiB in bytes
|
||||
}
|
||||
|
||||
def 'should get node memory used in bytes' () {
|
||||
given:
|
||||
def METRICS_JSON = '''
|
||||
{
|
||||
"kind": "NodeMetrics",
|
||||
"apiVersion": "metrics.k8s.io/v1beta1",
|
||||
"metadata": {
|
||||
"name": "test-node"
|
||||
},
|
||||
"usage": {
|
||||
"cpu": "1500m",
|
||||
"memory": "2Gi"
|
||||
}
|
||||
}
|
||||
'''
|
||||
|
||||
def client = Spy(K8sClient)
|
||||
final NODE_NAME = 'test-node'
|
||||
def METRICS_RESP = Mock(K8sResponseApi)
|
||||
METRICS_RESP.getText() >> METRICS_JSON
|
||||
|
||||
when:
|
||||
def result = client.getNodeMemoryUsedBytes(NODE_NAME)
|
||||
then:
|
||||
1 * client.get('/apis/metrics.k8s.io/v1beta1/nodes/test-node') >> METRICS_RESP
|
||||
result == 2L * 1024L * 1024L * 1024L // 2 GiB in bytes
|
||||
}
|
||||
|
||||
def 'should handle fractional CPU values' () {
|
||||
expect:
|
||||
K8sClient.parseK8sQuantity('2.5', true) == 2500L // 2.5 cores = 2500 millicores
|
||||
K8sClient.parseK8sQuantity('0.25', true) == 250L // 0.25 cores = 250 millicores
|
||||
}
|
||||
|
||||
def 'should handle fractional memory values' () {
|
||||
expect:
|
||||
K8sClient.parseK8sQuantity('1.5Gi') == (long)(1.5 * 1024L * 1024L * 1024L)
|
||||
K8sClient.parseK8sQuantity('0.5Mi') == (long)(0.5 * 1024L * 1024L)
|
||||
}
|
||||
|
||||
def 'should handle decimal suffixes for memory' () {
|
||||
expect:
|
||||
K8sClient.parseK8sQuantity('1K') == 1000L
|
||||
K8sClient.parseK8sQuantity('1M') == 1000L * 1000L
|
||||
K8sClient.parseK8sQuantity('1G') == 1000L * 1000L * 1000L
|
||||
}
|
||||
|
||||
def 'should return null for invalid quantity strings' () {
|
||||
expect:
|
||||
K8sClient.parseK8sQuantity('invalid') == null
|
||||
K8sClient.parseK8sQuantity('16 GiB') == null // space not allowed
|
||||
K8sClient.parseK8sQuantity('Gi16') == null // suffix before number (doesn't match any suffix pattern)
|
||||
K8sClient.parseK8sQuantity('16.5.5') == null // multiple decimal points
|
||||
K8sClient.parseK8sQuantity('abc123') == null // non-numeric start
|
||||
K8sClient.parseK8sQuantity('123abc') == null // non-numeric end (no valid suffix)
|
||||
}
|
||||
|
||||
def 'should handle large memory values like 8058776Ki' () {
|
||||
expect:
|
||||
K8sClient.parseK8sQuantity('8058776Ki') == 8058776L * 1024L
|
||||
}
|
||||
|
||||
def 'should handle large memory values like 8058776ki' () {
|
||||
expect:
|
||||
K8sClient.parseK8sQuantity('8058776ki') == 8058776L * 1024L
|
||||
}
|
||||
|
||||
def 'should handle memory values with B suffix' () {
|
||||
expect:
|
||||
K8sClient.parseK8sQuantity('1KiB') == 1024L
|
||||
K8sClient.parseK8sQuantity('1MiB') == 1024L * 1024L
|
||||
K8sClient.parseK8sQuantity('1GiB') == 1024L * 1024L * 1024L
|
||||
}
|
||||
|
||||
def 'should handle decimal memory suffixes' () {
|
||||
expect:
|
||||
K8sClient.parseK8sQuantity('1K') == 1000L
|
||||
K8sClient.parseK8sQuantity('1KB') == 1000L
|
||||
K8sClient.parseK8sQuantity('1M') == 1000L * 1000L
|
||||
K8sClient.parseK8sQuantity('1MB') == 1000L * 1000L
|
||||
K8sClient.parseK8sQuantity('1G') == 1000L * 1000L * 1000L
|
||||
}
|
||||
|
||||
def 'should handle real world memory value 8058776Ki' () {
|
||||
expect:
|
||||
K8sClient.parseK8sQuantity('8058776Ki') == 8058776L * 1024L
|
||||
K8sClient.parseK8sQuantity('8058776ki') == 8058776L * 1024L
|
||||
K8sClient.parseK8sQuantity('8058776KI') == 8058776L * 1024L
|
||||
}
|
||||
|
||||
def 'should handle CPU values without suffix' () {
|
||||
expect:
|
||||
K8sClient.parseK8sQuantity('1', true) == 1000L
|
||||
K8sClient.parseK8sQuantity('2', true) == 2000L
|
||||
K8sClient.parseK8sQuantity('0.5', true) == 500L
|
||||
K8sClient.parseK8sQuantity('2.5', true) == 2500L
|
||||
}
|
||||
|
||||
def 'should handle fractional memory values with suffixes' () {
|
||||
expect:
|
||||
K8sClient.parseK8sQuantity('1.5Gi') == (long)(1.5 * 1024 * 1024 * 1024)
|
||||
K8sClient.parseK8sQuantity('0.5Mi') == (long)(0.5 * 1024 * 1024)
|
||||
K8sClient.parseK8sQuantity('2.25Ki') == (long)(2.25 * 1024)
|
||||
}
|
||||
|
||||
def 'should handle edge cases' () {
|
||||
expect:
|
||||
K8sClient.parseK8sQuantity('0') == 0L
|
||||
K8sClient.parseK8sQuantity('0Ki') == 0L
|
||||
K8sClient.parseK8sQuantity('0m', true) == 0L
|
||||
K8sClient.parseK8sQuantity('+100') == 100L
|
||||
K8sClient.parseK8sQuantity('-50') == -50L
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user