add nextflow d30e48d
This commit is contained in:
82
nextflow/plugins/nf-google/README.md
Normal file
82
nextflow/plugins/nf-google/README.md
Normal file
@@ -0,0 +1,82 @@
|
||||
# Google Cloud plugin for Nextflow
|
||||
|
||||
## Summary
|
||||
|
||||
The Google Cloud plugin provides support for Google Cloud Platform (GCP), including Google Cloud Batch as a compute executor and Google Cloud Storage as a file system.
|
||||
|
||||
## Get Started
|
||||
|
||||
To use this plugin, add it to your `nextflow.config`:
|
||||
|
||||
```groovy
|
||||
plugins {
|
||||
id 'nf-google'
|
||||
}
|
||||
```
|
||||
|
||||
Configure your Google Cloud credentials and project:
|
||||
|
||||
```groovy
|
||||
google {
|
||||
project = '<YOUR PROJECT ID>'
|
||||
location = 'us-central1'
|
||||
}
|
||||
|
||||
process.executor = 'google-batch'
|
||||
workDir = 'gs://<YOUR BUCKET>/work'
|
||||
```
|
||||
|
||||
Authentication can be done via:
|
||||
- Application Default Credentials
|
||||
- Service account JSON key file
|
||||
- Workload Identity (for GKE)
|
||||
|
||||
## Examples
|
||||
|
||||
### Basic Google Cloud Batch Configuration
|
||||
|
||||
```groovy
|
||||
plugins {
|
||||
id 'nf-google'
|
||||
}
|
||||
|
||||
google {
|
||||
project = 'my-gcp-project'
|
||||
location = 'europe-west1'
|
||||
batch {
|
||||
spot = true
|
||||
}
|
||||
}
|
||||
|
||||
process.executor = 'google-batch'
|
||||
workDir = 'gs://my-bucket/work'
|
||||
```
|
||||
|
||||
### Using Service Account
|
||||
|
||||
```groovy
|
||||
google {
|
||||
project = 'my-gcp-project'
|
||||
location = 'us-central1'
|
||||
credentials = '/path/to/service-account.json'
|
||||
}
|
||||
```
|
||||
|
||||
### Machine Type Configuration
|
||||
|
||||
```groovy
|
||||
process {
|
||||
executor = 'google-batch'
|
||||
machineType = 'n2-standard-4'
|
||||
disk = '100 GB'
|
||||
}
|
||||
```
|
||||
|
||||
## Resources
|
||||
|
||||
- [Google Cloud Batch Executor Documentation](https://nextflow.io/docs/latest/google.html)
|
||||
- [Google Cloud Storage Documentation](https://nextflow.io/docs/latest/google.html#google-cloud-storage)
|
||||
|
||||
## License
|
||||
|
||||
[Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0)
|
||||
1
nextflow/plugins/nf-google/VERSION
Normal file
1
nextflow/plugins/nf-google/VERSION
Normal file
@@ -0,0 +1 @@
|
||||
1.27.2
|
||||
74
nextflow/plugins/nf-google/build.gradle
Normal file
74
nextflow/plugins/nf-google/build.gradle
Normal file
@@ -0,0 +1,74 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
plugins {
|
||||
id 'io.nextflow.nextflow-plugin' version "${nextflowPluginVersion}"
|
||||
id 'java-test-fixtures'
|
||||
}
|
||||
|
||||
nextflowPlugin {
|
||||
nextflowVersion = '25.12.0-edge'
|
||||
|
||||
provider = "${nextflowPluginProvider}"
|
||||
description = 'Enables Google Cloud Platform execution through Batch service with native Cloud Storage access and intelligent machine type selection'
|
||||
className = 'nextflow.cloud.google.GoogleCloudPlugin'
|
||||
useDefaultDependencies = false
|
||||
generateSpec = false
|
||||
extensionPoints = [
|
||||
'nextflow.cloud.google.GoogleOpts',
|
||||
'nextflow.cloud.google.batch.GoogleBatchExecutor',
|
||||
'nextflow.cloud.google.util.GsPathFactory',
|
||||
'nextflow.cloud.google.util.GsPathSerializer',
|
||||
]
|
||||
}
|
||||
|
||||
sourceSets {
|
||||
main.java.srcDirs = []
|
||||
main.groovy.srcDirs = ['src/main']
|
||||
main.resources.srcDirs = ['src/resources']
|
||||
test.groovy.srcDirs = ['src/test']
|
||||
test.java.srcDirs = []
|
||||
test.resources.srcDirs = []
|
||||
}
|
||||
|
||||
configurations {
|
||||
// see https://docs.gradle.org/4.1/userguide/dependency_management.html#sub:exclude_transitive_dependencies
|
||||
runtimeClasspath.exclude group: 'org.slf4j', module: 'slf4j-api'
|
||||
}
|
||||
|
||||
dependencies {
|
||||
compileOnly project(':nextflow')
|
||||
compileOnly 'org.slf4j:slf4j-api:2.0.17'
|
||||
compileOnly 'org.pf4j:pf4j:3.14.1'
|
||||
|
||||
api 'com.google.auth:google-auth-library-oauth2-http:1.39.1'
|
||||
api 'com.google.cloud:google-cloud-batch:0.75.0'
|
||||
api 'com.google.cloud:google-cloud-logging:3.23.5'
|
||||
api 'com.google.cloud:google-cloud-nio:0.128.5'
|
||||
api 'com.google.cloud:google-cloud-storage:2.58.0'
|
||||
api 'io.seqera:lib-cloudinfo:1.0.0'
|
||||
// Force patched version to address CVE-2025-55163 (MadeYouReset HTTP/2 DDoS vulnerability)
|
||||
runtimeOnly 'io.grpc:grpc-netty-shaded:1.75.0'
|
||||
// Force patched version to address GHSA-72hv-8253-57qq (jackson-core Number Length Constraint Bypass DoS)
|
||||
runtimeOnly 'com.fasterxml.jackson.core:jackson-core:2.18.6'
|
||||
|
||||
testImplementation(testFixtures(project(":nextflow")))
|
||||
testImplementation "org.apache.groovy:groovy:4.0.31"
|
||||
testImplementation "org.apache.groovy:groovy-nio:4.0.31"
|
||||
}
|
||||
|
||||
test {
|
||||
useJUnitPlatform()
|
||||
}
|
||||
255
nextflow/plugins/nf-google/changelog.txt
Normal file
255
nextflow/plugins/nf-google/changelog.txt
Normal file
@@ -0,0 +1,255 @@
|
||||
nf-google changelog
|
||||
===================
|
||||
1.27.2 - 25 Apr 2026
|
||||
- Replace current cloud info URL call with cloudInfo client (#7065) [629184251]
|
||||
|
||||
1.27.1 - 26 Mar 2026
|
||||
- Fix netty and jackson vulnerabilities (#6955) [8dafdd95d]
|
||||
|
||||
1.27.0 - 17 Mar 2026
|
||||
- Add support for latest-generation Google Cloud machine families (#6841) [27785b171]
|
||||
- Add support for GCP Ops Agent (#6608) [247a53bc0]
|
||||
- Fix Google Batch exit code when spot claim is successfully retried (#6926) [76927c27e]
|
||||
|
||||
1.26.1 - 28 Feb 2026
|
||||
- Report actual GCP zone in Google Batch trace records (#6855) [465791294]
|
||||
|
||||
1.26.0 - 8 Feb 2026
|
||||
- Fix isCompleted check in getNumSpotInterruptions (#6805) [76558481a]
|
||||
- Remove isCompleted() from getNumSpotInterruptions (#6729) [24cc59e27]
|
||||
- Refactor GoogleBatchTaskHandler.newSubmitRequest for reduced complexity (#6687) [38c39108c]
|
||||
|
||||
1.25.0 - 19 Dec 2025
|
||||
- Add spot interruption tracking to trace records (#6606) [eecd81671]
|
||||
- Refactor Google Batch getExitCode to imperative style (#6649) [addd59e9f]
|
||||
- Add default maxSpotAttempts for fusion snapshots in Google Batch (#6652) [458ef97a9]
|
||||
|
||||
1.24.0 - 28 Nov 2025
|
||||
- Add stageFileEnabled flag to control .command.stage file creation (#6618) [2d117cb59]
|
||||
- Add Google Batch LogsPolicy PATH option for logging to GCS (#6431) [5b61afe01]
|
||||
|
||||
1.23.3 - 22 Oct 2025
|
||||
- Prioritize Google Batch API exit codes with fallback to .exitcode file (#6498) [6ac2efcba]
|
||||
|
||||
1.23.2 - 21 Oct 2025
|
||||
- Rename `config.schema` package to `config.spec` (#6485) [ef0d2d601]
|
||||
|
||||
1.23.1 - 8 Oct 2025
|
||||
- Fix CVE-2025-55163 in nf-google plugin [7d7061797]
|
||||
- Bump Google Cloud libraries to latest versions (#6438) [59a63f1ae]
|
||||
|
||||
1.22.2 - 15 Aug 2025
|
||||
- Fix NPE in GoogleBatchMachineTypeSelector when spotPrice is null [ci fast] [a797a795]
|
||||
- Unify nf-lang config scopes with runtime classes (#6271) [bfa67ca3]
|
||||
- Bump groovy 4.0.28 (#6304) [ci fast] [a468f8ef]
|
||||
|
||||
1.22.1 - 6 Jul 2025
|
||||
- Fix class not found exception Google Life science executor (#6193) [7bfb9358]
|
||||
- Update Google dependencies (#6229) [8bd42acb]
|
||||
- Upload Google Batch log on task exit (#6226) [78d9f473]
|
||||
- Bump Slf4j version 2.0.17 [93199e09]
|
||||
- Bump gson version 2.13.1 [ab8e36a2]
|
||||
|
||||
1.22.0 - 2 Jun 2025
|
||||
- Sunset the Google LS executor (#6070) [06e0d426]
|
||||
- Bump Groovy to version 4.0.27 (#6125) [258e1790]
|
||||
|
||||
1.21.0 - 8 May 2025
|
||||
- Add Support for Google Batch network tags (#5951) [d6e4d6c2]
|
||||
- Remove test constructors or mark as TestOnly (#5216) [d4fadd42]
|
||||
|
||||
1.20.0 - 23 Apr 2025
|
||||
- Add Google Batch gcsfuseOptions (#5991) [1d4dd574]
|
||||
|
||||
1.19.0 - 17 Mar 2025
|
||||
- Fix Google Batch autoRetryExitCodes bug (#5828) [cfeedd6f]
|
||||
- Improve Google Batch support for GPUs (#5406) [420fb17e]
|
||||
- Bump groovy 4.0.26 [f740bc56]
|
||||
|
||||
1.18.0 - 12 Feb 2025
|
||||
- Add DeadlineExceededException to Google Batch retryable exceptions [944f48f9]
|
||||
- Fix Google Batch task array causes process to fail (#5780) [7ad7a237]
|
||||
- Fix Mount input file buckets in task arrays for Google Batch (#5739) [ba171fd1]
|
||||
- Improve Google Batch executor stability and error handling (#5690) [b64087fc]
|
||||
- Ignore tests when smoke mode is enabled [3eb6efad]
|
||||
- Bump groovy 4.0.25 [19c40a4a]
|
||||
|
||||
1.17.0 - 20 Jan 2025
|
||||
- Ensure job is killed when exception in task status check (#5561) [9eefd207]
|
||||
- Fix Google Batch hang when internal error during scheduling (#5567) [18f7de13]
|
||||
- Bump groovy 4.0.24 missing deps [40670f7e]
|
||||
- Bump logback 1.5.13 + slf4j 2.0.16 [cc0163ac]
|
||||
|
||||
1.16.0 - 3 Dec 2024
|
||||
- Detecting errors in data unstaging (#5345) [3c8e602d]
|
||||
- Bump bouncycastle to jdk18on:1.78.1 (#5467) [cd8c385f]
|
||||
- Bump groovy 4.0.24 [dd71ad31]
|
||||
- Bump protobuf-java:3.25.5 to nf-google [488b7906]
|
||||
- Add NotFoundException to retry condition for Google Batch [aa4d19cc]
|
||||
|
||||
1.15.2 - 14 Oct 2024
|
||||
- Add Google LS deprecation notice (#5400) [0ee1d9bc]
|
||||
|
||||
1.15.1 - 13 Oct 2024
|
||||
- Add retry policy to google batch describe task (#5356) [64bb5a92]
|
||||
|
||||
1.15.0 - 4 Sep 2024
|
||||
- Add Google Batch warning when for conflicting disk image config (#5279) [96cb57cb]
|
||||
- Add support for Google Batch used specified boot images (#5268) [0aaa6482]
|
||||
- Disable Google Batch automatic spot retries (#5223) [aad21533]
|
||||
|
||||
1.14.0 - 5 Aug 2024
|
||||
- Bump pf4j to version 3.12.0 [96117b9a]
|
||||
- Make Google Batch auto retry codes configurable (#5148) [e562ce06]
|
||||
- Improve Google Batch 5000x error class handling (#5141) [61b2205f]
|
||||
|
||||
1.13.2-patch1 - 9 Jul 2024
|
||||
- Fix normalization of consecutive slashes in uri path (#5114) [3f366b7e]
|
||||
|
||||
1.13.4 - 8 Jul 2024
|
||||
- Fix normalization of consecutive slashes in uri path (#5114) [18ec484f]
|
||||
- Bump groovy 4.0.22 [284a6606]
|
||||
|
||||
1.13.3 - 17 Jun 2024
|
||||
- Improve retry strategy for Google cloud errors when writing task helper files (#5037) [f8b324ab]
|
||||
|
||||
1.13.2 - 20 May 2024
|
||||
- Fix nf-google plugin dependency [725e2860]
|
||||
|
||||
1.13.1 - 20 May 2024
|
||||
- Use protected visibility for updateStatus method [6871ba06]
|
||||
|
||||
1.13.0 - 13 May 2024
|
||||
- Add support for Job arrays (#3892) [ca9bc9d4]
|
||||
|
||||
1.12.0 - 15 Apr 2024
|
||||
- Add custom jobName for Google Batch [df40d55f]
|
||||
- Add retry policy to Google Batch client [c4981dcc]
|
||||
- Improve error message when Google creds file is corrupted [a550e52f]
|
||||
- Fix support for GCS requester pays bucket option [d9d61cff]
|
||||
- Fix failing CI tests (#4861) [1c0e648e]
|
||||
- Bump groovy 4.0.21 [9e08390b]
|
||||
|
||||
1.11.0 - 5 Feb 2024
|
||||
- Bump Groovy 4 (#4443) [9d32503b]
|
||||
|
||||
1.10.0 - 20 Dec 2023
|
||||
- Add ability to disable Cloudinfo service (#4606) [f7251895]
|
||||
- Add support for Instance template to Google Batch [df7ed294]
|
||||
- Improve GLS tests [58590b1c]
|
||||
|
||||
1.9.0 - 24 Nov 2023
|
||||
- Add labels field in Job request for Google Batch (#4538) [627c595e]
|
||||
- Add Google Batch native retry on spot termination (#4500) [ea1c1b70]
|
||||
- Add ability detect Google Batch spot interruption (#4462) [d49f02ae]
|
||||
- Add Retry policy to Google Storage (#4524) [c271bb18]
|
||||
- Fix security vulnerabilities (#4513) [a310c777]
|
||||
- Fix Bypass Google Batch Price query if task cpus and memory are defined (#4521) [7f8f20d3]
|
||||
- Update logging filter for Google Batch provider. (#4488) [66a3ed19]
|
||||
|
||||
1.8.3-patch2 - 11 Jun 2024
|
||||
- Fix security vulnerabilities (#5057) [6d8765b8]
|
||||
|
||||
1.8.3-patch1 - 28 May 2024
|
||||
- Bump dependency with Nextflow 23.10.2
|
||||
|
||||
1.8.3 - 10 Oct 2023
|
||||
- Add setting to enable the use of sync command [f0d5cc5c]
|
||||
- Fix Google Batch do not stop running jobs (#4381) [3d6b7358]
|
||||
|
||||
1.8.2 - 28 Sep 2023
|
||||
- Fix allow_other vulnerability preventing google-batch submissions (#4332) [9b3741e3]
|
||||
|
||||
1.8.1 - 22 Jul 2023
|
||||
- Wait for all child processes in nxf_parallel (#4050) [60a5f1a7]
|
||||
- Bump Groovy 3.0.18 [207eb535]
|
||||
|
||||
1.8.0 - 14 Jun 2023
|
||||
- Add httpConnectTimeout and httpReadTimeout to Google options (#3974) [49fa15f7]
|
||||
- Add disk resource with type option for Google Batch (#3861) [166b3638]
|
||||
- Prevent null exit code when Google batch is unable to access exit status [f68a39ec]
|
||||
- Fix S3 path normalization [b75ec444]
|
||||
- Fix invalid machine type setting when no valid machine type is found (#3961) [5eb93971]
|
||||
- Fix Google Batch default instance family types (#3960) [b5257cd7]
|
||||
|
||||
1.7.4 - 15 May 2023
|
||||
- Update logging libraries [d7eae86e]
|
||||
- Improve task out redirect remove the use of mkfifo (#3863) [efedec74]
|
||||
- Bump groovy 3.0.17 [cfe4ba56]
|
||||
|
||||
1.7.3 - 15 Apr 2023
|
||||
- Bump gson:2.10.1 [83ca1e32]
|
||||
|
||||
1.7.2 - 1 Apr 2023
|
||||
- Fix issue checking google batch script launcher type [39c3a517]
|
||||
- Fix NoSuchMethodError String.stripIndent with Java 11 [308eafe6]
|
||||
|
||||
1.7.1 - 23 Mar 2023
|
||||
- Fix Google Batch logging exception [d7e38e9e]
|
||||
- Add error message for missing container image with Google Batch (#3747) [6419e68f]
|
||||
- Bump groovy 3.0.16 [d3ff5dcb]
|
||||
|
||||
1.7.0 - 21 Feb 2023
|
||||
- Add Fusion support for Google Batch (#3577) [d5fbab4f]
|
||||
- Add Header provider to Google Batch client [20979929]
|
||||
- Bump groovy 3.0.15 [7a3ebc7d]
|
||||
|
||||
1.6.0 - 14 Jan 2023
|
||||
- Refactor Google Batch executor [c0a25fc2]
|
||||
- Improve container native executor configuration [03126371]
|
||||
- Bump groovy 3.0.14 [7c204236]
|
||||
|
||||
1.5.0 - 13 Dec 2022
|
||||
- Add allowedLocations option to google batch (#3453) [c619eb81]
|
||||
- Add warning on Google Logs failure [bdbcdde9]
|
||||
- Fix Quote the logName in the Cloud Logging filter (#3464) [b3975063]
|
||||
- Fix a few issues in BatchLogging.groovy (#3443) [e2bbcf15]
|
||||
- Fix Error & info messages, code comments language fixes (#3475) [29ae36ca]
|
||||
- Bump nf-google@1.5.0 [c07dcec2]
|
||||
|
||||
1.4.5 - 13 Nov 2022
|
||||
- Fix support for serviceAccountEmail and GPU accelerator [7f7007a8] #7f7007a8
|
||||
- Bump Google Batch sdk 0.5.0
|
||||
|
||||
1.4.4 - 26 Sep 2022
|
||||
- Update Google Batch mount point with the requirements [5aec28ac]
|
||||
- Apply GCP resourceLabels to the VirtualMachine (#3234) [2275c03c] <Doug Daniels>
|
||||
|
||||
1.4.3 - 22 Sep 2022
|
||||
- Add shutdown to Google Batch client [8f413cf7]
|
||||
- Add native_id to Google Batch handler [352b4239]
|
||||
- Bump groovy 3.0.13 [4a17e198]
|
||||
|
||||
1.4.2 - 1 Sep 2022
|
||||
- Add support for resource labels for google batch (#3168) (#3170) [2d24917b] (#2853) [5d0b7c35]
|
||||
- Add support for project resources [c2ad6566]
|
||||
- Bump google-cloud-batch 0.2.2 [2f5716da]
|
||||
- Get rid of remote bin dir [6cfb51e7]
|
||||
|
||||
1.4.1 - 11 Aug 2022
|
||||
- Change google batch disk directive to override boot disk size (#3097) [7e1c0686]
|
||||
- Fix Allow disabling scratch with Google Batch [e8e5c721]>
|
||||
|
||||
1.4.0 - 1 Aug 2022
|
||||
- Report warning when Google Batch quota is exceeded (#3066) [6b9c52ad] <Ben Sherman>
|
||||
- Add boot disk, cpu platform to google batch (#3058) [17a8483d] <Ben Sherman>
|
||||
- Add support for GPU accelerator to Google Batch (#3056) [f34ad7f6] <Ben Sherman>
|
||||
- Add disk directive to google batch (#3057) [ec6e290c] <Ben Sherman>
|
||||
- Refactor google batch executor to use java api (#3044) [31a6e85c] <Ben Sherman>
|
||||
- Bump google-cloud-nio:0.124.8 [dfaa9d19] <Paolo Di Tommaso>
|
||||
|
||||
1.3.0 - 13 Jul 2022
|
||||
- Add support for Google Batch API v1 [4c116d58] [e85d87ee]
|
||||
|
||||
1.2.0 - 25 May 2022
|
||||
- Add support for job timeout to Google LifeSciences executor
|
||||
|
||||
1.1.3 - 22 Nov 2021
|
||||
- Downgrade Google NIO library to version 0.121.2
|
||||
|
||||
1.1.2 - 28 Oct 2021
|
||||
- Fix task temporary files cleanup
|
||||
|
||||
1.1.1 - 12 Oct 2021
|
||||
- Fix NPE exception on configuration failure
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package nextflow.cloud.google
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.plugin.BasePlugin
|
||||
import org.pf4j.PluginWrapper
|
||||
/**
|
||||
* Implement the plugin entry point for Google Cloud support
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class GoogleCloudPlugin extends BasePlugin {
|
||||
|
||||
GoogleCloudPlugin(PluginWrapper wrapper) {
|
||||
super(wrapper)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud.google
|
||||
|
||||
import com.google.auth.oauth2.GoogleCredentials
|
||||
import groovy.json.JsonSlurper
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.Memoized
|
||||
import groovy.transform.ToString
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.Session
|
||||
import nextflow.SysEnv
|
||||
import nextflow.cloud.google.batch.client.BatchConfig
|
||||
import nextflow.cloud.google.config.GoogleStorageOpts
|
||||
import nextflow.config.spec.ConfigOption
|
||||
import nextflow.config.spec.ConfigScope
|
||||
import nextflow.config.spec.ScopeName
|
||||
import nextflow.script.dsl.Description
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.util.Duration
|
||||
/**
|
||||
* Model Google config options
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@ScopeName("google")
|
||||
@Description("""
|
||||
The `google` scope allows you to configure the interactions with Google Cloud, including Google Cloud Batch and Google Cloud Storage.
|
||||
""")
|
||||
@Slf4j
|
||||
@ToString(includeNames = true, includePackage = false)
|
||||
@CompileStatic
|
||||
class GoogleOpts implements ConfigScope {
|
||||
|
||||
static final public String DEFAULT_LOCATION = 'us-central1'
|
||||
|
||||
static Map<String,String> env = SysEnv.get()
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The Google Cloud project ID to use for pipeline execution.
|
||||
""")
|
||||
String project
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The Google Cloud location where jobs are executed (default: `us-central1`).
|
||||
""")
|
||||
final String location
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Use the given Google Cloud project ID as the billing project for storage access (default: `false`). Required when accessing data from [requester pays](https://cloud.google.com/storage/docs/requester-pays) buckets.
|
||||
""")
|
||||
final boolean enableRequesterPaysBuckets
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The HTTP connection timeout for Cloud Storage API requests (default: `'60s'`).
|
||||
""")
|
||||
final Duration httpConnectTimeout
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The HTTP read timeout for Cloud Storage API requests (default: `'60s'`).
|
||||
""")
|
||||
final Duration httpReadTimeout
|
||||
|
||||
final BatchConfig batch
|
||||
|
||||
final GoogleStorageOpts storage
|
||||
|
||||
private File credsFile
|
||||
|
||||
String getProjectId() { project }
|
||||
GoogleStorageOpts getStorageOpts() { storage }
|
||||
|
||||
/* required by extension point -- do not remove */
|
||||
GoogleOpts() {}
|
||||
|
||||
GoogleOpts(Map opts) {
|
||||
project = opts.project
|
||||
location = opts.location ?: DEFAULT_LOCATION
|
||||
enableRequesterPaysBuckets = opts.enableRequesterPaysBuckets as boolean
|
||||
httpConnectTimeout = opts.httpConnectTimeout ? opts.httpConnectTimeout as Duration : Duration.of('60s')
|
||||
httpReadTimeout = opts.httpReadTimeout ? opts.httpReadTimeout as Duration : Duration.of('60s')
|
||||
batch = new BatchConfig( opts.batch as Map ?: Collections.emptyMap() )
|
||||
storage = new GoogleStorageOpts( opts.storage as Map ?: Collections.emptyMap() )
|
||||
}
|
||||
|
||||
@Memoized
|
||||
static GoogleOpts fromSession(Session session) {
|
||||
try {
|
||||
return fromSession0(session.config)
|
||||
}
|
||||
catch (Exception e) {
|
||||
if(session) session.abort()
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
protected static GoogleOpts fromSession0(Map config) {
|
||||
final result = new GoogleOpts( config.google as Map ?: Collections.emptyMap() )
|
||||
|
||||
if( result.enableRequesterPaysBuckets && !result.projectId )
|
||||
throw new IllegalArgumentException("Config option 'google.enableRequesterPaysBuckets' cannot be honoured because the Google project Id has not been specified - Provide it by adding the option 'google.project' in the nextflow.config file")
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
static protected String getProjectIdFromCreds(String credsFilePath) {
|
||||
if( !credsFilePath )
|
||||
throw new AbortOperationException('Missing Google credentials -- make sure your environment defines the GOOGLE_APPLICATION_CREDENTIALS environment variable')
|
||||
|
||||
final file = new File(credsFilePath)
|
||||
try {
|
||||
final creds = (Map)new JsonSlurper().parse(file)
|
||||
if( creds.project_id )
|
||||
return creds.project_id
|
||||
else
|
||||
throw new AbortOperationException("Missing `project_id` in Google credentials file: $credsFilePath")
|
||||
}
|
||||
catch(FileNotFoundException e) {
|
||||
throw new AbortOperationException("Missing Google credentials file: $credsFilePath")
|
||||
}
|
||||
catch (Exception e) {
|
||||
throw new AbortOperationException("Invalid or corrupted Google credentials file: $credsFilePath", e)
|
||||
}
|
||||
}
|
||||
|
||||
static GoogleOpts create(Session session) {
|
||||
// Typically the credentials picked up are the "Application Default Credentials"
|
||||
// as described at:
|
||||
// https://github.com/googleapis/google-auth-library-java
|
||||
//
|
||||
// In that case, the project ID needs to be set in the nextflow config file.
|
||||
|
||||
// If instead, the GOOGLE_APPLICATION_CREDENTIALS environment variable is set,
|
||||
// then the project ID will be picked up (along with the credentials) from the
|
||||
// JSON file that environment variable points to.
|
||||
|
||||
final config = fromSession(session)
|
||||
|
||||
def projectId
|
||||
def credsPath = env.get('GOOGLE_APPLICATION_CREDENTIALS')
|
||||
if( credsPath && (projectId = getProjectIdFromCreds(credsPath)) ) {
|
||||
config.credsFile = new File(credsPath)
|
||||
if( !config.project )
|
||||
config.project = projectId
|
||||
else if( config.project != projectId )
|
||||
throw new AbortOperationException("Project Id `$config.project` declared in the nextflow config file does not match the one expected by credentials file: $credsPath")
|
||||
}
|
||||
|
||||
if( !config.project ) {
|
||||
throw new AbortOperationException("Missing Google project Id -- Specify it adding the setting `google.project='your-project-id'` in the nextflow.config file")
|
||||
}
|
||||
|
||||
return config
|
||||
}
|
||||
|
||||
@Memoized // make memoized to prevent multiple access to the creds file
|
||||
GoogleCredentials getCredentials() {
|
||||
return makeCreds(credsFile)
|
||||
}
|
||||
|
||||
static protected GoogleCredentials makeCreds(File credsFile) {
|
||||
GoogleCredentials result
|
||||
if( credsFile ) {
|
||||
log.debug "Google auth via application credentials file: $credsFile"
|
||||
result = GoogleCredentials.fromStream(new FileInputStream(credsFile))
|
||||
}
|
||||
else {
|
||||
log.debug "Google auth via application DEFAULT"
|
||||
result = GoogleCredentials.getApplicationDefault()
|
||||
}
|
||||
return result.createScoped("https://www.googleapis.com/auth/cloud-platform")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud.google.batch
|
||||
|
||||
import static nextflow.cloud.google.batch.GoogleBatchScriptLauncher.*
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
import com.google.cloud.storage.contrib.nio.CloudStoragePath
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.SysEnv
|
||||
import nextflow.cloud.google.GoogleOpts
|
||||
import nextflow.cloud.google.batch.client.BatchClient
|
||||
import nextflow.cloud.google.batch.client.BatchConfig
|
||||
import nextflow.cloud.google.batch.logging.BatchLogging
|
||||
import nextflow.exception.AbortOperationException
|
||||
import nextflow.executor.Executor
|
||||
import nextflow.executor.TaskArrayExecutor
|
||||
import nextflow.extension.FilesEx
|
||||
import nextflow.fusion.FusionHelper
|
||||
import nextflow.processor.TaskHandler
|
||||
import nextflow.processor.TaskMonitor
|
||||
import nextflow.processor.TaskPollingMonitor
|
||||
import nextflow.processor.TaskRun
|
||||
import nextflow.util.Duration
|
||||
import nextflow.util.Escape
|
||||
import nextflow.util.ServiceName
|
||||
import org.pf4j.ExtensionPoint
|
||||
/**
|
||||
* Implements support for Google Batch
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@ServiceName(value='google-batch')
|
||||
@CompileStatic
|
||||
class GoogleBatchExecutor extends Executor implements ExtensionPoint, TaskArrayExecutor {
|
||||
|
||||
private BatchClient client
|
||||
private GoogleOpts googleOpts
|
||||
private Path remoteBinDir
|
||||
private BatchLogging logging
|
||||
|
||||
private final Set<String> deletedJobs = new HashSet<>()
|
||||
|
||||
BatchClient getClient() { return client }
|
||||
GoogleOpts getGoogleOpts() { return googleOpts }
|
||||
BatchConfig getBatchConfig() { return googleOpts.batch }
|
||||
Path getRemoteBinDir() { return remoteBinDir }
|
||||
BatchLogging getLogging() { logging }
|
||||
|
||||
@Override
|
||||
final boolean isContainerNative() {
|
||||
return true
|
||||
}
|
||||
|
||||
@Override
|
||||
String containerConfigEngine() {
|
||||
return 'docker'
|
||||
}
|
||||
|
||||
@Override
|
||||
final Path getWorkDir() {
|
||||
return session.bucketDir ?: session.workDir
|
||||
}
|
||||
|
||||
protected void validateWorkDir() {
|
||||
if ( getWorkDir()?.scheme != 'gs' ) {
|
||||
session.abort()
|
||||
throw new AbortOperationException("Executor `google-batch` requires a Google Storage bucket to be specified as a working directory -- Add the option `-w gs://<your-bucket/path>` to your run command line or specify a workDir in your config file")
|
||||
}
|
||||
}
|
||||
|
||||
protected void uploadBinDir() {
|
||||
if( session.binDir && !session.binDir.empty() && !session.disableRemoteBinDir ) {
|
||||
final cloudPath = getTempDir()
|
||||
log.info "Uploading local `bin` scripts folder to ${cloudPath.toUriString()}/bin"
|
||||
this.remoteBinDir = FilesEx.copyTo(session.binDir, cloudPath)
|
||||
}
|
||||
}
|
||||
|
||||
protected void createConfig() {
|
||||
this.googleOpts = GoogleOpts.create(session)
|
||||
log.debug "[GOOGLE BATCH] Executor config=$googleOpts"
|
||||
}
|
||||
|
||||
protected void createClient() {
|
||||
this.client = new BatchClient(googleOpts)
|
||||
this.logging = new BatchLogging(googleOpts)
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void register() {
|
||||
super.register()
|
||||
createConfig()
|
||||
validateWorkDir()
|
||||
uploadBinDir()
|
||||
createClient()
|
||||
}
|
||||
|
||||
@Override
|
||||
protected TaskMonitor createTaskMonitor() {
|
||||
TaskPollingMonitor.create(session, config, name, 1000, Duration.of('10 sec'))
|
||||
}
|
||||
|
||||
@Override
|
||||
TaskHandler createTaskHandler(TaskRun task) {
|
||||
return new GoogleBatchTaskHandler(task, this)
|
||||
}
|
||||
|
||||
@Override
|
||||
void shutdown() {
|
||||
client.shutdown()
|
||||
logging.close()
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean isFusionEnabled() {
|
||||
return FusionHelper.isFusionEnabled(session)
|
||||
}
|
||||
|
||||
boolean isCloudinfoEnabled() {
|
||||
return Boolean.parseBoolean(SysEnv.get('NXF_CLOUDINFO_ENABLED', 'true') )
|
||||
}
|
||||
|
||||
boolean shouldDeleteJob(String jobId) {
|
||||
if( jobId in deletedJobs ) {
|
||||
// if the job is already in the list it has been already deleted
|
||||
return false
|
||||
}
|
||||
synchronized (deletedJobs) {
|
||||
// add the job id to the set of deleted jobs, if it's a new id, the `add` method
|
||||
// returns true therefore the job should be deleted
|
||||
return deletedJobs.add(jobId)
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
String getArrayIndexName() {
|
||||
return 'BATCH_TASK_INDEX'
|
||||
}
|
||||
|
||||
@Override
|
||||
int getArrayIndexStart() {
|
||||
return 0
|
||||
}
|
||||
|
||||
@Override
|
||||
String getArrayTaskId(String jobId, int index) {
|
||||
return index.toString()
|
||||
}
|
||||
|
||||
@Override
|
||||
String getArrayWorkDir(TaskHandler handler) {
|
||||
return isFusionEnabled() || isWorkDirDefaultFS()
|
||||
? TaskArrayExecutor.super.getArrayWorkDir(handler)
|
||||
: containerMountPath(handler.task.workDir as CloudStoragePath)
|
||||
}
|
||||
|
||||
@Override
|
||||
String getArrayLaunchCommand(String taskDir) {
|
||||
if( isFusionEnabled() || isWorkDirDefaultFS() ) {
|
||||
return TaskArrayExecutor.super.getArrayLaunchCommand(taskDir)
|
||||
}
|
||||
else {
|
||||
final cmd = List.of('/bin/bash','-o','pipefail','-c', launchCommand(taskDir))
|
||||
return Escape.cli(cmd as String[])
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud.google.batch
|
||||
|
||||
|
||||
import com.google.cloud.batch.v1.Volume
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.fusion.FusionAwareTask
|
||||
import nextflow.fusion.FusionScriptLauncher
|
||||
/**
|
||||
* A mere adapter for {@link GoogleBatchLauncherSpec} for fusion
|
||||
* launch
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class GoogleBatchFusionAdapter implements GoogleBatchLauncherSpec {
|
||||
|
||||
private FusionAwareTask task
|
||||
|
||||
private FusionScriptLauncher launcher
|
||||
|
||||
GoogleBatchFusionAdapter(FusionAwareTask task, FusionScriptLauncher launcher) {
|
||||
this.task = task
|
||||
this.launcher = launcher
|
||||
}
|
||||
|
||||
@Override
|
||||
List<String> getContainerMounts() {
|
||||
return List.of()
|
||||
}
|
||||
|
||||
@Override
|
||||
List<Volume> getVolumes() {
|
||||
return List.of()
|
||||
}
|
||||
|
||||
@Override
|
||||
String runCommand() {
|
||||
throw new UnsupportedOperationException()
|
||||
}
|
||||
|
||||
@Override
|
||||
List<String> launchCommand() {
|
||||
return task.fusionSubmitCli()
|
||||
}
|
||||
|
||||
@Override
|
||||
Map<String, String> getEnvironment() {
|
||||
return launcher.fusionEnv()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud.google.batch
|
||||
|
||||
import com.google.cloud.batch.v1.Volume
|
||||
|
||||
/**
|
||||
* Defines the operation supported by Google Batch tasks launcher
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
interface GoogleBatchLauncherSpec {
|
||||
|
||||
/**
|
||||
* @return
|
||||
* A list of string representing the container mounts. Each mount uses the docker
|
||||
* mount conventional syntax e.g. {@code /mnt/disks/foo/scratch:/mnt/disks/foo/scratch:rw}
|
||||
*/
|
||||
List<String> getContainerMounts()
|
||||
|
||||
/**
|
||||
* @return A list of Batch {@link Volume} to be made accessible to the container
|
||||
*/
|
||||
List<Volume> getVolumes()
|
||||
|
||||
/**
|
||||
* @return A string representing the command to be executed by the containerised task
|
||||
*/
|
||||
String runCommand()
|
||||
|
||||
default List<String> launchCommand() {
|
||||
return ['/bin/bash','-o','pipefail','-c', runCommand() ]
|
||||
}
|
||||
|
||||
default Map<String,String> getEnvironment() {
|
||||
return Map.of()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,382 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package nextflow.cloud.google.batch
|
||||
|
||||
import io.seqera.cloudinfo.api.CloudPrice
|
||||
import io.seqera.cloudinfo.client.CloudInfoClient
|
||||
|
||||
import java.math.RoundingMode
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.Immutable
|
||||
import groovy.transform.Memoized
|
||||
import nextflow.cloud.types.PriceModel
|
||||
import nextflow.util.MemoryUnit
|
||||
|
||||
/**
|
||||
* Choose best machine type that fits the requested resources and
|
||||
* reduces the estimated cost on current location.
|
||||
*
|
||||
* Average spot and on-demand prices are requested once per execution
|
||||
* from Seqera cloud info API.
|
||||
*
|
||||
* If cloud info service is not available it will fallback to use
|
||||
* the user provided machine type or default to Google Batch automatic
|
||||
* selection from the requested resource.
|
||||
*
|
||||
* @author Jordi Deu-Pons <jordi@jordeu.net>
|
||||
*/
|
||||
@CompileStatic
|
||||
class GoogleBatchMachineTypeSelector {
|
||||
|
||||
static GoogleBatchMachineTypeSelector INSTANCE = new GoogleBatchMachineTypeSelector()
|
||||
|
||||
/*
|
||||
* Some families CPUs are faster so this is a cost correction factor
|
||||
* for processes that request more than 2 CPUs or 2GB, smaller processes
|
||||
* we assume that do not have high CPU usage.
|
||||
* https://cloud.google.com/compute/docs/cpu-platforms
|
||||
*/
|
||||
private static final Map<String, BigDecimal> FAMILY_COST_CORRECTION = [
|
||||
'e2' : 1.0, // Mix of processors, tend to be similar in performance to N1
|
||||
|
||||
// INTEL
|
||||
'n1' : 1.0, // Skylake, Broadwell, Haswell, Sandy Bridge, and Ivy Bridge ~2.7 Ghz
|
||||
'n2' : 0.85, // Intel Xeon Gold 6268CL ~3.4 Ghz
|
||||
'c2' : 0.8, // Intel Xeon Gold 6253CL ~3.8 Ghz
|
||||
'm1' : 1.0, // Intel Xeon E7-8880V4 ~2.7 Ghz
|
||||
'm2' : 0.85, // Intel Xeon Platinum 8280L ~3.4 Ghz
|
||||
'm3' : 0.85, // Intel Xeon Platinum 8373C ~3.4 Ghz
|
||||
'a2' : 0.9, // Intel Xeon Platinum 8273CL ~2.9 Ghz
|
||||
|
||||
// AMD
|
||||
't2d': 1.0, // AMD EPYC Milan ~2.7 Ghz
|
||||
'n2d': 1.0, // AMD EPYC Milan ~2.7 Ghz
|
||||
]
|
||||
|
||||
/*
|
||||
* Families that will be use as default if Fusion is enabled but no list is provided
|
||||
* https://cloud.google.com/compute/docs/disks#local_ssd_machine_type_restrictions
|
||||
* LAST UPDATE 2023-02-17
|
||||
*/
|
||||
private static final List<String> DEFAULT_FAMILIES_FOR_FUSION = ['n1-*', 'n2-*', 'n2d-*', 'c2-*', 'c2d-*', 'm3-*']
|
||||
|
||||
private static final List<String> DEFAULT_FAMILIES = ['n1-*', 'n2-*', 'n2d-*', 'c2-*', 'c2d-*', 'm1-*', 'm2-*', 'm3-*', 'e2-*']
|
||||
|
||||
/*
|
||||
* Accelerator optimized families. See: https://cloud.google.com/compute/docs/accelerator-optimized-machines
|
||||
* LAST UPDATE 2024-10-16
|
||||
*/
|
||||
private static final List<String> ACCELERATOR_OPTIMIZED_FAMILIES = ['a2-*', 'a3-*', 'g2-*']
|
||||
|
||||
/*
|
||||
* Families that only support Hyperdisk disk types (not pd-standard, pd-balanced, pd-ssd).
|
||||
* These require 'hyperdisk-*' as boot disk type.
|
||||
* https://docs.cloud.google.com/compute/docs/general-purpose-machines?hl=en#supported_disk_types_for_c4
|
||||
*/
|
||||
private static final List<String> HYPERDISK_ONLY_FAMILIES = ['c4-*', 'c4a-*', 'c4d-*', 'n4-*', 'n4a-*', 'n4d-*', 'z3-*']
|
||||
/*
|
||||
* Families that do not support Local SSD
|
||||
*/
|
||||
private static final List<String> PD_ONLY_FAMILIES = ['e2-*']
|
||||
/*
|
||||
* Families that do not support Local SSD
|
||||
*/
|
||||
private static final List<String> NO_LOCAL_SSD_SUPPORT_FAMILIES = ['e2-*', 'h3-*', 'm2-*', 'm4-*', 'n4-*', 't2a-*', 't2d-*', 'x4-*']
|
||||
/*
|
||||
* Families that support local SSD with 'lssd' suffix
|
||||
*/
|
||||
private static final List<String> PARTIAL_LOCAL_SSD_SUPPORT_FAMILIES = ['c3-*', 'c3a-*', 'c3d-*', 'c4-*', 'c4a-*', 'c4d-*', 'h4d-*', 'z3-*']
|
||||
|
||||
private CloudInfoClient cloudInfo
|
||||
|
||||
GoogleBatchMachineTypeSelector(){
|
||||
cloudInfo = CloudInfoClient.create()
|
||||
}
|
||||
|
||||
@Immutable
|
||||
static class MachineType {
|
||||
String type
|
||||
String family
|
||||
String location
|
||||
float spotPrice
|
||||
float onDemandPrice
|
||||
int cpusPerVm
|
||||
int memPerVm
|
||||
int gpusPerVm
|
||||
PriceModel priceModel
|
||||
}
|
||||
|
||||
MachineType bestMachineType(int cpus, int memoryMB, String region, boolean spot, boolean fusionEnabled, List<String> families) {
|
||||
if (families == null)
|
||||
families = Collections.<String>emptyList()
|
||||
|
||||
// Check if a specific machine type was defined
|
||||
if (families.size() == 1) {
|
||||
final familyOrType = families.get(0)
|
||||
if (familyOrType.contains("custom-"))
|
||||
return new MachineType(type: familyOrType, family: 'custom', cpusPerVm: cpus, memPerVm: memoryMB, gpusPerVm: 0, location: region, priceModel: spot ? PriceModel.spot : PriceModel.standard)
|
||||
|
||||
final machineType = getAvailableMachineTypes(region, spot).find { it.type == familyOrType }
|
||||
if( machineType )
|
||||
return machineType
|
||||
}
|
||||
|
||||
final memoryGB = Math.ceil(memoryMB / 1024.0 as float) as int
|
||||
|
||||
if (!families ) {
|
||||
families = fusionEnabled
|
||||
? DEFAULT_FAMILIES_FOR_FUSION
|
||||
: DEFAULT_FAMILIES
|
||||
}
|
||||
|
||||
// All types are valid if no families are defined, otherwise at least it has to start with one of the given values
|
||||
final matchMachineType = {String type -> !families || families.find { matchType(it, type) }}
|
||||
|
||||
// find machines with enough resources and SSD local disk
|
||||
def validMachineTypes = getAvailableMachineTypes(region, spot).findAll {
|
||||
it.cpusPerVm >= cpus &&
|
||||
it.memPerVm >= memoryGB &&
|
||||
matchMachineType(it.type)
|
||||
}.collect()
|
||||
if (fusionEnabled)
|
||||
validMachineTypes = validMachineTypes.findAll { hasLocalSsd(it.type)}.collect()
|
||||
|
||||
final sortedByCost = validMachineTypes.sort {
|
||||
(it.cpusPerVm > 2 || it.memPerVm > 2 ? FAMILY_COST_CORRECTION.get(it.family, 1.0) : 1.0) * (spot ? it.spotPrice : it.onDemandPrice)
|
||||
}
|
||||
|
||||
return sortedByCost.first()
|
||||
}
|
||||
|
||||
protected static boolean matchType(String family, String vmType) {
|
||||
if (!family)
|
||||
return true
|
||||
if (family.contains('*'))
|
||||
family = family.toLowerCase().replaceAll(/\*/, '.*')
|
||||
if (family.contains('?'))
|
||||
family = family.toLowerCase().replaceAll(/\?/, '.{1}')
|
||||
|
||||
return vmType =~ /(?i)^${family}$/
|
||||
}
|
||||
|
||||
@Memoized
|
||||
protected List<MachineType> getAvailableMachineTypes(String region, boolean spot) {
|
||||
final priceModel = spot ? PriceModel.spot : PriceModel.standard
|
||||
final products = cloudInfo.getProducts('google', region)
|
||||
final averageSpotPrice = (List<CloudPrice> prices) -> prices ? prices.collect{it.price as float}.average() as float : 0.0f
|
||||
|
||||
products.collect {
|
||||
new MachineType(
|
||||
type: it.type,
|
||||
family: it.type.toString().split('-')[0],
|
||||
spotPrice: averageSpotPrice(it.spotPrice as List<CloudPrice>),
|
||||
onDemandPrice: it.onDemandPrice as float,
|
||||
cpusPerVm: it.cpusPerVm as int,
|
||||
memPerVm: it.memPerVm as int,
|
||||
gpusPerVm: it.gpusPerVm as int,
|
||||
location: region,
|
||||
priceModel: priceModel
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Find valid local SSD size. See: https://cloud.google.com/compute/docs/disks#local_ssd_machine_type_restrictions
|
||||
*
|
||||
* @param requested Amount of disk requested
|
||||
* @param machineType Machine type
|
||||
* @return Next greater multiple of 375 GB that is a valid size for the given machine type
|
||||
*/
|
||||
protected MemoryUnit findValidLocalSSDSize(MemoryUnit requested, MachineType machineType) {
|
||||
|
||||
if( machineType.family == "n1" )
|
||||
return findFirstValidSize(requested, [1,2,3,4,5,6,7,8,16,24])
|
||||
|
||||
if( machineType.family == "n2" ) {
|
||||
if( machineType.cpusPerVm < 12 )
|
||||
return findFirstValidSize(requested, [1,2,4,8,16,24])
|
||||
if( machineType.cpusPerVm < 22 )
|
||||
return findFirstValidSize(requested, [2,4,8,16,24])
|
||||
if( machineType.cpusPerVm < 42 )
|
||||
return findFirstValidSize(requested, [4,8,16,24])
|
||||
if( machineType.cpusPerVm < 82 )
|
||||
return findFirstValidSize(requested, [8,16,24])
|
||||
return findFirstValidSize(requested, [16,24])
|
||||
}
|
||||
|
||||
if( machineType.family == "n2d" ) {
|
||||
if( machineType.cpusPerVm < 32 )
|
||||
return findFirstValidSize(requested, [1,2,4,8,16,24])
|
||||
if( machineType.cpusPerVm < 64 )
|
||||
return findFirstValidSize(requested, [2,4,8,16,24])
|
||||
if( machineType.cpusPerVm < 96 )
|
||||
return findFirstValidSize(requested, [4,8,16,24])
|
||||
return findFirstValidSize(requested, [8,16,24])
|
||||
}
|
||||
|
||||
if( machineType.family == "c2" ) {
|
||||
if( machineType.cpusPerVm < 16 )
|
||||
return findFirstValidSize(requested, [1,2,4,8])
|
||||
if( machineType.cpusPerVm < 30 )
|
||||
return findFirstValidSize(requested, [2,4,8])
|
||||
if( machineType.cpusPerVm < 60 )
|
||||
return findFirstValidSize(requested, [4,8])
|
||||
return findFirstValidSize(requested, [8])
|
||||
}
|
||||
|
||||
if( machineType.family == "c2d" ) {
|
||||
if( machineType.cpusPerVm < 32 )
|
||||
return findFirstValidSize(requested, [1,2,4,8])
|
||||
if( machineType.cpusPerVm < 56 )
|
||||
return findFirstValidSize(requested, [2,4,8])
|
||||
if( machineType.cpusPerVm < 112 )
|
||||
return findFirstValidSize(requested, [4,8])
|
||||
return findFirstValidSize(requested, [8])
|
||||
}
|
||||
|
||||
if( machineType.family == "m3" ) {
|
||||
if ( machineType.type == 'm3-megamem-128' || machineType.type == 'm3-ultramem-128' )
|
||||
return findFirstValidSize(requested, [8])
|
||||
return findFirstValidSize(requested, [4,8])
|
||||
}
|
||||
|
||||
if( machineType.family == "a2" ) {
|
||||
if ( machineType.type == 'a2-highgpu-1g' )
|
||||
return findFirstValidSize(requested, [1, 2, 4, 8])
|
||||
if ( machineType.type == 'a2-highgpu-2g' )
|
||||
return findFirstValidSize(requested, [2, 4, 8])
|
||||
if ( machineType.type == 'a2-highgpu-4g' )
|
||||
return findFirstValidSize(requested, [4, 8])
|
||||
if ( machineType.type == 'a2-highgpu-8g' || machineType.type == 'a2-megagpu-16g' )
|
||||
return findFirstValidSize(requested, [8])
|
||||
}
|
||||
|
||||
if( machineType.family == "g2" ) {
|
||||
if( machineType.type == 'g2-standard-4' || machineType.type == 'g2-standard-8' ||
|
||||
machineType.type == 'g2-standard-12' || machineType.type == 'g2-standard-16' ||
|
||||
machineType.type == 'g2-standard-32' )
|
||||
return findFirstValidSize(requested, [1])
|
||||
if( machineType.type == 'g2-standard-24' )
|
||||
return findFirstValidSize(requested, [2])
|
||||
if( machineType.type == 'g2-standard-48' )
|
||||
return findFirstValidSize(requested, [4])
|
||||
if( machineType.type == 'g2-standard-96' )
|
||||
return findFirstValidSize(requested, [8])
|
||||
}
|
||||
|
||||
if( notConfigurableLocalSSD(machineType) )
|
||||
return new MemoryUnit( 0 )
|
||||
|
||||
// For other special families, the user must provide a valid size. If a family does not
|
||||
// support local disks, then Google Batch shall return an appropriate error.
|
||||
return requested
|
||||
}
|
||||
|
||||
protected notConfigurableLocalSSD(MachineType machineType) {
|
||||
// These families have a local SSD already attached and is not configurable.
|
||||
return ((machineType.family == "c3" || machineType.family == "c3d") && machineType.type.endsWith("-lssd")) ||
|
||||
((machineType.family == "c4" || machineType.family == "c4a" || machineType.family == "c4d") && machineType.type.endsWith("-lssd")) ||
|
||||
machineType.family == "a3" ||
|
||||
machineType.type.startsWith("a2-ultragpu-")
|
||||
}
|
||||
|
||||
/**
|
||||
* Find first valid disk size given the possible mounted partition
|
||||
*
|
||||
* @param requested Requested disk size
|
||||
* @param allowedPartitions Valid number of disks of 375.GB.
|
||||
* @return
|
||||
*/
|
||||
protected MemoryUnit findFirstValidSize(MemoryUnit requested, List<Integer> allowedPartitions) {
|
||||
|
||||
// Sort the possible number of disks
|
||||
allowedPartitions.sort()
|
||||
|
||||
// Minimum number of 375.GB disks to fulfill the requested size
|
||||
final disks = (requested.toGiga() / 375).setScale(0, RoundingMode.UP).toInteger()
|
||||
|
||||
// Find first valid number of disks
|
||||
def numberOfDisks = allowedPartitions.find { it >= disks}
|
||||
if( !numberOfDisks )
|
||||
numberOfDisks = allowedPartitions.last()
|
||||
|
||||
return new MemoryUnit( numberOfDisks * 375L * (1<<30) )
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the machine type belongs to a family that only supports Hyperdisk.
|
||||
*
|
||||
* @param machineType Machine type
|
||||
* @return Boolean value indicating if the machine type requires Hyperdisk.
|
||||
*/
|
||||
static boolean isHyperdiskOnly(String machineType) {
|
||||
return HYPERDISK_ONLY_FAMILIES.any { matchType(it, machineType) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the machine type belongs to a family that only supports pd-* disk.
|
||||
*
|
||||
* @param machineType Machine type
|
||||
* @return Boolean value indicating if the machine type requires pd-* disk type.
|
||||
*/
|
||||
static boolean isPdOnly(String machineType) {
|
||||
return PD_ONLY_FAMILIES.any { matchType(it, machineType) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the machine type allow to have a local-ssd .
|
||||
*
|
||||
* @param machineType Machine type
|
||||
* @return Boolean value indicating if the machine type can have local ssd disks.
|
||||
*/
|
||||
static boolean hasLocalSsd(String machineType) {
|
||||
if( machineType.contains('lssd') )
|
||||
return true
|
||||
|
||||
if( PARTIAL_LOCAL_SSD_SUPPORT_FAMILIES.any { matchType(it, machineType) } )
|
||||
return false
|
||||
|
||||
if( NO_LOCAL_SSD_SUPPORT_FAMILIES.any { matchType(it, machineType) } )
|
||||
return false
|
||||
|
||||
return true
|
||||
}
|
||||
/**
|
||||
* Check if a machine type doesn't support
|
||||
* @param machineTypeOrFamily
|
||||
* @return
|
||||
*/
|
||||
static boolean unsupportedLocalSSD(String machineTypeOrFamily) {
|
||||
return NO_LOCAL_SSD_SUPPORT_FAMILIES.any { matchType(it, machineTypeOrFamily) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine whether GPU drivers should be installed.
|
||||
*
|
||||
* @param machineType Machine type
|
||||
* @return Boolean value indicating if GPU drivers should be installed.
|
||||
*/
|
||||
protected boolean installGpuDrivers(MachineType machineType) {
|
||||
if ( machineType.gpusPerVm > 0 ) {
|
||||
return true
|
||||
}
|
||||
// Cloud Info service currently does not currently return gpusPerVm values (or the user
|
||||
// could have disabled use of the service) so also check against a known set of families.
|
||||
return ACCELERATOR_OPTIMIZED_FAMILIES.any { matchType(it, machineType.type) }
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud.google.batch
|
||||
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
|
||||
import com.google.cloud.batch.v1.GCS
|
||||
import com.google.cloud.batch.v1.Volume
|
||||
import com.google.cloud.storage.contrib.nio.CloudStoragePath
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.cloud.google.GoogleOpts
|
||||
import nextflow.executor.BashWrapperBuilder
|
||||
import nextflow.extension.FilesEx
|
||||
import nextflow.processor.TaskBean
|
||||
import nextflow.processor.TaskRun
|
||||
import nextflow.util.Escape
|
||||
import nextflow.util.PathTrie
|
||||
import nextflow.util.TestOnly
|
||||
|
||||
/**
|
||||
* Implement Nextflow task launcher script
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class GoogleBatchScriptLauncher extends BashWrapperBuilder implements GoogleBatchLauncherSpec {
|
||||
|
||||
private static final String MOUNT_ROOT = '/mnt/disks'
|
||||
|
||||
private GoogleOpts config
|
||||
private CloudStoragePath remoteWorkDir
|
||||
private Path remoteBinDir
|
||||
private Set<String> buckets = new HashSet<>()
|
||||
private PathTrie pathTrie = new PathTrie()
|
||||
private boolean isArray
|
||||
|
||||
@TestOnly
|
||||
protected GoogleBatchScriptLauncher() {}
|
||||
|
||||
GoogleBatchScriptLauncher(TaskBean bean, Path remoteBinDir) {
|
||||
super(bean)
|
||||
// keep track the google storage work dir
|
||||
this.remoteWorkDir = (CloudStoragePath) bean.workDir
|
||||
this.remoteBinDir = toContainerMount(remoteBinDir)
|
||||
|
||||
// map bean work and target dirs to container mount
|
||||
// this is needed to create the command launcher using container local file paths
|
||||
bean.workDir = toContainerMount(bean.workDir)
|
||||
bean.targetDir = toContainerMount(bean.targetDir)
|
||||
|
||||
// add all children work dir
|
||||
if( bean.arrayWorkDirs ) {
|
||||
for( Path it : bean.arrayWorkDirs )
|
||||
toContainerMount(it)
|
||||
}
|
||||
|
||||
// add input file mounts
|
||||
if( bean.arrayInputFiles ) {
|
||||
for( Path it : bean.arrayInputFiles )
|
||||
toContainerMount(it)
|
||||
}
|
||||
|
||||
// remap input files to container mounted paths
|
||||
for( Map.Entry<String,Path> entry : new HashMap<>(bean.inputFiles).entrySet() ) {
|
||||
bean.inputFiles.put( entry.key, toContainerMount(entry.value, true) )
|
||||
}
|
||||
|
||||
// include task script as an input to force its staging in the container work directory
|
||||
bean.inputFiles[TaskRun.CMD_SCRIPT] = bean.workDir.resolve(TaskRun.CMD_SCRIPT)
|
||||
// add the wrapper file when stats are enabled
|
||||
// NOTE: this must match the logic that uses the run script in BashWrapperBuilder
|
||||
if( isTraceRequired() ) {
|
||||
bean.inputFiles[TaskRun.CMD_RUN] = bean.workDir.resolve(TaskRun.CMD_RUN)
|
||||
}
|
||||
// include task stdin file
|
||||
if( bean.input != null ) {
|
||||
bean.inputFiles[TaskRun.CMD_INFILE] = bean.workDir.resolve(TaskRun.CMD_INFILE)
|
||||
}
|
||||
|
||||
// make it change to the task work dir
|
||||
bean.headerScript = headerScript(bean)
|
||||
// enable use of local scratch dir
|
||||
if( scratch==null )
|
||||
scratch = true
|
||||
}
|
||||
|
||||
protected String headerScript(TaskBean bean) {
|
||||
def result = "NXF_CHDIR=${Escape.path(bean.workDir)}\n"
|
||||
if( remoteBinDir ) {
|
||||
result += "cp -r $remoteBinDir \$HOME/.nextflow-bin\n"
|
||||
result += 'chmod +x $HOME/.nextflow-bin/*\n'
|
||||
result += 'export PATH=$HOME/.nextflow-bin:$PATH\n'
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
protected Path toContainerMount(Path path, boolean parent=false) {
|
||||
if( path instanceof CloudStoragePath ) {
|
||||
buckets.add(path.bucket())
|
||||
pathTrie.add( (parent ? "/${path.bucket()}${path.parent}" : "/${path.bucket()}${path}").toString() )
|
||||
final containerMount = containerMountPath(path)
|
||||
log.trace "Path ${FilesEx.toUriString(path)} to container mount: $containerMount"
|
||||
return Paths.get(containerMount)
|
||||
}
|
||||
else if( path==null )
|
||||
return null
|
||||
throw new IllegalArgumentException("Unexpected path for Google Batch task handler: ${path.toUriString()}")
|
||||
}
|
||||
|
||||
@Override
|
||||
String runCommand() {
|
||||
return isArray
|
||||
? launchArrayCommand(workDirMount)
|
||||
: launchCommand(workDirMount)
|
||||
}
|
||||
|
||||
@Override
|
||||
List<String> getContainerMounts() {
|
||||
final result = new ArrayList(10)
|
||||
for( String it : pathTrie.longest() ) {
|
||||
result.add( "${MOUNT_ROOT}${it}:${MOUNT_ROOT}${it}:rw".toString() )
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
@Override
|
||||
List<Volume> getVolumes() {
|
||||
final result = new ArrayList(10)
|
||||
for( String it : buckets ) {
|
||||
final mountOptions = new LinkedList<String>()
|
||||
if( config && config.batch.gcsfuseOptions )
|
||||
mountOptions.addAll(config.batch.gcsfuseOptions)
|
||||
if( config && config.enableRequesterPaysBuckets )
|
||||
mountOptions.add("--billing-project ${config.projectId}".toString())
|
||||
|
||||
result.add(
|
||||
Volume.newBuilder()
|
||||
.setGcs(
|
||||
GCS.newBuilder()
|
||||
.setRemotePath(it)
|
||||
)
|
||||
.setMountPath( "${MOUNT_ROOT}/${it}".toString() )
|
||||
.addAllMountOptions( mountOptions )
|
||||
.build()
|
||||
)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
String getWorkDirMount() {
|
||||
return workDir.toString()
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Path targetWrapperFile() {
|
||||
return remoteWorkDir.resolve(TaskRun.CMD_RUN)
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Path targetScriptFile() {
|
||||
return remoteWorkDir.resolve(TaskRun.CMD_SCRIPT)
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Path targetInputFile() {
|
||||
return remoteWorkDir.resolve(TaskRun.CMD_INFILE)
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Path targetStageFile() {
|
||||
return remoteWorkDir.resolve(TaskRun.CMD_STAGE)
|
||||
}
|
||||
|
||||
GoogleBatchScriptLauncher withConfig(GoogleOpts config) {
|
||||
this.config = config
|
||||
addLogsBucket(config.batch.logsPath())
|
||||
return this
|
||||
}
|
||||
|
||||
protected void addLogsBucket(Path path) {
|
||||
if( path instanceof CloudStoragePath )
|
||||
buckets.add(path.bucket())
|
||||
else if( path != null )
|
||||
throw new IllegalArgumentException("Unexpected value for Google Batch logs path: ${path.toUriString()}")
|
||||
}
|
||||
|
||||
GoogleBatchScriptLauncher withIsArray(boolean value) {
|
||||
this.isArray = value
|
||||
return this
|
||||
}
|
||||
|
||||
static String launchArrayCommand(String workDir ) {
|
||||
// when executing a job array run directly the command script
|
||||
// to prevent that all child jobs write on the same .command.*
|
||||
// control files, causing an issue with the gcsfuse mount
|
||||
// For the same reason the .command.log file is not uploaded
|
||||
// See https://github.com/nextflow-io/nextflow/issues/5777
|
||||
"/bin/bash ${workDir}/${TaskRun.CMD_SCRIPT}"
|
||||
}
|
||||
|
||||
static String launchCommand( String workDir ) {
|
||||
"trap \"{ cp ${TaskRun.CMD_LOG} ${workDir}/${TaskRun.CMD_LOG}; }\" EXIT; /bin/bash ${workDir}/${TaskRun.CMD_RUN} 2>&1 | tee ${TaskRun.CMD_LOG}"
|
||||
}
|
||||
|
||||
static String containerMountPath(CloudStoragePath path) {
|
||||
return "$MOUNT_ROOT/${path.bucket()}${path}"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,951 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud.google.batch
|
||||
|
||||
import java.nio.file.Path
|
||||
import java.util.regex.Pattern
|
||||
|
||||
import com.google.cloud.batch.v1.AllocationPolicy
|
||||
import com.google.cloud.batch.v1.ComputeResource
|
||||
import com.google.cloud.batch.v1.Environment
|
||||
import com.google.cloud.batch.v1.Job
|
||||
import com.google.cloud.batch.v1.JobStatus
|
||||
import com.google.cloud.batch.v1.LifecyclePolicy
|
||||
import com.google.cloud.batch.v1.LogsPolicy
|
||||
import com.google.cloud.batch.v1.Runnable
|
||||
import com.google.cloud.batch.v1.ServiceAccount
|
||||
import com.google.cloud.batch.v1.StatusEvent
|
||||
import com.google.cloud.batch.v1.TaskGroup
|
||||
import com.google.cloud.batch.v1.TaskSpec
|
||||
import com.google.cloud.batch.v1.Volume
|
||||
import com.google.cloud.storage.contrib.nio.CloudStoragePath
|
||||
import com.google.protobuf.Duration
|
||||
import groovy.transform.Canonical
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.PackageScope
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.cloud.google.batch.client.BatchClient
|
||||
import nextflow.cloud.google.batch.client.BatchConfig
|
||||
import nextflow.cloud.types.CloudMachineInfo
|
||||
import nextflow.cloud.types.PriceModel
|
||||
import nextflow.exception.ProcessException
|
||||
import nextflow.exception.ProcessUnrecoverableException
|
||||
import nextflow.executor.BashWrapperBuilder
|
||||
import nextflow.executor.res.DiskResource
|
||||
import nextflow.fusion.FusionAwareTask
|
||||
import nextflow.fusion.FusionConfig
|
||||
import nextflow.fusion.FusionScriptLauncher
|
||||
import nextflow.processor.TaskArrayRun
|
||||
import nextflow.processor.TaskConfig
|
||||
import nextflow.processor.TaskHandler
|
||||
import nextflow.processor.TaskProcessor
|
||||
import nextflow.processor.TaskRun
|
||||
import nextflow.processor.TaskStatus
|
||||
import nextflow.trace.TraceRecord
|
||||
import nextflow.util.TestOnly
|
||||
/**
|
||||
* Implements a task handler for Google Batch executor
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class GoogleBatchTaskHandler extends TaskHandler implements FusionAwareTask {
|
||||
|
||||
/**
|
||||
* Result of building the allocation policy, containing the policy and whether a scratch volume is needed
|
||||
*/
|
||||
@Canonical
|
||||
static class AllocationPolicyResult {
|
||||
AllocationPolicy policy
|
||||
boolean requiresScratchVolume
|
||||
}
|
||||
|
||||
private static final Pattern EXIT_CODE_REGEX = ~/exit code 500(\d\d)/
|
||||
|
||||
private static final Pattern BATCH_ERROR_REGEX = ~/Batch Error: code/
|
||||
|
||||
private GoogleBatchExecutor executor
|
||||
|
||||
private BatchConfig batchConfig
|
||||
|
||||
private Path exitFile
|
||||
|
||||
private Path outputFile
|
||||
|
||||
private Path errorFile
|
||||
|
||||
private BatchClient client
|
||||
|
||||
private BashWrapperBuilder launcher
|
||||
|
||||
/**
|
||||
* Job Id assigned by Nextflow
|
||||
*/
|
||||
private String jobId
|
||||
|
||||
/**
|
||||
* Task id assigned by Google Batch service
|
||||
*/
|
||||
private String taskId
|
||||
|
||||
/**
|
||||
* Job unique id assigned by Google Batch service
|
||||
*/
|
||||
private String uid
|
||||
|
||||
/**
|
||||
* Task state assigned by Google Batch service
|
||||
*/
|
||||
private String taskState
|
||||
|
||||
private volatile CloudMachineInfo machineInfo
|
||||
|
||||
private volatile long timestamp
|
||||
|
||||
/**
|
||||
* Flag to indicate that the zone has been resolved from the status events
|
||||
*/
|
||||
private volatile boolean zoneUpdated
|
||||
|
||||
/**
|
||||
* A flag to indicate that the job has failed without launching any tasks
|
||||
*/
|
||||
private volatile boolean noTaskJobfailure
|
||||
|
||||
GoogleBatchTaskHandler(TaskRun task, GoogleBatchExecutor executor) {
|
||||
super(task)
|
||||
this.client = executor.getClient()
|
||||
this.executor = executor
|
||||
this.batchConfig = executor.batchConfig
|
||||
this.jobId = customJobName(task) ?: "nf-${task.hashLog.replace('/','')}-${System.currentTimeMillis()}"
|
||||
// those files are access via NF runtime, keep based on CloudStoragePath
|
||||
this.outputFile = task.workDir.resolve(TaskRun.CMD_OUTFILE)
|
||||
this.errorFile = task.workDir.resolve(TaskRun.CMD_ERRFILE)
|
||||
this.exitFile = task.workDir.resolve(TaskRun.CMD_EXIT)
|
||||
}
|
||||
|
||||
@TestOnly
|
||||
protected GoogleBatchTaskHandler() {}
|
||||
|
||||
/**
|
||||
* Resolve the `jobName` property defined in the nextflow config file
|
||||
*
|
||||
* @param task The underlying task to be executed
|
||||
* @return The custom job name for the specified task or {@code null} if the `jobName` attribute has been specified
|
||||
*/
|
||||
protected String customJobName(TaskRun task) {
|
||||
try {
|
||||
final custom = (Closure) executor.config.getExecConfigProp(executor.name, 'jobName', null)
|
||||
if( !custom )
|
||||
return null
|
||||
|
||||
final ctx = [ (TaskProcessor.TASK_CONTEXT_PROPERTY_NAME): task.config ]
|
||||
return custom.cloneWith(ctx).call()?.toString()
|
||||
}
|
||||
catch( Exception e ) {
|
||||
log.debug "Unable to resolve job custom name", e
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
protected BashWrapperBuilder createTaskWrapper() {
|
||||
if( fusionEnabled() ) {
|
||||
return fusionLauncher()
|
||||
}
|
||||
else {
|
||||
final taskBean = task.toTaskBean()
|
||||
return new GoogleBatchScriptLauncher(taskBean, executor.remoteBinDir)
|
||||
.withConfig(executor.googleOpts)
|
||||
.withIsArray(task.isArray())
|
||||
}
|
||||
}
|
||||
|
||||
protected GoogleBatchLauncherSpec spec0(BashWrapperBuilder launcher) {
|
||||
if( launcher instanceof GoogleBatchLauncherSpec )
|
||||
return launcher
|
||||
if( launcher instanceof FusionScriptLauncher )
|
||||
return new GoogleBatchFusionAdapter(this, launcher)
|
||||
throw new IllegalArgumentException("Unexpected Google Batch launcher type: ${launcher?.getClass()?.getName()}")
|
||||
}
|
||||
|
||||
@Override
|
||||
void prepareLauncher() {
|
||||
launcher = createTaskWrapper()
|
||||
launcher.build()
|
||||
}
|
||||
|
||||
@Override
|
||||
void submit() {
|
||||
/*
|
||||
* create submit request
|
||||
*/
|
||||
final req = newSubmitRequest(task, spec0(launcher))
|
||||
log.trace "[GOOGLE BATCH] new job request > $req"
|
||||
final resp = client.submitJob(jobId, req)
|
||||
final uid = resp.getUid()
|
||||
updateStatus(jobId, '0', uid)
|
||||
log.debug "[GOOGLE BATCH] Process `${task.lazyName()}` submitted > job=$jobId; uid=$uid; work-dir=${task.getWorkDirStr()}"
|
||||
}
|
||||
|
||||
protected void updateStatus(String jobId, String taskId, String uid) {
|
||||
if( task instanceof TaskArrayRun ) {
|
||||
// update status for children
|
||||
for( int i=0; i<task.children.size(); i++ ) {
|
||||
final handler = task.children[i] as GoogleBatchTaskHandler
|
||||
final arrayTaskId = executor.getArrayTaskId(jobId, i)
|
||||
handler.updateStatus(jobId, arrayTaskId, uid)
|
||||
}
|
||||
}
|
||||
else {
|
||||
this.jobId = jobId
|
||||
this.taskId = taskId
|
||||
this.uid = uid
|
||||
this.status = TaskStatus.SUBMITTED
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the compute resource configuration for a task
|
||||
*
|
||||
* @param config The task configuration
|
||||
* @param disk The disk resource (may be null)
|
||||
* @return The compute resource with CPU, memory, and boot disk settings
|
||||
*/
|
||||
protected ComputeResource buildComputeResource(TaskConfig config, DiskResource disk) {
|
||||
final computeResource = ComputeResource.newBuilder()
|
||||
computeResource.setCpuMilli(config.getCpus() * 1000)
|
||||
|
||||
if( config.getMemory() )
|
||||
computeResource.setMemoryMib(config.getMemory().getMega())
|
||||
|
||||
// apply disk directive to boot disk if type is not specified
|
||||
if( disk && !disk.type )
|
||||
computeResource.setBootDiskMib(disk.request.getMega())
|
||||
// otherwise use config setting
|
||||
else if( batchConfig.bootDiskSize )
|
||||
computeResource.setBootDiskMib(batchConfig.bootDiskSize.getMega())
|
||||
|
||||
return computeResource.build()
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the lifecycle policy for spot instance retry
|
||||
*
|
||||
* @return The lifecycle policy for retrying on spot reclaim
|
||||
*/
|
||||
protected LifecyclePolicy buildSpotRetryPolicy() {
|
||||
return LifecyclePolicy.newBuilder()
|
||||
.setActionCondition(
|
||||
LifecyclePolicy.ActionCondition.newBuilder()
|
||||
.addAllExitCodes(batchConfig.autoRetryExitCodes)
|
||||
)
|
||||
.setAction(LifecyclePolicy.Action.RETRY_TASK)
|
||||
.build()
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the network policy for the allocation
|
||||
*
|
||||
* @return The network policy or null if no network configuration is present
|
||||
*/
|
||||
protected AllocationPolicy.NetworkPolicy buildNetworkPolicy() {
|
||||
final networkInterface = AllocationPolicy.NetworkInterface.newBuilder()
|
||||
def hasNetworkPolicy = false
|
||||
|
||||
if( batchConfig.network ) {
|
||||
hasNetworkPolicy = true
|
||||
networkInterface.setNetwork(batchConfig.network)
|
||||
}
|
||||
if( batchConfig.subnetwork ) {
|
||||
hasNetworkPolicy = true
|
||||
networkInterface.setSubnetwork(batchConfig.subnetwork)
|
||||
}
|
||||
if( batchConfig.usePrivateAddress ) {
|
||||
hasNetworkPolicy = true
|
||||
networkInterface.setNoExternalIpAddress(true)
|
||||
}
|
||||
|
||||
return hasNetworkPolicy
|
||||
? AllocationPolicy.NetworkPolicy.newBuilder().addNetworkInterfaces(networkInterface).build()
|
||||
: null
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the task group for the job
|
||||
*
|
||||
* @param taskSpec The task specification
|
||||
* @param task The task run
|
||||
* @return The task group
|
||||
*/
|
||||
protected TaskGroup buildTaskGroup(TaskSpec.Builder taskSpec, TaskRun task) {
|
||||
final taskGroup = TaskGroup.newBuilder()
|
||||
.setTaskSpec(taskSpec)
|
||||
|
||||
if( task instanceof TaskArrayRun ) {
|
||||
final arraySize = task.getArraySize()
|
||||
taskGroup.setTaskCount(arraySize)
|
||||
}
|
||||
|
||||
return taskGroup.build()
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the container runnable with environment
|
||||
*
|
||||
* @param task The task run
|
||||
* @param launcher The launcher specification
|
||||
* @return The runnable with container and environment configuration
|
||||
*/
|
||||
protected Runnable buildContainerRunnable(TaskRun task, GoogleBatchLauncherSpec launcher) {
|
||||
final cmd = launcher.launchCommand()
|
||||
final container = Runnable.Container.newBuilder()
|
||||
.setImageUri(task.container)
|
||||
.addAllCommands(cmd)
|
||||
.addAllVolumes(launcher.getContainerMounts())
|
||||
|
||||
def containerOptions = task.config.getContainerOptions() ?: ''
|
||||
if( fusionEnabled() ) {
|
||||
if( containerOptions ) containerOptions += ' '
|
||||
containerOptions += '--privileged'
|
||||
}
|
||||
|
||||
if( containerOptions )
|
||||
container.setOptions(containerOptions)
|
||||
|
||||
final env = Environment.newBuilder()
|
||||
.putAllVariables(launcher.getEnvironment())
|
||||
.build()
|
||||
|
||||
return Runnable.newBuilder()
|
||||
.setContainer(container)
|
||||
.setEnvironment(env)
|
||||
.build()
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the scratch volume for disk mounting
|
||||
*
|
||||
* @return The scratch volume for /tmp mount, or null if not needed
|
||||
*/
|
||||
protected Volume buildScratchVolume() {
|
||||
return Volume.newBuilder()
|
||||
.setDeviceName('scratch')
|
||||
.setMountPath('/tmp')
|
||||
.build()
|
||||
}
|
||||
|
||||
/**
|
||||
* Result of building the instance policy, containing the policy and whether a scratch volume is needed
|
||||
*/
|
||||
@Canonical
|
||||
static class InstancePolicyResult {
|
||||
AllocationPolicy.InstancePolicyOrTemplate policy
|
||||
boolean requiresScratchVolume
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the instance policy or template for job allocation.
|
||||
* Note: This method sets machineInfo field as a side effect.
|
||||
*
|
||||
* @param task The task run
|
||||
* @param disk The disk resource (may be null, may be modified for fusion/local-ssd)
|
||||
* @return The instance policy result containing the policy and scratch volume flag
|
||||
*/
|
||||
protected InstancePolicyResult buildInstancePolicyOrTemplate(TaskRun task, DiskResource disk) {
|
||||
final instancePolicyOrTemplate = AllocationPolicy.InstancePolicyOrTemplate.newBuilder()
|
||||
boolean requiresScratchVolume = false
|
||||
|
||||
// use instance template if specified
|
||||
if( task.config.getMachineType()?.startsWith('template://') ) {
|
||||
if( task.config.getAccelerator() )
|
||||
log.warn1 'Process directive `accelerator` ignored because an instance template was specified'
|
||||
|
||||
if( task.config.getDisk() )
|
||||
log.warn1 'Process directive `disk` ignored because an instance template was specified'
|
||||
|
||||
if( batchConfig.getBootDiskImage() )
|
||||
log.warn1 'Config option `google.batch.bootDiskImage` ignored because an instance template was specified'
|
||||
|
||||
if( batchConfig.cpuPlatform )
|
||||
log.warn1 'Config option `google.batch.cpuPlatform` ignored because an instance template was specified'
|
||||
|
||||
if( batchConfig.networkTags )
|
||||
log.warn1 'Config option `google.batch.networkTags` ignored because an instance template was specified'
|
||||
|
||||
if( batchConfig.preemptible )
|
||||
log.warn1 'Config option `google.batch.premptible` ignored because an instance template was specified'
|
||||
|
||||
if( batchConfig.spot )
|
||||
log.warn1 'Config option `google.batch.spot` ignored because an instance template was specified'
|
||||
|
||||
instancePolicyOrTemplate
|
||||
.setInstallGpuDrivers(batchConfig.getInstallGpuDrivers())
|
||||
.setInstanceTemplate(task.config.getMachineType().minus('template://'))
|
||||
}
|
||||
// otherwise create instance policy
|
||||
else {
|
||||
final instancePolicy = AllocationPolicy.InstancePolicy.newBuilder()
|
||||
|
||||
if( fusionEnabled() && !disk ) {
|
||||
final reqMachineType = task.config.getMachineType()
|
||||
disk = new DiskResource(
|
||||
request: '375 GB',
|
||||
type: reqMachineType ? chooseFusionDiskType(reqMachineType) : 'local-ssd'
|
||||
)
|
||||
log.debug "[GOOGLE BATCH] Process `${task.lazyName()}` - adding local volume as fusion scratch: $disk"
|
||||
}
|
||||
|
||||
final machineType = findBestMachineType(task.config, disk?.type == 'local-ssd')
|
||||
|
||||
if( machineType ) {
|
||||
instancePolicy.setMachineType(machineType.type)
|
||||
instancePolicyOrTemplate.setInstallGpuDrivers(
|
||||
GoogleBatchMachineTypeSelector.INSTANCE.installGpuDrivers(machineType)
|
||||
)
|
||||
machineInfo = new CloudMachineInfo(
|
||||
type: machineType.type,
|
||||
zone: machineType.location,
|
||||
priceModel: machineType.priceModel
|
||||
)
|
||||
}
|
||||
|
||||
// Configure boot disk
|
||||
final bootDisk = AllocationPolicy.Disk.newBuilder()
|
||||
boolean setBoot = false
|
||||
if( batchConfig.getBootDiskImage() ) {
|
||||
bootDisk.setImage(batchConfig.getBootDiskImage())
|
||||
setBoot = true
|
||||
}
|
||||
if( machineType && GoogleBatchMachineTypeSelector.INSTANCE.isHyperdiskOnly(machineType.type) ) {
|
||||
bootDisk.setType('hyperdisk-balanced')
|
||||
setBoot = true
|
||||
}
|
||||
if( setBoot )
|
||||
instancePolicy.setBootDisk(bootDisk)
|
||||
|
||||
if( task.config.getAccelerator() ) {
|
||||
final accelerator = AllocationPolicy.Accelerator.newBuilder()
|
||||
.setCount(task.config.getAccelerator().getRequest())
|
||||
|
||||
if( task.config.getAccelerator().getType() )
|
||||
accelerator.setType(task.config.getAccelerator().getType())
|
||||
|
||||
instancePolicy.addAccelerators(accelerator)
|
||||
instancePolicyOrTemplate.setInstallGpuDrivers(true)
|
||||
}
|
||||
|
||||
// When using local SSD not all the disk sizes are valid and depends on the machine type
|
||||
if( disk?.type == 'local-ssd' && machineType ) {
|
||||
final validSize = GoogleBatchMachineTypeSelector.INSTANCE.findValidLocalSSDSize(disk.request, machineType)
|
||||
if( validSize.toBytes() == 0 ) {
|
||||
disk = new DiskResource(request: 0)
|
||||
log.debug "[GOOGLE BATCH] Process `${task.lazyName()}` - ${machineType.type} does not allow configuring local disks"
|
||||
}
|
||||
if( validSize != disk.request ) {
|
||||
disk = new DiskResource(request: validSize, type: 'local-ssd')
|
||||
log.debug "[GOOGLE BATCH] Process `${task.lazyName()}` - adjusting local disk size to: $validSize"
|
||||
}
|
||||
}
|
||||
|
||||
// use disk directive for an attached disk if type is specified
|
||||
if( disk?.type ) {
|
||||
instancePolicy.addDisks(
|
||||
AllocationPolicy.AttachedDisk.newBuilder()
|
||||
.setNewDisk(
|
||||
AllocationPolicy.Disk.newBuilder()
|
||||
.setType(disk.type)
|
||||
.setSizeGb(disk.request.toGiga())
|
||||
)
|
||||
.setDeviceName('scratch')
|
||||
)
|
||||
requiresScratchVolume = true
|
||||
}
|
||||
|
||||
if( batchConfig.cpuPlatform )
|
||||
instancePolicy.setMinCpuPlatform(batchConfig.cpuPlatform)
|
||||
|
||||
if( batchConfig.preemptible )
|
||||
instancePolicy.setProvisioningModel(AllocationPolicy.ProvisioningModel.PREEMPTIBLE)
|
||||
|
||||
if( batchConfig.spot )
|
||||
instancePolicy.setProvisioningModel(AllocationPolicy.ProvisioningModel.SPOT)
|
||||
|
||||
instancePolicyOrTemplate.setPolicy(instancePolicy)
|
||||
}
|
||||
|
||||
if( batchConfig.installOpsAgent ) {
|
||||
if( !batchConfig.bootDiskImage?.toLowerCase()?.contains('debian') )
|
||||
log.warn1 "The Ops Agent requires a compatible boot disk image. Set 'google.batch.bootDiskImage' to a batch-debian image."
|
||||
instancePolicyOrTemplate.setInstallOpsAgent( true )
|
||||
}
|
||||
|
||||
return new InstancePolicyResult(instancePolicyOrTemplate.build(), requiresScratchVolume)
|
||||
}
|
||||
|
||||
/**
|
||||
* Choose the disk type for Fusion according to the machine or family.
|
||||
* Preference is 'local-ssd', 'hyperdisk-balanced' and 'pd-balanced' other types can be set by setting disk directive
|
||||
* @param machineTypeOrFamily
|
||||
* @return Disk type
|
||||
*/
|
||||
protected String chooseFusionDiskType(String machineTypeOrFamily){
|
||||
if( !GoogleBatchMachineTypeSelector.unsupportedLocalSSD(machineTypeOrFamily) ){
|
||||
return 'local-ssd'
|
||||
} else if( GoogleBatchMachineTypeSelector.isPdOnly(machineTypeOrFamily) ){
|
||||
return 'pd-balanced'
|
||||
} else {
|
||||
return 'hyperdisk-balanced'
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the allocation policy for the job
|
||||
*
|
||||
* @param task The task run
|
||||
* @param disk The disk resource
|
||||
* @return The allocation policy result containing the policy and scratch volume flag
|
||||
*/
|
||||
protected AllocationPolicyResult buildAllocationPolicy(TaskRun task, DiskResource disk) {
|
||||
final allocationPolicy = AllocationPolicy.newBuilder()
|
||||
|
||||
if( batchConfig.getAllowedLocations() )
|
||||
allocationPolicy.setLocation(
|
||||
AllocationPolicy.LocationPolicy.newBuilder()
|
||||
.addAllAllowedLocations(batchConfig.getAllowedLocations())
|
||||
)
|
||||
|
||||
if( batchConfig.serviceAccountEmail )
|
||||
allocationPolicy.setServiceAccount(
|
||||
ServiceAccount.newBuilder()
|
||||
.setEmail(batchConfig.serviceAccountEmail)
|
||||
)
|
||||
|
||||
allocationPolicy.putAllLabels(task.config.getResourceLabels())
|
||||
|
||||
if( batchConfig.networkTags )
|
||||
allocationPolicy.addAllTags(batchConfig.networkTags)
|
||||
|
||||
// instance policy or template
|
||||
final instanceResult = buildInstancePolicyOrTemplate(task, disk)
|
||||
allocationPolicy.addInstances(instanceResult.policy)
|
||||
|
||||
// network policy
|
||||
final networkPolicy = buildNetworkPolicy()
|
||||
if( networkPolicy )
|
||||
allocationPolicy.setNetwork(networkPolicy)
|
||||
|
||||
return new AllocationPolicyResult(allocationPolicy.build(), instanceResult.requiresScratchVolume)
|
||||
}
|
||||
|
||||
protected Job newSubmitRequest(TaskRun task, GoogleBatchLauncherSpec launcher) {
|
||||
// container validation
|
||||
if( !task.container )
|
||||
throw new ProcessUnrecoverableException("Process `${task.lazyName()}` failed because the container image was not specified")
|
||||
|
||||
// resource requirements
|
||||
final disk = task.config.getDiskResource()
|
||||
final taskSpec = TaskSpec.newBuilder()
|
||||
.setComputeResource(buildComputeResource(task.config, disk))
|
||||
|
||||
if( task.config.getTime() )
|
||||
taskSpec.setMaxRunDuration(
|
||||
Duration.newBuilder()
|
||||
.setSeconds( task.config.getTime().toSeconds() )
|
||||
)
|
||||
|
||||
taskSpec
|
||||
.addRunnables(buildContainerRunnable(task, launcher))
|
||||
.addAllVolumes(launcher.getVolumes())
|
||||
|
||||
// retry on spot reclaim
|
||||
final attempts = maxSpotAttempts()
|
||||
if( attempts > 0 ) {
|
||||
taskSpec
|
||||
.setMaxRetryCount(attempts)
|
||||
.addLifecyclePolicies(buildSpotRetryPolicy())
|
||||
}
|
||||
|
||||
// allocation policy
|
||||
final allocationResult = buildAllocationPolicy(task, disk)
|
||||
|
||||
// add scratch volume if needed by instance policy
|
||||
if( allocationResult.requiresScratchVolume )
|
||||
taskSpec.addVolumes(buildScratchVolume())
|
||||
|
||||
// create the job
|
||||
return Job.newBuilder()
|
||||
.addTaskGroups(buildTaskGroup(taskSpec, task))
|
||||
.setAllocationPolicy(allocationResult.policy)
|
||||
.setLogsPolicy(createLogsPolicy())
|
||||
.putAllLabels(task.config.getResourceLabels())
|
||||
.build()
|
||||
}
|
||||
|
||||
protected LogsPolicy createLogsPolicy() {
|
||||
final logsPath = executor.batchConfig.logsPath()
|
||||
if( logsPath instanceof CloudStoragePath ) {
|
||||
return LogsPolicy.newBuilder()
|
||||
.setDestination(LogsPolicy.Destination.PATH)
|
||||
.setLogsPath(GoogleBatchScriptLauncher.containerMountPath(logsPath))
|
||||
.build()
|
||||
}
|
||||
else {
|
||||
return LogsPolicy.newBuilder()
|
||||
.setDestination(LogsPolicy.Destination.CLOUD_LOGGING)
|
||||
.build()
|
||||
}
|
||||
}
|
||||
|
||||
protected int maxSpotAttempts() {
|
||||
final result = batchConfig.maxSpotAttempts
|
||||
if( result > 0 )
|
||||
return result
|
||||
// when fusion snapshot is enabled max attempt should be > 0
|
||||
// to enable to allow snapshot retry the job execution in a new compute instance
|
||||
return fusionEnabled() && fusionConfig().snapshotsEnabled() ? FusionConfig.DEFAULT_SNAPSHOT_MAX_SPOT_ATTEMPTS : 0
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Retrieve the submitted task state
|
||||
*/
|
||||
protected String getTaskState() {
|
||||
return isArrayChild
|
||||
? getStateFromTaskStatus()
|
||||
: getStateFromJobStatus()
|
||||
}
|
||||
|
||||
protected String getStateFromTaskStatus() {
|
||||
final now = System.currentTimeMillis()
|
||||
final delta = now - timestamp;
|
||||
if( !taskState || delta >= 1_000) {
|
||||
final status = client.getTaskInArrayStatus(jobId, taskId)
|
||||
if( status ) {
|
||||
inspectTaskStatus(status)
|
||||
} else {
|
||||
// If no task status retrieved check job status
|
||||
final jobStatus = client.getJobStatus(jobId)
|
||||
inspectJobStatus(jobStatus)
|
||||
}
|
||||
}
|
||||
return taskState
|
||||
}
|
||||
|
||||
protected String getStateFromJobStatus() {
|
||||
final now = System.currentTimeMillis()
|
||||
final delta = now - timestamp;
|
||||
if( !taskState || delta >= 1_000) {
|
||||
final status = client.getJobStatus(jobId)
|
||||
inspectJobStatus(status)
|
||||
}
|
||||
return taskState
|
||||
}
|
||||
|
||||
private void inspectTaskStatus(com.google.cloud.batch.v1.TaskStatus status) {
|
||||
final newState = status?.state as String
|
||||
if (newState) {
|
||||
log.trace "[GOOGLE BATCH] Get job=$jobId task=$taskId state=$newState"
|
||||
taskState = newState
|
||||
timestamp = System.currentTimeMillis()
|
||||
}
|
||||
if (newState == 'PENDING') {
|
||||
final eventsCount = status.getStatusEventsCount()
|
||||
final lastEvent = eventsCount > 0 ? status.getStatusEvents(eventsCount - 1) : null
|
||||
if (lastEvent?.getDescription()?.contains('CODE_GCE_QUOTA_EXCEEDED'))
|
||||
log.warn1 "Batch job cannot be run: ${lastEvent.getDescription()}"
|
||||
}
|
||||
}
|
||||
|
||||
protected String inspectJobStatus(JobStatus status) {
|
||||
final newState = status?.state as String
|
||||
if (newState) {
|
||||
log.trace "[GOOGLE BATCH] Get job=$jobId state=$newState"
|
||||
taskState = newState
|
||||
timestamp = System.currentTimeMillis()
|
||||
if (newState == "FAILED") {
|
||||
noTaskJobfailure = true
|
||||
}
|
||||
}
|
||||
if (newState == 'SCHEDULED') {
|
||||
final eventsCount = status.getStatusEventsCount()
|
||||
final lastEvent = eventsCount > 0 ? status.getStatusEvents(eventsCount - 1) : null
|
||||
if (lastEvent?.getDescription()?.contains('CODE_GCE_QUOTA_EXCEEDED'))
|
||||
log.warn1 "Batch job cannot be run: ${lastEvent.getDescription()}"
|
||||
}
|
||||
}
|
||||
|
||||
static private final List<String> RUNNING_OR_COMPLETED = ['RUNNING', 'SUCCEEDED', 'FAILED', 'DELETION_IN_PROGRESS']
|
||||
|
||||
static private final List<String> COMPLETED = ['SUCCEEDED', 'FAILED', 'DELETION_IN_PROGRESS']
|
||||
|
||||
@Override
|
||||
boolean checkIfRunning() {
|
||||
if(isSubmitted()) {
|
||||
// include `terminated` state to allow the handler status to progress
|
||||
if( getTaskState() in RUNNING_OR_COMPLETED ) {
|
||||
status = TaskStatus.RUNNING
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean checkIfCompleted() {
|
||||
final state = getTaskState()
|
||||
if( state in COMPLETED ) {
|
||||
log.debug "[GOOGLE BATCH] Process `${task.lazyName()}` - terminated job=$jobId; task=$taskId; state=$state"
|
||||
// finalize the task
|
||||
task.exitStatus = getExitCode()
|
||||
if( state == 'FAILED' ) {
|
||||
// When no exit code or 500XX codes, get the jobError reason from events
|
||||
if( task.exitStatus == Integer.MAX_VALUE || task.exitStatus >= 50000)
|
||||
task.error = getJobError()
|
||||
task.stdout = executor.logging.stdout(uid, taskId) ?: outputFile
|
||||
task.stderr = executor.logging.stderr(uid, taskId) ?: errorFile
|
||||
}
|
||||
else {
|
||||
// Retried spot instances could keep the 500xx exit code event when the automatic retied succeeds. In this case, we need to read the exit code from .exitcode
|
||||
// https://github.com/nextflow-io/nextflow/issues/6779
|
||||
if( task.exitStatus >= 50000 )
|
||||
task.exitStatus = readExitFile()
|
||||
task.stdout = outputFile
|
||||
task.stderr = errorFile
|
||||
}
|
||||
status = TaskStatus.COMPLETED
|
||||
if( isArrayChild )
|
||||
client.removeFromArrayTasks(jobId, taskId)
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
/**
|
||||
* Try to get the latest exit code form the task status events list.
|
||||
* Fallback to read .exitcode file generated by Nextflow if not found (null).
|
||||
* The rationale of this is that, in case of error, the exit code return by the batch API is more reliable.
|
||||
*
|
||||
* @return exit code if found, otherwise Integer.MAX_VALUE
|
||||
*/
|
||||
protected Integer getExitCode(){
|
||||
final events = client.getTaskStatus(jobId, taskId)?.getStatusEventsList()
|
||||
if( events ) {
|
||||
// Find the most recent event that contains a TaskExecution with an exit code.
|
||||
// Events are not guaranteed to be in chronological order, so we iterate through
|
||||
// all of them and track the one with the highest timestamp.
|
||||
StatusEvent latestEvent = null
|
||||
long latestTime = Long.MIN_VALUE
|
||||
for( StatusEvent ev : events ) {
|
||||
// Only consider events that have task execution info (which contains exit code)
|
||||
if( ev.hasTaskExecution() ) {
|
||||
final long eventTime = ev.getEventTime().getSeconds()
|
||||
if( eventTime > latestTime ) {
|
||||
latestTime = eventTime
|
||||
latestEvent = ev
|
||||
}
|
||||
}
|
||||
}
|
||||
// Return the exit code from the most recent task execution event
|
||||
if( latestEvent?.getTaskExecution()?.getExitCode() != null ) {
|
||||
return latestEvent.getTaskExecution().getExitCode()
|
||||
}
|
||||
}
|
||||
// Fallback: if no exit code found from the Batch API (either no events or none with
|
||||
// TaskExecution), read the .exitcode file that Nextflow generates in the work directory
|
||||
log.debug("[GOOGLE BATCH] Exit code not found from API. Checking .exitcode file...")
|
||||
return readExitFile()
|
||||
}
|
||||
|
||||
protected Throwable getJobError() {
|
||||
try {
|
||||
final events = noTaskJobfailure
|
||||
? client.getJobStatus(jobId).getStatusEventsList()
|
||||
: client.getTaskStatus(jobId, taskId).getStatusEventsList()
|
||||
final lastEvent = events?.get(events.size() - 1)
|
||||
log.debug "[GOOGLE BATCH] Process `${task.lazyName()}` - last event: ${lastEvent}; exit code: ${lastEvent?.taskExecution?.exitCode}"
|
||||
|
||||
final error = lastEvent?.description
|
||||
if( error && (EXIT_CODE_REGEX.matcher(error).find() || BATCH_ERROR_REGEX.matcher(error).find())) {
|
||||
return new ProcessException(error)
|
||||
}
|
||||
}
|
||||
catch (Throwable t) {
|
||||
log.debug "[GOOGLE BATCH] Unable to fetch task `${task.lazyName()}` exit code - cause: ${t.message}"
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
@PackageScope Integer readExitFile() {
|
||||
try {
|
||||
exitFile.text as Integer
|
||||
}
|
||||
catch (Exception e) {
|
||||
log.debug "[GOOGLE BATCH] Cannot read exit status for task: `${task.lazyName()}` - ${e.message}"
|
||||
// return MAX_VALUE to signal it was unable to retrieve the exit code
|
||||
return Integer.MAX_VALUE
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void killTask() {
|
||||
if( isActive() ) {
|
||||
log.trace "[GOOGLE BATCH] Process `${task.lazyName()}` - deleting job name=$jobId"
|
||||
if( executor.shouldDeleteJob(jobId) )
|
||||
client.deleteJob(jobId)
|
||||
}
|
||||
else {
|
||||
log.debug "[GOOGLE BATCH] Process `${task.lazyName()}` - invalid delete action"
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Regex pattern to extract zone from StatusEvent description.
|
||||
* Matches the pattern: zones/<zone-name>/instances/<instance-id>
|
||||
*/
|
||||
private static final Pattern ZONE_PATTERN = ~/zones\/([^\/]+)\/instances\//
|
||||
|
||||
protected CloudMachineInfo getMachineInfo() {
|
||||
if( machineInfo!=null && !zoneUpdated && isCompleted() )
|
||||
updateZoneFromEvents()
|
||||
return machineInfo
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the actual execution zone from StatusEvent descriptions and update
|
||||
* the machineInfo zone field accordingly.
|
||||
*
|
||||
* Google Batch status events include zone info in the description field with
|
||||
* the format: {@code zones/<zone-name>/instances/<instance-id>}
|
||||
*/
|
||||
private void updateZoneFromEvents() {
|
||||
try {
|
||||
final events = client.getTaskStatus(jobId, taskId)?.statusEventsList
|
||||
final zone = resolveZoneFromEvents(events)
|
||||
if( zone ) {
|
||||
machineInfo = new CloudMachineInfo(
|
||||
type: machineInfo.type,
|
||||
zone: zone,
|
||||
priceModel: machineInfo.priceModel
|
||||
)
|
||||
}
|
||||
}
|
||||
catch( Exception e ) {
|
||||
log.debug "[GOOGLE BATCH] Unable to resolve zone from events for task: `${task.lazyName()}` - ${e.message}"
|
||||
}
|
||||
finally {
|
||||
zoneUpdated = true
|
||||
}
|
||||
}
|
||||
|
||||
@PackageScope
|
||||
static String resolveZoneFromEvents(List<StatusEvent> events) {
|
||||
if( !events )
|
||||
return null
|
||||
for( def event : events ) {
|
||||
final matcher = ZONE_PATTERN.matcher(event.description ?: '')
|
||||
if( matcher.find() )
|
||||
return matcher.group(1)
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Count the number of spot instance reclamations for this task by examining
|
||||
* the task status events and checking for preemption exit codes
|
||||
*
|
||||
* @param jobId The Google Batch Job Id
|
||||
* @return The number of times this task was retried due to spot instance reclamation
|
||||
*/
|
||||
|
||||
protected Integer getNumSpotInterruptions(String jobId) {
|
||||
if (!jobId || !taskId || !isCompleted()) {
|
||||
return null
|
||||
}
|
||||
|
||||
try {
|
||||
final status = client.getTaskStatus(jobId, taskId)
|
||||
|
||||
if (!status)
|
||||
return null
|
||||
|
||||
// valid status but no events present means no interruptions occurred
|
||||
if (!status?.statusEventsList)
|
||||
return 0
|
||||
|
||||
int count = 0
|
||||
for (def event : status.statusEventsList) {
|
||||
// Google Batch uses exit code 50001 for spot preemption
|
||||
// Check if the event has a task execution with exit code 50001
|
||||
if (event.hasTaskExecution() && event.taskExecution.exitCode == 50001) {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
|
||||
} catch (Exception e) {
|
||||
log.debug "[GOOGLE BATCH] Unable to count spot interruptions for job=$jobId task=$taskId - ${e.message}"
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
TraceRecord getTraceRecord() {
|
||||
def result = super.getTraceRecord()
|
||||
if( jobId && uid )
|
||||
result.put('native_id', "$jobId/$taskId/$uid")
|
||||
result.machineInfo = getMachineInfo()
|
||||
result.numSpotInterruptions = getNumSpotInterruptions(jobId)
|
||||
return result
|
||||
}
|
||||
|
||||
protected GoogleBatchMachineTypeSelector.MachineType bestMachineType0(int cpus, int memory, String location, boolean spot, boolean localSSD, List<String> families) {
|
||||
return GoogleBatchMachineTypeSelector.INSTANCE.bestMachineType(cpus, memory, location, spot, localSSD, families)
|
||||
}
|
||||
|
||||
protected GoogleBatchMachineTypeSelector.MachineType findBestMachineType(TaskConfig config, boolean localSSD) {
|
||||
final location = client.location
|
||||
final cpus = config.getCpus()
|
||||
final memory = config.getMemory() ? config.getMemory().toMega().toInteger() : 1024
|
||||
final spot = batchConfig.spot ?: batchConfig.preemptible
|
||||
final machineType = config.getMachineType()
|
||||
final families = machineType ? machineType.tokenize(',') : List.<String>of()
|
||||
final priceModel = spot ? PriceModel.spot : PriceModel.standard
|
||||
|
||||
try {
|
||||
if( executor.isCloudinfoEnabled() ) {
|
||||
return bestMachineType0(cpus, memory, location, spot, localSSD, families)
|
||||
}
|
||||
}
|
||||
catch (Exception e) {
|
||||
log.warn "Cannot determine the machine type to be used for task: `${task.lazyName()}` - If this problem persists disable disable the Cloudinfo service by setting the variable NXF_CLOUDINFO_ENABLED=false in your environment", e
|
||||
}
|
||||
|
||||
// Check if a specific machine type was provided by the user
|
||||
if( machineType && !machineType.contains(',') && !machineType.contains('*') )
|
||||
return new GoogleBatchMachineTypeSelector.MachineType(
|
||||
type: machineType,
|
||||
location: location,
|
||||
priceModel: priceModel
|
||||
)
|
||||
|
||||
// Fallback to Google Batch automatically deduce from requested resources
|
||||
return null
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud.google.batch.client
|
||||
|
||||
import java.time.temporal.ChronoUnit
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.TimeoutException
|
||||
import java.util.function.Predicate
|
||||
|
||||
import com.google.api.gax.core.CredentialsProvider
|
||||
import com.google.api.gax.rpc.DeadlineExceededException
|
||||
import com.google.api.gax.rpc.FixedHeaderProvider
|
||||
import com.google.api.gax.rpc.NotFoundException
|
||||
import com.google.api.gax.rpc.UnavailableException
|
||||
import com.google.auth.Credentials
|
||||
import com.google.cloud.batch.v1.BatchServiceClient
|
||||
import com.google.cloud.batch.v1.BatchServiceSettings
|
||||
import com.google.cloud.batch.v1.Job
|
||||
import com.google.cloud.batch.v1.JobName
|
||||
import com.google.cloud.batch.v1.JobStatus
|
||||
import com.google.cloud.batch.v1.LocationName
|
||||
import com.google.cloud.batch.v1.Task
|
||||
import com.google.cloud.batch.v1.TaskGroupName
|
||||
import com.google.cloud.batch.v1.TaskName
|
||||
import com.google.cloud.batch.v1.TaskStatus
|
||||
import dev.failsafe.Failsafe
|
||||
import dev.failsafe.RetryPolicy
|
||||
import dev.failsafe.event.EventListener
|
||||
import dev.failsafe.event.ExecutionAttemptedEvent
|
||||
import dev.failsafe.function.CheckedSupplier
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.cloud.google.GoogleOpts
|
||||
import nextflow.util.TestOnly
|
||||
/**
|
||||
* Implements Google Batch HTTP client
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class BatchClient {
|
||||
private final static long TASK_STATE_INVALID_TIME = 1_000
|
||||
protected String projectId
|
||||
protected String location
|
||||
protected BatchServiceClient batchServiceClient
|
||||
protected GoogleOpts config
|
||||
private Map<String, TaskStatusRecord> arrayTaskStatus = new ConcurrentHashMap<String, TaskStatusRecord>()
|
||||
|
||||
BatchClient(GoogleOpts config) {
|
||||
this.config = config
|
||||
this.projectId = config.projectId
|
||||
this.location = config.location
|
||||
this.batchServiceClient = createBatchService(config)
|
||||
}
|
||||
|
||||
@TestOnly
|
||||
protected BatchClient() {}
|
||||
|
||||
protected CredentialsProvider createCredentialsProvider(GoogleOpts config) {
|
||||
if( !config.getCredentials() )
|
||||
return null
|
||||
return new CredentialsProvider() {
|
||||
@Override
|
||||
Credentials getCredentials() throws IOException {
|
||||
return config.getCredentials()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected BatchServiceClient createBatchService(GoogleOpts config) {
|
||||
final provider = createCredentialsProvider(config)
|
||||
if( provider ) {
|
||||
log.debug "[GOOGLE BATCH] Creating service client with config credentials"
|
||||
final userAgent = FixedHeaderProvider.create('user-agent', 'Nextflow')
|
||||
final settings = BatchServiceSettings
|
||||
.newBuilder()
|
||||
.setHeaderProvider(userAgent)
|
||||
.setCredentialsProvider(provider)
|
||||
.build()
|
||||
return BatchServiceClient.create(settings)
|
||||
}
|
||||
else {
|
||||
log.debug "[GOOGLE BATCH] Creating service client with default settings"
|
||||
return BatchServiceClient.create()
|
||||
}
|
||||
}
|
||||
|
||||
Job submitJob(String jobId, Job job) {
|
||||
final parent = LocationName.of(projectId, location)
|
||||
return apply(()-> batchServiceClient.createJob(parent, job, jobId))
|
||||
}
|
||||
|
||||
Job describeJob(String jobId) {
|
||||
final name = JobName.of(projectId, location, jobId)
|
||||
return apply(()-> batchServiceClient.getJob(name))
|
||||
}
|
||||
|
||||
Iterable<Task> listTasks(String jobId) {
|
||||
final parent = TaskGroupName.of(projectId, location, jobId, 'group0')
|
||||
return apply(()-> batchServiceClient.listTasks(parent).iterateAll())
|
||||
}
|
||||
|
||||
Task describeTask(String jobId, String taskId) {
|
||||
final name = generateTaskName(jobId, taskId)
|
||||
return apply(()-> batchServiceClient.getTask(name))
|
||||
}
|
||||
|
||||
void deleteJob(String jobId) {
|
||||
final name = JobName.of(projectId, location, jobId).toString()
|
||||
apply(()-> batchServiceClient.deleteJobAsync(name))
|
||||
}
|
||||
|
||||
TaskStatus getTaskStatus(String jobId, String taskId) {
|
||||
return describeTask(jobId, taskId).getStatus()
|
||||
}
|
||||
|
||||
JobStatus getJobStatus(String jobId) {
|
||||
return describeJob(jobId).getStatus()
|
||||
}
|
||||
|
||||
String getTaskState(String jobId, String taskId) {
|
||||
final status = getTaskStatus(jobId, taskId)
|
||||
return status ? status.getState().toString() : null
|
||||
}
|
||||
|
||||
void shutdown() {
|
||||
batchServiceClient.close()
|
||||
}
|
||||
|
||||
String getLocation() {
|
||||
return location
|
||||
}
|
||||
|
||||
String generateTaskName(String jobId, String taskId) {
|
||||
TaskName.of(projectId, location, jobId, 'group0', taskId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a retry policy using the configuration specified by {@link BatchRetryConfig}
|
||||
*
|
||||
* @param cond A predicate that determines when a retry should be triggered
|
||||
* @return The {@link dev.failsafe.RetryPolicy} instance
|
||||
*/
|
||||
protected <T> RetryPolicy<T> retryPolicy(Predicate<? extends Throwable> cond) {
|
||||
final cfg = config.batch.getRetryConfig()
|
||||
final listener = new EventListener<ExecutionAttemptedEvent<T>>() {
|
||||
@Override
|
||||
void accept(ExecutionAttemptedEvent<T> event) throws Throwable {
|
||||
log.debug("[GOOGLE BATCH] response error - attempt: ${event.attemptCount}; reason: ${event.lastFailure.message}")
|
||||
}
|
||||
}
|
||||
return RetryPolicy.<T>builder()
|
||||
.handleIf(cond)
|
||||
.withBackoff(cfg.delay.toMillis(), cfg.maxDelay.toMillis(), ChronoUnit.MILLIS)
|
||||
.withMaxAttempts(cfg.maxAttempts)
|
||||
.withJitter(cfg.jitter)
|
||||
.onRetry(listener)
|
||||
.build()
|
||||
}
|
||||
|
||||
/**
|
||||
* Carry out the invocation of the specified action using a retry policy
|
||||
* when an API UnavailableException is thrown
|
||||
*
|
||||
* see also https://github.com/nextflow-io/nextflow/issues/4537
|
||||
*
|
||||
* @param action A {@link dev.failsafe.function.CheckedSupplier} instance modeling the action to be performed in a safe manner
|
||||
* @return The result of the supplied action
|
||||
*/
|
||||
protected <T> T apply(CheckedSupplier<T> action) {
|
||||
// define the retry condition
|
||||
final cond = new Predicate<? extends Throwable>() {
|
||||
@Override
|
||||
boolean test(Throwable t) {
|
||||
if( t instanceof UnavailableException )
|
||||
return true
|
||||
if( t instanceof DeadlineExceededException )
|
||||
return true
|
||||
if( t instanceof IOException || t.cause instanceof IOException )
|
||||
return true
|
||||
if( t instanceof TimeoutException || t.cause instanceof TimeoutException )
|
||||
return true
|
||||
if( t instanceof NotFoundException || t.cause instanceof NotFoundException )
|
||||
return true
|
||||
return false
|
||||
}
|
||||
}
|
||||
// create the retry policy object
|
||||
final policy = retryPolicy(cond)
|
||||
// apply the action with
|
||||
return Failsafe.with(policy).get(action)
|
||||
}
|
||||
|
||||
|
||||
TaskStatus getTaskInArrayStatus(String jobId, String taskId) {
|
||||
final taskName = generateTaskName(jobId,taskId)
|
||||
final now = System.currentTimeMillis()
|
||||
TaskStatusRecord record = arrayTaskStatus.get(taskName)
|
||||
if( !record || now - record.timestamp > TASK_STATE_INVALID_TIME ){
|
||||
log.trace("[GOOGLE BATCH] Updating tasks status for job $jobId")
|
||||
updateArrayTasks(jobId, now)
|
||||
record = arrayTaskStatus.get(taskName)
|
||||
}
|
||||
return record?.status
|
||||
}
|
||||
|
||||
private void updateArrayTasks(String jobId, long now){
|
||||
for( Task t: listTasks(jobId) ){
|
||||
arrayTaskStatus.put(t.name, new TaskStatusRecord(t.status, now))
|
||||
}
|
||||
}
|
||||
|
||||
void removeFromArrayTasks(String jobId, String taskId){
|
||||
final taskName = generateTaskName(jobId,taskId)
|
||||
TaskStatusRecord record = arrayTaskStatus.remove(taskName)
|
||||
}
|
||||
}
|
||||
|
||||
@CompileStatic
|
||||
class TaskStatusRecord {
|
||||
protected TaskStatus status
|
||||
protected long timestamp
|
||||
|
||||
TaskStatusRecord(TaskStatus status, long timestamp) {
|
||||
this.status = status
|
||||
this.timestamp = timestamp
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud.google.batch.client
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.config.spec.ConfigOption
|
||||
import nextflow.config.spec.ConfigScope
|
||||
import nextflow.script.dsl.Description
|
||||
import nextflow.util.MemoryUnit
|
||||
/**
|
||||
* Model Google Batch config settings
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class BatchConfig implements ConfigScope {
|
||||
|
||||
static final private int DEFAULT_MAX_SPOT_ATTEMPTS = 0
|
||||
|
||||
static final private List<Integer> DEFAULT_RETRY_LIST = List.of(50001)
|
||||
|
||||
static final private List<String> DEFAULT_GCSFUSE_OPTS = List.<String>of('-o rw', '-implicit-dirs')
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The set of [allowed locations](https://cloud.google.com/batch/docs/reference/rest/v1/projects.locations.jobs#locationpolicy) for VMs to be provisioned (default: no restriction).
|
||||
""")
|
||||
final List<String> allowedLocations
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The list of exit codes that should be automatically retried by Google Batch when `google.batch.maxSpotAttempts` is greater than 0 (default: `[50001]`).
|
||||
|
||||
[Read more](https://cloud.google.com/batch/docs/troubleshooting#reserved-exit-codes)
|
||||
""")
|
||||
final List<Integer> autoRetryExitCodes
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The image URI of the virtual machine boot disk, e.g `batch-debian` (default: none).
|
||||
|
||||
[Read more](https://cloud.google.com/batch/docs/vm-os-environment-overview#vm-os-image-options)
|
||||
""")
|
||||
final String bootDiskImage
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The size of the virtual machine boot disk, e.g `50.GB` (default: none).
|
||||
""")
|
||||
final MemoryUnit bootDiskSize
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The [minimum CPU Platform](https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform#specifications), e.g. `'Intel Skylake'` (default: none).
|
||||
""")
|
||||
final String cpuPlatform
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
List of custom mount options for `gcsfuse` (default: `['-o rw', '-implicit-dirs']`).
|
||||
""")
|
||||
final List<String> gcsfuseOptions
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
""")
|
||||
final boolean installGpuDrivers
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Enable the installation of the Ops Agent on Google Batch instances for enhanced monitoring and logging (default: `false`).
|
||||
""")
|
||||
final boolean installOpsAgent
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The Google Cloud Storage path where job logs should be stored, e.g. `gs://my-logs-bucket/logs`.
|
||||
""")
|
||||
final String logsPath
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Max number of execution attempts of a job interrupted by a Compute Engine Spot reclaim event (default: `0`).
|
||||
""")
|
||||
final int maxSpotAttempts
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The URL of an existing network resource to which the VM will be attached.
|
||||
""")
|
||||
final String network
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The [network tags](https://cloud.google.com/vpc/docs/add-remove-network-tags) to be applied to the instances created by Google Batch jobs (e.g., `['allow-ssh', 'allow-http']`).
|
||||
""")
|
||||
final List<String> networkTags
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
""")
|
||||
final boolean preemptible
|
||||
|
||||
final BatchRetryConfig retry
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The Google service account email to use for the pipeline execution. If not specified, the default Compute Engine service account for the project will be used.
|
||||
""")
|
||||
final String serviceAccountEmail
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Enable the use of spot virtual machines (default: `false`).
|
||||
""")
|
||||
final boolean spot
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
The URL of an existing subnetwork resource in the network to which the VM will be attached.
|
||||
""")
|
||||
final String subnetwork
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Do not provision public IP addresses for VMs, such that they only have an internal IP address (default: `false`).
|
||||
""")
|
||||
final boolean usePrivateAddress
|
||||
|
||||
BatchConfig(Map opts) {
|
||||
allowedLocations = opts.allowedLocations as List<String> ?: Collections.emptyList()
|
||||
autoRetryExitCodes = opts.autoRetryExitCodes as List<Integer> ?: DEFAULT_RETRY_LIST
|
||||
bootDiskImage = opts.bootDiskImage
|
||||
bootDiskSize = opts.bootDiskSize as MemoryUnit
|
||||
cpuPlatform = opts.cpuPlatform
|
||||
gcsfuseOptions = opts.gcsfuseOptions as List<String> ?: DEFAULT_GCSFUSE_OPTS
|
||||
installGpuDrivers = opts.installGpuDrivers as boolean
|
||||
installOpsAgent = opts.installOpsAgent as boolean
|
||||
logsPath = opts.logsPath
|
||||
maxSpotAttempts = opts.maxSpotAttempts != null ? opts.maxSpotAttempts as int : DEFAULT_MAX_SPOT_ATTEMPTS
|
||||
network = opts.network
|
||||
networkTags = opts.networkTags as List<String> ?: Collections.emptyList()
|
||||
preemptible = opts.preemptible as boolean
|
||||
retry = new BatchRetryConfig( opts.retryPolicy as Map ?: Collections.emptyMap() )
|
||||
serviceAccountEmail = opts.serviceAccountEmail
|
||||
spot = opts.spot as boolean
|
||||
subnetwork = opts.subnetwork
|
||||
usePrivateAddress = opts.usePrivateAddress as boolean
|
||||
}
|
||||
|
||||
BatchRetryConfig getRetryConfig() { retry }
|
||||
|
||||
Path logsPath() {
|
||||
return logsPath as Path
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud.google.batch.client
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.EqualsAndHashCode
|
||||
import groovy.transform.ToString
|
||||
import nextflow.util.Duration
|
||||
|
||||
/**
|
||||
* Model retry policy configuration
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@ToString(includePackage = false, includeNames = true)
|
||||
@EqualsAndHashCode
|
||||
@CompileStatic
|
||||
class BatchRetryConfig {
|
||||
Duration delay = Duration.of('350ms')
|
||||
Duration maxDelay = Duration.of('90s')
|
||||
int maxAttempts = 5
|
||||
double jitter = 0.25
|
||||
|
||||
BatchRetryConfig() {
|
||||
this(Collections.emptyMap())
|
||||
}
|
||||
|
||||
BatchRetryConfig(Map config) {
|
||||
if( config.delay )
|
||||
delay = config.delay as Duration
|
||||
if( config.maxDelay )
|
||||
maxDelay = config.maxDelay as Duration
|
||||
if( config.maxAttempts )
|
||||
maxAttempts = config.maxAttempts as int
|
||||
if( config.jitter )
|
||||
jitter = config.jitter as double
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud.google.batch.logging
|
||||
|
||||
import com.google.cloud.logging.LogEntry
|
||||
import com.google.cloud.logging.Logging
|
||||
import com.google.cloud.logging.LoggingOptions
|
||||
import com.google.cloud.logging.Severity
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.transform.Memoized
|
||||
import groovy.transform.PackageScope
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.cloud.google.GoogleOpts
|
||||
/**
|
||||
* Batch logging client
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@CompileStatic
|
||||
class BatchLogging implements Closeable {
|
||||
private LoggingOptions opts
|
||||
private String projectId
|
||||
private volatile Logging logging0
|
||||
|
||||
BatchLogging(GoogleOpts config) {
|
||||
final creds = config.credentials
|
||||
this.projectId = config.projectId
|
||||
this.opts = LoggingOptions .newBuilder() .setCredentials(creds) .setProjectId(this.projectId) .build()
|
||||
}
|
||||
|
||||
String stdout(String uid, String taskId) {
|
||||
return safeLogs(uid, taskId, 0)
|
||||
}
|
||||
|
||||
String stderr(String uid, String taskId) {
|
||||
return safeLogs(uid, taskId, 1)
|
||||
}
|
||||
|
||||
protected String safeLogs(String uid, String taskId, int index) {
|
||||
try {
|
||||
return fetchLogs(uid, taskId)[index]
|
||||
}
|
||||
catch (Exception e) {
|
||||
log.warn("Cannot read logs for Batch job '$uid/$taskId' - cause: ${e.message}", e)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
@Memoized(maxCacheSize = 1000)
|
||||
@PackageScope List<String> fetchLogs(String uid, String taskId) {
|
||||
final stdout = new StringBuilder()
|
||||
final stderr = new StringBuilder()
|
||||
final filter = "resource.type=generic_task OR resource.type=\"batch.googleapis.com/Job\" AND logName=\"projects/${projectId}/logs/batch_task_logs\" AND labels.job_uid=$uid AND labels.task_id=$uid-group0-$taskId"
|
||||
final entries = loggingService().listLogEntries(
|
||||
Logging.EntryListOption.filter(filter),
|
||||
Logging.EntryListOption.pageSize(1000) )
|
||||
|
||||
final page = entries.getValues()
|
||||
for (LogEntry logEntry : page.iterator()) {
|
||||
parseOutput(logEntry, stdout, stderr)
|
||||
}
|
||||
return [ stdout.toString(), stderr.toString() ]
|
||||
}
|
||||
|
||||
protected static void parseOutput(LogEntry logEntry, StringBuilder stdout, StringBuilder stderr) {
|
||||
final output = logEntry.payload.data.toString()
|
||||
if (logEntry.severity == Severity.ERROR) {
|
||||
stderr.append(output)
|
||||
} else {
|
||||
stdout.append(output)
|
||||
}
|
||||
}
|
||||
|
||||
synchronized protected Logging loggingService() {
|
||||
if( logging0==null ) {
|
||||
logging0 = opts.getService()
|
||||
}
|
||||
return logging0
|
||||
}
|
||||
|
||||
@Override
|
||||
void close() throws IOException {
|
||||
if( logging0==null )
|
||||
return
|
||||
try {
|
||||
logging0.close()
|
||||
}
|
||||
catch (Exception e) {
|
||||
log.debug "Unexpected error closing Google Logging service", e
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud.google.config
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.config.spec.ConfigOption
|
||||
import nextflow.config.spec.ConfigScope
|
||||
import nextflow.script.dsl.Description
|
||||
import nextflow.util.Duration
|
||||
|
||||
/**
|
||||
* Model Google storage retry settings
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class GoogleRetryOpts implements ConfigScope {
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Max attempts when retrying failed API requests to Cloud Storage (default: `10`).
|
||||
""")
|
||||
final int maxAttempts
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Delay multiplier when retrying failed API requests to Cloud Storage (default: `2.0`).
|
||||
""")
|
||||
final double multiplier
|
||||
|
||||
@ConfigOption
|
||||
@Description("""
|
||||
Max delay when retrying failed API requests to Cloud Storage (default: `'90s'`).
|
||||
""")
|
||||
final Duration maxDelay
|
||||
|
||||
GoogleRetryOpts(Map opts) {
|
||||
maxAttempts = opts.maxAttempts ? opts.maxAttempts as int : 10
|
||||
multiplier = opts.multiplier ? opts.multiplier as double : 2d
|
||||
maxDelay = opts.maxDelay ? opts.maxDelay as Duration : Duration.of('90s')
|
||||
}
|
||||
|
||||
long maxDelaySecs() {
|
||||
return maxDelay.seconds
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud.google.config
|
||||
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.config.spec.ConfigScope
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class GoogleStorageOpts implements ConfigScope {
|
||||
|
||||
final GoogleRetryOpts retryPolicy
|
||||
|
||||
GoogleStorageOpts(Map opts) {
|
||||
retryPolicy = new GoogleRetryOpts( opts.retryPolicy as Map ?: Collections.emptyMap() )
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud.google.util
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
import com.google.api.gax.retrying.RetrySettings
|
||||
import com.google.cloud.storage.StorageOptions
|
||||
import com.google.cloud.storage.contrib.nio.CloudStorageConfiguration
|
||||
import com.google.cloud.storage.contrib.nio.CloudStorageFileSystem
|
||||
import com.google.cloud.storage.contrib.nio.CloudStoragePath
|
||||
import groovy.transform.CompileStatic
|
||||
import nextflow.Global
|
||||
import nextflow.Session
|
||||
import nextflow.cloud.google.GoogleOpts
|
||||
import nextflow.file.FileSystemPathFactory
|
||||
/**
|
||||
* Implements FileSystemPathFactory interface for Google storage
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@CompileStatic
|
||||
class GsPathFactory extends FileSystemPathFactory {
|
||||
|
||||
private GoogleOpts googleOpts;
|
||||
|
||||
private CloudStorageConfiguration storageConfig
|
||||
|
||||
private StorageOptions storageOptions
|
||||
|
||||
static private GoogleOpts getGoogleOpts() {
|
||||
final session = (Session) Global.getSession()
|
||||
if (!session)
|
||||
throw new IllegalStateException("Cannot initialize GsPathFactory: missing session")
|
||||
return GoogleOpts.fromSession(session)
|
||||
}
|
||||
|
||||
static protected CloudStorageConfiguration getCloudStorageConfig(GoogleOpts googleOpts) {
|
||||
final builder = CloudStorageConfiguration.builder()
|
||||
if (googleOpts.enableRequesterPaysBuckets) {
|
||||
builder.userProject(googleOpts.getProjectId())
|
||||
}
|
||||
return builder.build()
|
||||
}
|
||||
|
||||
static protected StorageOptions getCloudStorageOptions(GoogleOpts opts) {
|
||||
final transportOptions = StorageOptions.getDefaultHttpTransportOptions().toBuilder()
|
||||
if( opts.httpConnectTimeout )
|
||||
transportOptions.setConnectTimeout( (int)opts.httpConnectTimeout.toMillis() )
|
||||
if( opts.httpReadTimeout )
|
||||
transportOptions.setReadTimeout( (int)opts.httpReadTimeout.toMillis() )
|
||||
|
||||
RetrySettings retrySettings =
|
||||
StorageOptions.getDefaultRetrySettings()
|
||||
.toBuilder()
|
||||
.setMaxAttempts(opts.storageOpts.retryPolicy.maxAttempts)
|
||||
.setRetryDelayMultiplier(opts.storageOpts.retryPolicy.multiplier)
|
||||
.setTotalTimeout(org.threeten.bp.Duration.ofSeconds(opts.storageOpts.retryPolicy.maxDelaySecs()))
|
||||
.build()
|
||||
|
||||
return StorageOptions.getDefaultInstance()
|
||||
.toBuilder()
|
||||
.setTransportOptions(transportOptions.build())
|
||||
.setRetrySettings(retrySettings)
|
||||
.build()
|
||||
}
|
||||
|
||||
private void init() {
|
||||
synchronized (this) {
|
||||
if( googleOpts!=null )
|
||||
return
|
||||
this.googleOpts = getGoogleOpts()
|
||||
this.storageConfig = getCloudStorageConfig(googleOpts)
|
||||
this.storageOptions = getCloudStorageOptions(googleOpts)
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Path parseUri(String uri) {
|
||||
if( !uri.startsWith('gs://') )
|
||||
return null
|
||||
init()
|
||||
final str = uri.substring(5)
|
||||
final p = str.indexOf('/')
|
||||
final ret = p == -1
|
||||
? CloudStorageFileSystem.forBucket(str, storageConfig, storageOptions).getPath('')
|
||||
: CloudStorageFileSystem.forBucket(str.substring(0,p), storageConfig, storageOptions).getPath(str.substring(p))
|
||||
return ret.normalize()
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String toUriString(Path path) {
|
||||
if( path instanceof CloudStoragePath ) {
|
||||
return "gs://${path.bucket()}$path".toString()
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String getBashLib(Path path) {
|
||||
return null
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String getUploadCmd(String source, Path target) {
|
||||
return null
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud.google.util
|
||||
|
||||
import com.esotericsoftware.kryo.Kryo
|
||||
import com.esotericsoftware.kryo.Serializer
|
||||
import com.esotericsoftware.kryo.io.Input
|
||||
import com.esotericsoftware.kryo.io.Output
|
||||
import com.google.cloud.storage.contrib.nio.CloudStorageFileSystem
|
||||
import com.google.cloud.storage.contrib.nio.CloudStoragePath
|
||||
import groovy.transform.CompileStatic
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.util.SerializerRegistrant
|
||||
import org.pf4j.Extension
|
||||
/**
|
||||
* Serializer for a {@link CloudStoragePath}
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
@Extension
|
||||
@CompileStatic
|
||||
class GsPathSerializer extends Serializer<CloudStoragePath> implements SerializerRegistrant {
|
||||
|
||||
@Override
|
||||
void write(Kryo kryo, Output output, CloudStoragePath target) {
|
||||
def path = target.toString()
|
||||
if( !path.startsWith('/') ) // <-- it looks a bug in the google nio library, in some case the path returned is not absolute
|
||||
path = '/' + path
|
||||
path = target.bucket() + path
|
||||
log.trace "Google CloudStoragePath serialisation > path=$path"
|
||||
output.writeString(path)
|
||||
}
|
||||
|
||||
@Override
|
||||
CloudStoragePath read(Kryo kryo, Input input, Class<CloudStoragePath> type) {
|
||||
final path = input.readString()
|
||||
log.trace "Google CloudStoragePath de-serialization > path=$path"
|
||||
def uri = CloudStorageFileSystem.URI_SCHEME + '://' + path
|
||||
(CloudStoragePath) GsPathFactory.parse(uri)
|
||||
}
|
||||
|
||||
@Override
|
||||
void register(Map<Class, Object> serializers) {
|
||||
serializers.put(CloudStoragePath, GsPathSerializer)
|
||||
}
|
||||
}
|
||||
31
nextflow/plugins/nf-google/src/resources/logback-test.xml
Normal file
31
nextflow/plugins/nf-google/src/resources/logback-test.xml
Normal file
@@ -0,0 +1,31 @@
|
||||
<!--
|
||||
~ Copyright 2013-2026, Seqera Labs
|
||||
~
|
||||
~ Licensed under the Apache License, Version 2.0 (the "License");
|
||||
~ you may not use this file except in compliance with the License.
|
||||
~ You may obtain a copy of the License at
|
||||
~
|
||||
~ http://www.apache.org/licenses/LICENSE-2.0
|
||||
~
|
||||
~ Unless required by applicable law or agreed to in writing, software
|
||||
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
|
||||
<configuration>
|
||||
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
|
||||
<encoder>
|
||||
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} – %m%n</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<logger name="io.grpc.netty" level="INFO" />
|
||||
<logger name="reactor" level="INFO" />
|
||||
<logger name="io.netty" level="INFO" />
|
||||
|
||||
<root level="DEBUG">
|
||||
<appender-ref ref="STDOUT"/>
|
||||
</root>
|
||||
</configuration>
|
||||
@@ -0,0 +1,79 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud.google
|
||||
|
||||
import java.nio.file.Paths
|
||||
|
||||
import com.google.cloud.storage.contrib.nio.CloudStoragePath
|
||||
import spock.lang.Specification
|
||||
|
||||
import java.nio.file.FileSystem
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.attribute.BasicFileAttributes
|
||||
import java.nio.file.spi.FileSystemProvider
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
abstract class GoogleSpecification extends Specification {
|
||||
|
||||
protected Path mockGsPath(String path, boolean isDir=false) {
|
||||
assert path.startsWith('gs://')
|
||||
|
||||
def tokens = path.tokenize('/')
|
||||
def bucket = tokens[1]
|
||||
def file = '/' + tokens[2..-1].join('/')
|
||||
|
||||
def attr = Mock(BasicFileAttributes)
|
||||
attr.isDirectory() >> isDir
|
||||
attr.isRegularFile() >> !isDir
|
||||
attr.isSymbolicLink() >> false
|
||||
|
||||
def provider = Mock(FileSystemProvider)
|
||||
provider.getScheme() >> 'gs'
|
||||
provider.readAttributes(_, _, _) >> attr
|
||||
|
||||
def fs = Mock(FileSystem)
|
||||
fs.provider() >> provider
|
||||
fs.toString() >> ('gs://' + bucket)
|
||||
def uri = GroovyMock(URI)
|
||||
uri.toString() >> path
|
||||
|
||||
|
||||
def result = GroovyMock(Path)
|
||||
result.bucket() >> bucket
|
||||
result.toUriString() >> path
|
||||
result.toString() >> file
|
||||
result.getFileSystem() >> fs
|
||||
result.toUri() >> uri
|
||||
result.resolve(_) >> { mockGsPath("$path/${it[0]}") }
|
||||
result.toAbsolutePath() >> result
|
||||
result.asBoolean() >> true
|
||||
result.getParent() >> { def p=path.lastIndexOf('/'); p!=-1 ? mockGsPath("${path.substring(0,p)}", true) : null }
|
||||
result.getFileName() >> { Paths.get(tokens[-1]) }
|
||||
result.getName() >> tokens[1]
|
||||
return result
|
||||
}
|
||||
|
||||
|
||||
static CloudStoragePath gsPath(String path) {
|
||||
assert path.startsWith('gs://')
|
||||
(CloudStoragePath) Paths.get( new URI(null,null,path,null,null))
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud.google.batch
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
import com.google.cloud.storage.contrib.nio.CloudStorageFileSystem
|
||||
import nextflow.Session
|
||||
import nextflow.SysEnv
|
||||
import nextflow.processor.TaskHandler
|
||||
import nextflow.processor.TaskRun
|
||||
import spock.lang.Specification
|
||||
import spock.lang.Unroll
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class GoogleBatchExecutorTest extends Specification {
|
||||
|
||||
def 'should check is fusion' () {
|
||||
given:
|
||||
SysEnv.push(ENV)
|
||||
and:
|
||||
def sess = Mock(Session) {
|
||||
getConfig() >> CONFIG
|
||||
}
|
||||
def executor = new GoogleBatchExecutor(session: sess)
|
||||
|
||||
expect:
|
||||
executor.isFusionEnabled() == EXPECTED
|
||||
|
||||
cleanup:
|
||||
SysEnv.pop()
|
||||
|
||||
where:
|
||||
CONFIG | ENV | EXPECTED
|
||||
[:] | [:] | false
|
||||
[fusion:[enabled: true]] | [:] | true
|
||||
[fusion:[enabled: false]] | [FUSION_ENABLED:'true'] | false // <-- config has priority
|
||||
[:] | [FUSION_ENABLED:'true'] | true
|
||||
[:] | [FUSION_ENABLED:'false'] | false
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should check cloudinfo enabled' () {
|
||||
given:
|
||||
SysEnv.push(ENV)
|
||||
and:
|
||||
def sess = Mock(Session) { getConfig() >> [:] }
|
||||
def executor = new GoogleBatchExecutor(session: sess)
|
||||
|
||||
expect:
|
||||
executor.isCloudinfoEnabled() == EXPECTED
|
||||
|
||||
cleanup:
|
||||
SysEnv.pop()
|
||||
|
||||
where:
|
||||
ENV | EXPECTED
|
||||
[:] | true
|
||||
[NXF_CLOUDINFO_ENABLED:'true'] | true
|
||||
[NXF_CLOUDINFO_ENABLED:'false'] | false
|
||||
}
|
||||
|
||||
def 'should get array index variable and start' () {
|
||||
given:
|
||||
def executor = Spy(GoogleBatchExecutor)
|
||||
expect:
|
||||
executor.getArrayIndexName() == 'BATCH_TASK_INDEX'
|
||||
executor.getArrayIndexStart() == 0
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should get array task id' () {
|
||||
given:
|
||||
def executor = Spy(GoogleBatchExecutor)
|
||||
expect:
|
||||
executor.getArrayTaskId(JOB_ID, TASK_INDEX) == EXPECTED
|
||||
|
||||
where:
|
||||
JOB_ID | TASK_INDEX | EXPECTED
|
||||
'foo' | 1 | '1'
|
||||
'bar' | 2 | '2'
|
||||
}
|
||||
|
||||
protected Path gs(String str) {
|
||||
def b = str.tokenize('/')[0]
|
||||
def p = str.tokenize('/')[1..-1].join('/')
|
||||
CloudStorageFileSystem.forBucket(b).getPath('/'+p)
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should get array task id' () {
|
||||
given:
|
||||
def executor = Spy(GoogleBatchExecutor) {
|
||||
isFusionEnabled()>>FUSION
|
||||
isWorkDirDefaultFS()>>DEFAULT_FS
|
||||
}
|
||||
and:
|
||||
def handler = Mock(TaskHandler) {
|
||||
getTask() >> Mock(TaskRun) { getWorkDir() >> WORK_DIR }
|
||||
}
|
||||
expect:
|
||||
executor.getArrayWorkDir(handler) == EXPECTED
|
||||
|
||||
where:
|
||||
FUSION | DEFAULT_FS | WORK_DIR | EXPECTED
|
||||
false | false | gs('/foo/work/dir') | '/mnt/disks/foo/work/dir'
|
||||
true | false | gs('/foo/work/dir') | '/fusion/gs/foo/work/dir'
|
||||
false | true | Path.of('/nfs/work') | '/nfs/work'
|
||||
}
|
||||
|
||||
def 'should get array launch command' (){
|
||||
given:
|
||||
def executor = Spy(GoogleBatchExecutor) {
|
||||
isFusionEnabled()>>FUSION
|
||||
isWorkDirDefaultFS()>>DEFAULT_FS
|
||||
}
|
||||
expect:
|
||||
executor.getArrayLaunchCommand(TASK_DIR) == EXPECTED
|
||||
|
||||
where:
|
||||
FUSION | DEFAULT_FS | TASK_DIR | EXPECTED
|
||||
false | false | 'gs://foo/work/dir' | '/bin/bash -o pipefail -c \'trap "{ cp .command.log gs://foo/work/dir/.command.log; }" EXIT; /bin/bash gs://foo/work/dir/.command.run 2>&1 | tee .command.log\''
|
||||
true | false | '/fusion/work/dir' | 'bash /fusion/work/dir/.command.run'
|
||||
false | true | '/nfs/work/dir' | 'bash /nfs/work/dir/.command.run 2>&1 > /nfs/work/dir/.command.log'
|
||||
}
|
||||
|
||||
def 'should validate shouldDeleteJob method' () {
|
||||
given:
|
||||
def executor = Spy(GoogleBatchExecutor)
|
||||
|
||||
expect:
|
||||
executor.shouldDeleteJob('job-1')
|
||||
executor.shouldDeleteJob('job-2')
|
||||
executor.shouldDeleteJob('job-3')
|
||||
and:
|
||||
!executor.shouldDeleteJob('job-1')
|
||||
!executor.shouldDeleteJob('job-1')
|
||||
!executor.shouldDeleteJob('job-2')
|
||||
!executor.shouldDeleteJob('job-2')
|
||||
!executor.shouldDeleteJob('job-3')
|
||||
!executor.shouldDeleteJob('job-3')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud.google.batch
|
||||
|
||||
import com.google.cloud.batch.v1.Volume
|
||||
import groovy.transform.Canonical
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Canonical
|
||||
class GoogleBatchLauncherSpecMock implements GoogleBatchLauncherSpec {
|
||||
|
||||
String runCommand
|
||||
List<String> containerMounts = List.of()
|
||||
List<Volume> volumes = List.of()
|
||||
Map<String,String> environment = Map.of()
|
||||
|
||||
@Override
|
||||
List<String> getContainerMounts() {
|
||||
return containerMounts
|
||||
}
|
||||
|
||||
@Override
|
||||
List<Volume> getVolumes() {
|
||||
return volumes
|
||||
}
|
||||
|
||||
@Override
|
||||
String runCommand() {
|
||||
return runCommand
|
||||
}
|
||||
|
||||
Map<String,String> getEnvironment() {
|
||||
return environment
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud.google.batch
|
||||
|
||||
import nextflow.cloud.google.batch.GoogleBatchMachineTypeSelector.MachineType
|
||||
import nextflow.util.MemoryUnit
|
||||
import spock.lang.IgnoreIf
|
||||
import spock.lang.Specification
|
||||
|
||||
class GoogleBatchMachineTypeSelectorTest extends Specification {
|
||||
|
||||
static final MACHINE_TYPES = [
|
||||
new MachineType(type: 'e2-type01', family: 'e2', 'spotPrice': 0.001, 'onDemandPrice': 0.01, 'cpusPerVm': 1, 'memPerVm': 1),
|
||||
new MachineType(type: 'n1-type02', family: 'n1', 'spotPrice': 0.002, 'onDemandPrice': 0.02, 'cpusPerVm': 2, 'memPerVm': 2),
|
||||
new MachineType(type: 'e2-type03', family: 'e2', 'spotPrice': 0.010, 'onDemandPrice': 0.05, 'cpusPerVm': 4, 'memPerVm': 4),
|
||||
new MachineType(type: 'n2-type04', family: 'n2', 'spotPrice': 0.011, 'onDemandPrice': 0.15, 'cpusPerVm': 4, 'memPerVm': 4),
|
||||
new MachineType(type: 'e2-type05', family: 'e2', 'spotPrice': 0.020, 'onDemandPrice': 0.20, 'cpusPerVm': 6, 'memPerVm': 6),
|
||||
new MachineType(type: 'n1-type06', family: 'n1', 'spotPrice': 0.025, 'onDemandPrice': 0.25, 'cpusPerVm': 6, 'memPerVm': 6),
|
||||
new MachineType(type: 'm1-type07', family: 'm1', 'spotPrice': 0.030, 'onDemandPrice': 0.30, 'cpusPerVm': 8, 'memPerVm': 8),
|
||||
new MachineType(type: 'm2-type08', family: 'm2', 'spotPrice': 0.036, 'onDemandPrice': 0.35, 'cpusPerVm': 8, 'memPerVm': 8),
|
||||
new MachineType(type: 'n2-type09', family: 'n2', 'spotPrice': 0.040, 'onDemandPrice': 0.40, 'cpusPerVm': 10, 'memPerVm': 10),
|
||||
new MachineType(type: 'c2-type10', family: 'c2', 'spotPrice': 0.045, 'onDemandPrice': 0.45, 'cpusPerVm': 10, 'memPerVm': 10),
|
||||
new MachineType(type: 'c4-type11', family: 'c4', 'spotPrice': 0.040, 'onDemandPrice': 0.40, 'cpusPerVm': 8, 'memPerVm': 8),
|
||||
new MachineType(type: 'c4a-type12', family: 'c4a', 'spotPrice': 0.038, 'onDemandPrice': 0.38, 'cpusPerVm': 8, 'memPerVm': 8),
|
||||
new MachineType(type: 'c4d-type13', family: 'c4d', 'spotPrice': 0.039, 'onDemandPrice': 0.39, 'cpusPerVm': 8, 'memPerVm': 8),
|
||||
new MachineType(type: 'n4-type14', family: 'n4', 'spotPrice': 0.035, 'onDemandPrice': 0.35, 'cpusPerVm': 8, 'memPerVm': 8),
|
||||
new MachineType(type: 'n4a-type15', family: 'n4a', 'spotPrice': 0.033, 'onDemandPrice': 0.33, 'cpusPerVm': 8, 'memPerVm': 8),
|
||||
new MachineType(type: 'n4d-type16', family: 'n4d', 'spotPrice': 0.034, 'onDemandPrice': 0.34, 'cpusPerVm': 8, 'memPerVm': 8),
|
||||
]
|
||||
|
||||
def 'should select best machine type'() {
|
||||
given:
|
||||
final selector = Spy(GoogleBatchMachineTypeSelector) {
|
||||
getAvailableMachineTypes(REGION, SPOT) >> MACHINE_TYPES
|
||||
}
|
||||
expect:
|
||||
selector.bestMachineType(CPUS, MEM, REGION, SPOT, FUSION, FAMILIES).type == EXPECTED
|
||||
|
||||
where:
|
||||
CPUS | MEM | REGION | SPOT | FUSION | FAMILIES | EXPECTED
|
||||
1 | 1000 | 'reg' | true | false | null | 'e2-type01'
|
||||
1 | 1000 | 'reg' | false | true | null | 'n1-type02'
|
||||
4 | 4000 | 'reg' | false | false | [] | 'e2-type03'
|
||||
4 | 4000 | 'reg' | true | false | [] | 'n2-type04'
|
||||
4 | 4000 | 'reg' | false | true | [] | 'n2-type04'
|
||||
6 | 6000 | 'reg' | true | false | null | 'e2-type05'
|
||||
6 | 6000 | 'reg' | true | true | null | 'n1-type06'
|
||||
6 | 6000 | 'reg' | true | false | ['n1-*', 'm1-*'] | 'n1-type06'
|
||||
8 | 8000 | 'reg' | true | false | null | 'm1-type07'
|
||||
8 | 8000 | 'reg' | false | false | ['m?-*', 'c2-*'] | 'm2-type08'
|
||||
8 | 8000 | 'reg' | false | false | ['m1-type07', 'm2-type66'] | 'm1-type07'
|
||||
8 | 8000 | 'reg' | true | false | ['c4-*'] | 'c4-type11'
|
||||
8 | 8000 | 'reg' | true | false | ['c4a-*'] | 'c4a-type12'
|
||||
8 | 8000 | 'reg' | true | false | ['c4d-*'] | 'c4d-type13'
|
||||
8 | 8000 | 'reg' | true | false | ['n4-*'] | 'n4-type14'
|
||||
8 | 8000 | 'reg' | true | false | ['n4a-*'] | 'n4a-type15'
|
||||
8 | 8000 | 'reg' | true | false | ['n4d-*'] | 'n4d-type16'
|
||||
|
||||
|
||||
}
|
||||
|
||||
def 'should not select a machine type'() {
|
||||
given:
|
||||
final selector = Spy(GoogleBatchMachineTypeSelector) {
|
||||
getAvailableMachineTypes(REGION, SPOT) >> MACHINE_TYPES
|
||||
}
|
||||
when:
|
||||
selector.bestMachineType(CPUS, MEM, REGION, SPOT, SSD, FAMILIES)
|
||||
|
||||
then:
|
||||
thrown(NoSuchElementException)
|
||||
|
||||
where:
|
||||
CPUS | MEM | REGION | SPOT | SSD | FAMILIES
|
||||
8 | 9000 | 'reg' | true | true | ['s2-*', 'e2-*']
|
||||
12 | 1000 | 'reg' | true | false | null
|
||||
|
||||
}
|
||||
|
||||
@IgnoreIf({System.getenv('NXF_SMOKE')})
|
||||
def 'should parse Seqera cloud info API'() {
|
||||
when:
|
||||
GoogleBatchMachineTypeSelector.INSTANCE.getAvailableMachineTypes("europe-west2", true)
|
||||
|
||||
then:
|
||||
noExceptionThrown()
|
||||
}
|
||||
|
||||
def 'should find first valid disk size'() {
|
||||
expect:
|
||||
GoogleBatchMachineTypeSelector.INSTANCE.findFirstValidSize(MemoryUnit.of(REQUESTED), ALLOWED) == MemoryUnit.of(EXPECTED)
|
||||
|
||||
where:
|
||||
REQUESTED | ALLOWED | EXPECTED
|
||||
'100 GB' | [2, 4, 8] | '750 GB'
|
||||
'100 GB' | [1, 2, 4] | '375 GB'
|
||||
'500 GB' | [1, 2] | '750 GB'
|
||||
'1 TB' | [1, 2] | '750 GB'
|
||||
}
|
||||
|
||||
def 'should find valid local disk size given the machine type'() {
|
||||
expect:
|
||||
final machineType = new MachineType(type: TYPE, family: FAMILY, cpusPerVm: CPUS)
|
||||
GoogleBatchMachineTypeSelector.INSTANCE.findValidLocalSSDSize(MemoryUnit.of(REQUESTED), machineType) == MemoryUnit.of(EXPECTED)
|
||||
|
||||
where:
|
||||
REQUESTED | TYPE | FAMILY | CPUS | EXPECTED
|
||||
'100 GB' | 'n1-highmem-8' | 'n1' | 8 | '375 GB'
|
||||
'375 GB' | 'n2-highcpu-16' | 'n2' | 16 | '750 GB'
|
||||
'780 GB' | 'n2d-standard-48' | 'n2d' | 48 | '1500 GB'
|
||||
'200 GB' | 'c2-standard-4' | 'c2' | 4 | '375 GB'
|
||||
'50 GB' | 'c2d-highmem-56' | 'c2d' | 56 | '1500 GB'
|
||||
'750 GB' | 'm3-megamem-64' | 'm3' | 64 | '1500 GB'
|
||||
'100 GB' | 'c4-standard-8-lssd' | 'c4' | 8 | '0'
|
||||
'100 GB' | 'c4a-standard-8-lssd' | 'c4a' | 8 | '0'
|
||||
'100 GB' | 'c4d-standard-8-lssd' | 'c4d' | 8 | '0'
|
||||
}
|
||||
|
||||
def 'should know when hyperdisk is required'() {
|
||||
expect:
|
||||
GoogleBatchMachineTypeSelector.INSTANCE.isHyperdiskOnly(TYPE) == EXPECTED
|
||||
|
||||
where:
|
||||
TYPE | EXPECTED
|
||||
'c4-standard-8' | true
|
||||
'c4a-standard-8' | true
|
||||
'c4d-standard-8' | true
|
||||
'n4-standard-8' | true
|
||||
'n4a-standard-8' | true
|
||||
'n4d-standard-8' | true
|
||||
'n1-standard-8' | false
|
||||
'n2-standard-8' | false
|
||||
'e2-standard-8' | false
|
||||
'c2-standard-8' | false
|
||||
}
|
||||
|
||||
def 'should know when to install GPU drivers'() {
|
||||
expect:
|
||||
final machineType = new MachineType(type: TYPE, gpusPerVm: GPUS)
|
||||
GoogleBatchMachineTypeSelector.INSTANCE.installGpuDrivers(machineType) == EXPECTED
|
||||
|
||||
where:
|
||||
TYPE | GPUS | EXPECTED
|
||||
'n2-standard-4' | 0 | false
|
||||
'n2-standard-4' | 1 | true
|
||||
'a2-highgpu-1g' | 0 | true
|
||||
'a3-highgpu-1g' | 0 | true
|
||||
'g2-standard-4' | 0 | true
|
||||
}
|
||||
|
||||
def 'should detect non-configurable local SSD'() {
|
||||
expect:
|
||||
final machineType = new MachineType(type: TYPE, family: FAMILY)
|
||||
GoogleBatchMachineTypeSelector.INSTANCE.notConfigurableLocalSSD(machineType) == EXPECTED
|
||||
|
||||
where:
|
||||
TYPE | FAMILY | EXPECTED
|
||||
// c3/c3d with -lssd suffix → true
|
||||
'c3-standard-8-lssd' | 'c3' | true
|
||||
'c3d-standard-8-lssd' | 'c3d' | true
|
||||
// c4/c4a/c4d with -lssd suffix → true
|
||||
'c4-standard-8-lssd' | 'c4' | true
|
||||
'c4a-standard-8-lssd' | 'c4a' | true
|
||||
'c4d-standard-8-lssd' | 'c4d' | true
|
||||
// a3 family → always true regardless of type
|
||||
'a3-highgpu-8g' | 'a3' | true
|
||||
'a3-megagpu-64g' | 'a3' | true
|
||||
// a2-ultragpu- prefix → true regardless of family
|
||||
'a2-ultragpu-1g' | 'a2' | true
|
||||
'a2-ultragpu-8g' | 'a2' | true
|
||||
// c3/c4 without -lssd suffix → false
|
||||
'c3-standard-8' | 'c3' | false
|
||||
'c4-standard-8' | 'c4' | false
|
||||
// a2 non-ultragpu → false
|
||||
'a2-highgpu-1g' | 'a2' | false
|
||||
// unrelated families → false
|
||||
'n2-standard-4' | 'n2' | false
|
||||
'e2-standard-8' | 'e2' | false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud.google.batch
|
||||
|
||||
import java.nio.file.Paths
|
||||
|
||||
import com.google.cloud.storage.contrib.nio.CloudStorageFileSystem
|
||||
import nextflow.cloud.google.GoogleOpts
|
||||
import nextflow.cloud.google.batch.client.BatchConfig
|
||||
import nextflow.processor.TaskRun
|
||||
import spock.lang.Specification
|
||||
import spock.lang.Unroll
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class GoogleBatchScriptLauncherTest extends Specification{
|
||||
|
||||
@Unroll
|
||||
def 'should convert to container path' () {
|
||||
given:
|
||||
def launcher = new GoogleBatchScriptLauncher()
|
||||
|
||||
expect:
|
||||
def path = CloudStorageFileSystem.forBucket(BUCKET).getPath(PATH)
|
||||
launcher.toContainerMount(path, PARENT) == EXPECTED
|
||||
and:
|
||||
launcher.getContainerMounts() == [MOUNTS]
|
||||
where:
|
||||
BUCKET | PATH | PARENT | EXPECTED | MOUNTS
|
||||
'foo' | '/' | false | Paths.get('/mnt/disks/foo') | '/mnt/disks/foo:/mnt/disks/foo:rw'
|
||||
'foo' | '/some/dir' | false | Paths.get('/mnt/disks/foo/some/dir') | '/mnt/disks/foo/some/dir:/mnt/disks/foo/some/dir:rw'
|
||||
'foo' | '/some/dir' | true | Paths.get('/mnt/disks/foo/some/dir') | '/mnt/disks/foo/some:/mnt/disks/foo/some:rw'
|
||||
}
|
||||
|
||||
def 'should compute volume mounts' () {
|
||||
given:
|
||||
def launcher = new GoogleBatchScriptLauncher()
|
||||
launcher.config = Mock(GoogleOpts) {
|
||||
getBatch() >> Mock(BatchConfig) {
|
||||
getGcsfuseOptions() >> ['-o rw', '-implicit-dirs', '-o allow_other', '--uid=1000']
|
||||
}
|
||||
getProjectId() >> 'my-project'
|
||||
enableRequesterPaysBuckets >> true
|
||||
}
|
||||
and:
|
||||
def PATH1 = CloudStorageFileSystem.forBucket('alpha').getPath('/data/sample1.bam')
|
||||
def PATH2 = CloudStorageFileSystem.forBucket('alpha').getPath('/data/sample2.bam')
|
||||
def PATH3 = CloudStorageFileSystem.forBucket('omega').getPath('/data/sample3.bam')
|
||||
|
||||
expect:
|
||||
launcher.toContainerMount(PATH1) == Paths.get('/mnt/disks/alpha/data/sample1.bam')
|
||||
launcher.toContainerMount(PATH2) == Paths.get('/mnt/disks/alpha/data/sample2.bam')
|
||||
launcher.toContainerMount(PATH3) == Paths.get('/mnt/disks/omega/data/sample3.bam')
|
||||
|
||||
and:
|
||||
def containerMounts = launcher.getContainerMounts()
|
||||
and:
|
||||
containerMounts.size() == 2
|
||||
containerMounts[0] == '/mnt/disks/alpha/data:/mnt/disks/alpha/data:rw'
|
||||
containerMounts[1] == '/mnt/disks/omega/data/sample3.bam:/mnt/disks/omega/data/sample3.bam:rw'
|
||||
|
||||
and:
|
||||
def volumes = launcher.getVolumes()
|
||||
and:
|
||||
volumes.size() == 2
|
||||
volumes[0].getGcs().getRemotePath() == 'alpha'
|
||||
volumes[0].getMountPath() == '/mnt/disks/alpha'
|
||||
volumes[0].getMountOptionsList() == ['-o rw', '-implicit-dirs', '-o allow_other', '--uid=1000', '--billing-project my-project']
|
||||
volumes[1].getGcs().getRemotePath() == 'omega'
|
||||
volumes[1].getMountPath() == '/mnt/disks/omega'
|
||||
volumes[1].getMountOptionsList() == ['-o rw', '-implicit-dirs', '-o allow_other', '--uid=1000', '--billing-project my-project']
|
||||
}
|
||||
|
||||
def 'should return target files in remote work dir' () {
|
||||
given:
|
||||
def launcher = new GoogleBatchScriptLauncher()
|
||||
def workDir = CloudStorageFileSystem.forBucket('my-bucket').getPath('/work/dir')
|
||||
launcher.@remoteWorkDir = workDir
|
||||
|
||||
expect:
|
||||
launcher.targetInputFile() == workDir.resolve(TaskRun.CMD_INFILE)
|
||||
launcher.targetScriptFile() == workDir.resolve(TaskRun.CMD_SCRIPT)
|
||||
launcher.targetWrapperFile() == workDir.resolve(TaskRun.CMD_RUN)
|
||||
launcher.targetStageFile() == workDir.resolve(TaskRun.CMD_STAGE)
|
||||
}
|
||||
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,78 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package nextflow.cloud.google.batch.client
|
||||
|
||||
import com.google.cloud.batch.v1.Task
|
||||
import com.google.cloud.batch.v1.TaskName
|
||||
import com.google.cloud.batch.v1.TaskStatus
|
||||
import spock.lang.Specification
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
|
||||
*/
|
||||
class BatchClientTest extends Specification{
|
||||
|
||||
def 'should return task status with getTaskInArray' () {
|
||||
given:
|
||||
def project = 'project-id'
|
||||
def location = 'location-id'
|
||||
def job1 = 'job1-id'
|
||||
def task1 = 'task1-id'
|
||||
def task1Name = TaskName.of(project, location, job1, 'group0', task1).toString()
|
||||
def job2 = 'job2-id'
|
||||
def task2 = 'task2-id'
|
||||
def task2Name = TaskName.of(project, location, job2, 'group0', task2).toString()
|
||||
def job3 = 'job3-id'
|
||||
def task3 = 'task3-id'
|
||||
def task3Name = TaskName.of(project, location, job3, 'group0', task3).toString()
|
||||
def arrayTasks = new HashMap<String,TaskStatusRecord>()
|
||||
def client = Spy( new BatchClient( projectId: project, location: location, arrayTaskStatus: arrayTasks ) )
|
||||
|
||||
when:
|
||||
client.listTasks(job2) >> {
|
||||
def list = new LinkedList<>()
|
||||
list.add(makeTask(task2Name, TaskStatus.State.FAILED))
|
||||
return list
|
||||
}
|
||||
client.listTasks(job3) >> {
|
||||
def list = new LinkedList<>()
|
||||
list.add(makeTask(task3Name, TaskStatus.State.SUCCEEDED))
|
||||
return list
|
||||
}
|
||||
arrayTasks.put(task1Name, makeTaskStatusRecord(TaskStatus.State.RUNNING, System.currentTimeMillis()))
|
||||
arrayTasks.put(task2Name, makeTaskStatusRecord(TaskStatus.State.PENDING, System.currentTimeMillis() - 1_001))
|
||||
|
||||
then:
|
||||
// recent cached task
|
||||
client.getTaskInArrayStatus(job1, task1).state == TaskStatus.State.RUNNING
|
||||
// Outdated cached task
|
||||
client.getTaskInArrayStatus(job2, task2).state == TaskStatus.State.FAILED
|
||||
// no cached task
|
||||
client.getTaskInArrayStatus(job3, task3).state == TaskStatus.State.SUCCEEDED
|
||||
}
|
||||
|
||||
TaskStatusRecord makeTaskStatusRecord(TaskStatus.State state, long timestamp) {
|
||||
return new TaskStatusRecord(TaskStatus.newBuilder().setState(state).build(), timestamp)
|
||||
}
|
||||
|
||||
def makeTask(String name, TaskStatus.State state){
|
||||
Task.newBuilder().setName(name)
|
||||
.setStatus(TaskStatus.newBuilder().setState(state).build())
|
||||
.build()
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud.google.batch.client
|
||||
|
||||
import nextflow.util.MemoryUnit
|
||||
import spock.lang.Specification
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class BatchConfigTest extends Specification {
|
||||
|
||||
def 'should create batch config' () {
|
||||
when:
|
||||
def config = new BatchConfig([:])
|
||||
then:
|
||||
!config.getSpot()
|
||||
and:
|
||||
config.retryConfig.maxAttempts == 5
|
||||
config.maxSpotAttempts == 0
|
||||
config.autoRetryExitCodes == [50001]
|
||||
and:
|
||||
!config.bootDiskImage
|
||||
!config.bootDiskSize
|
||||
!config.logsPath
|
||||
}
|
||||
|
||||
def 'should create batch config with custom settings' () {
|
||||
given:
|
||||
def opts = [
|
||||
spot: true,
|
||||
maxSpotAttempts: 8,
|
||||
autoRetryExitCodes: [50001, 50003, 50005],
|
||||
retryPolicy: [maxAttempts: 10],
|
||||
bootDiskImage: 'batch-foo',
|
||||
bootDiskSize: '100GB',
|
||||
logsPath: 'gs://my-logs-bucket/logs',
|
||||
installOpsAgent: true
|
||||
]
|
||||
|
||||
when:
|
||||
def config = new BatchConfig(opts)
|
||||
then:
|
||||
config.getSpot()
|
||||
and:
|
||||
config.retryConfig.maxAttempts == 10
|
||||
config.maxSpotAttempts == 8
|
||||
config.autoRetryExitCodes == [50001, 50003, 50005]
|
||||
and:
|
||||
config.bootDiskImage == 'batch-foo'
|
||||
config.bootDiskSize == MemoryUnit.of('100GB')
|
||||
and:
|
||||
config.logsPath == 'gs://my-logs-bucket/logs'
|
||||
and:
|
||||
config.installOpsAgent == true
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud.google.batch.client
|
||||
|
||||
import nextflow.util.Duration
|
||||
import spock.lang.Specification
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class BatchRetryConfigTest extends Specification {
|
||||
|
||||
def 'should create retry config' () {
|
||||
|
||||
expect:
|
||||
new BatchRetryConfig().delay == Duration.of('350ms')
|
||||
new BatchRetryConfig().maxDelay == Duration.of('90s')
|
||||
new BatchRetryConfig().maxAttempts == 5
|
||||
new BatchRetryConfig().jitter == 0.25d
|
||||
|
||||
and:
|
||||
new BatchRetryConfig([maxAttempts: 20]).maxAttempts == 20
|
||||
new BatchRetryConfig([delay: '1s']).delay == Duration.of('1s')
|
||||
new BatchRetryConfig([maxDelay: '1m']).maxDelay == Duration.of('1m')
|
||||
new BatchRetryConfig([jitter: '0.5']).jitter == 0.5d
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud.google.batch.logging
|
||||
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
import com.google.cloud.batch.v1.Job
|
||||
import com.google.cloud.batch.v1.LogsPolicy
|
||||
import com.google.cloud.batch.v1.Runnable
|
||||
import com.google.cloud.batch.v1.TaskGroup
|
||||
import com.google.cloud.batch.v1.TaskSpec
|
||||
import com.google.cloud.logging.LogEntry
|
||||
import com.google.cloud.logging.Payload.StringPayload
|
||||
import com.google.cloud.logging.Severity
|
||||
import groovy.util.logging.Slf4j
|
||||
import nextflow.Session
|
||||
import nextflow.cloud.google.GoogleOpts
|
||||
import nextflow.cloud.google.batch.client.BatchClient
|
||||
import spock.lang.IgnoreIf
|
||||
import spock.lang.Requires
|
||||
import spock.lang.Specification
|
||||
import spock.lang.Timeout
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
@Slf4j
|
||||
class BatchLoggingTest extends Specification {
|
||||
|
||||
def 'should parse stdout and stderr' () {
|
||||
given:
|
||||
def OUT_ENTRY1 = LogEntry.newBuilder(StringPayload.of('No user sessions are running outdated binaries.\n')).setSeverity(Severity.INFO).build()
|
||||
def OUT_ENTRY2 = LogEntry.newBuilder(StringPayload.of('Hello world')).setSeverity(Severity.INFO).build()
|
||||
def ERR_ENTRY1 = LogEntry.newBuilder(StringPayload.of('Oops something has failed. We are sorry.\n')).setSeverity(Severity.ERROR).build()
|
||||
def ERR_ENTRY2 = LogEntry.newBuilder(StringPayload.of('blah blah')).setSeverity(Severity.ERROR).build()
|
||||
|
||||
when:
|
||||
def stdout = new StringBuilder()
|
||||
def stderr = new StringBuilder()
|
||||
and:
|
||||
BatchLogging.parseOutput(OUT_ENTRY1, stdout, stderr)
|
||||
then:
|
||||
stdout.toString() == 'No user sessions are running outdated binaries.\n'
|
||||
and:
|
||||
stderr.toString() == ''
|
||||
|
||||
when:
|
||||
BatchLogging.parseOutput(ERR_ENTRY1, stdout, stderr)
|
||||
then:
|
||||
stderr.toString() == 'Oops something has failed. We are sorry.\n'
|
||||
|
||||
when:
|
||||
BatchLogging.parseOutput(ERR_ENTRY2, stdout, stderr)
|
||||
then:
|
||||
// the message is appended to the stderr because not prefix is provided
|
||||
stderr.toString() == 'Oops something has failed. We are sorry.\nblah blah'
|
||||
and:
|
||||
// no change to the stdout
|
||||
stdout.toString() == 'No user sessions are running outdated binaries.\n'
|
||||
|
||||
when:
|
||||
BatchLogging.parseOutput(OUT_ENTRY2, stdout, stderr)
|
||||
then:
|
||||
// the message is added to the stdout
|
||||
stdout.toString() == 'No user sessions are running outdated binaries.\nHello world'
|
||||
and:
|
||||
// no change to the stderr
|
||||
stderr.toString() == 'Oops something has failed. We are sorry.\nblah blah'
|
||||
|
||||
}
|
||||
|
||||
@Timeout(value = 10, unit = TimeUnit.MINUTES)
|
||||
@IgnoreIf({System.getenv('NXF_SMOKE')})
|
||||
@Requires({System.getenv('GOOGLE_APPLICATION_CREDENTIALS')})
|
||||
def 'should fetch logs' () {
|
||||
given:
|
||||
def sess = Mock(Session) { getConfig() >> [:] }
|
||||
def config = GoogleOpts.create(sess)
|
||||
and:
|
||||
def batchClient = new BatchClient(config)
|
||||
def logClient = new BatchLogging(config)
|
||||
|
||||
when:
|
||||
def imageUri = 'quay.io/nextflow/bash'
|
||||
def cmd = ['/bin/bash','-c','echo "Hello world!" && echo "Oops something went wrong" >&2']
|
||||
def req = Job.newBuilder()
|
||||
.addTaskGroups(
|
||||
TaskGroup.newBuilder()
|
||||
.setTaskSpec(
|
||||
TaskSpec.newBuilder()
|
||||
.addRunnables(
|
||||
Runnable.newBuilder()
|
||||
.setContainer(
|
||||
Runnable.Container.newBuilder()
|
||||
.setImageUri(imageUri)
|
||||
.addAllCommands(cmd)
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
.setLogsPolicy(
|
||||
LogsPolicy.newBuilder()
|
||||
.setDestination(LogsPolicy.Destination.CLOUD_LOGGING)
|
||||
)
|
||||
.build()
|
||||
def jobId = 'nf-test-' + System.currentTimeMillis()
|
||||
def resp = batchClient.submitJob(jobId, req)
|
||||
def uid = resp.getUid()
|
||||
log.debug "Test job uid=$uid"
|
||||
then:
|
||||
uid
|
||||
|
||||
when:
|
||||
def state=null
|
||||
do {
|
||||
if( batchClient.listTasks(jobId).iterator().hasNext() )
|
||||
state = batchClient.getTaskState(jobId, '0')
|
||||
else
|
||||
state = 'PENDING'
|
||||
log.debug "Test task state=$state"
|
||||
sleep 10_000
|
||||
} while( state !in ['SUCCEEDED', 'FAILED'] )
|
||||
then:
|
||||
state in ['SUCCEEDED', 'FAILED']
|
||||
|
||||
when:
|
||||
def stdout = logClient.stdout(uid, '0')
|
||||
def stderr = logClient.stderr(uid, '0')
|
||||
log.debug "STDOUT: $stdout"
|
||||
log.debug "STDERR: $stderr"
|
||||
then:
|
||||
stdout.contains('Hello world!')
|
||||
stderr.contains('Oops something went wrong')
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud.google.config
|
||||
|
||||
import nextflow.util.Duration
|
||||
import spock.lang.Specification
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class GoogleRetryOptsTest extends Specification {
|
||||
|
||||
def 'should get retry opts' () {
|
||||
when:
|
||||
def opts1 = new GoogleRetryOpts([:])
|
||||
then:
|
||||
opts1.maxAttempts == 10
|
||||
opts1.multiplier == 2.0d
|
||||
opts1.maxDelay == Duration.of('90s')
|
||||
|
||||
when:
|
||||
def opts2 = new GoogleRetryOpts([maxAttempts: 5, maxDelay: '5s', multiplier: 10])
|
||||
then:
|
||||
opts2.maxAttempts == 5
|
||||
opts2.multiplier == 10d
|
||||
opts2.maxDelay == Duration.of('5s')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud.google.util
|
||||
|
||||
import com.google.cloud.storage.StorageOptions
|
||||
import nextflow.Global
|
||||
import nextflow.Session
|
||||
import nextflow.cloud.google.GoogleOpts
|
||||
import nextflow.cloud.google.config.GoogleRetryOpts
|
||||
import spock.lang.Specification
|
||||
import spock.lang.Unroll
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class GsPathFactoryTest extends Specification {
|
||||
|
||||
@Unroll
|
||||
def 'should create gs path #PATH' () {
|
||||
given:
|
||||
Global.session = Mock(Session) {
|
||||
getConfig() >> [google:[project:'foo', region:'x']]
|
||||
}
|
||||
and:
|
||||
def factory = new GsPathFactory()
|
||||
|
||||
expect:
|
||||
factory.parseUri(PATH).toUriString() == PATH
|
||||
factory.parseUri(PATH).toString() == STR
|
||||
|
||||
where:
|
||||
_ | PATH | STR
|
||||
_ | 'gs://foo' | ''
|
||||
_ | 'gs://foo/bar' | '/bar'
|
||||
_ | 'gs://foo/bar/' | '/bar/' // <-- bug or feature ?
|
||||
_ | 'gs://foo/b a r' | '/b a r'
|
||||
_ | 'gs://f o o/bar' | '/bar'
|
||||
_ | 'gs://f_o_o/bar' | '/bar'
|
||||
}
|
||||
|
||||
def 'should use requester pays' () {
|
||||
given:
|
||||
def session = Mock(Session) {
|
||||
getConfig() >> [google:[project:'foo', region:'x', enableRequesterPaysBuckets:true]]
|
||||
}
|
||||
and:
|
||||
def opts = GoogleOpts.fromSession(session)
|
||||
|
||||
when:
|
||||
def storageConfig = GsPathFactory.getCloudStorageConfig(opts)
|
||||
|
||||
then:
|
||||
storageConfig.userProject() == 'foo'
|
||||
}
|
||||
|
||||
def 'should not use requester pays' () {
|
||||
given:
|
||||
def session = new Session()
|
||||
session.config = [google:[project:'foo', region:'x', lifeSciences: [:]]]
|
||||
and:
|
||||
def opts = GoogleOpts.fromSession(session)
|
||||
|
||||
when:
|
||||
def storageConfig = GsPathFactory.getCloudStorageConfig(opts)
|
||||
|
||||
then:
|
||||
storageConfig.userProject() == null
|
||||
}
|
||||
|
||||
def 'should apply http timeout settings from config' () {
|
||||
given:
|
||||
def session = Mock(Session) {
|
||||
getConfig() >> [google:[httpConnectTimeout: CONNECT, httpReadTimeout: READ]]
|
||||
}
|
||||
and:
|
||||
def policy = new GoogleRetryOpts([:])
|
||||
def retrySettings = StorageOptions.getDefaultRetrySettings()
|
||||
.toBuilder()
|
||||
.setMaxAttempts(policy.maxAttempts)
|
||||
.setRetryDelayMultiplier(policy.multiplier)
|
||||
.setTotalTimeout(org.threeten.bp.Duration.ofSeconds(policy.maxDelaySecs()))
|
||||
.build()
|
||||
and:
|
||||
def opts = GoogleOpts.fromSession(session)
|
||||
and:
|
||||
def storageOptions = GsPathFactory.getCloudStorageOptions(opts)
|
||||
and:
|
||||
def transportOptions = StorageOptions.getDefaultHttpTransportOptions().toBuilder()
|
||||
if( CONNECT ) transportOptions.setConnectTimeout( CONNECT_MILLIS )
|
||||
if( READ ) transportOptions.setReadTimeout( READ_MILLIS )
|
||||
|
||||
expect:
|
||||
storageOptions == StorageOptions.getDefaultInstance().toBuilder()
|
||||
.setTransportOptions(transportOptions.build())
|
||||
.setRetrySettings(retrySettings)
|
||||
.build()
|
||||
|
||||
where:
|
||||
CONNECT | CONNECT_MILLIS | READ | READ_MILLIS
|
||||
null | 60000 | null | 60000
|
||||
'30s' | 30000 | '30s' | 30000
|
||||
'60s' | 60000 | '60s' | 60000
|
||||
}
|
||||
|
||||
def 'should apply retry settings' () {
|
||||
given:
|
||||
def session = Mock(Session) {
|
||||
getConfig() >> [google:[storage:[retryPolicy: [maxAttempts: 5, maxDelay:'50s', multiplier: 500]]]]
|
||||
}
|
||||
|
||||
when:
|
||||
def opts = GoogleOpts.fromSession(session)
|
||||
then:
|
||||
opts.storageOpts.retryPolicy.maxAttempts == 5
|
||||
opts.storageOpts.retryPolicy.maxDelaySecs() == 50
|
||||
opts.storageOpts.retryPolicy.multiplier == 500d
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.cloud.google.util
|
||||
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
|
||||
import com.google.cloud.storage.contrib.nio.CloudStoragePath
|
||||
import nextflow.Global
|
||||
import nextflow.Session
|
||||
import nextflow.util.KryoHelper
|
||||
import spock.lang.Specification
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class GsPathSerializerTest extends Specification {
|
||||
|
||||
def 'should serialize a google cloud path'() {
|
||||
given:
|
||||
Global.session = Mock(Session) {
|
||||
getConfig() >> [google:[project:'foo', region:'x']]
|
||||
}
|
||||
|
||||
when:
|
||||
def uri = URI.create("gs://my-seq/data/ggal/sample.fq")
|
||||
def path = Paths.get(uri)
|
||||
def buffer = KryoHelper.serialize(path)
|
||||
def copy = (Path)KryoHelper.deserialize(buffer)
|
||||
then:
|
||||
copy instanceof CloudStoragePath
|
||||
copy.toUri() == uri
|
||||
copy.toUriString() == "gs://my-seq/data/ggal/sample.fq"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.executor
|
||||
|
||||
|
||||
import com.google.api.client.http.HttpResponseException
|
||||
import spock.lang.Specification
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class BashWrapperBuilderWithGoogleTest extends Specification {
|
||||
|
||||
def 'should check retryable errors' () {
|
||||
expect:
|
||||
BashWrapperBuilder.isRetryable0(ERROR) == EXPECTED
|
||||
where:
|
||||
ERROR | EXPECTED
|
||||
new HttpResponseException(GroovyMock(HttpResponseException.Builder)) | true
|
||||
new Exception() | false
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.extension
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
import com.google.cloud.storage.contrib.nio.CloudStorageFileSystem
|
||||
import nextflow.util.Escape
|
||||
import spock.lang.Specification
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class EscapeTest2 extends Specification {
|
||||
|
||||
|
||||
Path asPath(String bucket, String path) {
|
||||
CloudStorageFileSystem.forBucket(bucket).getPath(path)
|
||||
}
|
||||
|
||||
|
||||
def 'should escape gs path'() {
|
||||
expect:
|
||||
Escape.uriPath(PATH) == EXPECTED
|
||||
where:
|
||||
PATH | EXPECTED
|
||||
asPath('foo','/work') | 'gs://foo/work'
|
||||
asPath('foo','/work/') | 'gs://foo/work'
|
||||
asPath('foo','/b a r') | 'gs://foo/b\\ a\\ r'
|
||||
asPath('f_o o','/bar') | 'gs://f_o\\ o/bar'
|
||||
asPath('f_o o','/bar/') | 'gs://f_o\\ o/bar'
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.extension
|
||||
|
||||
import spock.lang.Specification
|
||||
import spock.lang.Unroll
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
import com.google.cloud.storage.contrib.nio.CloudStoragePath
|
||||
import nextflow.Global
|
||||
import nextflow.Session
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class FilesExTest2 extends Specification {
|
||||
|
||||
@Unroll
|
||||
def 'should return uri string for #PATH' () {
|
||||
given:
|
||||
Global.session = Mock(Session) {
|
||||
getConfig() >> [google:[project:'foo', region:'x']]
|
||||
}
|
||||
|
||||
when:
|
||||
def path = PATH as Path
|
||||
then:
|
||||
path instanceof CloudStoragePath
|
||||
println FilesEx.toUriString(path)
|
||||
FilesEx.toUriString(path) == PATH
|
||||
|
||||
where:
|
||||
PATH | _
|
||||
'gs://foo/bar' | _
|
||||
'gs://foo' | _
|
||||
'gs://foo/' | _
|
||||
'gs://foo/bar/baz' | _
|
||||
'gs://foo/bar/baz/' | _
|
||||
'gs://foo/bar - baz/' | _
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.file
|
||||
|
||||
import java.nio.file.FileAlreadyExistsException
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
|
||||
import com.google.cloud.storage.contrib.nio.CloudStorageFileSystem
|
||||
import nextflow.Global
|
||||
import nextflow.Session
|
||||
import nextflow.SysEnv
|
||||
import spock.lang.Ignore
|
||||
import spock.lang.Specification
|
||||
import spock.lang.Unroll
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class FileHelperGsTest extends Specification {
|
||||
|
||||
def 'should parse google storage path' () {
|
||||
|
||||
given:
|
||||
Global.session = Mock(Session) {
|
||||
getConfig() >> [google:[project:'foo', region:'x']]
|
||||
}
|
||||
|
||||
expect:
|
||||
FileHelper.asPath('file.txt') ==
|
||||
Paths.get('file.txt')
|
||||
and:
|
||||
FileHelper.asPath('gs://foo') ==
|
||||
CloudStorageFileSystem.forBucket('foo').getPath('')
|
||||
|
||||
and:
|
||||
FileHelper.asPath('gs://foo/this/and/that.txt') ==
|
||||
CloudStorageFileSystem.forBucket('foo').getPath('/this/and/that.txt')
|
||||
|
||||
and:
|
||||
FileHelper.asPath('gs://foo/b a r.txt') ==
|
||||
CloudStorageFileSystem.forBucket('foo').getPath('/b a r.txt')
|
||||
|
||||
and:
|
||||
FileHelper.asPath('gs://f o o/bar.txt') ==
|
||||
CloudStorageFileSystem.forBucket('f o o').getPath('/bar.txt')
|
||||
|
||||
and:
|
||||
FileHelper.asPath('gs://f_o_o/bar.txt') ==
|
||||
CloudStorageFileSystem.forBucket('f_o_o').getPath('/bar.txt')
|
||||
}
|
||||
|
||||
|
||||
def 'should strip ending slash' () {
|
||||
given:
|
||||
Global.session = Mock(Session) { getConfig() >> [google:[project:'foo', region:'x']] }
|
||||
def nxFolder = Paths.get('/my-bucket/foo')
|
||||
def nxNested = Paths.get('/my-bucket/foo/bar/')
|
||||
and:
|
||||
def gsFolder = 'gs://my-bucket/foo' as Path
|
||||
def gsNested = 'gs://my-bucket/foo/bar/' as Path
|
||||
|
||||
expect:
|
||||
nxFolder.relativize(nxNested).toString() == 'bar'
|
||||
gsFolder.relativize(gsNested).toString() == 'bar/' // <-- gs adds a slash that mess-up things
|
||||
and:
|
||||
FileHelper.relativize0(nxFolder,nxNested).toString() == 'bar'
|
||||
FileHelper.relativize0(gsFolder,gsNested).toString() == 'bar'
|
||||
|
||||
}
|
||||
|
||||
@Ignore
|
||||
def 'should throw FileAlreadyExistsException'() {
|
||||
given:
|
||||
def foo = CloudStorageFileSystem.forBucket('nf-bucket').getPath('foo.txt')
|
||||
def bar = CloudStorageFileSystem.forBucket('nf-bucket').getPath('bar.txt')
|
||||
and:
|
||||
if( !Files.exists(foo) ) Files.createFile(foo)
|
||||
if( !Files.exists(bar) ) Files.createFile(bar)
|
||||
when:
|
||||
Files.copy(foo, bar)
|
||||
then:
|
||||
thrown(FileAlreadyExistsException)
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should convert to canonical path with base' () {
|
||||
given:
|
||||
Global.session = Mock(Session) { getConfig() >> [google:[project:'foo', region:'x']] }
|
||||
and:
|
||||
SysEnv.push(NXF_FILE_ROOT: 'gs://host.com/work')
|
||||
|
||||
expect:
|
||||
FileHelper.toCanonicalPath(VALUE) == (EXPECTED ? FileHelper.asPath(EXPECTED) : null)
|
||||
|
||||
cleanup:
|
||||
SysEnv.pop()
|
||||
Global.session = null
|
||||
|
||||
where:
|
||||
VALUE | EXPECTED
|
||||
null | null
|
||||
'file.txt' | 'gs://host.com/work/file.txt'
|
||||
Path.of('file.txt') | 'gs://host.com/work/file.txt'
|
||||
and:
|
||||
'./file.txt' | 'gs://host.com/work/file.txt'
|
||||
'.' | 'gs://host.com/work'
|
||||
'./' | 'gs://host.com/work'
|
||||
'../file.txt' | 'gs://host.com/file.txt'
|
||||
and:
|
||||
'/file.txt' | '/file.txt'
|
||||
Path.of('/file.txt') | '/file.txt'
|
||||
}
|
||||
|
||||
def 'should convert to a canonical path' () {
|
||||
given:
|
||||
Global.session = Mock(Session) { getConfig() >> [google:[project:'foo', region:'x']] }
|
||||
|
||||
expect:
|
||||
FileHelper.toCanonicalPath(VALUE).toUri() == EXPECTED
|
||||
|
||||
where:
|
||||
VALUE | EXPECTED
|
||||
'gs://foo/some/file.txt' | new URI('gs://foo/some/file.txt')
|
||||
'gs://foo/some///file.txt' | new URI('gs://foo/some/file.txt')
|
||||
}
|
||||
|
||||
@Unroll
|
||||
def 'should remove consecutive slashes in the path' () {
|
||||
given:
|
||||
Global.session = Mock(Session) { getConfig() >> [google:[project:'foo', region:'x']] }
|
||||
|
||||
expect:
|
||||
FileHelper.asPath(STR).toUri() == EXPECTED
|
||||
where:
|
||||
STR | EXPECTED
|
||||
'gs://foo//this/that' | new URI('gs://foo/this/that')
|
||||
'gs://foo//this///that' | new URI('gs://foo/this/that')
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
/*
|
||||
* Copyright 2013-2026, Seqera Labs
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package nextflow.file
|
||||
|
||||
import com.google.cloud.storage.contrib.nio.CloudStoragePath
|
||||
import nextflow.Global
|
||||
import nextflow.Session
|
||||
import spock.lang.Specification
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
|
||||
*/
|
||||
class GsPathTest extends Specification {
|
||||
|
||||
def 'should check equals and hashcode' () {
|
||||
given:
|
||||
Global.session = Mock(Session) {
|
||||
getConfig() >> [google:[project:'foo', region:'x']]
|
||||
}
|
||||
and:
|
||||
|
||||
def path1 = FileHelper.asPath('gs://foo/some/foo.txt')
|
||||
def path2 = FileHelper.asPath('gs://foo/some/foo.txt')
|
||||
def path3 = FileHelper.asPath('gs://foo/some/bar.txt')
|
||||
def path4 = FileHelper.asPath('gs://bar/some/foo.txt')
|
||||
|
||||
expect:
|
||||
path1 instanceof CloudStoragePath
|
||||
path2 instanceof CloudStoragePath
|
||||
path3 instanceof CloudStoragePath
|
||||
path4 instanceof CloudStoragePath
|
||||
|
||||
and:
|
||||
path1 == path2
|
||||
path1 != path3
|
||||
path3 != path4
|
||||
|
||||
and:
|
||||
path1.hashCode() == path2.hashCode()
|
||||
path1.hashCode() != path3.hashCode()
|
||||
path3.hashCode() != path4.hashCode()
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user