kevin/ma
1
0
Fork 0
Code Issues 1 Packages Activity

add nextflow d30e48d

Browse Source
This commit is contained in:
Kevin
2026-04-29 23:01:54 +02:00
parent d0b12d668d
commit 97cc9058d3
2840 changed files with 730250 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
nextflow/plugins/nf-azure/README.md Normal file
View File

@@ -0,0 +1,85 @@
# Microsoft Azure plugin for Nextflow
## Summary
The Microsoft Azure plugin provides support for Azure Blob Storage as a file system, and Azure Batch as a compute executor for Nextflow pipelines.
## Get Started
To use this plugin, add it to your `nextflow.config`:
```groovy
plugins {
id 'nf-azure'
}
```
Configure your Azure credentials and services:
```groovy
azure {
storage {
accountName = '<YOUR STORAGE ACCOUNT NAME>'
accountKey = '<YOUR STORAGE ACCOUNT KEY>'
}
batch {
endpoint = 'https://<YOUR BATCH ACCOUNT NAME>.<REGION>.batch.azure.com'
accountName = '<YOUR BATCH ACCOUNT NAME>'
accountKey = '<YOUR BATCH ACCOUNT KEY>'
}
}
```
Set the executor and work directory:
```groovy
process.executor = 'azurebatch'
workDir = 'az://<YOUR CONTAINER>/work'
```
## Examples
### Basic Azure Batch Configuration
```groovy
plugins {
id 'nf-azure'
}
azure {
storage {
accountName = 'mystorageaccount'
accountKey = System.getenv('AZURE_STORAGE_KEY')
}
batch {
endpoint = 'https://mybatchaccount.westeurope.batch.azure.com'
accountName = 'mybatchaccount'
accountKey = System.getenv('AZURE_BATCH_KEY')
autoPoolMode = true
deletePoolsOnCompletion = true
}
}
process.executor = 'azurebatch'
workDir = 'az://mycontainer/work'
```
### Using Managed Identity
```groovy
azure {
managedIdentity {
clientId = '<YOUR MANAGED IDENTITY CLIENT ID>'
}
}
```
## Resources
- [Azure Batch Executor Documentation](https://nextflow.io/docs/latest/azure.html)
## License
[Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0)

1
nextflow/plugins/nf-azure/VERSION Normal file
View File

@@ -0,0 +1 @@
1.22.2

116
nextflow/plugins/nf-azure/azure-login.json Normal file
View File

@@ -0,0 +1,116 @@
[
{
"cloudName": "AzureCloud",
"homeTenantId": "7005851f-400b-4acb-8bc1-12c44a7d39e5",
"id": "cb4ff255-ac8c-4721-83bd-2d98e75b50d7",
"isDefault": true,
"managedByTenants": [],
"name": "Free Trial",
"state": "Enabled",
"tenantId": "7005851f-400b-4acb-8bc1-12c44a7d39e5",
"user": {
"name": "paolo@seqera.io",
"type": "user"
}
}
]
$ az login
$ az group create --name my-storage-group --location westeurope
$ az storage account create --resource-group my-resource-group --name nfaccount --location westeurope
{- Finished ..
"accessTier": "Hot",
"azureFilesIdentityBasedAuthentication": null,
"blobRestoreStatus": null,
"creationTime": "2020-05-15T20:42:17.206927+00:00",
"customDomain": null,
"enableHttpsTrafficOnly": true,
"encryption": {
"keySource": "Microsoft.Storage",
"keyVaultProperties": null,
"services": {
"blob": {
"enabled": true,
"keyType": "Account",
"lastEnabledTime": "2020-05-15T20:42:17.300678+00:00"
},
"file": {
"enabled": true,
"keyType": "Account",
"lastEnabledTime": "2020-05-15T20:42:17.300678+00:00"
},
"queue": null,
"table": null
}
},
"failoverInProgress": null,
"geoReplicationStats": null,
"id": "/subscriptions/cb4ff255-ac8c-4721-83bd-2d98e75b50d7/resourceGroups/my-resource-group/providers/Microsoft.Storage/storageAccounts/nfaccount",
"identity": null,
"isHnsEnabled": null,
"kind": "StorageV2",
"largeFileSharesState": null,
"lastGeoFailoverTime": null,
"location": "westeurope",
"name": "nfaccount",
"networkRuleSet": {
"bypass": "AzureServices",
"defaultAction": "Allow",
"ipRules": [],
"virtualNetworkRules": []
},
"primaryEndpoints": {
"blob": "https://nfaccount.blob.core.windows.net/",
"dfs": "https://nfaccount.dfs.core.windows.net/",
"file": "https://nfaccount.file.core.windows.net/",
"internetEndpoints": null,
"microsoftEndpoints": null,
"queue": "https://nfaccount.queue.core.windows.net/",
"table": "https://nfaccount.table.core.windows.net/",
"web": "https://nfaccount.z6.web.core.windows.net/"
},
"primaryLocation": "westeurope",
"privateEndpointConnections": [],
"provisioningState": "Succeeded",
"resourceGroup": "my-resource-group",
"routingPreference": null,
"secondaryEndpoints": {
"blob": "https://nfaccount-secondary.blob.core.windows.net/",
"dfs": "https://nfaccount-secondary.dfs.core.windows.net/",
"file": null,
"internetEndpoints": null,
"microsoftEndpoints": null,
"queue": "https://nfaccount-secondary.queue.core.windows.net/",
"table": "https://nfaccount-secondary.table.core.windows.net/",
"web": "https://nfaccount-secondary.z6.web.core.windows.net/"
},
"secondaryLocation": "northeurope",
"sku": {
"name": "Standard_RAGRS",
"tier": "Standard"
},
"statusOfPrimary": "available",
"statusOfSecondary": "available",
"tags": {},
"type": "Microsoft.Storage/storageAccounts"
}
Connection string
BlobEndpoint=https://nfaccount.blob.core.windows.net/;QueueEndpoint=https://nfaccount.queue.core.windows.net/;FileEndpoint=https://nfaccount.file.core.windows.net/;TableEndpoint=https://nfaccount.table.core.windows.net/;SharedAccessSignature=sv=2019-10-10&ss=bfqt&srt=sco&sp=rwdlacupx&se=2020-05-16T04:48:12Z&st=2020-05-15T20:48:12Z&spr=https&sig=9xCn8O%2FxjKroc7YOc9fHffiNOtRaY46spv9VJa4D8pU%3D
SAS token
?sv=2019-10-10&ss=bfqt&srt=sco&sp=rwdlacupx&se=2020-05-16T04:48:12Z&st=2020-05-15T20:48:12Z&spr=https&sig=9xCn8O%2FxjKroc7YOc9fHffiNOtRaY46spv9VJa4D8pU%3D
Blob service SAS URL
https://nfaccount.blob.core.windows.net/?sv=2019-10-10&ss=bfqt&srt=sco&sp=rwdlacupx&se=2020-05-16T04:48:12Z&st=2020-05-15T20:48:12Z&spr=https&sig=9xCn8O%2FxjKroc7YOc9fHffiNOtRaY46spv9VJa4D8pU%3D
File service SAS URL
https://nfaccount.file.core.windows.net/?sv=2019-10-10&ss=bfqt&srt=sco&sp=rwdlacupx&se=2020-05-16T04:48:12Z&st=2020-05-15T20:48:12Z&spr=https&sig=9xCn8O%2FxjKroc7YOc9fHffiNOtRaY46spv9VJa4D8pU%3D
Queue service SAS URL
https://nfaccount.queue.core.windows.net/?sv=2019-10-10&ss=bfqt&srt=sco&sp=rwdlacupx&se=2020-05-16T04:48:12Z&st=2020-05-15T20:48:12Z&spr=https&sig=9xCn8O%2FxjKroc7YOc9fHffiNOtRaY46spv9VJa4D8pU%3D
Table service SAS URL
https://nfaccount.table.core.windows.net/?sv=2019-10-10&ss=bfqt&srt=sco&sp=rwdlacupx&se=2020-05-16T04:48:12Z&st=2020-05-15T20:48:12Z&spr=https&sig=9xCn8O%2FxjKroc7YOc9fHffiNOtRaY46spv9VJa4D8pU%3D

104
nextflow/plugins/nf-azure/azure-login.txt Normal file
View File

@@ -0,0 +1,104 @@
» az login
» az group create --name nf-storage-group --location westeurope
{
"id": "/subscriptions/f7ef67b9-51f5-4fc2-91a8-0f9cce0c6598/resourceGroups/nf-storage-group",
"location": "westeurope",
"managedBy": null,
"name": "nf-storage-group",
"properties": {
"provisioningState": "Succeeded"
},
"tags": null,
"type": "Microsoft.Resources/resourceGroups"
}
» az storage account create --resource-group nf-storage-group --name nfstore --location westeurope
{- Finished ..
"accessTier": "Hot",
"allowBlobPublicAccess": null,
"azureFilesIdentityBasedAuthentication": null,
"blobRestoreStatus": null,
"creationTime": "2020-07-18T07:52:22.585318+00:00",
"customDomain": null,
"enableHttpsTrafficOnly": true,
"encryption": {
"keySource": "Microsoft.Storage",
"keyVaultProperties": null,
"requireInfrastructureEncryption": null,
"services": {
"blob": {
"enabled": true,
"keyType": "Account",
"lastEnabledTime": "2020-07-18T07:52:22.679222+00:00"
},
"file": {
"enabled": true,
"keyType": "Account",
"lastEnabledTime": "2020-07-18T07:52:22.679222+00:00"
},
"queue": null,
"table": null
}
},
"failoverInProgress": null,
"geoReplicationStats": null,
"id": "/subscriptions/f7ef67b9-51f5-4fc2-91a8-0f9cce0c6598/resourceGroups/nf-storage-group/providers/Microsoft.Storage/storageAccounts/nfstore",
"identity": null,
"isHnsEnabled": null,
"kind": "StorageV2",
"largeFileSharesState": null,
"lastGeoFailoverTime": null,
"location": "westeurope",
"minimumTlsVersion": null,
"name": "nfstore",
"networkRuleSet": {
"bypass": "AzureServices",
"defaultAction": "Allow",
"ipRules": [],
"virtualNetworkRules": []
},
"primaryEndpoints": {
"blob": "https://nfstore.blob.core.windows.net/",
"dfs": "https://nfstore.dfs.core.windows.net/",
"file": "https://nfstore.file.core.windows.net/",
"internetEndpoints": null,
"microsoftEndpoints": null,
"queue": "https://nfstore.queue.core.windows.net/",
"table": "https://nfstore.table.core.windows.net/",
"web": "https://nfstore.z6.web.core.windows.net/"
},
"primaryLocation": "westeurope",
"privateEndpointConnections": [],
"provisioningState": "Succeeded",
"resourceGroup": "nf-storage-group",
"routingPreference": null,
"secondaryEndpoints": {
"blob": "https://nfstore-secondary.blob.core.windows.net/",
"dfs": "https://nfstore-secondary.dfs.core.windows.net/",
"file": null,
"internetEndpoints": null,
"microsoftEndpoints": null,
"queue": "https://nfstore-secondary.queue.core.windows.net/",
"table": "https://nfstore-secondary.table.core.windows.net/",
"web": "https://nfstore-secondary.z6.web.core.windows.net/"
},
"secondaryLocation": "northeurope",
"sku": {
"name": "Standard_RAGRS",
"tier": "Standard"
},
"statusOfPrimary": "available",
"statusOfSecondary": "available",
"tags": {},
"type": "Microsoft.Storage/storageAccounts"
}
az storage blob generate-sas \
--account-name nfstore \
--container-name my-data \
--name MyBlob \
--permissions racdw \
--expiry 2021-06-15

74
nextflow/plugins/nf-azure/build.gradle Normal file
View File

@@ -0,0 +1,74 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
plugins {
id 'io.nextflow.nextflow-plugin' version "${nextflowPluginVersion}"
id 'java-test-fixtures'
}
nextflowPlugin {
nextflowVersion = '25.11.0-edge'
provider = "${nextflowPluginProvider}"
description = 'Enables Azure cloud execution through Batch service with native Blob storage access and comprehensive authentication options'
className = 'nextflow.cloud.azure.AzurePlugin'
useDefaultDependencies = false
generateSpec = false
extensionPoints = [
'nextflow.cloud.azure.batch.AzBatchExecutor',
'nextflow.cloud.azure.config.AzConfig',
'nextflow.cloud.azure.file.AzPathFactory',
'nextflow.cloud.azure.file.AzPathSerializer',
'nextflow.cloud.azure.fusion.AzFusionEnv',
]
}
sourceSets {
main.java.srcDirs = []
main.groovy.srcDirs = ['src/main']
main.resources.srcDirs = ['src/resources']
test.groovy.srcDirs = ['src/test']
test.java.srcDirs = ['src/testResources']
test.resources.srcDirs = []
}
configurations {
// see https://docs.gradle.org/4.1/userguide/dependency_management.html#sub:exclude_transitive_dependencies
runtimeClasspath.exclude group: 'org.slf4j', module: 'slf4j-api'
}
dependencies {
compileOnly project(':nextflow')
compileOnly 'org.slf4j:slf4j-api:2.0.17'
compileOnly 'org.pf4j:pf4j:3.14.1'
api('com.azure:azure-storage-blob:12.33.2') {
exclude group: 'org.slf4j', module: 'slf4j-api'
}
api('com.azure:azure-compute-batch:1.0.0-beta.3') {
exclude group: 'org.slf4j', module: 'slf4j-api'
exclude group: 'com.google.guava', module: 'guava'
}
api('com.azure:azure-identity:1.18.2') {
exclude group: 'org.slf4j', module: 'slf4j-api'
}
// Force patched version to address GHSA-72hv-8253-57qq (jackson-core Number Length Constraint Bypass DoS)
runtimeOnly 'com.fasterxml.jackson.core:jackson-core:2.18.6'
testImplementation(testFixtures(project(":nextflow")))
testImplementation project(':nextflow')
testImplementation "org.apache.groovy:groovy:4.0.31"
testImplementation "org.apache.groovy:groovy-nio:4.0.31"
}

254
nextflow/plugins/nf-azure/changelog.txt Normal file
View File

@@ -0,0 +1,254 @@
nf-azure changelog
===================
1.22.2 - 26 Mar 2026
- Fix netty and jackson vulnerabilities (#6955) [8dafdd95d]
- Fix security vulnerabilities (#6938) [8b1ab40c4]
1.22.1 - 17 Mar 2026
- Record types (#6679) [d54ff29af]
1.22.0 - 28 Feb 2026
- Handle Azure Batch ActiveJobAndScheduleQuotaReached with retry (#6874) [6e66aaa58]
- Update default Azure Batch VM image to Ubuntu 24.04 (#6844) [b621fc7cb]
1.21.0 - 28 Nov 2025
- Optimize exit code handling by relying on scheduler status for successful executions (#6484) [454a2ae85]
1.20.2 - 21 Oct 2025
- Rename `config.schema` package to `config.spec` (#6485) [ef0d2d601]
1.20.1 - 8 Oct 2025
- Fix unstage controls in command.run when using storeDir (#6364) [a5756da3e]
1.19.0 - 15 Aug 2025
- Fix Azure Batch startTask concatenation issue (#6300) (#6305) [ci fast] [81d5c0dc]
- Unify nf-lang config scopes with runtime classes (#6271) [bfa67ca3]
- Update Azure and AWS deps (#6343) [ci fast] [ff00e2de]
- Bump groovy 4.0.28 (#6304) [ci fast] [a468f8ef]
- Bump netty-codec-http2:4.1.124.Final [7e690b44]
1.18.0 - 6 Jun 2025
- Allow users to provide implicit managed identity to Azure Batch (#6144) [d1f70f50]
- Minor Azure Batch disk slot calculation demoted to debug (#6234) [ci skip] [c65955ce]
- Bump Slf4j version 2.0.17 [93199e09]
1.17.0 - 2 Jun 2025
- Add support for Azure Managed identities on Azure worker nodes with Fusion (#6118) [37981a5f]
- Bump Groovy to version 4.0.27 (#6125) [258e1790]
1.16.0 - 8 May 2025
- Add azure.batch.jobMaxWallClockTime config option (#5996) [74963fdc]
- Remove test constructors or mark as TestOnly (#5216) [d4fadd42]
1.15.0 - 23 Apr 2025
- Update Azure Batch VM sizes and regions (#5985) [297150b8]
1.14.1 - 19 Mar 2025
- Fix handling of exit status with Azure Batch and Fusion (#5806) [7085862d]
- Removing Azure vmList from log [67ffc8ab]
1.14.0 - 17 Mar 2025
- Add cpu-shares and memory limits to Azure Batch tasks (#5799) [f9c0cbfd]
- Add disk directive support in Azure Batch (#5784) [113d7250]
- Validates Azure region before checking available VMs (#5108) [080893a2]
- Fix Ignore Azure pool already exists error (#5721) [e267961b]
- Bump Ubuntu 22.04 as default SKU for Azure Batch (#5804) [e0ba536d]
- Bump groovy 4.0.26 [f740bc56]
1.13.0 - 12 Feb 2025
- Allow Azure Batch tasks to be submitted to different pools (#5766) [76790d2a]
- Fix Check for number of low priority nodes in Azure Batch before raising a pool resize error (#5576) [9b528c11]
- Update azure deps [b163da95]
- Bump groovy 4.0.25 [19c40a4a]
- Bump io.netty:netty-handler:4.1.118.Final [db4a9037]
- Bump net.minidev:json-smart:2.5.2 [b5c4faf4]
- Bump netty-common:4.1.118.Final [8574e243]
1.12.0 - 20 Jan 2025
- Ensure job is killed when exception in task status check (#5561) [9eefd207]
- Bump logback 1.5.13 + slf4j 2.0.16 [cc0163ac]
- Bump groovy 4.0.24 missing deps [40670f7e]
1.11.0 - 3 Dec 2024
- Detecting errors in data unstaging (#5345) [3c8e602d]
- Bump netty-common to version 4.1.115.Final [d1bbd3d0]
- Bump groovy 4.0.24 [dd71ad31]
- Bump com.azure:azure-identity from 1.11.3 to 1.12.2 (#5449) [cb70f1df]
- Target Java 17 as minimal Java version (#5045) [0140f954]
1.10.1 - 27 Oct 2024
- Demote azure batch task status log level to trace (#5416) [ci skip] [d6c684bb]
1.10.0 - 2 Oct 2024
- Fix Azure Fusion env misses credentials when no key or SAS provided (#5328) [e11382c8]
- Bump groovy 4.0.23 (#5303) [fe3e3ac7]
1.9.0 - 4 Sep 2024
- Support Azure Managed Identities in Fusion configuration logic (#5278) [a0bf8b40]
1.8.1 - 5 Aug 2024
- Bump pf4j to version 3.12.0 [96117b9a]
1.8.0 - 8 Jul 2024
- Update Azure VMs and regions for 2024-07-01 (#5100) [12b027ee]
- Add retry options to Azure Blob client (#5098) [7d5e5d2b]
- Bump groovy 4.0.22 [284a6606]
1.7.0 - 17 Jun 2024
- Add support for Azure managed identity (#4897) [21ca16e6]
- Fix Azure system-assigned managed identity [a639a17d]
- Fix support for Azure managed identity clientId [306814e7]
- Bump azure-compute-batch:1.0.0-beta.2 [c08dc49b]
- Bump azure-storage-blob 12.26.1 [c76ff5e7]
1.6.1 - 1 Aug 2024
- Update Azure batch deps [72576648]
- Bump pf4j to version 3.12.0 [1a8f086a]
1.6.0 - 15 Apr 2024
- Add support for Azure custom startTask (#4913) [27d01e3a]
- Improve control on azcopy install (#4883) [01447d5c]
- Fix Azure pool creation [2ee4d11e]
- Bump groovy 4.0.21 [9e08390b]
1.5.1 - 10 Mar 2024
- Update Azure dependencies [1bcbaf0d]
- Bump groovy 4.0.19 [854dc1f0]
1.5.0 - 5 Feb 2024
- Fix azure retry policy (#4638) [85bab699]
- Use AZURE_STORAGE_SAS_TOKEN environment variable (#4627) [2e02afbf]
- Bump Groovy 4 (#4443) [9d32503b]
1.4.0 - 24 Nov 2023
- Fix security vulnerabilities (#4513) [a310c777]
- Add support for Azure low-priority pool (#4527) [8320ea10]
1.3.3-patch3 - 31 Jul 2024
- Update Azure batch deps [e0c6d77d]
1.3.3-patch2 - 11 Jun 2024
- Fix security vulnerabilities (#5057) [6d8765b8]
1.3.3-patch1 - 28 May 2024
- Bump dependency with Nextflow 23.10.2
1.3.3 - 12 Jan 2023
- Use AZURE_STORAGE_SAS_TOKEN environment variable (#4627) [2e1cb413]
- Fix azure retry policy (#4638) [2bc3cf0e]
1.3.2 - 28 Sep 2023
- Retry TimeoutException in azure file system (#4295) [79248355]
1.3.1 - 10 Sep 2023
- Disable staging script for remote work dir (#4282) [80f7cd46]
- Fix IOException should be thrown when failing to create Azure directory [b0bdfd79]
- Fix security deps in nf-azure plugin [c30d5211]
- Bump groovy 3.0.19 [cb411208]
1.3.0 - 17 Aug 2023
- Add resource labels support for Azure Batch (#4178) [7b5e50a1]
- Fix typos in source code comments (#4173) [e78bc37e]
1.2.0 - 5 Aug 2023
- Add deleteTasksOnCompletion to Azure Batch configuration (#4114) [b14674dc]
1.1.4 - 22 Jul 2023
- Fix failing test [9a52f848]
- Fix Improve error message for invalid Azure URI [0f4d8867]
- Fix invalid detection of hierarchical namespace stub blobs as files (#4046) [ce06c877]
- Wait for all child processes in nxf_parallel (#4050) [60a5f1a7]
- Bump Groovy 3.0.18 [207eb535]
1.1.3 - 19 Jum 2023
- Increase Azure min retry delay to 250ms [2e77e5e4]
- Fix AzFileSystem retry policy (2) [c2f3cc96]
1.1.2 - 19 Jun 2023
- Fix AzFileSystem retry policy [ba9b6d18]
- Improve Azure retry logging [de58697a]
1.1.1 - 14 Jun 2023
- Add retry policy on Az blob operations [295bc1ff]
- Bump azure-storage-blob:12.22.1 [2a36fa77]
- Fix S3 path normalization [b75ec444]
1.1.0 - 15 May 2023
- Add support for `time` directive in Azure Batch (#3869) [5c11a0d4]
- Increase Azure default maxRetries to 10 [a017139f]
- Fix Azure jobs correctly deleted after completion (#3927) [b173a983]
- Fix missing SAS token fusion env for Azure [43015029]
- Fix failing tests [06337962]
- Fix Azure pool creation when using scaling formula (#3868) [79984a87]
- Security fixes [973b7bea]
- Update logging libraries [d7eae86e]
- Bump groovy 3.0.17 [cfe4ba56]
1.0.1 - 15 Apr 2023
- Security fixes [83e8fd6a]
- Fix Azure pool creation when using scaling formula (#3868) [84a808a5]
1.0.0 - 1 Apr 2023
- Add support for Fusion to Azure Batch executor (#3209) [3d3cbfa2]
- Fix NoSuchMethodError String.stripIndent with Java 11 [308eafe6]
0.16.0 - 19 Mar 2023
- Add azure batch pool virtualNetwork option (#3723) [e3917b8e]
- Update Azure VM sizes (#3751) [1d06e9a6]
- Increase pwd obfuscation min length [ba23d036]
- Bump groovy 3.0.16 [d3ff5dcb]
0.15.1 - 14 Jan 2023
- Improve container native executor configuration [03126371]
- Minor logging change [646776a8]
- Bump groovy 3.0.14 [7c204236]
0.15.0 - 23 Nov 2022
- Allow identity based authentication on Azure Batch (#3132) [a08611be]
- Add Azure SAS token validation [e2244b48]
0.14.1 - 10 Sep 2022
- Fix Azure NPE on missing pool opts [d5c0aabd]
- Fix shutdown/cleanup hooks invocation [f4185070
0.14.0 - 7 Sep 2022
- Fix thread pool race condition on shutdown [8d2b0587]
- Update Azure vm types [80f5fbe4]
0.13.5 - 1 Sep 2022
- Get rid of remote bin dir [6cfb51e7]
- Fix typos in log messages [76a87c72]
- Improve Az Batch err handling and testing [85d31e8d]
0.13.4 - 1 Aug 2022
- Add retry when Azure submit fails with OperationTimedOut [6a3f9742]
0.13.3 - 13 Jul 2022
- Fix escape unstage outputs with double quotes #2912 #2904 #2790 [49ff02a6]
0.13.2 - 15 May 2022
- Update default SKU for Azure Batch 'batch.node.ubuntu 20.04' [be60fc14]
0.13.1 - 2 Apr 2022
- Add retry policy Azure create job [792820a2]
0.13.0 - 27 Mar 2022
- Add azcopy fine grain config settings [3998a56b]
- Add retry policy to Az Batch operations [991c6175]
0.12.0 - 6 Feb 2022
- Generate "account" token instead of container token when not SAS token is not provided [d5125975d]
- Fix upload of nested directory outputs on azure [85ad55225] [744447155]
0.11.2 - 22 Nov 2021
- Fix Azure executor fail to cleanup jobs on completion [533448be4]
- Make Azure executor logging less verbose [e0b2117ad]
0.11.1 - 18 Nov 2021
- Fix NPE when pool is not configured and auto pool mode is not specified
0.11.0 - 12 Oct 2021
- Add Azure pool node SKU selection #2360 [9afcac756]
- Add Built-in support for Azure File Shares #2285 [a4c3e0ad5]
- Add missing information for pulling images from private registry in Azure Batch #2355 [040e190bd]

42
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/AzurePlugin.groovy Normal file
View File

@@ -0,0 +1,42 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure
import groovy.transform.CompileStatic
import nextflow.cloud.azure.nio.AzFileSystemProvider
import nextflow.file.FileHelper
import nextflow.plugin.BasePlugin
import org.pf4j.PluginWrapper
/**
* Azure cloud plugin for Nextflow
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class AzurePlugin extends BasePlugin {
AzurePlugin(PluginWrapper wrapper) {
super(wrapper)
}
@Override
void start() {
super.start()
// register Azure file system
FileHelper.getOrInstallProvider(AzFileSystemProvider)
}
}

151
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchExecutor.groovy Normal file
View File

@@ -0,0 +1,151 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.batch
import java.nio.file.Path
import groovy.transform.CompileStatic
import groovy.transform.PackageScope
import groovy.util.logging.Slf4j
import nextflow.Global
import nextflow.cloud.azure.config.AzConfig
import nextflow.cloud.azure.nio.AzPath
import nextflow.exception.AbortOperationException
import nextflow.executor.Executor
import nextflow.extension.FilesEx
import nextflow.fusion.FusionHelper
import nextflow.processor.TaskHandler
import nextflow.processor.TaskMonitor
import nextflow.processor.TaskPollingMonitor
import nextflow.processor.TaskRun
import nextflow.util.Duration
import nextflow.util.ServiceName
import org.pf4j.ExtensionPoint
/**
* Nextflow executor for Azure batch service
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@ServiceName('azurebatch')
@CompileStatic
class AzBatchExecutor extends Executor implements ExtensionPoint {
private Path remoteBinDir
private AzConfig azConfig
private AzBatchService batchService
/**
* @return {@code true} to signal containers are managed directly the AWS Batch service
*/
final boolean isContainerNative() {
return true
}
@Override
String containerConfigEngine() {
return 'docker'
}
@Override
Path getWorkDir() {
session.bucketDir ?: session.workDir
}
protected void validateWorkDir() {
/*
* make sure the work dir is an Azure bucket
*/
if( !(workDir instanceof AzPath) ) {
session.abort()
throw new AbortOperationException("When using `$name` executor an Azure bucket must be provided as working directory using either the `-bucket-dir` or `-work-dir` command line option")
}
}
protected void validatePathDir() {
def path = session.config.navigate('env.PATH')
if( path ) {
log.warn "Environment PATH defined in config file is ignored by Azure Batch executor"
}
}
protected void uploadBinDir() {
/*
* upload local binaries
*/
if( session.binDir && !session.binDir.empty() && !session.disableRemoteBinDir ) {
final remote = getTempDir()
log.info "Uploading local `bin` scripts folder to ${remote.toUriString()}/bin"
remoteBinDir = FilesEx.copyTo(session.binDir, remote)
}
}
protected void initBatchService() {
azConfig = AzConfig.getConfig(session)
batchService = new AzBatchService(this)
// Generate an account SAS token using either activeDirectory configs or storage account keys
if (!azConfig.storage().sasToken) {
azConfig.storage().sasToken = azConfig.activeDirectory().isConfigured() || azConfig.managedIdentity().isConfigured()
? AzHelper.generateContainerSasWithActiveDirectory(workDir, azConfig.storage().tokenDuration)
: AzHelper.generateAccountSasWithAccountKey(workDir, azConfig.storage().tokenDuration)
}
Global.onCleanup((it) -> batchService.close())
}
/**
* Initialise the Azure Batch executor.
*/
@Override
protected void register() {
super.register()
initBatchService()
validateWorkDir()
validatePathDir()
uploadBinDir()
}
@PackageScope AzConfig getAzConfig() {
return azConfig
}
@Override
protected TaskMonitor createTaskMonitor() {
TaskPollingMonitor.create(session, config, name, 1000, Duration.of('10 sec'))
}
@Override
TaskHandler createTaskHandler(TaskRun task) {
return new AzBatchTaskHandler(task, this)
}
AzBatchService getBatchService() {
return batchService
}
Path getRemoteBinDir() { return remoteBinDir }
@Override
boolean isFusionEnabled() {
return FusionHelper.isFusionEnabled(session)
}
}

41
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchScriptLauncher.groovy Normal file
View File

@@ -0,0 +1,41 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.batch
import nextflow.executor.BashWrapperBuilder
import nextflow.processor.TaskBean
/**
* Custom bash wrapper builder for Azure batch tasks
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
class AzBatchScriptLauncher extends BashWrapperBuilder {
AzBatchScriptLauncher(TaskBean bean, AzBatchExecutor executor) {
super(bean, new AzFileCopyStrategy(bean, executor))
}
@Override
protected boolean shouldUnstageOutputs() {
return true
}
@Override
protected boolean shouldUnstageControls() {
return true
}
}

1159
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchService.groovy Normal file
View File

File diff suppressed because it is too large Load Diff

215
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchTaskHandler.groovy Normal file
View File

@@ -0,0 +1,215 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.batch
import nextflow.exception.ProcessException
import nextflow.util.TestOnly
import java.nio.file.Path
import com.azure.compute.batch.models.BatchTaskState
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.cloud.types.CloudMachineInfo
import nextflow.exception.ProcessUnrecoverableException
import nextflow.executor.BashWrapperBuilder
import nextflow.fusion.FusionAwareTask
import nextflow.processor.TaskHandler
import nextflow.processor.TaskRun
import nextflow.processor.TaskStatus
import nextflow.trace.TraceRecord
/**
* Implements a task handler for Azure Batch service
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
class AzBatchTaskHandler extends TaskHandler implements FusionAwareTask {
AzBatchExecutor executor
private Path exitFile
private Path outputFile
private Path errorFile
private volatile AzTaskKey taskKey
private volatile long timestamp
private volatile BatchTaskState taskState
private CloudMachineInfo machineInfo
AzBatchTaskHandler(TaskRun task, AzBatchExecutor executor) {
super(task)
this.executor = executor
this.outputFile = task.workDir.resolve(TaskRun.CMD_OUTFILE)
this.errorFile = task.workDir.resolve(TaskRun.CMD_ERRFILE)
this.exitFile = task.workDir.resolve(TaskRun.CMD_EXIT)
validateConfiguration()
}
AzBatchService getBatchService() {
return executor.batchService
}
void validateConfiguration() {
if (!task.container) {
throw new ProcessUnrecoverableException("No container image specified for process $task.name -- Either specify the container to use in the process definition or with 'process.container' value in your config")
}
}
protected BashWrapperBuilder createBashWrapper() {
return fusionEnabled()
? fusionLauncher()
: new AzBatchScriptLauncher(task.toTaskBean(), executor)
}
@Override
void submit() {
log.debug "[AZURE BATCH] Submitting task $task.name - work-dir=${task.workDirStr}"
createBashWrapper().build()
// submit the task execution
this.taskKey = batchService.submitTask(task)
log.debug "[AZURE BATCH] Submitted task $task.name with taskId=$taskKey"
// update the status
this.status = TaskStatus.SUBMITTED
}
@Override
boolean checkIfRunning() {
if( !taskKey || !isSubmitted() )
return false
final state = taskState0(taskKey)
// note, include complete status otherwise it hangs if the task
// completes before reaching this check
final running = state==BatchTaskState.RUNNING || state==BatchTaskState.COMPLETED
log.trace "[AZURE BATCH] Task status $task.name taskId=$taskKey; running=$running"
if( running )
this.status = TaskStatus.RUNNING
return running
}
@Override
boolean checkIfCompleted() {
assert taskKey
if( !isRunning() )
return false
final done = taskState0(taskKey)==BatchTaskState.COMPLETED
if( done ) {
// finalize the task
final info = batchService.getTask(taskKey).executionInfo
// Try to get exit code from Azure batch API and fallback to .exitcode
task.exitStatus = info?.exitCode != null ? info.exitCode : readExitFile()
task.stdout = outputFile
task.stderr = errorFile
status = TaskStatus.COMPLETED
if( task.exitStatus == Integer.MAX_VALUE && info.failureInfo.message) {
final reason = info.failureInfo.message
final unrecoverable = reason.contains('CannotPullContainer') && reason.contains('unauthorized')
// when task exist code is not defined and there is a Azure Batch task failure raise an exception with Azure's failure message
task.error = unrecoverable ? new ProcessUnrecoverableException(reason) : new ProcessException(reason)
}
deleteTask(taskKey, task)
return true
}
return false
}
private Boolean shouldDelete() {
executor.azConfig.batch().deleteTasksOnCompletion
}
protected void deleteTask(AzTaskKey taskKey, TaskRun task) {
if( !taskKey || shouldDelete()==Boolean.FALSE )
return
if( !task.isSuccess() && shouldDelete()==null ) {
// preserve failed tasks for debugging purposes, unless deletion is explicitly enabled
return
}
try {
batchService.deleteTask(taskKey)
}
catch( Exception e ) {
log.warn "Unable to cleanup batch task: $taskKey -- see the log file for details", e
}
}
/**
* @return Retrieve the task status caching the result for at lest one second
*/
protected BatchTaskState taskState0(AzTaskKey key) {
final now = System.currentTimeMillis()
final delta = now - timestamp;
if( !taskState || delta >= 1_000) {
def newState = batchService.getTask(key).state
log.trace "[AZURE BATCH] Task: $key state=$newState"
if( newState ) {
taskState = newState
timestamp = now
}
}
return taskState
}
protected int readExitFile() {
try {
exitFile.text as Integer
}
catch( Exception e ) {
log.debug "[AZURE BATCH] Cannot read exit status for task: `$task.name` | ${e.message}"
return Integer.MAX_VALUE
}
}
@Override
protected void killTask() {
if( !taskKey )
return
batchService.terminate(taskKey)
}
@Override
TraceRecord getTraceRecord() {
def result = super.getTraceRecord()
if( taskKey ) {
result.put('native_id', taskKey.keyPair())
result.machineInfo = getMachineInfo()
}
return result
}
protected CloudMachineInfo getMachineInfo() {
if( machineInfo )
return machineInfo
if( taskKey ) {
machineInfo = batchService.machineInfo(taskKey)
log.trace "[AZURE BATCH] task=$taskKey => machineInfo=$machineInfo"
}
return machineInfo
}
@TestOnly
protected setTaskKey(AzTaskKey key){
this.taskKey = key
}
}

177
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzFileCopyStrategy.groovy Normal file
View File

@@ -0,0 +1,177 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.batch
import java.nio.file.Path
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.cloud.azure.config.AzConfig
import nextflow.cloud.azure.file.AzBashLib
import nextflow.executor.SimpleFileCopyStrategy
import nextflow.processor.TaskBean
import nextflow.processor.TaskRun
import nextflow.util.Duration
import nextflow.util.Escape
/**
* Implements file copy strategy for Azure Batch
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
class AzFileCopyStrategy extends SimpleFileCopyStrategy {
private AzConfig config
private int maxTransferAttempts
private int maxParallelTransfers
private Duration delayBetweenAttempts
private String sasToken
private Path remoteBinDir
protected AzFileCopyStrategy() {}
AzFileCopyStrategy(TaskBean bean, AzBatchExecutor executor) {
super(bean)
this.config = executor.azConfig
this.remoteBinDir = executor.remoteBinDir
this.sasToken = config.storage().sasToken
this.maxParallelTransfers = config.batch().maxParallelTransfers
this.maxTransferAttempts = config.batch().maxTransferAttempts
this.delayBetweenAttempts = config.batch().delayBetweenAttempts
}
@Override
String getEnvScript(Map environment, boolean container) {
if( container )
throw new IllegalArgumentException("Parameter `container` not supported by ${this.class.simpleName}")
final result = new StringBuilder()
final copy = environment ? new LinkedHashMap<String,String>(environment) : new LinkedHashMap<String,String>()
copy.remove('PATH')
copy.put('PATH', '$PWD/.nextflow-bin:$AZ_BATCH_NODE_SHARED_DIR/bin/:$PATH')
copy.put('AZCOPY_LOG_LOCATION', '$PWD/.azcopy_log')
copy.put('AZ_SAS', sasToken)
// finally render the environment
final envSnippet = super.getEnvScript(copy,false)
if( envSnippet )
result << envSnippet
return result.toString()
}
static String uploadCmd(String source, Path targetDir) {
"nxf_az_upload ${Escape.path(source)} '${AzHelper.toHttpUrl(targetDir)}'"
}
@Override
String getBeforeStartScript() {
AzBashLib.script(config.azcopy(), maxParallelTransfers, maxTransferAttempts, delayBetweenAttempts)
}
@Override
String getStageInputFilesScript(Map<String, Path> inputFiles) {
String result = ( remoteBinDir ? """\
nxf_az_download '${AzHelper.toHttpUrl(remoteBinDir)}' \$PWD/.nextflow-bin
chmod +x \$PWD/.nextflow-bin/* || true
""".stripIndent(true) : '' )
result += 'downloads=(true)\n'
result += super.getStageInputFilesScript(inputFiles) + '\n'
result += 'nxf_parallel "${downloads[@]}"\n'
return result
}
/**
* {@inheritDoc}
*/
@Override
String stageInputFile( Path path, String targetName ) {
// third param should not be escaped, because it's used in the grep match rule
def stage_cmd = maxTransferAttempts > 1
? "downloads+=(\"nxf_cp_retry nxf_az_download '${AzHelper.toHttpUrl(path)}' ${Escape.path(targetName)}\")"
: "downloads+=(\"nxf_az_download '${AzHelper.toHttpUrl(path)}' ${Escape.path(targetName)}\")"
return stage_cmd
}
/**
* {@inheritDoc}
*/
@Override
String getUnstageOutputFilesScript(List<String> outputFiles, Path targetDir) {
final patterns = normalizeGlobStarPaths(outputFiles)
// create a bash script that will copy the out file to the working directory
log.trace "[AZURE BATCH] Unstaging file path: $patterns"
if( !patterns )
return null
final escape = new ArrayList(outputFiles.size())
for( String it : patterns )
escape.add( Escape.path(it) )
return """\
uploads=()
IFS=\$'\\n'
for name in \$(eval "ls -1d ${escape.join(' ')}" | sort | uniq); do
uploads+=("nxf_az_upload '\$name' '${AzHelper.toHttpUrl(targetDir)}'")
done
unset IFS
nxf_parallel "\${uploads[@]}"
""".stripIndent(true)
}
/**
* {@inheritDoc}
*/
@Override
String touchFile( Path file ) {
"echo start > .command.begin"
}
/**
* {@inheritDoc}
*/
@Override
String fileStr( Path path ) {
Escape.path(path.getFileName())
}
/**
* {@inheritDoc}
*/
@Override
String copyFile( String name, Path target ) {
"nxf_az_upload ${Escape.path(name)} '${AzHelper.toHttpUrl(target.parent)}'"
}
/**
* {@inheritDoc}
*/
String exitFile( Path path ) {
" > ${TaskRun.CMD_EXIT}"
}
/**
* {@inheritDoc}
*/
@Override
String pipeInputFile( Path path ) {
" < ${Escape.path(path.getFileName())}"
}
}

263
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzHelper.groovy Normal file
View File

@@ -0,0 +1,263 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.batch
import java.nio.file.Path
import java.time.OffsetDateTime
import com.azure.identity.ClientSecretCredentialBuilder
import com.azure.identity.ManagedIdentityCredentialBuilder
import com.azure.storage.blob.BlobContainerClient
import com.azure.storage.blob.BlobServiceClient
import com.azure.storage.blob.BlobServiceClientBuilder
import com.azure.storage.blob.models.UserDelegationKey
import com.azure.storage.blob.sas.BlobContainerSasPermission
import com.azure.storage.blob.sas.BlobSasPermission
import com.azure.storage.blob.sas.BlobServiceSasSignatureValues
import com.azure.storage.common.StorageSharedKeyCredential
import com.azure.storage.common.policy.RequestRetryOptions
import com.azure.storage.common.policy.RetryPolicyType
import com.azure.storage.common.sas.AccountSasPermission
import com.azure.storage.common.sas.AccountSasResourceType
import com.azure.storage.common.sas.AccountSasService
import com.azure.storage.common.sas.AccountSasSignatureValues
import groovy.transform.CompileStatic
import groovy.transform.Memoized
import groovy.util.logging.Slf4j
import nextflow.cloud.azure.config.AzConfig
import nextflow.cloud.azure.config.AzRetryConfig
import nextflow.cloud.azure.nio.AzPath
import nextflow.util.Duration
/**
* Azure helper functions
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
class AzHelper {
static private AzPath az0(Path path){
if( path !instanceof AzPath )
throw new IllegalArgumentException("Not a valid Azure path: $path [${path?.getClass()?.getName()}]")
return (AzPath)path
}
static String toHttpUrl(Path path, String sas=null) {
def url = az0(path).blobClient().getBlobUrl()
url = URLDecoder.decode(url, 'UTF-8').stripEnd('/')
return !sas ? url : "${url}?${sas}"
}
static String toContainerUrl(Path path, String sas) {
def url = az0(path).containerClient().getBlobContainerUrl()
url = URLDecoder.decode(url, 'UTF-8').stripEnd('/')
return !sas ? url : "${url}?${sas}"
}
static BlobContainerSasPermission CONTAINER_PERMS = new BlobContainerSasPermission()
.setAddPermission(true)
.setCreatePermission(true)
.setDeletePermission(true)
.setListPermission(true)
.setMovePermission(true)
.setReadPermission(true)
.setTagsPermission(true)
.setWritePermission(true)
static BlobSasPermission BLOB_PERMS = new BlobSasPermission()
.setAddPermission(true)
.setCreatePermission(true)
.setDeletePermission(true)
.setListPermission(true)
.setMovePermission(true)
.setReadPermission(true)
.setTagsPermission(true)
.setWritePermission(true)
static AccountSasPermission ACCOUNT_PERMS = new AccountSasPermission()
.setAddPermission(true)
.setCreatePermission(true)
.setDeletePermission(true)
.setListPermission(true)
.setReadPermission(true)
.setTagsPermission(true)
.setWritePermission(true)
.setUpdatePermission(true)
static AccountSasService ACCOUNT_SERVICES = new AccountSasService()
.setBlobAccess(true)
.setFileAccess(true)
static AccountSasResourceType ACCOUNT_RESOURCES = new AccountSasResourceType()
.setContainer(true)
.setObject(true)
.setService(true)
static String generateContainerSasWithActiveDirectory(Path path, Duration duration) {
final key = generateUserDelegationKey(az0(path), duration)
return generateContainerUserDelegationSas(az0(path).containerClient(), duration, key)
}
static String generateAccountSasWithAccountKey(Path path, Duration duration) {
generateAccountSas(az0(path).getFileSystem().getBlobServiceClient(), duration)
}
static UserDelegationKey generateUserDelegationKey(Path path, Duration duration) {
final client = az0(path).getFileSystem().getBlobServiceClient()
final startTime = OffsetDateTime.now()
final indicatedExpiryTime = startTime.plusHours(duration.toHours())
// The maximum lifetime for user delegation key (and therefore delegation SAS) is 7 days
// Reference https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blob-user-delegation-sas-create-cli
final maxExpiryTime = startTime.plusDays(7)
final expiryTime = (indicatedExpiryTime.toEpochSecond() <= maxExpiryTime.toEpochSecond()) ? indicatedExpiryTime : maxExpiryTime
final delegationKey = client.getUserDelegationKey(startTime, expiryTime)
return delegationKey
}
static String generateContainerUserDelegationSas(BlobContainerClient client, Duration duration, UserDelegationKey key) {
final startTime = OffsetDateTime.now()
final indicatedExpiryTime = startTime.plusHours(duration.toHours())
// The maximum lifetime for user delegation key (and therefore delegation SAS) is 7 days
// Reference https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blob-user-delegation-sas-create-cli
final maxExpiryTime = startTime.plusDays(7)
final expiryTime = (indicatedExpiryTime.toEpochSecond() <= maxExpiryTime.toEpochSecond()) ? indicatedExpiryTime : maxExpiryTime
final signature = new BlobServiceSasSignatureValues()
.setPermissions(BLOB_PERMS)
.setPermissions(CONTAINER_PERMS)
.setStartTime(startTime)
.setExpiryTime(expiryTime)
final generatedSas = client.generateUserDelegationSas(signature, key)
return generatedSas
}
static String generateAccountSas(BlobServiceClient client, Duration duration) {
final expiryTime = OffsetDateTime.now().plusSeconds(duration.toSeconds())
final signature = new AccountSasSignatureValues(
expiryTime,
ACCOUNT_PERMS,
ACCOUNT_SERVICES,
ACCOUNT_RESOURCES)
return client.generateAccountSas(signature)
}
static String generateAccountSas(String accountName, String accountKey, Duration duration) {
final client = getOrCreateBlobServiceWithKey(accountName, accountKey)
return generateAccountSas(client, duration)
}
@Memoized
static synchronized BlobServiceClient getOrCreateBlobServiceWithKey(String accountName, String accountKey) {
log.debug "Creating Azure blob storage client -- accountName=$accountName; accountKey=${accountKey?.substring(0,5)}.."
final credential = new StorageSharedKeyCredential(accountName, accountKey)
final endpoint = String.format(Locale.ROOT, "https://%s.blob.core.windows.net", accountName)
return new BlobServiceClientBuilder()
.endpoint(endpoint)
.credential(credential)
.retryOptions(requestRetryOptions())
.buildClient()
}
@Memoized
static synchronized BlobServiceClient getOrCreateBlobServiceWithToken(String accountName, String sasToken) {
if( !sasToken )
throw new IllegalArgumentException("Missing Azure blob SAS token")
if( sasToken.length()<100 )
throw new IllegalArgumentException("Invalid Azure blob SAS token -- offending value: $sasToken")
log.debug "Creating Azure blob storage client -- accountName: $accountName; sasToken: ${sasToken?.substring(0,10)}.."
final endpoint = String.format(Locale.ROOT, "https://%s.blob.core.windows.net", accountName)
return new BlobServiceClientBuilder()
.endpoint(endpoint)
.sasToken(sasToken)
.retryOptions(requestRetryOptions())
.buildClient()
}
@Memoized
static synchronized BlobServiceClient getOrCreateBlobServiceWithServicePrincipal(String accountName, String clientId, String clientSecret, String tenantId) {
log.debug "Creating Azure Blob storage client using Service Principal credentials"
final endpoint = String.format(Locale.ROOT, "https://%s.blob.core.windows.net", accountName)
final credential = new ClientSecretCredentialBuilder()
.clientId(clientId)
.clientSecret(clientSecret)
.tenantId(tenantId)
.build()
return new BlobServiceClientBuilder()
.credential(credential)
.endpoint(endpoint)
.retryOptions(requestRetryOptions())
.buildClient()
}
@Memoized
static synchronized BlobServiceClient getOrCreateBlobServiceWithManagedIdentity(String accountName, String clientId) {
log.debug "Creating Azure blob storage client using Managed Identity ${clientId ?: '<system-assigned identity>'}"
final endpoint = String.format(Locale.ROOT, "https://%s.blob.core.windows.net", accountName)
final credentialBuilder = new ManagedIdentityCredentialBuilder()
if( clientId )
credentialBuilder.clientId(clientId)
return new BlobServiceClientBuilder()
.credential(credentialBuilder.build())
.endpoint(endpoint)
.retryOptions(requestRetryOptions())
.buildClient()
}
@Memoized
static protected RequestRetryOptions requestRetryOptions() {
final cfg = AzConfig.getConfig().retryConfig()
return requestRetryOptions0(cfg)
}
static protected RequestRetryOptions requestRetryOptions0(AzRetryConfig cfg) {
final retryDelay = java.time.Duration.ofMillis(cfg.getDelay().millis)
final maxRetryDelay = java.time.Duration.ofMillis(cfg.getMaxDelay().millis)
new RequestRetryOptions(
RetryPolicyType.EXPONENTIAL,
cfg.maxAttempts,
null,
retryDelay,
maxRetryDelay,
null)
}
}

32
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzJobKey.groovy Normal file
View File

@@ -0,0 +1,32 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.batch
import groovy.transform.Canonical
import groovy.transform.CompileStatic
import nextflow.processor.TaskProcessor
/**
* Model a Batch job key for caching purposes
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Canonical
@CompileStatic
class AzJobKey {
final TaskProcessor processor
final String poolId
}

36
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzTaskKey.groovy Normal file
View File

@@ -0,0 +1,36 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.batch
import groovy.transform.Canonical
import groovy.transform.CompileStatic
/**
* Model a fully qualified taskId ie. JobId + TaskId
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Canonical
@CompileStatic
class AzTaskKey {
String jobId
String taskId
String keyPair() {
"$jobId/$taskId"
}
}

40
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzVmPoolSpec.groovy Normal file
View File

@@ -0,0 +1,40 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.batch
import groovy.transform.CompileStatic
import groovy.transform.EqualsAndHashCode
import groovy.transform.ToString
import groovy.transform.builder.Builder
import nextflow.cloud.azure.config.AzPoolOpts
/**
* Model the spec of Azure VM pool
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Builder
@ToString(includeNames = true, includePackage = false)
@EqualsAndHashCode
@CompileStatic
class AzVmPoolSpec {
String poolId
AzVmType vmType
AzPoolOpts opts
Map<String,String> metadata
}

49
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzVmType.groovy Normal file
View File

@@ -0,0 +1,49 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.batch
import groovy.transform.CompileStatic
import groovy.transform.EqualsAndHashCode
import groovy.transform.ToString
import nextflow.util.MemoryUnit
/**
* Model the size of a Azure VM
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@ToString(includeNames = true, includePackage = false)
@EqualsAndHashCode
@CompileStatic
class AzVmType {
String name
Integer maxDataDiskCount
MemoryUnit memory
Integer numberOfCores
MemoryUnit osDiskSize
MemoryUnit resourceDiskSize
AzVmType() {}
AzVmType(Map map) {
this.name = map.name
this.maxDataDiskCount = map.maxDataDiskCount as Integer
this.memory = map.memoryInMB ? MemoryUnit.of( "$map.memoryInMB MB" ) : null
this.numberOfCores = map.numberOfCores as Integer
this.osDiskSize = map.osDiskSizeInMB ? MemoryUnit.of( "$map.osDiskSizeInMB MB" ) : MemoryUnit.of("0 MB")
this.resourceDiskSize = map.resourceDiskSizeInMB ? MemoryUnit.of( "$map.resourceDiskSizeInMB MB" ) : MemoryUnit.of("0 MB")
}
}

74
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzActiveDirectoryOpts.groovy Normal file
View File

@@ -0,0 +1,74 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.config
import groovy.transform.CompileStatic
import nextflow.SysEnv
import nextflow.cloud.azure.nio.AzFileSystemProvider
import nextflow.config.spec.ConfigOption
import nextflow.config.spec.ConfigScope
import nextflow.script.dsl.Description
/**
* Model Azure Entra (formerly Active Directory) config options
*
* @author Abhinav Sharma <abhi18av@outlook.com>
*/
@CompileStatic
class AzActiveDirectoryOpts implements ConfigScope {
@ConfigOption
@Description("""
The service principal client ID. Defaults to environment variable `AZURE_CLIENT_ID`.
""")
final String servicePrincipalId
@ConfigOption
@Description("""
The service principal client secret. Defaults to environment variable `AZURE_CLIENT_SECRET`.
""")
final String servicePrincipalSecret
@ConfigOption
@Description("""
The Azure tenant ID. Defaults to environment variable `AZURE_TENANT_ID`.
""")
final String tenantId
AzActiveDirectoryOpts(Map config, Map<String, String> env = null) {
assert config != null
this.servicePrincipalId = config.servicePrincipalId ?: SysEnv.get('AZURE_CLIENT_ID')
this.servicePrincipalSecret = config.servicePrincipalSecret ?: SysEnv.get('AZURE_CLIENT_SECRET')
this.tenantId = config.tenantId ?: SysEnv.get('AZURE_TENANT_ID')
}
Map<String, Object> getEnv() {
Map<String, Object> props = new HashMap<>();
props.put(AzFileSystemProvider.AZURE_CLIENT_ID, servicePrincipalId)
props.put(AzFileSystemProvider.AZURE_CLIENT_SECRET, servicePrincipalSecret)
props.put(AzFileSystemProvider.AZURE_TENANT_ID, tenantId)
return props
}
boolean isConfigured() {
if (servicePrincipalId && servicePrincipalSecret && tenantId)
return true
if (!servicePrincipalId && !servicePrincipalSecret && !tenantId)
return false
throw new IllegalArgumentException("Invalid Service Principal configuration - Make sure servicePrincipalId and servicePrincipalClient are set in nextflow.config or configured via environment variables")
}
}

237
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzBatchOpts.groovy Normal file
View File

@@ -0,0 +1,237 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.config
import java.util.regex.Matcher
import java.util.regex.Pattern
import groovy.transform.CompileStatic
import nextflow.Global
import nextflow.Session
import nextflow.cloud.CloudTransferOptions
import nextflow.config.spec.ConfigOption
import nextflow.config.spec.ConfigScope
import nextflow.config.spec.PlaceholderName
import nextflow.fusion.FusionHelper
import nextflow.script.dsl.Description
import nextflow.util.Duration
import nextflow.util.StringUtils
/**
* Model Azure Batch pool config settings
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class AzBatchOpts implements ConfigScope, CloudTransferOptions {
static final private Pattern ENDPOINT_PATTERN = ~/https:\/\/(\w+)\.(\w+)\.batch\.azure\.com/
private Map<String,String> sysEnv
int maxParallelTransfers
int maxTransferAttempts
Duration delayBetweenAttempts
@ConfigOption
@Description("""
The batch service account name. Defaults to environment variable `AZURE_BATCH_ACCOUNT_NAME`.
""")
final String accountName
@ConfigOption
@Description("""
The batch service account key. Defaults to environment variable `AZURE_BATCH_ACCOUNT_KEY`.
""")
final String accountKey
@ConfigOption
@Description("""
Enable the automatic creation of batch pools specified in the Nextflow configuration file (default: `false`).
""")
final Boolean allowPoolCreation
@ConfigOption
@Description("""
Enable the automatic creation of batch pools depending on the pipeline resources demand (default: `true`).
""")
final Boolean autoPoolMode
@ConfigOption(types=[String])
@Description("""
The mode in which the `azcopy` tool is installed by Nextflow (default: `'node'`).
""")
final CopyToolInstallMode copyToolInstallMode
@ConfigOption
@Description("""
Delete all jobs when the workflow completes (default: `false`).
""")
final Boolean deleteJobsOnCompletion
@ConfigOption
@Description("""
Delete all compute node pools when the workflow completes (default: `false`).
""")
final Boolean deletePoolsOnCompletion
@ConfigOption
@Description("""
Delete each task when it completes (default: `true`).
""")
final Boolean deleteTasksOnCompletion
@ConfigOption
@Description("""
The batch service endpoint e.g. `https://nfbatch1.westeurope.batch.azure.com`.
""")
final String endpoint
@ConfigOption
@Description("""
The maximum elapsed time that jobs may run, measured from the time they are created (default: `30d`).
""")
final Duration jobMaxWallClockTime
@ConfigOption
@Description("""
The name of the batch service region, e.g. `westeurope` or `eastus2`. Not needed when the endpoint is specified.
""")
final String location
@ConfigOption
@Description("""
The client ID for an Azure [managed identity](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview) that is available on all Azure Batch node pools. This identity is used by Fusion to authenticate to Azure storage. If set to `'auto'`, Fusion will use the first available managed identity.
""")
final String poolIdentityClientId
@PlaceholderName("<name>")
final Map<String, AzPoolOpts> pools
@ConfigOption
@Description("""
When the workflow completes, set all jobs to terminate on task completion (default: `true`).
""")
final Boolean terminateJobsOnCompletion
@ConfigOption
@Description("""
The maximum number of attempts to create a job when the active job quota has been reached (default: `3`). Set to `0` to fail immediately without retrying.
""")
final int maxJobQuotaRetries
@ConfigOption
@Description("""
The delay between attempts to create a job when the active job quota has been reached (default: `'2 min'`).
""")
final Duration jobQuotaRetryDelay
AzBatchOpts(Map config, Map<String,String> env=null) {
assert config!=null
sysEnv = env==null ? new HashMap<String,String>(System.getenv()) : env
accountName = config.accountName ?: sysEnv.get('AZURE_BATCH_ACCOUNT_NAME')
accountKey = config.accountKey ?: sysEnv.get('AZURE_BATCH_ACCOUNT_KEY')
endpoint = config.endpoint
location = config.location
autoPoolMode = config.autoPoolMode as Boolean
allowPoolCreation = config.allowPoolCreation as Boolean
terminateJobsOnCompletion = config.terminateJobsOnCompletion != Boolean.FALSE
deleteJobsOnCompletion = config.deleteJobsOnCompletion as Boolean
deletePoolsOnCompletion = config.deletePoolsOnCompletion as Boolean
deleteTasksOnCompletion = config.deleteTasksOnCompletion as Boolean
jobMaxWallClockTime = config.jobMaxWallClockTime ? config.jobMaxWallClockTime as Duration : Duration.of('30d')
poolIdentityClientId = config.poolIdentityClientId
pools = parsePools(config.pools instanceof Map ? config.pools as Map<String,Map> : Collections.<String,Map>emptyMap())
maxParallelTransfers = config.maxParallelTransfers ? config.maxParallelTransfers as int : MAX_TRANSFER
maxTransferAttempts = config.maxTransferAttempts ? config.maxTransferAttempts as int : MAX_TRANSFER_ATTEMPTS
delayBetweenAttempts = config.delayBetweenAttempts ? config.delayBetweenAttempts as Duration : DEFAULT_DELAY_BETWEEN_ATTEMPTS
copyToolInstallMode = config.copyToolInstallMode as CopyToolInstallMode
maxJobQuotaRetries = config.maxJobQuotaRetries != null ? config.maxJobQuotaRetries as int : 3
jobQuotaRetryDelay = config.jobQuotaRetryDelay ? config.jobQuotaRetryDelay as Duration : Duration.of('2 min')
}
static Map<String,AzPoolOpts> parsePools(Map<String,Map> pools) {
final result = new LinkedHashMap<String,AzPoolOpts>()
for( Map.Entry<String,Map> entry : pools ) {
result[entry.key] = new AzPoolOpts( entry.value )
}
if( !result.keySet().contains('auto') )
result.put('auto', new AzPoolOpts())
return result
}
AzPoolOpts pool(String name) {
return pools.get(name)
}
AzPoolOpts autoPoolOpts() {
pool('auto')
}
String toString() {
"endpoint=$endpoint; account-name=$accountName; account-key=${StringUtils.redact(accountKey)}"
}
private List<String> endpointParts() {
// try to infer the account name from the endpoint
Matcher m
if( endpoint && (m = ENDPOINT_PATTERN.matcher(endpoint)).matches() ) {
return [ m.group(1), m.group(2) ]
}
else {
return Collections.emptyList()
}
}
String getAccountName() {
if( accountName )
return accountName
return endpointParts()[0]
}
String getLocation() {
if( location )
return location
// try to infer the location name from the endpoint
return endpointParts()[1]
}
String getEndpoint() {
if( endpoint )
return endpoint
if( accountName && location )
return "https://${accountName}.${location}.batch.azure.com"
return null
}
boolean canCreatePool() {
allowPoolCreation || autoPoolMode
}
CopyToolInstallMode getCopyToolInstallMode() {
// if the `installAzCopy` is not specified
// `true` is returned when the pool is not create by Nextflow
// since it can be a pool provided by the user which does not
// provide the required `azcopy` tool
if( copyToolInstallMode )
return copyToolInstallMode
if( FusionHelper.isFusionEnabled((Session) Global.session) )
return CopyToolInstallMode.off
canCreatePool() ? CopyToolInstallMode.node : CopyToolInstallMode.task
}
}

89
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzConfig.groovy Normal file
View File

@@ -0,0 +1,89 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.config
import groovy.transform.CompileStatic
import nextflow.Global
import nextflow.Session
import nextflow.config.spec.ConfigScope
import nextflow.config.spec.ScopeName
import nextflow.script.dsl.Description
/**
* Model Azure settings defined in the nextflow.config file
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@ScopeName("azure")
@Description("""
The `azure` scope allows you to configure the interactions with Azure, including Azure Batch and Azure Blob Storage.
""")
@CompileStatic
class AzConfig implements ConfigScope {
private AzCopyOpts azcopy
private AzStorageOpts storage
private AzBatchOpts batch
private AzRegistryOpts registry
private AzRetryConfig retryPolicy
private AzActiveDirectoryOpts activeDirectory
private AzManagedIdentityOpts managedIdentity
/* required by extension point -- do not remove */
AzConfig() {}
AzConfig(Map azure) {
this.batch = new AzBatchOpts( (Map)azure.batch ?: Collections.emptyMap() )
this.storage = new AzStorageOpts( (Map)azure.storage ?: Collections.emptyMap() )
this.registry = new AzRegistryOpts( (Map)azure.registry ?: Collections.emptyMap() )
this.azcopy = new AzCopyOpts( (Map)azure.azcopy ?: Collections.emptyMap() )
this.retryPolicy = new AzRetryConfig( (Map)azure.retryPolicy ?: Collections.emptyMap() )
this.activeDirectory = new AzActiveDirectoryOpts((Map) azure.activeDirectory ?: Collections.emptyMap())
this.managedIdentity = new AzManagedIdentityOpts((Map) azure.managedIdentity ?: Collections.emptyMap())
}
AzCopyOpts azcopy() { azcopy }
AzBatchOpts batch() { batch }
AzStorageOpts storage() { storage }
AzRegistryOpts registry() { registry }
AzRetryConfig retryConfig() { retryPolicy }
AzActiveDirectoryOpts activeDirectory() { activeDirectory }
AzManagedIdentityOpts managedIdentity() { managedIdentity }
static AzConfig getConfig(Session session) {
if( !session )
throw new IllegalStateException("Missing Nextflow session")
new AzConfig( (Map)session.config.azure ?: Collections.emptyMap() )
}
static AzConfig getConfig() {
getConfig(Global.session as Session)
}
}

58
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzCopyOpts.groovy Normal file
View File

@@ -0,0 +1,58 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.config
import groovy.transform.CompileStatic
import nextflow.config.spec.ConfigOption
import nextflow.config.spec.ConfigScope
import nextflow.script.dsl.Description
/**
* Model Azure azcopy tool config settings from nextflow config file
*
* @author Abhinav Sharma <abhi18av@outlook.com>
*/
@CompileStatic
class AzCopyOpts implements ConfigScope {
static public final String DEFAULT_BLOCK_SIZE = "4"
static public final String DEFAULT_BLOB_TIER = "None"
@ConfigOption
@Description("""
The block size (in MB) used by `azcopy` to transfer files between Azure Blob Storage and compute nodes (default: `4`).
""")
final String blockSize
@ConfigOption
@Description("""
The blob [access tier](https://learn.microsoft.com/en-us/azure/storage/blobs/access-tiers-overview) used by `azcopy` to upload files to Azure Blob Storage. Valid options are `None`, `Hot`, or `Cool` (default: `None`).
""")
final String blobTier
AzCopyOpts() {
this.blockSize = DEFAULT_BLOCK_SIZE
this.blobTier = DEFAULT_BLOB_TIER
}
AzCopyOpts(Map config) {
assert config!=null
this.blockSize = config.blockSize ?: DEFAULT_BLOCK_SIZE
this.blobTier = config.blobTier ?: DEFAULT_BLOB_TIER
}
}

68
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzFileShareOpts.groovy Normal file
View File

@@ -0,0 +1,68 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.config
import com.google.common.hash.Hasher
import groovy.transform.CompileStatic
import groovy.transform.EqualsAndHashCode
import groovy.transform.ToString
import nextflow.config.spec.ConfigOption
import nextflow.config.spec.ConfigScope
import nextflow.script.dsl.Description
import nextflow.util.CacheFunnel
import nextflow.util.CacheHelper
/**
* Model the settings to access to an Azure File Share.
*
* @author Manuele Simi <manuele.simi@gmail.com>
*/
@ToString(includeNames = true, includePackage = false)
@EqualsAndHashCode
@CompileStatic
class AzFileShareOpts implements CacheFunnel, ConfigScope {
static public final String DEFAULT_MOUNT_OPTIONS = '-o vers=3.0,dir_mode=0777,file_mode=0777,sec=ntlmssp'
@ConfigOption
@Description("""
The file share mount path.
""")
final String mountPath
@ConfigOption
@Description("""
The file share mount options.
""")
final String mountOptions
AzFileShareOpts(Map opts) {
assert opts != null
this.mountPath = opts.mountPath ?: ''
this.mountOptions = opts.mountOptions ?: DEFAULT_MOUNT_OPTIONS
}
AzFileShareOpts() {
this(Collections.emptyMap())
}
@Override
Hasher funnel(Hasher hasher, CacheHelper.HashMode mode) {
hasher.putUnencodedChars(mountPath)
hasher.putUnencodedChars(mountOptions)
return hasher
}
}

71
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzManagedIdentityOpts.groovy Normal file
View File

@@ -0,0 +1,71 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.config
import groovy.transform.CompileStatic
import groovy.transform.EqualsAndHashCode
import groovy.transform.ToString
import nextflow.cloud.azure.nio.AzFileSystemProvider
import nextflow.config.spec.ConfigOption
import nextflow.config.spec.ConfigScope
import nextflow.script.dsl.Description
/**
* Model Azure managed identity config options
*
* @author Ben Sherman <bentshermann@gmail.com>
*/
@ToString(includePackage = false, includeNames = true)
@EqualsAndHashCode
@CompileStatic
class AzManagedIdentityOpts implements ConfigScope {
@ConfigOption
@Description("""
The client ID for an Azure [managed identity](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview). Defaults to environment variable `AZURE_MANAGED_IDENTITY_USER`.
""")
final String clientId
@ConfigOption
@Description("""
When `true`, use the system-assigned [managed identity](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview) to authenticate Azure resources. Defaults to environment variable `AZURE_MANAGED_IDENTITY_SYSTEM`.
""")
final boolean system
AzManagedIdentityOpts(Map config) {
assert config != null
this.clientId = config.clientId
this.system = Boolean.parseBoolean(config.system as String)
}
Map<String, Object> getEnv() {
Map<String, Object> props = new HashMap<>();
props.put(AzFileSystemProvider.AZURE_MANAGED_IDENTITY_USER, clientId)
props.put(AzFileSystemProvider.AZURE_MANAGED_IDENTITY_SYSTEM, system)
return props
}
boolean isConfigured() {
if( clientId && !system )
return true
if( !clientId && system )
return true
if( !clientId && !system )
return false
throw new IllegalArgumentException("Invalid Managed Identity configuration - Make sure the `clientId` or `system` is set in the nextflow config")
}
}

211
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzPoolOpts.groovy Normal file
View File

@@ -0,0 +1,211 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.config
import com.azure.compute.batch.models.ImageVerificationType
import com.azure.compute.batch.models.OSType
import com.google.common.hash.Hasher
import groovy.transform.CompileStatic
import groovy.transform.EqualsAndHashCode
import groovy.transform.ToString
import nextflow.config.spec.ConfigOption
import nextflow.config.spec.ConfigScope
import nextflow.script.dsl.Description
import nextflow.util.CacheFunnel
import nextflow.util.CacheHelper
import nextflow.util.Duration
/**
* Model the settings of a VM pool
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@ToString(includeNames = true, includePackage = false)
@EqualsAndHashCode
@CompileStatic
class AzPoolOpts implements CacheFunnel, ConfigScope {
static public final String DEFAULT_PUBLISHER = "microsoft-dsvm"
static public final String DEFAULT_OFFER = "ubuntu-hpc"
static public final String DEFAULT_SKU = "batch.node.ubuntu 24.04"
static public final String DEFAULT_VM_TYPE = "Standard_D4a_v4"
static public final OSType DEFAULT_OS_TYPE = OSType.LINUX
static public final String DEFAULT_SHARE_ROOT_PATH = "/mnt/batch/tasks/fsmounts"
static public final Duration DEFAULT_SCALE_INTERVAL = Duration.of('5 min')
@ConfigOption
@Description("""
Enable autoscaling feature for the pool identified with `<name>`.
""")
final boolean autoScale
@ConfigOption
@Description("""
The internal root mount point when mounting File Shares. Must be `/mnt/resource/batch/tasks/fsmounts` for CentOS nodes or `/mnt/batch/tasks/fsmounts` for Ubuntu nodes (default: CentOS).
""")
final String fileShareRootPath
@ConfigOption
@Description("""
Enable the use of low-priority VMs (default: `false`).
""")
final boolean lowPriority
@ConfigOption
@Description("""
The max number of virtual machines when using auto scaling.
""")
final Integer maxVmCount
@ConfigOption
@Description("""
The mount options for mounting the file shares (default: `-o vers=3.0,dir_mode=0777,file_mode=0777,sec=ntlmssp`).
""")
final String mountOptions
@ConfigOption
@Description("""
The offer type of the virtual machine type used by the pool identified with `<name>` (default: `centos-container`).
""")
final String offer
@ConfigOption
@Description("""
Enable the task to run with elevated access. Ignored if `runAs` is set (default: `false`).
""")
final boolean privileged
@ConfigOption
@Description("""
The publisher of virtual machine type used by the pool identified with `<name>` (default: `microsoft-azure-batch`).
""")
final String publisher
@ConfigOption
@Description("""
The username under which the task is run. The user must already exist on each node of the pool.
""")
final String runAs
@ConfigOption
@Description("""
The [scale formula](https://docs.microsoft.com/en-us/azure/batch/batch-automatic-scaling) for the pool identified with `<name>`.
""")
final String scaleFormula
@ConfigOption
@Description("""
The interval at which to automatically adjust the Pool size according to the autoscale formula. Must be at least 5 minutes and at most 168 hours (default: `10 mins`).
""")
final Duration scaleInterval
@ConfigOption
@Description("""
The scheduling policy for the pool identified with `<name>`. Can be either `spread` or `pack` (default: `spread`).
""")
final String schedulePolicy
@ConfigOption
@Description("""
The ID of the Compute Node agent SKU which the pool identified with `<name>` supports (default: `batch.node.centos 8`).
""")
final String sku
final AzStartTaskOpts startTask
@ConfigOption
@Description("""
The subnet ID of a virtual network in which to create the pool.
""")
final String virtualNetwork
@ConfigOption
@Description("""
The number of virtual machines provisioned by the pool identified with `<name>`.
""")
final Integer vmCount
@ConfigOption
@Description("""
The virtual machine type used by the pool identified with `<name>`.
""")
final String vmType
OSType osType = DEFAULT_OS_TYPE
ImageVerificationType verification = ImageVerificationType.VERIFIED
String registry
String userName
String password
AzPoolOpts() {
this(Collections.emptyMap())
}
AzPoolOpts(Map opts) {
this.runAs = opts.runAs ?: ''
this.privileged = opts.privileged ?: false
this.publisher = opts.publisher ?: DEFAULT_PUBLISHER
this.offer = opts.offer ?: DEFAULT_OFFER
this.sku = opts.sku ?: DEFAULT_SKU
this.vmType = opts.vmType ?: DEFAULT_VM_TYPE
this.fileShareRootPath = opts.fileShareRootPath ?: buildFileShareRootPath()
this.vmCount = opts.vmCount as Integer ?: 1
this.autoScale = opts.autoScale as boolean
this.scaleFormula = opts.scaleFormula
this.schedulePolicy = opts.schedulePolicy
this.scaleInterval = opts.scaleInterval as Duration ?: DEFAULT_SCALE_INTERVAL
this.maxVmCount = opts.maxVmCount as Integer ?: vmCount *3
this.startTask = new AzStartTaskOpts( opts.startTask ? opts.startTask as Map : Map.of() )
this.registry = opts.registry
this.userName = opts.userName
this.password = opts.password
this.virtualNetwork = opts.virtualNetwork
this.lowPriority = opts.lowPriority as boolean
}
@Override
Hasher funnel(Hasher hasher, CacheHelper.HashMode mode) {
hasher.putUnencodedChars(runAs)
hasher.putBoolean(privileged)
hasher.putUnencodedChars(publisher)
hasher.putUnencodedChars(offer)
hasher.putUnencodedChars(sku)
hasher.putUnencodedChars(vmType)
hasher.putUnencodedChars(fileShareRootPath)
hasher.putUnencodedChars(registry ?: '')
hasher.putUnencodedChars(userName ?: '')
hasher.putUnencodedChars(password ?: '')
hasher.putInt(vmCount)
hasher.putBoolean(autoScale)
hasher.putUnencodedChars(scaleFormula ?: '')
hasher.putUnencodedChars(schedulePolicy ?: '')
hasher.putUnencodedChars(virtualNetwork ?: '')
hasher.putBoolean(lowPriority)
hasher.putUnencodedChars(startTask.script ?: '')
hasher.putBoolean(startTask.privileged)
return hasher
}
String buildFileShareRootPath() {
if (this.sku ==~ /.*centos.*/)
return "/mnt/resource/batch/tasks/fsmounts"
else if (this.sku ==~ /.*ubuntu.*/)
return "/mnt/batch/tasks/fsmounts"
else
return ''
}
}

70
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzRegistryOpts.groovy Normal file
View File

@@ -0,0 +1,70 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.config
import groovy.transform.CompileStatic
import nextflow.SysEnv
import nextflow.config.spec.ConfigOption
import nextflow.config.spec.ConfigScope
import nextflow.script.dsl.Description
/**
* Model Azure Batch registry config settings from nextflow config file
*
* @author Manuele Simi <manuele.simi@gmail.com>
*/
@CompileStatic
class AzRegistryOpts implements ConfigScope {
@ConfigOption
@Description("""
The container registry from which to pull the Docker images (default: `docker.io`).
""")
final String server
@ConfigOption
@Description("""
The username to connect to a private container registry.
""")
final String userName
@ConfigOption
@Description("""
The password to connect to a private container registry.
""")
final String password
AzRegistryOpts() {
this(Collections.emptyMap())
}
AzRegistryOpts(Map config, Map<String,String> env=SysEnv.get()) {
assert config!=null
this.server = config.server ?: 'docker.io'
this.userName = config.userName ?: env.get('AZURE_REGISTRY_USER_NAME')
this.password = config.password ?: env.get('AZURE_REGISTRY_PASSWORD')
}
boolean isConfigured() {
if( userName && password )
return true
if( !userName && !password )
return false
throw new IllegalArgumentException("Invalid Container Registry configuration - Make sure userName and password are set for Container Registry")
}
}

75
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzRetryConfig.groovy Normal file
View File

@@ -0,0 +1,75 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.config
import groovy.transform.CompileStatic
import groovy.transform.EqualsAndHashCode
import groovy.transform.ToString
import nextflow.config.spec.ConfigOption
import nextflow.config.spec.ConfigScope
import nextflow.script.dsl.Description
import nextflow.util.Duration
/**
* Model retry policy configuration
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@ToString(includePackage = false, includeNames = true)
@EqualsAndHashCode
@CompileStatic
class AzRetryConfig implements ConfigScope {
@ConfigOption
@Description("""
Delay when retrying failed API requests (default: `250ms`).
""")
Duration delay = Duration.of('250ms')
@ConfigOption
@Description("""
Max delay when retrying failed API requests (default: `90s`).
""")
Duration maxDelay = Duration.of('90s')
@ConfigOption
@Description("""
Max attempts when retrying failed API requests (default: `10`).
""")
int maxAttempts = 10
@ConfigOption
@Description("""
Jitter value when retrying failed API requests (default: `0.25`).
""")
double jitter = 0.25
AzRetryConfig() {
this(Collections.emptyMap())
}
AzRetryConfig(Map config) {
if( config.delay )
delay = config.delay as Duration
if( config.maxDelay )
maxDelay = config.maxDelay as Duration
if( config.maxAttempts )
maxAttempts = config.maxAttempts as int
if( config.jitter )
jitter = config.jitter as double
}
}

49
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzStartTaskOpts.groovy Normal file
View File

@@ -0,0 +1,49 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.config
import groovy.transform.CompileStatic
import nextflow.config.spec.ConfigOption
import nextflow.config.spec.ConfigScope
import nextflow.script.dsl.Description
/**
* Model Azure pool start task options
*/
@CompileStatic
class AzStartTaskOpts implements ConfigScope {
@ConfigOption
@Description("""
The `startTask` that is executed as the node joins the Azure Batch node pool.
""")
final String script
@ConfigOption
@Description("""
Enable the `startTask` to run with elevated access (default: `false`).
""")
final boolean privileged
AzStartTaskOpts() {
this(Collections.emptyMap())
}
AzStartTaskOpts(Map config) {
this.script = config.script
this.privileged = Boolean.parseBoolean(config.privileged as String)
}
}

90
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzStorageOpts.groovy Normal file
View File

@@ -0,0 +1,90 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.config
import groovy.transform.CompileStatic
import nextflow.SysEnv
import nextflow.cloud.azure.batch.AzHelper
import nextflow.cloud.azure.nio.AzFileSystemProvider
import nextflow.config.spec.ConfigOption
import nextflow.config.spec.ConfigScope
import nextflow.config.spec.PlaceholderName
import nextflow.script.dsl.Description
import nextflow.util.Duration
/**
* Parse Azure settings from nextflow config file
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class AzStorageOpts implements ConfigScope {
@ConfigOption
@Description("""
The blob storage account key. Defaults to environment variable `AZURE_STORAGE_ACCOUNT_KEY`.
""")
final String accountKey
@ConfigOption
@Description("""
The blob storage account name. Defaults to environment variable `AZURE_STORAGE_ACCOUNT_NAME`.
""")
final String accountName
@ConfigOption
@Description("""
The blob storage shared access signature (SAS) token, which can be provided instead of an account key. Defaults to environment variable `AZURE_STORAGE_SAS_TOKEN`.
""")
String sasToken
@ConfigOption
@Description("""
The duration of the SAS token generated by Nextflow when the `sasToken` option is *not* specified (default: `48h`).
""")
final Duration tokenDuration
@PlaceholderName("<name>")
final Map<String,AzFileShareOpts> fileShares
AzStorageOpts(Map config, Map<String,String> env=SysEnv.get()) {
assert config!=null
this.accountKey = config.accountKey ?: env.get('AZURE_STORAGE_ACCOUNT_KEY')
this.accountName = config.accountName ?: env.get('AZURE_STORAGE_ACCOUNT_NAME')
this.sasToken = config.sasToken ?: env.get('AZURE_STORAGE_SAS_TOKEN')
this.tokenDuration = (config.tokenDuration as Duration) ?: Duration.of('48h')
this.fileShares = parseFileShares(config.fileShares instanceof Map ? config.fileShares as Map<String, Map>
: Collections.<String,Map> emptyMap())
}
Map<String,Object> getEnv() {
Map<String, Object> props = new HashMap<>();
props.put(AzFileSystemProvider.AZURE_STORAGE_ACCOUNT_KEY, accountKey)
props.put(AzFileSystemProvider.AZURE_STORAGE_ACCOUNT_NAME, accountName)
props.put(AzFileSystemProvider.AZURE_STORAGE_SAS_TOKEN, sasToken)
return props
}
static Map<String,AzFileShareOpts> parseFileShares(Map<String,Map> shares) {
final result = new LinkedHashMap<String,AzFileShareOpts>()
shares.each { Map.Entry<String, Map> entry ->
result[entry.key] = new AzFileShareOpts(entry.value)
}
return result
}
}

27
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/config/CopyToolInstallMode.groovy Normal file
View File

@@ -0,0 +1,27 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.config
/**
* Define the strategy to install the azcopy tool
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
enum CopyToolInstallMode {
node,
task,
off
}

116
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzBashLib.groovy Normal file
View File

@@ -0,0 +1,116 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.file
import groovy.transform.Memoized
import nextflow.cloud.azure.config.AzCopyOpts
import nextflow.executor.BashFunLib
import nextflow.util.Duration
/**
* Azure Bash helper functions
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
class AzBashLib extends BashFunLib<AzBashLib> {
private String blockSize = AzCopyOpts.DEFAULT_BLOCK_SIZE
private String blobTier = AzCopyOpts.DEFAULT_BLOB_TIER
AzBashLib withBlockSize(String value) {
if( value )
this.blockSize = value
return this
}
AzBashLib withBlobTier(String value) {
if( value )
this.blobTier = value
return this
}
protected String setupAzCopyOpts() {
"""
# custom env variables used for azcopy opts
export AZCOPY_BLOCK_SIZE_MB=${blockSize}
export AZCOPY_BLOCK_BLOB_TIER=${blobTier}
""".stripIndent(true)
}
protected String azLib() {
'''
nxf_az_upload() {
local name=$1
local target=${2%/} ## remove ending slash
local base_name="$(basename "$name")"
local dir_name="$(dirname "$name")"
if [[ -d $name ]]; then
if [[ "$base_name" == "$name" ]]; then
azcopy cp "$name" "$target?$AZ_SAS" --recursive --block-blob-tier $AZCOPY_BLOCK_BLOB_TIER --block-size-mb $AZCOPY_BLOCK_SIZE_MB
else
azcopy cp "$name" "$target/$dir_name?$AZ_SAS" --recursive --block-blob-tier $AZCOPY_BLOCK_BLOB_TIER --block-size-mb $AZCOPY_BLOCK_SIZE_MB
fi
else
azcopy cp "$name" "$target/$name?$AZ_SAS" --block-blob-tier $AZCOPY_BLOCK_BLOB_TIER --block-size-mb $AZCOPY_BLOCK_SIZE_MB
fi
}
nxf_az_download() {
local source=$1
local target=$2
local basedir=$(dirname $2)
local ret
mkdir -p "$basedir"
ret=$(azcopy cp "$source?$AZ_SAS" "$target" 2>&1) || {
## if fails check if it was trying to download a directory
mkdir -p $target
azcopy cp "$source/*?$AZ_SAS" "$target" --recursive >/dev/null || {
rm -rf $target
>&2 echo "Unable to download path: $source"
exit 1
}
}
}
'''.stripIndent(true)
}
String render() {
super.render() + setupAzCopyOpts() + azLib()
}
@Memoized
static String script(AzCopyOpts opts, Integer maxParallelTransfers, Integer maxTransferAttempts, Duration delayBetweenAttempts) {
new AzBashLib()
.includeCoreFun(true)
.withMaxParallelTransfers(maxParallelTransfers)
.withMaxTransferAttempts(maxTransferAttempts)
.withDelayBetweenAttempts(delayBetweenAttempts)
.withBlobTier(opts.blobTier)
.withBlockSize(opts.blockSize)
.render()
}
@Memoized
static String script() {
new AzBashLib().render()
}
}

103
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzPathFactory.groovy Normal file
View File

@@ -0,0 +1,103 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.file
import java.nio.file.FileSystem
import java.nio.file.Path
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.cloud.azure.AzurePlugin
import nextflow.cloud.azure.batch.AzFileCopyStrategy
import nextflow.cloud.azure.config.AzConfig
import nextflow.cloud.azure.nio.AzPath
import nextflow.file.FileHelper
import nextflow.file.FileSystemPathFactory
/**
* Create Azure path objects for az:// prefixed URIs
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
class AzPathFactory extends FileSystemPathFactory {
AzPathFactory() {
log.debug "Creating Azure path factory"
}
@Override
protected Path parseUri(String uri) {
if( !uri.startsWith('az://') )
return null
if( uri.startsWith('az:///') )
throw new IllegalArgumentException("Invalid Azure path URI - make sure the schema prefix does not container more than two slash characters - offending value: $uri")
final storageConfigEnv = AzConfig.getConfig().storage().getEnv()
final activeDirectoryConfigEnv = AzConfig.getConfig().activeDirectory().getEnv()
final managedIdentityConfigEnv = AzConfig.getConfig().managedIdentity().getEnv()
final configEnv = storageConfigEnv + activeDirectoryConfigEnv + managedIdentityConfigEnv
// find the related file system
final fs = getFileSystem(uri0(uri), configEnv)
// resulting az path
return fs.getPath(uri.substring(4))
}
private URI uri0(String uri) {
// note: this is needed to allow URI to handle curly brackets characters
// see https://github.com/nextflow-io/nextflow/issues/1969
new URI(null, null, uri, null, null)
}
protected FileSystem getFileSystem(URI uri, Map env) {
final bak = Thread.currentThread().getContextClassLoader()
// NOTE: setting the context classloader to allow loading azure deps via java ServiceLoader
// see
// com.azure.core.http.HttpClientProvider
// com.azure.core.http.netty.implementation.ReactorNettyClientProvider
//
try {
final loader = AzurePlugin.class.getClassLoader()
log.trace "+ Setting context class loader to=$loader - previous=$bak"
Thread.currentThread().setContextClassLoader(loader)
return FileHelper.getOrCreateFileSystemFor(uri, env)
}
finally {
Thread.currentThread().setContextClassLoader(bak)
}
}
@Override
protected String toUriString(Path path) {
return path instanceof AzPath ? ((AzPath)path).toUriString() : null
}
@Override
protected String getBashLib(Path path) {
return path instanceof AzPath ? AzBashLib.script() : null
}
@Override
protected String getUploadCmd(String source, Path target) {
return target instanceof AzPath ? AzFileCopyStrategy.uploadCmd(source, target) : null
}
}

54
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzPathSerializer.groovy Normal file
View File

@@ -0,0 +1,54 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.file
import com.esotericsoftware.kryo.Kryo
import com.esotericsoftware.kryo.Serializer
import com.esotericsoftware.kryo.io.Input
import com.esotericsoftware.kryo.io.Output
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.cloud.azure.nio.AzPath
import nextflow.file.FileHelper
import nextflow.util.SerializerRegistrant
/**
* Implements Serializer for {@link AzPath} objects
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
class AzPathSerializer extends Serializer<AzPath> implements SerializerRegistrant {
@Override
void write(Kryo kryo, Output output, AzPath path) {
log.trace "Azure Blob storage path serialisation > path=$path"
output.writeString(path.toUriString())
}
@Override
AzPath read(Kryo kryo, Input input, Class<AzPath> type) {
final path = input.readString()
log.trace "Azure Blob storage path > path=$path"
return (AzPath)FileHelper.asPath(path)
}
@Override
void register(Map<Class, Object> serializers) {
serializers.put(AzPath, AzPathSerializer)
}
}

109
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/fusion/AzFusionEnv.groovy Normal file
View File

@@ -0,0 +1,109 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.fusion
import groovy.util.logging.Slf4j
import nextflow.Global
import nextflow.cloud.azure.batch.AzHelper
import groovy.transform.CompileStatic
import nextflow.cloud.azure.config.AzConfig
import nextflow.fusion.FusionConfig
import nextflow.fusion.FusionEnv
import org.pf4j.Extension
/**
* Implement environment provider for Azure specific variables
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Extension
@CompileStatic
@Slf4j
class AzFusionEnv implements FusionEnv {
/**
* Get the environment variables for Fusion with Azure, taking into
* account a managed identity ID if provided
*
* @param scheme The storage scheme ('az' for Azure)
* @param config The Fusion configuration
* @param managedIdentityId Optional managed identity client ID from pool options
* @return Map of environment variables
*/
@Override
Map<String, String> getEnvironment(String scheme, FusionConfig config) {
if (scheme != 'az') {
return Collections.<String, String> emptyMap()
}
final cfg = AzConfig.config
final managedIdentityId = cfg.batch().poolIdentityClientId
final result = new LinkedHashMap(10)
if (!cfg.storage().accountName) {
throw new IllegalArgumentException("Missing Azure Storage account name")
}
result.AZURE_STORAGE_ACCOUNT = cfg.storage().accountName
// If pool has a managed identity, ONLY add the MSI client ID
// DO NOT add any SAS token or reference cfg.storage().sasToken
if (managedIdentityId) {
// Fusion will try and pick up a managed identity that is available.
// We recommend explicitly setting the config item to the managed ID so you know which one is being used.
// However if set to 'true' it will use whichever is available.
// This can be helpful if the pools have different managed identities.
if (managedIdentityId != 'auto') {
result.FUSION_AZ_MSI_CLIENT_ID = managedIdentityId
}
// No SAS token is added or generated
return result
}
// If no managed identity, use the standard environment with SAS token
result.AZURE_STORAGE_SAS_TOKEN = getOrCreateSasToken()
return result
}
/**
* Return the SAS token if it is defined in the configuration, otherwise generate one based on the requested
* authentication method.
*/
synchronized String getOrCreateSasToken() {
final cfg = AzConfig.config
// Check for incompatible configuration
if (cfg.storage().accountKey && cfg.storage().sasToken) {
throw new IllegalArgumentException("Azure Storage Access key and SAS token detected. Only one is allowed")
}
// If a SAS token is already defined in the configuration, just return it
if (cfg.storage().sasToken) {
return cfg.storage().sasToken
}
// For Active Directory and Managed Identity, we cannot generate an *account* SAS token, but we can generate
// a *container* SAS token for the work directory.
if (cfg.activeDirectory().isConfigured() || cfg.managedIdentity().isConfigured()) {
return AzHelper.generateContainerSasWithActiveDirectory(Global.session.workDir, cfg.storage().tokenDuration)
}
// Shared Key authentication can use an account SAS token
return AzHelper.generateAccountSasWithAccountKey(Global.session.workDir, cfg.storage().tokenDuration)
}
}

169
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/nio/AzFileAttributes.groovy Normal file
View File

@@ -0,0 +1,169 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.nio
import java.nio.file.attribute.BasicFileAttributes
import java.nio.file.attribute.FileTime
import java.time.OffsetDateTime
import java.util.concurrent.TimeUnit
import com.azure.storage.blob.BlobClient
import com.azure.storage.blob.BlobContainerClient
import com.azure.storage.blob.models.BlobContainerItem
import com.azure.storage.blob.models.BlobItem
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
/**
* Implements {@link BasicFileAttributes} for Azure blob path
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
class AzFileAttributes implements BasicFileAttributes {
private FileTime updateTime
private FileTime creationTime
private boolean directory
private long size
private String objectId
static AzFileAttributes root() {
new AzFileAttributes(size: 0, objectId: '/', directory: true)
}
AzFileAttributes() {}
AzFileAttributes(BlobClient client) {
final props = client.getProperties()
objectId = "/${client.containerName}/${client.blobName}"
creationTime = time(props.getCreationTime())
updateTime = time(props.getLastModified())
directory = client.blobName.endsWith('/')
size = props.getBlobSize()
// Support for Azure Data Lake Storage Gen2 with hierarchical namespace enabled
final meta = props.getMetadata()
if( meta.containsKey("hdi_isfolder") && size == 0 ){
directory = meta.get("hdi_isfolder")
}
}
AzFileAttributes(String containerName, BlobItem item) {
objectId = "/${containerName}/${item.name}"
directory = item.name.endsWith('/')
if( !directory ) {
creationTime = time(item.properties.getCreationTime())
updateTime = time(item.properties.getLastModified())
size = item.properties.getContentLength()
}
}
protected AzFileAttributes(BlobContainerClient client) {
objectId = "/$client.blobContainerName"
updateTime = time(client.getProperties().getLastModified())
directory = true
}
protected AzFileAttributes(BlobContainerItem client) {
objectId = "/$client.name"
updateTime = time(client.getProperties().getLastModified())
directory = true
}
protected AzFileAttributes(BlobContainerClient client, String blobName) {
objectId = "/$client.blobContainerName/$blobName"
directory = blobName.endsWith('/')
}
static protected FileTime time(Long millis) {
millis ? FileTime.from(millis, TimeUnit.MILLISECONDS) : null
}
static protected FileTime time(OffsetDateTime offset) {
time(offset.toInstant().toEpochMilli())
}
@Override
FileTime lastModifiedTime() {
updateTime
}
@Override
FileTime lastAccessTime() {
return null
}
@Override
FileTime creationTime() {
creationTime
}
@Override
boolean isRegularFile() {
return !directory
}
@Override
boolean isDirectory() {
return directory
}
@Override
boolean isSymbolicLink() {
return false
}
@Override
boolean isOther() {
return false
}
@Override
long size() {
return size
}
@Override
Object fileKey() {
return objectId
}
@Override
boolean equals( Object obj ) {
if( this.class != obj?.class ) return false
def other = (AzFileAttributes)obj
if( creationTime() != other.creationTime() ) return false
if( lastModifiedTime() != other.lastModifiedTime() ) return false
if( isRegularFile() != other.isRegularFile() ) return false
if( size() != other.size() ) return false
return true
}
@Override
int hashCode() {
Objects.hash( creationTime(), lastModifiedTime(), isRegularFile(), size() )
}
}

61
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/nio/AzFileAttributesView.groovy Normal file
View File

@@ -0,0 +1,61 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.nio
import java.nio.file.attribute.BasicFileAttributeView
import java.nio.file.attribute.BasicFileAttributes
import java.nio.file.attribute.FileTime
import com.azure.storage.blob.BlobClient
/**
* Implements {@link BasicFileAttributeView} for Azure storage blob
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
class AzFileAttributesView implements BasicFileAttributeView {
private BlobClient client
AzFileAttributesView( BlobClient client ) {
this.client = client
}
@Override
String name() {
return 'basic'
}
@Override
BasicFileAttributes readAttributes() throws IOException {
return new AzFileAttributes(client)
}
/**
* This API is implemented is not supported but instead of throwing an exception just do nothing
* to not break the method {@link java.nio.file.CopyMoveHelper#copyToForeignTarget(java.nio.file.Path, java.nio.file.Path, java.nio.file.CopyOption...)}
*
* @param lastModifiedTime
* @param lastAccessTime
* @param createTime
* @throws IOException
*/
@Override
void setTimes(FileTime lastModifiedTime, FileTime lastAccessTime, FileTime createTime) throws IOException {
// TBD
}
}

566
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/nio/AzFileSystem.groovy Normal file
View File

@@ -0,0 +1,566 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.nio
import java.nio.channels.Channels
import java.nio.channels.SeekableByteChannel
import java.nio.file.DirectoryNotEmptyException
import java.nio.file.DirectoryStream
import java.nio.file.FileStore
import java.nio.file.FileSystem
import java.nio.file.NoSuchFileException
import java.nio.file.Path
import java.nio.file.PathMatcher
import java.nio.file.WatchService
import java.nio.file.attribute.UserPrincipalLookupService
import java.time.Duration
import java.time.temporal.ChronoUnit
import java.util.concurrent.TimeoutException
import java.util.function.Predicate
import com.azure.core.util.polling.SyncPoller
import com.azure.storage.blob.BlobServiceClient
import com.azure.storage.blob.models.BlobContainerItem
import com.azure.storage.blob.models.BlobCopyInfo
import com.azure.storage.blob.models.BlobItem
import com.azure.storage.blob.models.BlobStorageException
import com.azure.storage.blob.models.ListBlobsOptions
import dev.failsafe.Failsafe
import dev.failsafe.RetryPolicy
import dev.failsafe.event.EventListener
import dev.failsafe.event.ExecutionAttemptedEvent
import dev.failsafe.function.CheckedSupplier
import groovy.transform.CompileStatic
import groovy.transform.Memoized
import groovy.transform.PackageScope
import groovy.util.logging.Slf4j
import nextflow.cloud.azure.config.AzConfig
/**
* Implements a file system for Azure Blob Storage service
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
class AzFileSystem extends FileSystem {
static class GuessPath {
boolean exists
boolean directory
boolean empty
}
private static String EMPTY_DIR_MARKER = '.azure_blob_dir'
private static String SLASH = '/'
private AzFileSystemProvider provider
private BlobServiceClient storageClient
private String containerName
private int maxCopyDurationSecs = 3600
@PackageScope AzFileSystem() {}
@PackageScope
AzFileSystem(AzFileSystemProvider provider, BlobServiceClient storageClient, String bucket) {
this.provider = provider
this.containerName = bucket
this.storageClient = storageClient
}
String getContainerName() { containerName }
BlobServiceClient getBlobServiceClient() { storageClient }
@Override
AzFileSystemProvider provider() {
return provider
}
@Override
void close() throws IOException {
// nothing to do
}
@Override
boolean isOpen() {
return true
}
@Override
boolean isReadOnly() {
return containerName == SLASH
}
@Override
String getSeparator() {
return SLASH
}
Iterable<? extends Path> getRootDirectories() {
return containerName == SLASH ? listContainers() : [new AzPath(this, "/$containerName/") ]
}
private Iterable<? extends Path> listContainers() {
return apply(()-> listContainers0())
}
private Iterable<? extends Path> listContainers0() {
final containers = new ArrayList()
storageClient
.listBlobContainers()
.forEach { BlobContainerItem it -> provider.getPath(it.getName()) }
return containers
}
@Override
Iterable<FileStore> getFileStores() {
throw new UnsupportedOperationException("Operation 'getFileStores' is not supported by AzFileSystem")
}
@Override
Set<String> supportedFileAttributeViews() {
return Collections.unmodifiableSet( ['basic'] as Set )
}
/**
* Get a {@link AzPath} given a string path. When a path starts with `/`
* is interpreted as an absolute path in which the first component
* is the bucket Azure storage bucket name.
*
* Otherwise it is interpreted as relative object name in the current
* file system.
*
*/
@Override
AzPath getPath(String path, String... more) {
assert path
if( more ) {
path = concat(path,more)
}
if( path.startsWith('/') ) {
return provider.getPath(path.substring(1))
}
else {
return new AzPath(this, path)
}
}
private String concat(String path, String... more) {
def concat = []
while( path.length()>1 && path.endsWith('/') )
path = path.substring(0,path.length()-1)
concat << path
concat.addAll( more.collect {String it -> trimSlash(it)} )
return concat.join('/')
}
private String trimSlash(String str) {
while( str.startsWith('/') )
str = str.substring(1)
while( str.endsWith('/') )
str = str.substring(0,str.length()-1)
return str
}
@Override
PathMatcher getPathMatcher(String syntaxAndPattern) {
throw new UnsupportedOperationException("Operation 'getPathMatcher' is not supported by AzFileSystem")
}
@Override
UserPrincipalLookupService getUserPrincipalLookupService() {
throw new UnsupportedOperationException("Operation 'getUserPrincipalLookupService' is not supported by AzFileSystem")
}
@Override
WatchService newWatchService() throws IOException {
throw new UnsupportedOperationException("Operation 'newWatchService' is not supported by AzFileSystem")
}
@PackageScope
SeekableByteChannel newReadableByteChannel(AzPath path) {
if( path.isContainer() )
throw new IllegalArgumentException("Operation not allowed on blob container path: ${path.toUriString()}")
final client = path.blobClient()
try {
final size = client.getProperties().getBlobSize()
final channel = Channels.newChannel( client.openInputStream() )
return new AzReadableByteChannel(channel, size)
}
catch (BlobStorageException e) {
if( e.statusCode == 404 )
throw new NoSuchFileException(path.toUriString())
else
throw new IOException("Unexpected exception processing Azure container blob: ${path.toUriString()}", e)
}
}
@PackageScope
SeekableByteChannel newWritableByteChannel(AzPath path) {
if( path.isContainer() )
throw new IllegalArgumentException("Operation not allowed on blob container path: ${path.toUriString()}")
try {
final client = path.blobClient()
final outStream = client.getBlockBlobClient().getBlobOutputStream(true)
final writer = Channels.newChannel(outStream)
return new AzWriteableByteChannel(writer)
}
catch (BlobStorageException e) {
if( e.statusCode == 404 )
throw new NoSuchFileException(path.toUriString())
else
throw new IOException("Unexpected exception processing Azure container blob: ${path.toUriString()}", e)
}
}
@PackageScope
DirectoryStream<Path> newDirectoryStream(AzPath dir, DirectoryStream.Filter<? super Path> filter) {
if( !dir.containerName )
throw new NoSuchFileException("Missing Azure storage container name: ${dir.toUriString()}")
if( dir.containerName == SLASH ) {
return listContainers(dir, filter)
}
else {
listFiles(dir, filter)
}
}
/**
* Create a {@link DirectoryStream} that allows the iteration over the files in a Azure Blob Storage
* file system system
*
* @param fs The underlying {@link AzFileSystem} instance
* @param filter A {@link java.nio.file.DirectoryStream.Filter} object to select which files to include in the file traversal
* @return A {@link DirectoryStream} object to traverse the associated objects
*/
private DirectoryStream<Path> listFiles(AzPath dir, DirectoryStream.Filter<? super Path> filter) {
return apply(()-> listFiles0(dir,filter))
}
private DirectoryStream<Path> listFiles0(AzPath dir, DirectoryStream.Filter<? super Path> filter) {
// -- create the list operation options
def prefix = dir.blobName()
if( prefix && !prefix.endsWith('/') ) {
prefix += '/'
}
// -- list the bucket content
final blobs = dir.containerClient()
.listBlobsByHierarchy(prefix)
.iterator()
// wrap the result with a directory stream
return new DirectoryStream<Path>() {
@Override
Iterator<Path> iterator() {
return new AzPathIterator.ForBlobs(dir, blobs, filter)
}
@Override void close() throws IOException { }
}
}
/**
* Create a {@link DirectoryStream} that allows the iteration over the buckets in a Azure Blob Storage Storage
* file system system
*
* @param fs The underlying {@link AzFileSystem} instance
* @param filter A {@link java.nio.file.DirectoryStream.Filter} object to select which buckets to include in the file traversal
* @return A {@link DirectoryStream} object to traverse the associated objects
*/
private DirectoryStream<Path> listContainers(AzPath path, DirectoryStream.Filter<? super Path> filter ) {
return apply(()-> listContainers0(path, filter))
}
private DirectoryStream<Path> listContainers0(AzPath path, DirectoryStream.Filter<? super Path> filter) {
Iterator<BlobContainerItem> containers = storageClient.listBlobContainers().iterator()
return new DirectoryStream<Path>() {
@Override
Iterator<Path> iterator() {
return new AzPathIterator.ForContainers(path, containers, filter)
}
@Override void close() throws IOException { }
}
}
@PackageScope
void createDirectory(AzPath path) {
if( isReadOnly() )
throw new UnsupportedOperationException("Operation 'createDirectory' not supported in root path")
if( !path.containerName )
throw new IllegalArgumentException("Missing Azure storage blob container name")
if( path.isContainer() ) {
storageClient.createBlobContainer(path.checkContainerName())
}
else {
path.directory = true
// Create an hidden file blob to as a placeholder for a new empty directory
// NOTE: this is *not* required by this implementation however third party tools
// such as azcopy get confused creation a blob name ending with a slash
// therefore this library creates an hidden file to simulate the creation
// of a directory
path.resolve(EMPTY_DIR_MARKER)
.blobClient()
.getAppendBlobClient()
.create()
}
}
@PackageScope
void delete(AzPath path) {
if( !path.containerName )
throw new IllegalArgumentException("Missing Azure blob container name")
if( path.isContainer() ) {
deleteBucket(path)
}
else {
deleteFile(path)
}
}
private void checkContainerExistsOrEmpty(AzPath path) {
try {
final container = path.containerClient()
final opts = new ListBlobsOptions().setMaxResultsPerPage(10)
final blobs = apply(()-> container.listBlobs(opts, null))
if( apply(()-> blobs.iterator().hasNext()) ) {
throw new DirectoryNotEmptyException(path.toUriString())
}
}
catch (BlobStorageException e) {
if( e.statusCode == 404 )
throw new NoSuchFileException(path.toUriString())
else
throw new IOException("Unexpected exception accessing blob storage path: ${path.toUriString()}")
}
}
private void checkPathExistOrEmpty(AzPath path) {
final result = guessPath(path)
if( result.directory && !result.empty )
throw new DirectoryNotEmptyException(path.toUriString())
if( !result.exists )
throw new NoSuchFileException(path.toUriString())
}
private GuessPath guessPath(AzPath path) {
boolean exists = false
boolean isDirectory = false
final opts = new ListBlobsOptions()
.setPrefix(path.blobName())
.setMaxResultsPerPage(10)
try {
final values = apply(()-> path.containerClient().listBlobs(opts,null).iterator())
final char SLASH = '/'
final String name = path.blobName()
int count=0
while( apply(()-> values.hasNext()) ) {
BlobItem blob = apply(()-> values.next())
if( blob.name == name )
exists = true
else if( blob.name.startsWith(name) && blob.name.charAt(name.length())==SLASH ) {
exists = true
isDirectory = true
}
// ignore empty dir marker file
if( blob.name.endsWith('/'+EMPTY_DIR_MARKER))
continue
count++
}
if( isDirectory && count>=1 )
return new GuessPath(exists: true, directory: true, empty: false)
else
return new GuessPath(exists: exists, directory: isDirectory, empty: true)
}
catch (BlobStorageException e) {
if( e.statusCode==404 )
return new GuessPath(exists: false)
throw e
}
}
private void deleteFile(AzPath path) {
checkPathExistOrEmpty(path)
apply(()-> path.blobClient().delete())
}
private void deleteBucket(AzPath path) {
checkContainerExistsOrEmpty(path)
apply(()-> path.containerClient().delete())
}
@PackageScope
void copy(AzPath source, AzPath target) {
final sasToken = provider.getSasToken()
String sourceUrl = source.blobClient().getBlobUrl()
if (sasToken != null) {
if (sourceUrl.contains('?')){
sourceUrl = String.format("%s&%s", sourceUrl, sasToken);
} else {
sourceUrl = String.format("%s?%s", sourceUrl, sasToken);
}
}
SyncPoller<BlobCopyInfo, Void> pollResponse =
target.blobClient().beginCopy( sourceUrl, null )
pollResponse.waitForCompletion(Duration.ofSeconds(maxCopyDurationSecs))
}
@PackageScope
AzFileAttributes readAttributes(AzPath path) {
final cache = path.attributesCache()
if( cache )
return cache
if( path.toString() == SLASH ) {
return AzFileAttributes.root()
}
if( path.containerName && !path.blobName()) {
return readContainerAttrs0(path)
}
else
return readBlobAttrs0(path)
}
private AzFileAttributes readBlobAttrs0(AzPath path) {
try {
return new AzFileAttributes(path.blobClient())
}
catch (BlobStorageException e) {
if( e.statusCode != 404 )
throw e
// if the previous check failed and
// the following pass, it can only be a directory
final result = guessPath(path)
if( result.exists ) {
def name = path.blobName()
if( !name.endsWith('/') ) name += '/'
return new AzFileAttributes(path.containerClient(), name)
}
else {
return null
}
}
}
private AzFileAttributes readContainerAttrs0(AzPath path) {
try {
return new AzFileAttributes(path.containerClient())
}
catch (BlobStorageException e) {
if( e.statusCode==404 )
throw new NoSuchFileException(path.toUriString())
throw e
}
}
@PackageScope
AzFileAttributesView getFileAttributeView(AzPath path) {
try {
return new AzFileAttributesView(path.blobClient())
}
catch (BlobStorageException e) {
if( e.statusCode==404 )
throw new NoSuchFileException(path.toUriString())
else
throw new IOException("Unable to get attributes for file: ${path.toUriString()}", e)
}
}
@PackageScope
boolean exists( AzPath path ) {
try {
return readAttributes(path) != null
}
catch( IOException e ){
return false
}
}
/**
* Creates a retry policy using the configuration specified by {@link nextflow.cloud.azure.config.AzRetryConfig}
*
* @param cond A predicate that determines when a retry should be triggered
* @return The {@link dev.failsafe.RetryPolicy} instance
*/
protected <T> RetryPolicy<T> retryPolicy(Predicate<? extends Throwable> cond) {
// this is needed because apparently bytebuddy used by testing framework is not able
// to handle properly this method signature using both generics and `@Memoized` annotation.
// therefore the `@Memoized` has been moved to the inner method invocation
return (RetryPolicy<T>) retryPolicy0(cond)
}
@Memoized
protected RetryPolicy retryPolicy0(Predicate<? extends Throwable> cond) {
final cfg = AzConfig.getConfig().retryConfig()
final listener = new EventListener<ExecutionAttemptedEvent>() {
@Override
void accept(ExecutionAttemptedEvent event) throws Throwable {
log.debug("Azure I/O exception - attempt: ${event.attemptCount}; cause: ${event.lastFailure?.message}")
}
}
return RetryPolicy.builder()
.handleIf(cond)
.withBackoff(cfg.delay.toMillis(), cfg.maxDelay.toMillis(), ChronoUnit.MILLIS)
.withMaxAttempts(cfg.maxAttempts)
.withJitter(cfg.jitter)
.onRetry(listener)
.build()
}
/**
* Carry out the invocation of the specified action using a retry policy
* when {@code TooManyRequests} Azure Batch error is returned
*
* @param action A {@link dev.failsafe.function.CheckedSupplier} instance modeling the action to be performed in a safe manner
* @return The result of the supplied action
*/
protected <T> T apply(CheckedSupplier<T> action) {
final policy = retryPolicy((Throwable t) -> t instanceof IOException || t.cause instanceof IOException || t instanceof TimeoutException || t.cause instanceof TimeoutException)
return Failsafe.with(policy).get(action)
}
}

523
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/nio/AzFileSystemProvider.groovy Normal file
View File

@@ -0,0 +1,523 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.nio
import static java.nio.file.StandardCopyOption.*
import static java.nio.file.StandardOpenOption.*
import java.nio.channels.SeekableByteChannel
import java.nio.file.AccessDeniedException
import java.nio.file.AccessMode
import java.nio.file.CopyOption
import java.nio.file.DirectoryStream
import java.nio.file.FileAlreadyExistsException
import java.nio.file.FileStore
import java.nio.file.FileSystem
import java.nio.file.FileSystemAlreadyExistsException
import java.nio.file.FileSystemNotFoundException
import java.nio.file.LinkOption
import java.nio.file.NoSuchFileException
import java.nio.file.OpenOption
import java.nio.file.Path
import java.nio.file.attribute.BasicFileAttributeView
import java.nio.file.attribute.BasicFileAttributes
import java.nio.file.attribute.FileAttribute
import java.nio.file.attribute.FileAttributeView
import java.nio.file.spi.FileSystemProvider
import com.azure.storage.blob.BlobServiceClient
import com.azure.storage.blob.models.BlobStorageException
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.cloud.azure.batch.AzHelper
/**
* Implements NIO File system provider for Azure Blob Storage
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
class AzFileSystemProvider extends FileSystemProvider {
public static final String AZURE_STORAGE_ACCOUNT_NAME = 'AZURE_STORAGE_ACCOUNT_NAME'
public static final String AZURE_STORAGE_ACCOUNT_KEY = 'AZURE_STORAGE_ACCOUNT_KEY'
public static final String AZURE_STORAGE_SAS_TOKEN = 'AZURE_STORAGE_SAS_TOKEN'
public static final String AZURE_CLIENT_ID = 'AZURE_CLIENT_ID'
public static final String AZURE_CLIENT_SECRET = 'AZURE_CLIENT_SECRET'
public static final String AZURE_TENANT_ID = 'AZURE_TENANT_ID'
public static final String AZURE_MANAGED_IDENTITY_USER = 'AZURE_MANAGED_IDENTITY_USER'
public static final String AZURE_MANAGED_IDENTITY_SYSTEM = 'AZURE_MANAGED_IDENTITY_SYSTEM'
public static final String SCHEME = 'az'
private Map<String,String> env = new HashMap<>(System.getenv())
private Map<String,AzFileSystem> fileSystems = [:]
private String sasToken = null
private String accountKey = null
/**
* @inheritDoc
*/
@Override
String getScheme() {
return SCHEME
}
String getSasToken() {
return this.sasToken
}
String getAccountKey() {
return this.accountKey
}
static private AzPath asAzPath(Path path) {
if( path !instanceof AzPath )
throw new IllegalArgumentException("Not a valid Azure blob storage path object: `$path` [${path?.class?.name?:'-'}]" )
return (AzPath)path
}
protected String getContainerName(URI uri) {
assert uri
if( !uri.scheme )
throw new IllegalArgumentException("Missing URI scheme")
if( uri.scheme.toLowerCase() != SCHEME )
throw new IllegalArgumentException("Mismatch provider URI scheme: `$scheme`")
if( !uri.authority ) {
if( uri.path == '/' )
return '/'
else
throw new IllegalArgumentException("Missing Azure blob storage container name")
}
return uri.authority.toLowerCase()
}
protected BlobServiceClient createBlobServiceWithKey(String accountName, String accountKey) {
AzHelper.getOrCreateBlobServiceWithKey(accountName, accountKey)
}
protected BlobServiceClient createBlobServiceWithToken(String accountName, String sasToken) {
AzHelper.getOrCreateBlobServiceWithToken(accountName, sasToken)
}
protected BlobServiceClient createBlobServiceWithServicePrincipal(String accountName, String clientId, String clientSecret, String tenantId) {
AzHelper.getOrCreateBlobServiceWithServicePrincipal(accountName, clientId, clientSecret, tenantId)
}
protected BlobServiceClient createBlobServiceWithManagedIdentity(String accountName, String clientId) {
AzHelper.getOrCreateBlobServiceWithManagedIdentity(accountName, clientId)
}
/**
* Constructs a new {@code FileSystem} object identified by a URI. This
* method is invoked by the {@link java.nio.file.FileSystems#newFileSystem(URI, Map)}
* method to open a new file system identified by a URI.
*
* <p> The {@code uri} parameter is an absolute, hierarchical URI, with a
* scheme equal (without regard to case) to the scheme supported by this
* provider. The exact form of the URI is highly provider dependent. The
* {@code env} parameter is a map of provider specific properties to configure
* the file system.
*
* <p> This method throws {@link java.nio.file.FileSystemAlreadyExistsException} if the
* file system already exists because it was previously created by an
* invocation of this method. Once a file system is {@link
* java.nio.file.FileSystem#close closed} it is provider-dependent if the
* provider allows a new file system to be created with the same URI as a
* file system it previously created.
*
* @param uri
* URI reference
* @param config
* A map of provider specific properties to configure the file system;
* may be empty
*
* @return A new file system
*
* @throws IllegalArgumentException
* If the pre-conditions for the {@code uri} parameter aren't met,
* or the {@code env} parameter does not contain properties required
* by the provider, or a property value is invalid
* @throws IOException
* An I/O error occurs creating the file system
* @throws SecurityException
* If a security manager is installed and it denies an unspecified
* permission required by the file system provider implementation
* @throws java.nio.file.FileSystemAlreadyExistsException
* If the file system has already been created
*/
@Override
AzFileSystem newFileSystem(URI uri, Map<String, ?> config) throws IOException {
final bucket = getContainerName(uri)
newFileSystem0(bucket, config)
}
/**
* Creates a new {@link AzFileSystem} for the given `bucket`.
*
* @param bucket The bucket name for which the file system will be created
* @param config
* A {@link Map} object holding the file system configuration settings. Valid keys:
* - credentials: path of the file
* - projectId
* - location
* - storageClass
* @return
* @throws IOException
*/
synchronized AzFileSystem newFileSystem0(String bucket, Map<String, ?> config) throws IOException {
if( fileSystems.containsKey(bucket) )
throw new FileSystemAlreadyExistsException("File system already exists for Azure blob container: `$bucket`")
final accountName = config.get(AZURE_STORAGE_ACCOUNT_NAME) as String
final accountKey = config.get(AZURE_STORAGE_ACCOUNT_KEY) as String
final sasToken = config.get(AZURE_STORAGE_SAS_TOKEN) as String
final servicePrincipalId = config.get(AZURE_CLIENT_ID) as String
final servicePrincipalSecret = config.get(AZURE_CLIENT_SECRET) as String
final tenantId = config.get(AZURE_TENANT_ID) as String
final managedIdentityUser = config.get(AZURE_MANAGED_IDENTITY_USER) as String
final managedIdentitySystem = config.get(AZURE_MANAGED_IDENTITY_SYSTEM) as Boolean
if( !accountName )
throw new IllegalArgumentException("Missing AZURE_STORAGE_ACCOUNT_NAME")
BlobServiceClient client
if( managedIdentityUser || managedIdentitySystem ) {
client = createBlobServiceWithManagedIdentity(accountName, managedIdentityUser)
}
else if( servicePrincipalSecret && servicePrincipalId && tenantId ) {
client = createBlobServiceWithServicePrincipal(accountName, servicePrincipalId, servicePrincipalSecret, tenantId)
}
else if( sasToken ) {
client = createBlobServiceWithToken(accountName, sasToken)
this.sasToken = sasToken
}
else if( accountKey ) {
client = createBlobServiceWithKey(accountName, accountKey)
this.accountKey = accountKey
}
else {
throw new IllegalArgumentException("Missing Azure storage credentials: please specify a managed identity, service principal, or storage account key")
}
final result = createFileSystem(client, bucket, config)
fileSystems[bucket] = result
return result
}
/**
* Creates a new {@link AzFileSystem} object.
*
* @param client
* @param bucket
* @param config
* @return
*/
protected AzFileSystem createFileSystem(BlobServiceClient client, String bucket, Map<String,?> config) {
def result = new AzFileSystem(this, client, bucket)
return result
}
/**
* Returns an existing {@code FileSystem} created by this provider.
*
* <p> This method returns a reference to a {@code FileSystem} that was
* created by invoking the {@link #newFileSystem(URI,Map) newFileSystem(URI,Map)}
* method. File systems created the {@link #newFileSystem(Path,Map)
* newFileSystem(Path,Map)} method are not returned by this method.
* The file system is identified by its {@code URI}. Its exact form
* is highly provider dependent. In the case of the default provider the URI's
* path component is {@code "/"} and the authority, query and fragment components
* are undefined (Undefined components are represented by {@code null}).
*
* <p> Once a file system created by this provider is {@link
* java.nio.file.FileSystem#close closed} it is provider-dependent if this
* method returns a reference to the closed file system or throws {@link
* java.nio.file.FileSystemNotFoundException}. If the provider allows a new file system to
* be created with the same URI as a file system it previously created then
* this method throws the exception if invoked after the file system is
* closed (and before a new instance is created by the {@link #newFileSystem
* newFileSystem} method).
*
* @param uri
* URI reference
*
* @return The file system
*
* @throws IllegalArgumentException
* If the pre-conditions for the {@code uri} parameter aren't met
* @throws java.nio.file.FileSystemNotFoundException
* If the file system does not exist
* @throws SecurityException
* If a security manager is installed and it denies an unspecified
* permission.
*/
@Override
FileSystem getFileSystem(URI uri) {
final bucket = getContainerName(uri)
getFileSystem0(bucket,false)
}
protected AzFileSystem getFileSystem0(String bucket, boolean canCreate) {
def fs = fileSystems.get(bucket)
if( !fs ) {
if( canCreate )
fs = newFileSystem0(bucket, env)
else
throw new FileSystemNotFoundException("Missing Azure storage blob file system for bucket: `$bucket`")
}
return fs
}
/**
* Return a {@code Path} object by converting the given {@link URI}. The
* resulting {@code Path} is associated with a {@link FileSystem} that
* already exists or is constructed automatically.
*
* <p> The exact form of the URI is file system provider dependent. In the
* case of the default provider, the URI scheme is {@code "file"} and the
* given URI has a non-empty path component, and undefined query, and
* fragment components. The resulting {@code Path} is associated with the
* default {@link java.nio.file.FileSystems#getDefault default} {@code FileSystem}.
*
* @param uri
* The URI to convert
*
* @return The resulting {@code Path}
*
* @throws IllegalArgumentException
* If the URI scheme does not identify this provider or other
* preconditions on the uri parameter do not hold
* @throws java.nio.file.FileSystemNotFoundException
* The file system, identified by the URI, does not exist and
* cannot be created automatically
* @throws SecurityException
* If a security manager is installed and it denies an unspecified
* permission.
*/
@Override
AzPath getPath(URI uri) {
final bucket = getContainerName(uri)
bucket=='/' ? getPath('/') : getPath("$bucket/${uri.path}")
}
/**
* Get a {@link AzPath} from an object path string
*
* @param path A path in the form {@code containerName/blobName}
* @return A {@link AzPath} object
*/
AzPath getPath(String path) {
assert path
// -- special root bucket
if( path == '/' ) {
final fs = getFileSystem0('/',true)
return new AzPath(fs, "/")
}
// -- remove first slash, if any
while( path.startsWith("/") )
path = path.substring(1)
// -- find the first component ie. the container name
int p = path.indexOf('/')
final bucket = p==-1 ? path : path.substring(0,p)
// -- get the file system
final fs = getFileSystem0(bucket,true)
// create a new path
new AzPath(fs, "/$path")
}
static private FileSystemProvider provider( Path path ) {
path.getFileSystem().provider()
}
@Deprecated
static private BlobServiceClient storage( Path path ) {
((AzPath)path).getFileSystem().getBlobServiceClient()
}
private void checkRoot(Path path) {
if( path.toString() == '/' )
throw new UnsupportedOperationException("Operation 'checkRoot' not supported on root path")
}
@Override
SeekableByteChannel newByteChannel(Path obj, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException {
checkRoot(obj)
final modeWrite = options.contains(WRITE) || options.contains(APPEND)
final modeRead = options.contains(READ) || !modeWrite
if( modeRead && modeWrite ) {
throw new IllegalArgumentException("Azure Blob Storage file cannot be opened in R/W mode at the same time")
}
if( options.contains(APPEND) ) {
throw new IllegalArgumentException("Azure Blob Storage file system does not support `APPEND` mode")
}
if( options.contains(SYNC) ) {
throw new IllegalArgumentException("Azure Blob Storage file system does not support `SYNC` mode")
}
if( options.contains(DSYNC) ) {
throw new IllegalArgumentException("Azure Blob Storage file system does not support `DSYNC` mode")
}
final path = asAzPath(obj)
final fs = path.getFileSystem()
if( modeRead ) {
return fs.newReadableByteChannel(path)
}
// -- mode write
if( options.contains(CREATE_NEW) ) {
if( fs.exists(path) )
throw new FileAlreadyExistsException(path.toUriString())
}
else if( !options.contains(CREATE) ) {
if( !fs.exists(path) )
throw new NoSuchFileException(path.toUriString())
}
if( options.contains(APPEND) ) {
throw new IllegalArgumentException("File APPEND mode is not supported by Azure Blob Storage")
}
return fs.newWritableByteChannel(path)
}
@Override
DirectoryStream<Path> newDirectoryStream(Path obj, DirectoryStream.Filter<? super Path> filter) throws IOException {
final path = asAzPath(obj)
path.fileSystem.newDirectoryStream(path, filter)
}
@Override
void createDirectory(Path dir, FileAttribute<?>... attrs) throws IOException {
checkRoot(dir)
final path = asAzPath(dir)
try {
path.fileSystem.createDirectory(path)
}
catch (BlobStorageException e) {
// 409 (CONFLICT) is returned when the path already
// exists, ignore it
if( e.statusCode!=409 )
throw new IOException("Unable to create Azure blob directory: ${dir.toUriString()} - cause: ${e.message}", e)
}
}
@Override
void delete(Path obj) throws IOException {
checkRoot(obj)
final path = asAzPath(obj)
path.fileSystem.delete(path)
}
@Override
void copy(Path from, Path to, CopyOption... options) throws IOException {
assert provider(from) == provider(to)
if( from == to )
return // nothing to do -- just return
checkRoot(from); checkRoot(to)
final source = asAzPath(from)
final target = asAzPath(to)
final fs = source.getFileSystem()
if( options.contains(REPLACE_EXISTING) && fs.exists(target) ) {
delete(target)
}
fs.copy(source, target)
}
@Override
void move(Path source, Path target, CopyOption... options) throws IOException {
copy(source,target,options)
delete(source)
}
@Override
boolean isSameFile(Path path, Path path2) throws IOException {
return path == path2
}
@Override
boolean isHidden(Path path) throws IOException {
return path.getFileName()?.toString()?.startsWith('.')
}
@Override
FileStore getFileStore(Path path) throws IOException {
throw new UnsupportedOperationException("Operation 'getFileStore' is not supported by AzFileSystem")
}
@Override
void checkAccess(Path path, AccessMode... modes) throws IOException {
checkRoot(path)
final az = asAzPath(path)
readAttributes(az, AzFileAttributes.class)
if( AccessMode.EXECUTE in modes)
throw new AccessDeniedException(az.toUriString(), null, 'Execute permission not allowed')
}
@Override
def <V extends FileAttributeView> V getFileAttributeView(Path path, Class<V> type, LinkOption... options) {
checkRoot(path)
if( type == BasicFileAttributeView || type == AzFileAttributesView ) {
def azPath = asAzPath(path)
return (V)azPath.fileSystem.getFileAttributeView(azPath)
}
throw new UnsupportedOperationException("Not a valid Azure Blob Storage file attribute view: $type")
}
@Override
def <A extends BasicFileAttributes> A readAttributes(Path path, Class<A> type, LinkOption... options) throws IOException {
if( type == BasicFileAttributes || type == AzFileAttributes ) {
def azPath = asAzPath(path)
def result = (A)azPath.fileSystem.readAttributes(azPath)
if( result )
return result
throw new NoSuchFileException(azPath.toUriString())
}
throw new UnsupportedOperationException("Not a valid Azure Blob Storage file attribute type: $type")
}
@Override
Map<String, Object> readAttributes(Path path, String attributes, LinkOption... options) throws IOException {
throw new UnsupportedOperationException("Operation 'readAttributes' is not supported by AzFileSystem")
}
@Override
void setAttribute(Path path, String attribute, Object value, LinkOption... options) throws IOException {
throw new UnsupportedOperationException("Operation 'setAttribute' is not supported by AzFileSystem")
}
}

336
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/nio/AzPath.groovy Normal file
View File

@@ -0,0 +1,336 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.nio
import java.nio.file.LinkOption
import java.nio.file.Path
import java.nio.file.Paths
import java.nio.file.ProviderMismatchException
import java.nio.file.WatchEvent
import java.nio.file.WatchKey
import java.nio.file.WatchService
import com.azure.storage.blob.BlobClient
import com.azure.storage.blob.BlobContainerClient
import com.azure.storage.blob.models.BlobItem
import groovy.transform.CompileStatic
import groovy.transform.EqualsAndHashCode
import groovy.transform.PackageScope
/**
* Implements Azure path object
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
@EqualsAndHashCode(includes = 'fs,path,directory', includeFields = true)
class AzPath implements Path {
private AzFileSystem fs
private Path path
private AzFileAttributes attributes
@PackageScope
boolean directory
@PackageScope
AzPath() {}
@PackageScope
AzPath( AzFileSystem fs, String path ) {
this(fs, Paths.get(path), path.endsWith("/") || path=="/$fs.containerName".toString())
}
@PackageScope
AzPath( AzFileSystem fs, BlobClient client ) {
this(fs, "/${client.getContainerName()}/${client.getBlobName()}")
this.attributes = new AzFileAttributes(client)
}
AzPath(AzFileSystem fs, BlobItem item) {
this(fs, "/${fs.containerName}/${item.name}")
this.attributes = new AzFileAttributes(fs.containerName, item)
}
@PackageScope
AzPath(AzFileSystem fs, BlobContainerClient client) {
this(fs, "/${client.blobContainerName}")
this.attributes = new AzFileAttributes(client)
}
@PackageScope
AzPath(AzFileSystem fs, Path path, boolean directory) {
// make sure that the path bucket match the file system bucket
if( path.isAbsolute() && path.nameCount>0 ) {
def container = path.getName(0).toString()
if( container != fs.containerName )
throw new IllegalArgumentException("Azure path `$container` does not match file system bucket: `${fs.containerName}`")
}
this.fs = fs
this.path = path
this.directory = directory
}
@PackageScope
AzPath setAttributes(AzFileAttributes attrs) {
this.attributes = attrs
return this
}
boolean isDirectory() {
return directory
}
String checkContainerName() {
if( !isAbsolute() )
throw new IllegalArgumentException("Azure blob container name is not available on relative blob path: $path")
return path.subpath(0,1)
}
BlobContainerClient containerClient() {
return fs.getBlobServiceClient().getBlobContainerClient(checkContainerName())
}
String blobName() {
if( !path.isAbsolute() )
return path.toString()
if( path.nameCount>1 )
return path.subpath(1, path.nameCount).toString()
return null
}
BlobClient blobClient() {
def name = blobName()
if( !name || isContainer() )
throw new IllegalArgumentException("Azure blob client is not available for container path $path")
if( directory && !name.endsWith('/') )
name += '/'
containerClient().getBlobClient(name)
}
@Override
AzFileSystem getFileSystem() {
return fs
}
@Override
boolean isAbsolute() {
path.isAbsolute()
}
@Override
Path getRoot() {
path.isAbsolute() ? new AzPath(fs, "/${path.getName(0)}/") : null
}
@Override
Path getFileName() {
final name = path.getFileName()
name ? new AzPath(fs, name, directory) : null
}
@Override
Path getParent() {
if( path.isAbsolute() && path.nameCount>1 ) {
new AzPath(fs, path.parent, true)
}
else {
null
}
}
@Override
int getNameCount() {
path.getNameCount()
}
@Override
Path getName(int index) {
final dir = index < path.getNameCount()-1
new AzPath(fs, path.getName(index), dir)
}
@Override
Path subpath(int beginIndex, int endIndex) {
final dir = endIndex < path.getNameCount()-1
new AzPath(fs, path.subpath(beginIndex,endIndex), dir)
}
@Override
boolean startsWith(Path other) {
path.startsWith(other.toString())
}
@Override
boolean startsWith(String other) {
path.startsWith(other)
}
@Override
boolean endsWith(Path other) {
path.endsWith(other.toString())
}
@Override
boolean endsWith(String other) {
path.endsWith(other)
}
@Override
Path normalize() {
new AzPath(fs, path.normalize(), directory)
}
@Override
AzPath resolve(Path other) {
if( other.class != AzPath )
throw new ProviderMismatchException()
final that = (AzPath)other
if( other.isAbsolute() )
return that
def newPath = path.resolve(that.path)
new AzPath(fs, newPath, false)
}
@Override
AzPath resolve(String other) {
if( other.startsWith('/') )
return (AzPath)fs.provider().getPath(new URI("$AzFileSystemProvider.SCHEME:/$other"))
def dir = other.endsWith('/')
def newPath = path.resolve(other)
new AzPath(fs, newPath, dir)
}
@Override
Path resolveSibling(Path other) {
if( other.class != AzPath )
throw new ProviderMismatchException()
final that = (AzPath)other
def newPath = path.resolveSibling(that.path)
if( newPath.isAbsolute() )
fs.getPath(newPath.toString())
else
new AzPath(fs, newPath, false)
}
@Override
Path resolveSibling(String other) {
def newPath = path.resolveSibling(other)
if( newPath.isAbsolute() )
fs.getPath(newPath.toString())
else
new AzPath(fs, newPath, false)
}
@Override
Path relativize(Path other) {
if( other.class != AzPath )
throw new ProviderMismatchException()
def newPath = path.relativize( ((AzPath)other).path )
new AzPath(fs,newPath,false)
}
@Override
String toString() {
path.toString()
}
@Override
URI toUri() {
return new URI(toUriString())
}
@Override
Path toAbsolutePath() {
if(isAbsolute()) return this
throw new UnsupportedOperationException("Operation 'toAbsolutePath' is not supported by AzPath")
}
@Override
Path toRealPath(LinkOption... options) throws IOException {
return toAbsolutePath()
}
@Override
File toFile() {
throw new UnsupportedOperationException("Operation 'toFile' is not supported by AzPath")
}
@Override
WatchKey register(WatchService watcher, WatchEvent.Kind<?>[] events, WatchEvent.Modifier... modifiers) throws IOException {
throw new UnsupportedOperationException("Operation 'register' is not supported by AzPath")
}
@Override
WatchKey register(WatchService watcher, WatchEvent.Kind<?>... events) throws IOException {
throw new UnsupportedOperationException("Operation 'register' is not supported by AzPath")
}
@Override
Iterator<Path> iterator() {
final count = path.nameCount
List<Path> paths = new ArrayList<>()
for( int i=0; i<count; i++ ) {
def dir = i<count-1
paths.add(i, new AzPath(fs, path.getName(i), dir))
}
paths.iterator()
}
@Override
int compareTo(Path other) {
return this.toString() <=> other.toString()
}
String getContainerName() {
if( path.isAbsolute() ) {
path.nameCount==0 ? '/' : path.getName(0)
}
else
return null
}
boolean isContainer() {
path.isAbsolute() && path.nameCount<2
}
String toUriString() {
if( path.isAbsolute() ) {
return "${AzFileSystemProvider.SCHEME}:/${path.toString()}"
}
else {
return "${AzFileSystemProvider.SCHEME}:${path.toString()}"
}
}
AzFileAttributes attributesCache() {
def result = attributes
attributes = null
return result
}
}

121
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/nio/AzPathIterator.groovy Normal file
View File

@@ -0,0 +1,121 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.nio
import java.nio.file.DirectoryStream
import java.nio.file.Path
import com.azure.storage.blob.models.BlobContainerItem
import com.azure.storage.blob.models.BlobItem
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
/**
* Iterator for Azure blob storage
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@Slf4j
@CompileStatic
abstract class AzPathIterator<T> implements Iterator<Path> {
private AzFileSystem fs
private Iterator<T> itr
private DirectoryStream.Filter<? super Path> filter
private AzPath next
private AzPath origin
AzPathIterator(AzPath origin, Iterator<T> itr, DirectoryStream.Filter<? super Path> filter) {
this.origin = origin
this.fs = origin.fileSystem
this.itr = itr
this.filter = filter
advance()
}
abstract AzPath createPath(AzFileSystem fs, T item)
private void advance() {
AzPath result = null
while( result == null && itr.hasNext() ) {
def item = itr.next()
def path = createPath(fs, item)
if( path == origin ) // make sure to skip the origin path
result = null
else if( filter )
result = filter.accept(path) ? path : null
else
result = path
}
next = result
}
@Override
boolean hasNext() {
return next != null
}
@Override
Path next() {
def result = next
if( result == null )
throw new NoSuchElementException()
advance()
return result
}
@Override
void remove() {
throw new UnsupportedOperationException("Operation 'remove' is not supported by AzPathIterator")
}
/**
* Implements a path iterator for blob object i.e. files and path in a
* Azure cloud storage file system
*/
static class ForBlobs extends AzPathIterator<BlobItem> {
ForBlobs(AzPath path, Iterator<BlobItem> itr, DirectoryStream.Filter<? super Path> filter) {
super(path, itr, filter)
}
@Override
AzPath createPath(AzFileSystem fs, BlobItem item) {
return new AzPath(fs, item)
}
}
/**
* Implements an iterator for buckets in a Google Cloud storage file system
*/
static class ForContainers extends AzPathIterator<BlobContainerItem> {
ForContainers(AzPath path, Iterator<BlobContainerItem> itr, DirectoryStream.Filter<? super Path> filter) {
super(path, itr, filter)
}
@Override
AzPath createPath(AzFileSystem fs, BlobContainerItem item) {
return fs.getPath("/${item.name}").setAttributes( new AzFileAttributes(item) )
}
}
}

83
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/nio/AzReadableByteChannel.groovy Normal file
View File

@@ -0,0 +1,83 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.nio
import java.nio.ByteBuffer
import java.nio.channels.ReadableByteChannel
import java.nio.channels.SeekableByteChannel
import groovy.transform.CompileStatic
/**
* Implements a {@link SeekableByteChannel} for a given {@link ReadableByteChannel}
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class AzReadableByteChannel implements SeekableByteChannel {
private long _position
private ReadableByteChannel channel
private long size
AzReadableByteChannel(ReadableByteChannel channel, long size) {
this.channel = channel
this.size = size
}
@Override
int read(ByteBuffer dst) throws IOException {
final len = channel.read(dst)
_position += len
return len
}
@Override
int write(ByteBuffer src) throws IOException {
throw new UnsupportedOperationException("Operation 'write(ByteBuffer)' is not supported by AzReadableByteChannel")
}
@Override
long position() throws IOException {
return _position
}
@Override
SeekableByteChannel position(long newPosition) throws IOException {
throw new UnsupportedOperationException("Operation 'position(long)' is not supported by AzReadableByteChannel")
}
@Override
long size() throws IOException {
return size
}
@Override
SeekableByteChannel truncate(long dummy) throws IOException {
throw new UnsupportedOperationException("Operation 'truncate(long)' is not supported by AzReadableByteChannel")
}
@Override
boolean isOpen() {
return true
}
@Override
void close() throws IOException {
channel.close()
}
}

81
nextflow/plugins/nf-azure/src/main/nextflow/cloud/azure/nio/AzWriteableByteChannel.groovy Normal file
View File

@@ -0,0 +1,81 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.nio
import java.nio.ByteBuffer
import java.nio.channels.SeekableByteChannel
import java.nio.channels.WritableByteChannel
import groovy.transform.CompileStatic
/**
* Implements a {@link SeekableByteChannel} for a given {@link java.nio.channels.WritableByteChannel}
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class AzWriteableByteChannel implements SeekableByteChannel {
private long _pos
private WritableByteChannel writer
AzWriteableByteChannel(WritableByteChannel channel) {
this.writer = channel
}
@Override
int read(ByteBuffer dst) throws IOException {
throw new UnsupportedOperationException("Operation 'read(ByteBuffer)' is not supported by AzWriteableByteChannel")
}
@Override
int write(ByteBuffer src) throws IOException {
def len = writer.write(src)
_pos += len
return len
}
@Override
long position() throws IOException {
return _pos
}
@Override
SeekableByteChannel position(long newPosition) throws IOException {
throw new UnsupportedOperationException("Operation 'position(long)' is not supported by AzWriteableByteChannel")
}
@Override
long size() throws IOException {
return _pos
}
@Override
SeekableByteChannel truncate(long size) throws IOException {
throw new UnsupportedOperationException("Operation 'truncate(long)' is not supported by AzWriteableByteChannel")
}
@Override
boolean isOpen() {
return true
}
@Override
void close() throws IOException {
writer.close()
}
}

17
nextflow/plugins/nf-azure/src/resources/META-INF/services/java.nio.file.spi.FileSystemProvider Normal file
View File

@@ -0,0 +1,17 @@
#
# Copyright 2013-2026, Seqera Labs
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
nextflow.cloud.azure.nio.AzFileSystemProvider

2117
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/az-locations.json Normal file
View File

File diff suppressed because it is too large Load Diff

17
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/az-locations.sh Normal file
View File

@@ -0,0 +1,17 @@
#
# Copyright 2013-2026, Seqera Labs
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
az account list-locations > az-locations.json

4074
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-australiacentral.json Normal file
View File

File diff suppressed because it is too large Load Diff

7154
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-australiaeast.json Normal file
View File

File diff suppressed because it is too large Load Diff

4498
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-australiasoutheast.json Normal file
View File

File diff suppressed because it is too large Load Diff

4962
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-brazilsouth.json Normal file
View File

File diff suppressed because it is too large Load Diff

6274
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-canadacentral.json Normal file
View File

File diff suppressed because it is too large Load Diff

4498
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-canadaeast.json Normal file
View File

File diff suppressed because it is too large Load Diff

6218
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-centralindia.json Normal file
View File

File diff suppressed because it is too large Load Diff

7386
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-centralus.json Normal file
View File

File diff suppressed because it is too large Load Diff

5594
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-eastasia.json Normal file
View File

File diff suppressed because it is too large Load Diff

8106
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-eastus.json Normal file
View File

File diff suppressed because it is too large Load Diff

7618
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-eastus2.json Normal file
View File

File diff suppressed because it is too large Load Diff

6506
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-francecentral.json Normal file
View File

File diff suppressed because it is too large Load Diff

7146
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-germanywestcentral.json Normal file
View File

File diff suppressed because it is too large Load Diff

3618
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-indonesiacentral.json Normal file
View File

File diff suppressed because it is too large Load Diff

3906
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-israelcentral.json Normal file
View File

File diff suppressed because it is too large Load Diff

6474
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-italynorth.json Normal file
View File

File diff suppressed because it is too large Load Diff

6970
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-japaneast.json Normal file
View File

File diff suppressed because it is too large Load Diff

4570
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-japanwest.json Normal file
View File

File diff suppressed because it is too large Load Diff

3922
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-jioindiawest.json Normal file
View File

File diff suppressed because it is too large Load Diff

4498
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-koreacentral.json Normal file
View File

File diff suppressed because it is too large Load Diff

4202
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-koreasouth.json Normal file
View File

File diff suppressed because it is too large Load Diff

4346
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-mexicocentral.json Normal file
View File

File diff suppressed because it is too large Load Diff

2786
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-newzealandnorth.json Normal file
View File

File diff suppressed because it is too large Load Diff

5778
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-northcentralus.json Normal file
View File

File diff suppressed because it is too large Load Diff

8106
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-northeurope.json Normal file
View File

File diff suppressed because it is too large Load Diff

4034
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-norwayeast.json Normal file
View File

File diff suppressed because it is too large Load Diff

4754
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-polandcentral.json Normal file
View File

File diff suppressed because it is too large Load Diff

3154
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-qatarcentral.json Normal file
View File

File diff suppressed because it is too large Load Diff

4154
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-southafricanorth.json Normal file
View File

File diff suppressed because it is too large Load Diff

7474
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-southcentralus.json Normal file
View File

File diff suppressed because it is too large Load Diff

7354
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-southeastasia.json Normal file
View File

File diff suppressed because it is too large Load Diff

4554
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-southindia.json Normal file
View File

File diff suppressed because it is too large Load Diff

5074
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-spaincentral.json Normal file
View File

File diff suppressed because it is too large Load Diff

6650
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-swedencentral.json Normal file
View File

File diff suppressed because it is too large Load Diff

6026
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-switzerlandnorth.json Normal file
View File

File diff suppressed because it is too large Load Diff

5330
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-uaenorth.json Normal file
View File

File diff suppressed because it is too large Load Diff

7074
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-uksouth.json Normal file
View File

File diff suppressed because it is too large Load Diff

4754
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-ukwest.json Normal file
View File

File diff suppressed because it is too large Load Diff

5074
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-westcentralus.json Normal file
View File

File diff suppressed because it is too large Load Diff

8362
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-westeurope.json Normal file
View File

File diff suppressed because it is too large Load Diff

1714
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-westindia.json Normal file
View File

File diff suppressed because it is too large Load Diff

6890
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-westus.json Normal file
View File

File diff suppressed because it is too large Load Diff

6842
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-westus2.json Normal file
View File

File diff suppressed because it is too large Load Diff

6746
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size-westus3.json Normal file
View File

File diff suppressed because it is too large Load Diff

22
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size.ps1 Normal file
View File

@@ -0,0 +1,22 @@
$scriptPath = Split-Path -Parent $MyInvocation.MyCommand.Path
# Get all supported locations for VM sizes using the compute provider
$supportedLocations = az provider show --namespace Microsoft.Compute `
--query "resourceTypes[?resourceType=='virtualMachines'].locations[]" | ConvertFrom-Json
Write-Host "Found $($supportedLocations.Count) supported locations"
foreach ($region in $supportedLocations) {
$normalizedRegion = $region.Replace(' ', '').ToLower()
Write-Host "Listing VM size for $normalizedRegion"
try {
$output = az vm list-sizes -l $normalizedRegion
if ($output) {
$output | Out-File -FilePath "$scriptPath\vm-list-size-$normalizedRegion.json"
Write-Host "Successfully saved VM sizes for $normalizedRegion"
}
}
catch {
Write-Warning "Failed to get VM sizes for ${normalizedRegion}: $_"
}
}

4
nextflow/plugins/nf-azure/src/resources/nextflow/cloud/azure/vm-list-size.sh Normal file
View File

@@ -0,0 +1,4 @@
for region in $(az account list-locations | jq -r '.[].name'); do
echo "Listing mv size for $region"
az vm list-sizes -l $region > "vm-list-size-$region.json"
done

1188
nextflow/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzBatchServiceTest.groovy Normal file
View File

File diff suppressed because it is too large Load Diff

277
nextflow/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzBatchTaskHandlerTest.groovy Normal file
View File

@@ -0,0 +1,277 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.batch
import com.azure.compute.batch.models.BatchTask
import com.azure.compute.batch.models.BatchTaskExecutionInfo
import com.azure.compute.batch.models.BatchTaskFailureInfo
import com.azure.compute.batch.models.BatchTaskState
import com.azure.compute.batch.models.ErrorCategory
import com.sun.jna.platform.unix.X11
import nextflow.processor.TaskStatus
import java.nio.file.Path
import nextflow.cloud.types.CloudMachineInfo
import nextflow.cloud.types.PriceModel
import nextflow.exception.ProcessUnrecoverableException
import nextflow.executor.BashWrapperBuilder
import nextflow.executor.Executor
import nextflow.processor.TaskConfig
import nextflow.processor.TaskProcessor
import nextflow.processor.TaskRun
import nextflow.script.BaseScript
import nextflow.script.ProcessConfig
import spock.lang.Specification
/**
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
class AzBatchTaskHandlerTest extends Specification {
def createTaskRun() {
Mock(TaskRun) {
name >> 'foo'
workDir >> Path.of('/work/dir')
container >> 'ubuntu'
}
}
def 'should validate config' () {
given:
def exec = Mock(AzBatchExecutor)
when:
def task = Mock(TaskRun) {
name >> 'foo'
workDir >> Path.of('/work/dir')
}
and:
new AzBatchTaskHandler(task, exec)
then:
def e = thrown(ProcessUnrecoverableException)
e.message.startsWith('No container image specified for process foo')
when:
task = createTaskRun()
and:
new AzBatchTaskHandler(task, exec)
then:
noExceptionThrown()
}
def 'should submit task'() {
given:
def azure = Mock(AzBatchService)
def executor = Mock(AzBatchExecutor)
def processor = Mock(TaskProcessor) {
getExecutor() >> executor
}
def task = createTaskRun()
task.getProcessor() >> processor
task.getConfig() >> Mock(TaskConfig)
and:
def handler = Spy(new AzBatchTaskHandler(task, executor)) {
getBatchService() >> azure
}
when:
handler.submit()
then:
1 * handler.createBashWrapper() >> Mock(BashWrapperBuilder)
1 * handler.getBatchService() >> Mock(AzBatchService)
}
def 'should create the trace record' () {
given:
def exec = Mock(AzBatchExecutor) { getName() >> 'azurebatch' }
def processor = Mock(TaskProcessor)
processor.getExecutor() >> exec
processor.getName() >> 'foo'
processor.getConfig() >> new ProcessConfig(Mock(BaseScript))
def task = createTaskRun()
task.getProcessor() >> processor
task.getConfig() >> GroovyMock(TaskConfig)
def handler = Spy(new AzBatchTaskHandler(task, exec))
handler.@taskKey = new AzTaskKey('job-123', 'nf-456')
when:
def trace = handler.getTraceRecord()
then:
1 * handler.isCompleted() >> false
1 * handler.getMachineInfo() >> new CloudMachineInfo('Standard1', 'west-eu', PriceModel.standard)
and:
trace.native_id == 'job-123/nf-456'
trace.executorName == 'azurebatch'
trace.machineInfo.type == 'Standard1'
trace.machineInfo.zone == 'west-eu'
trace.machineInfo.priceModel == PriceModel.standard
}
def 'should check if completed with exit code from scheduler'() {
given:
def task = Spy(new TaskRun()){
getContainer() >> 'ubuntu'
}
task.name = 'foo'
task.workDir = Path.of('/tmp/wdir')
def taskKey = new AzTaskKey('pool-123', 'job-456')
def azTask = new BatchTask()
def execInfo = new BatchTaskExecutionInfo(0,0)
execInfo.exitCode = 0
azTask.executionInfo = execInfo
azTask.state = BatchTaskState.COMPLETED
def batchService = Mock(AzBatchService){
getTask(taskKey) >> azTask
}
def executor = Mock(AzBatchExecutor){
getBatchService() >> batchService
}
def handler = Spy(new AzBatchTaskHandler(task, executor)){
deleteTask(_,_) >> null
}
handler.status = TaskStatus.RUNNING
handler.taskKey = taskKey
when:
def result = handler.checkIfCompleted()
then:
0 * handler.readExitFile() // Should NOT read exit file when scheduler provides exit code
and:
result == true
handler.task.exitStatus == 0
handler.status == TaskStatus.COMPLETED
}
def 'should check if completed with non-zero exit code from scheduler'() {
given:
def task = Spy(new TaskRun()){
getContainer() >> 'ubuntu'
}
task.name = 'foo'
task.workDir = Path.of('/tmp/wdir')
def taskKey = new AzTaskKey('pool-123', 'job-456')
def azTask = new BatchTask()
def execInfo = new BatchTaskExecutionInfo(0,0)
execInfo.exitCode = 137
azTask.executionInfo = execInfo
azTask.state = BatchTaskState.COMPLETED
def batchService = Mock(AzBatchService){
getTask(taskKey) >> azTask
}
def executor = Mock(AzBatchExecutor){
getBatchService() >> batchService
}
def handler = Spy(new AzBatchTaskHandler(task, executor)){
deleteTask(_,_) >> null
}
handler.status = TaskStatus.RUNNING
handler.taskKey = taskKey
when:
def result = handler.checkIfCompleted()
then:
0 * handler.readExitFile() // Should NOT read exit file when scheduler provides exit code
and:
result == true
handler.task.exitStatus == 137
handler.status == TaskStatus.COMPLETED
}
def 'should check if completed and fallback to exit file when scheduler exit code is null'() {
given:
def task = Spy(new TaskRun()){
getContainer() >> 'ubuntu'
}
task.name = 'foo'
task.workDir = Path.of('/tmp/wdir')
def taskKey = new AzTaskKey('pool-123', 'job-456')
def azTask = new BatchTask()
def execInfo = new BatchTaskExecutionInfo(0,0)
azTask.executionInfo = execInfo
azTask.state = BatchTaskState.COMPLETED
def batchService = Mock(AzBatchService){
getTask(taskKey) >> azTask
}
def executor = Mock(AzBatchExecutor){
getBatchService() >> batchService
}
def handler = Spy(new AzBatchTaskHandler(task, executor)){
deleteTask(_,_) >> null
}
handler.status = TaskStatus.RUNNING
handler.taskKey = taskKey
when:
def result = handler.checkIfCompleted()
then:
1 * handler.readExitFile() >> 0 // Should read exit file as fallback
and:
result == true
handler.task.exitStatus == 0
handler.status == TaskStatus.COMPLETED
}
def 'should check if completed and no scheduler exit code neither .exitcode file'() {
given:
def task = Spy(new TaskRun()){
getContainer() >> 'ubuntu'
}
task.name = 'foo'
task.workDir = Path.of('/tmp/wdir')
def taskKey = new AzTaskKey('pool-123', 'job-456')
def azTask = new BatchTask()
def execInfo = new BatchTaskExecutionInfo(0,0)
def failureInfo = new BatchTaskFailureInfo(ErrorCategory.USER_ERROR)
failureInfo.message = 'Unknown error'
execInfo.failureInfo = failureInfo
azTask.executionInfo = execInfo
azTask.state = BatchTaskState.COMPLETED
def batchService = Mock(AzBatchService){
getTask(taskKey) >> azTask
}
def executor = Mock(AzBatchExecutor){
getBatchService() >> batchService
}
def handler = Spy(new AzBatchTaskHandler(task, executor)){
deleteTask(_,_) >> null
}
handler.status = TaskStatus.RUNNING
handler.taskKey = taskKey
when:
def result = handler.checkIfCompleted()
then:
1 * handler.readExitFile() >> Integer.MAX_VALUE // Should read exit file as fallback
and:
result == true
handler.task.exitStatus == Integer.MAX_VALUE
handler.status == TaskStatus.COMPLETED
handler.task.error.message == 'Unknown error'
}
}

514
nextflow/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy Normal file
View File

@@ -0,0 +1,514 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.batch
import java.nio.file.Path
import java.nio.file.Paths
import java.nio.file.attribute.BasicFileAttributes
import java.nio.file.spi.FileSystemProvider
import com.azure.storage.blob.BlobClient
import nextflow.Session
import nextflow.cloud.azure.config.AzConfig
import nextflow.cloud.azure.nio.AzFileSystem
import nextflow.cloud.azure.nio.AzPath
import nextflow.processor.TaskBean
import spock.lang.Specification
/**
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
class AzFileCopyStrategyTest extends Specification {
protected Path mockAzPath(String path, boolean isDir=false) {
assert path.startsWith('az://')
def tokens = path.tokenize('/')
def bucket = tokens[1]
def file = '/' + tokens[2..-1].join('/')
def attr = Mock(BasicFileAttributes)
attr.isDirectory() >> isDir
attr.isRegularFile() >> !isDir
attr.isSymbolicLink() >> false
def provider = Mock(FileSystemProvider)
provider.getScheme() >> 'az'
provider.readAttributes(_, _, _) >> attr
def fs = Mock(AzFileSystem)
fs.provider() >> provider
fs.toString() >> ('az://' + bucket)
def uri = GroovyMock(URI)
uri.toString() >> path
def BLOB_CLIENT = Mock(BlobClient) {
getBlobUrl() >> { path.replace('az://', 'http://account.blob.core.windows.net/') }
}
def result = GroovyMock(AzPath)
result.toUriString() >> path
result.toString() >> file
result.getFileSystem() >> fs
result.toUri() >> uri
result.resolve(_) >> { mockAzPath("$path/${it[0]}") }
result.toAbsolutePath() >> result
result.asBoolean() >> true
result.getParent() >> { def p=path.lastIndexOf('/'); p!=-1 ? mockAzPath("${path.substring(0,p)}", true) : null }
result.getFileName() >> { Paths.get(tokens[-1]) }
result.getName() >> tokens[1]
result.blobClient() >> BLOB_CLIENT
return result
}
def setup() {
new Session()
}
def 'test bash wrapper with input'() {
given:
def workDir = mockAzPath( 'az://my-data/work/dir' )
def token = '12345'
def config = new AzConfig([storage:[sasToken: token]])
def executor = Mock(AzBatchExecutor) { getAzConfig() >> config }
when:
def binding = new AzBatchScriptLauncher([
name: 'Hello 1',
workDir: workDir,
script: 'echo Hello world!',
environment: [FOO: 1, BAR:'any'],
input: 'Ciao ciao' ] as TaskBean, executor) .makeBinding()
then:
binding.stage_inputs == '''\
# stage input files
downloads=(true)
nxf_parallel "${downloads[@]}"
'''.stripIndent()
binding.unstage_controls == '''\
nxf_az_upload .command.out 'http://account.blob.core.windows.net/my-data/work/dir' || true
nxf_az_upload .command.err 'http://account.blob.core.windows.net/my-data/work/dir' || true
'''.stripIndent()
binding.launch_cmd == '/bin/bash -ue .command.sh < .command.in'
binding.unstage_outputs == null
binding.helpers_script == '''\
# bash helper functions
nxf_cp_retry() {
local max_attempts=1
local timeout=10
local attempt=0
local exitCode=0
while (( \$attempt < \$max_attempts ))
do
if "\$@"
then
return 0
else
exitCode=\$?
fi
if [[ \$exitCode == 0 ]]
then
break
fi
nxf_sleep \$timeout
attempt=\$(( attempt + 1 ))
timeout=\$(( timeout * 2 ))
done
}
nxf_parallel() {
IFS=$'\\n\'
local cmd=("$@")
local cpus=$(nproc 2>/dev/null || < /proc/cpuinfo grep '^process' -c)
local max=$(if (( cpus>4 )); then echo 4; else echo $cpus; fi)
local i=0
local pid=()
(
set +u
while ((i<${#cmd[@]})); do
local copy=()
for x in "${pid[@]}"; do
# if the process exist, keep in the 'copy' array, otherwise wait on it to capture the exit code
# see https://github.com/nextflow-io/nextflow/pull/4050
[[ -e /proc/$x ]] && copy+=($x) || wait $x
done
pid=("${copy[@]}")
if ((${#pid[@]}>=$max)); then
nxf_sleep 0.2
else
eval "${cmd[$i]}" &
pid+=($!)
((i+=1))
fi
done
for p in "${pid[@]}"; do
wait $p
done
)
unset IFS
}
# custom env variables used for azcopy opts
export AZCOPY_BLOCK_SIZE_MB=4
export AZCOPY_BLOCK_BLOB_TIER=None
nxf_az_upload() {
local name=$1
local target=${2%/} ## remove ending slash
local base_name="$(basename "$name")"
local dir_name="$(dirname "$name")"
if [[ -d $name ]]; then
if [[ "$base_name" == "$name" ]]; then
azcopy cp "$name" "$target?$AZ_SAS" --recursive --block-blob-tier $AZCOPY_BLOCK_BLOB_TIER --block-size-mb $AZCOPY_BLOCK_SIZE_MB
else
azcopy cp "$name" "$target/$dir_name?$AZ_SAS" --recursive --block-blob-tier $AZCOPY_BLOCK_BLOB_TIER --block-size-mb $AZCOPY_BLOCK_SIZE_MB
fi
else
azcopy cp "$name" "$target/$name?$AZ_SAS" --block-blob-tier $AZCOPY_BLOCK_BLOB_TIER --block-size-mb $AZCOPY_BLOCK_SIZE_MB
fi
}
nxf_az_download() {
local source=$1
local target=$2
local basedir=$(dirname $2)
local ret
mkdir -p "$basedir"
ret=$(azcopy cp "$source?$AZ_SAS" "$target" 2>&1) || {
## if fails check if it was trying to download a directory
mkdir -p $target
azcopy cp "$source/*?$AZ_SAS" "$target" --recursive >/dev/null || {
rm -rf $target
>&2 echo "Unable to download path: $source"
exit 1
}
}
}
'''.stripIndent(true)
}
def 'should include remote bind dir' () {
given:
def remoteBin = mockAzPath( 'az://my-data/work/remote/bin' )
def workDir = mockAzPath( 'az://my-data/work/dir' )
def token = '12345'
def config = new AzConfig([storage:[sasToken: token]])
def executor = Mock(AzBatchExecutor) {
getAzConfig() >> config
getRemoteBinDir() >> remoteBin
}
when:
def binding = new AzBatchScriptLauncher([
name: 'Hello 1',
workDir: workDir,
script: 'echo Hello world!',
environment: [FOO: 1, BAR:'any'],
input: 'Ciao ciao' ] as TaskBean, executor) .makeBinding()
then:
binding.stage_inputs == '''\
# stage input files
nxf_az_download 'http://account.blob.core.windows.net/my-data/work/remote/bin' $PWD/.nextflow-bin
chmod +x $PWD/.nextflow-bin/* || true
downloads=(true)
nxf_parallel "${downloads[@]}"
'''.stripIndent()
binding.task_env == '''\
export FOO="1"
export BAR="any"
export PATH="$PWD/.nextflow-bin:$AZ_BATCH_NODE_SHARED_DIR/bin/:$PATH"
export AZCOPY_LOG_LOCATION="$PWD/.azcopy_log"
export AZ_SAS="12345"
'''.stripIndent()
binding.helpers_script == '''\
# bash helper functions
nxf_cp_retry() {
local max_attempts=1
local timeout=10
local attempt=0
local exitCode=0
while (( \$attempt < \$max_attempts ))
do
if "\$@"
then
return 0
else
exitCode=\$?
fi
if [[ \$exitCode == 0 ]]
then
break
fi
nxf_sleep \$timeout
attempt=\$(( attempt + 1 ))
timeout=\$(( timeout * 2 ))
done
}
nxf_parallel() {
IFS=$'\\n\'
local cmd=("$@")
local cpus=$(nproc 2>/dev/null || < /proc/cpuinfo grep '^process' -c)
local max=$(if (( cpus>4 )); then echo 4; else echo $cpus; fi)
local i=0
local pid=()
(
set +u
while ((i<${#cmd[@]})); do
local copy=()
for x in "${pid[@]}"; do
# if the process exist, keep in the 'copy' array, otherwise wait on it to capture the exit code
# see https://github.com/nextflow-io/nextflow/pull/4050
[[ -e /proc/$x ]] && copy+=($x) || wait $x
done
pid=("${copy[@]}")
if ((${#pid[@]}>=$max)); then
nxf_sleep 0.2
else
eval "${cmd[$i]}" &
pid+=($!)
((i+=1))
fi
done
for p in "${pid[@]}"; do
wait $p
done
)
unset IFS
}
# custom env variables used for azcopy opts
export AZCOPY_BLOCK_SIZE_MB=4
export AZCOPY_BLOCK_BLOB_TIER=None
nxf_az_upload() {
local name=$1
local target=${2%/} ## remove ending slash
local base_name="$(basename "$name")"
local dir_name="$(dirname "$name")"
if [[ -d $name ]]; then
if [[ "$base_name" == "$name" ]]; then
azcopy cp "$name" "$target?$AZ_SAS" --recursive --block-blob-tier $AZCOPY_BLOCK_BLOB_TIER --block-size-mb $AZCOPY_BLOCK_SIZE_MB
else
azcopy cp "$name" "$target/$dir_name?$AZ_SAS" --recursive --block-blob-tier $AZCOPY_BLOCK_BLOB_TIER --block-size-mb $AZCOPY_BLOCK_SIZE_MB
fi
else
azcopy cp "$name" "$target/$name?$AZ_SAS" --block-blob-tier $AZCOPY_BLOCK_BLOB_TIER --block-size-mb $AZCOPY_BLOCK_SIZE_MB
fi
}
nxf_az_download() {
local source=$1
local target=$2
local basedir=$(dirname $2)
local ret
mkdir -p "$basedir"
ret=$(azcopy cp "$source?$AZ_SAS" "$target" 2>&1) || {
## if fails check if it was trying to download a directory
mkdir -p $target
azcopy cp "$source/*?$AZ_SAS" "$target" --recursive >/dev/null || {
rm -rf $target
>&2 echo "Unable to download path: $source"
exit 1
}
}
}
'''.stripIndent(true)
}
def 'test bash wrapper with outputs and stats'() {
/*
* simple bash run
*/
given:
def workDir = mockAzPath( 'az://my-data/work/dir' )
def input1 = mockAzPath('az://my-data/work/dir/file1.txt')
def input2 = mockAzPath('az://my-data/work/dir/file2.txt')
def token = '12345'
def config = new AzConfig([storage:[sasToken: token]])
def executor = Mock(AzBatchExecutor) { getAzConfig() >> config }
when:
def binding = new AzBatchScriptLauncher([
name: 'Hello 1',
workDir: workDir,
targetDir: workDir,
statsEnabled: true,
inputFiles: ['file1.txt': input1, 'file2.txt': input2],
outputFiles: ['foo.txt', 'bar.fastq'],
script: 'echo Hello world!',
input: 'Ciao ciao' ] as TaskBean, executor) .makeBinding()
then:
binding.unstage_controls == '''\
nxf_az_upload .command.out 'http://account.blob.core.windows.net/my-data/work/dir' || true
nxf_az_upload .command.err 'http://account.blob.core.windows.net/my-data/work/dir' || true
nxf_az_upload .command.trace 'http://account.blob.core.windows.net/my-data/work/dir' || true
'''.stripIndent()
binding.stage_inputs == '''\
# stage input files
downloads=(true)
rm -f file1.txt
rm -f file2.txt
downloads+=("nxf_az_download 'http://account.blob.core.windows.net/my-data/work/dir/file1.txt' file1.txt")
downloads+=("nxf_az_download 'http://account.blob.core.windows.net/my-data/work/dir/file2.txt' file2.txt")
nxf_parallel "${downloads[@]}"
'''.stripIndent()
binding.unstage_outputs == '''
uploads=()
IFS=$'\\n'
for name in $(eval "ls -1d foo.txt bar.fastq" | sort | uniq); do
uploads+=("nxf_az_upload '$name' 'http://account.blob.core.windows.net/my-data/work/dir'")
done
unset IFS
nxf_parallel "${uploads[@]}"
'''.stripIndent().leftTrim()
binding.launch_cmd == '/bin/bash .command.run nxf_trace'
binding.task_env == '''\
export PATH="$PWD/.nextflow-bin:$AZ_BATCH_NODE_SHARED_DIR/bin/:$PATH"
export AZCOPY_LOG_LOCATION="$PWD/.azcopy_log"
export AZ_SAS="12345"
'''.stripIndent()
binding.helpers_script == '''\
# bash helper functions
nxf_cp_retry() {
local max_attempts=1
local timeout=10
local attempt=0
local exitCode=0
while (( \$attempt < \$max_attempts ))
do
if "\$@"
then
return 0
else
exitCode=\$?
fi
if [[ \$exitCode == 0 ]]
then
break
fi
nxf_sleep \$timeout
attempt=\$(( attempt + 1 ))
timeout=\$(( timeout * 2 ))
done
}
nxf_parallel() {
IFS=$'\\n\'
local cmd=("$@")
local cpus=$(nproc 2>/dev/null || < /proc/cpuinfo grep '^process' -c)
local max=$(if (( cpus>4 )); then echo 4; else echo $cpus; fi)
local i=0
local pid=()
(
set +u
while ((i<${#cmd[@]})); do
local copy=()
for x in "${pid[@]}"; do
# if the process exist, keep in the 'copy' array, otherwise wait on it to capture the exit code
# see https://github.com/nextflow-io/nextflow/pull/4050
[[ -e /proc/$x ]] && copy+=($x) || wait $x
done
pid=("${copy[@]}")
if ((${#pid[@]}>=$max)); then
nxf_sleep 0.2
else
eval "${cmd[$i]}" &
pid+=($!)
((i+=1))
fi
done
for p in "${pid[@]}"; do
wait $p
done
)
unset IFS
}
# custom env variables used for azcopy opts
export AZCOPY_BLOCK_SIZE_MB=4
export AZCOPY_BLOCK_BLOB_TIER=None
nxf_az_upload() {
local name=$1
local target=${2%/} ## remove ending slash
local base_name="$(basename "$name")"
local dir_name="$(dirname "$name")"
if [[ -d $name ]]; then
if [[ "$base_name" == "$name" ]]; then
azcopy cp "$name" "$target?$AZ_SAS" --recursive --block-blob-tier $AZCOPY_BLOCK_BLOB_TIER --block-size-mb $AZCOPY_BLOCK_SIZE_MB
else
azcopy cp "$name" "$target/$dir_name?$AZ_SAS" --recursive --block-blob-tier $AZCOPY_BLOCK_BLOB_TIER --block-size-mb $AZCOPY_BLOCK_SIZE_MB
fi
else
azcopy cp "$name" "$target/$name?$AZ_SAS" --block-blob-tier $AZCOPY_BLOCK_BLOB_TIER --block-size-mb $AZCOPY_BLOCK_SIZE_MB
fi
}
nxf_az_download() {
local source=$1
local target=$2
local basedir=$(dirname $2)
local ret
mkdir -p "$basedir"
ret=$(azcopy cp "$source?$AZ_SAS" "$target" 2>&1) || {
## if fails check if it was trying to download a directory
mkdir -p $target
azcopy cp "$source/*?$AZ_SAS" "$target" --recursive >/dev/null || {
rm -rf $target
>&2 echo "Unable to download path: $source"
exit 1
}
}
}
'''.stripIndent(true)
}
}

51
nextflow/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzHelperTest.groovy Normal file
View File

@@ -0,0 +1,51 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.batch
import java.time.Duration
import com.azure.storage.common.policy.RetryPolicyType
import nextflow.cloud.azure.config.AzRetryConfig
import spock.lang.Specification
import spock.lang.Unroll
/**
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
class AzHelperTest extends Specification {
@Unroll
def 'should create retry options' () {
given:
def opts = AzHelper.requestRetryOptions0(CONFIG)
expect:
opts.@retryPolicyType == RetryPolicyType.EXPONENTIAL
opts.maxRetryDelay.toMillis() == CONFIG.maxDelay.millis
opts.retryDelay.toMillis() == CONFIG.delay.millis
opts.maxTries == CONFIG.maxAttempts
and:
opts.secondaryHost == null
opts.tryTimeoutDuration == Duration.ofSeconds(Integer.MAX_VALUE)
where:
_ | CONFIG
_ | new AzRetryConfig()
_ | new AzRetryConfig([delay: '10s', maxAttempts: 20, maxDelay: '200s'])
}
}

47
nextflow/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzJobKeyTest.groovy Normal file
View File

@@ -0,0 +1,47 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.batch
import nextflow.processor.TaskProcessor
import spock.lang.Specification
/**
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
class AzJobKeyTest extends Specification {
def 'should validate equals and hashcode' () {
given:
def p1 = Mock(TaskProcessor)
def p2 = Mock(TaskProcessor)
def k1 = new AzJobKey(p1, 'foo')
def k2 = new AzJobKey(p1, 'foo')
def k3 = new AzJobKey(p2, 'foo')
def k4 = new AzJobKey(p1, 'bar')
expect:
k1 == k2
k1 != k3
k1 != k4
and:
k1.hashCode() == k2.hashCode()
k1.hashCode() != k3.hashCode()
k1.hashCode() != k4.hashCode()
}
}

32
nextflow/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzTaskKeyTest.groovy Normal file
View File

@@ -0,0 +1,32 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.batch
import spock.lang.Specification
/**
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
class AzTaskKeyTest extends Specification {
def 'should return the key pair' () {
expect:
new AzTaskKey('foo','bar').keyPair() == 'foo/bar'
}
}

124
nextflow/plugins/nf-azure/src/test/nextflow/cloud/azure/config/AzBatchOptsTest.groovy Normal file
View File

@@ -0,0 +1,124 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.config
import nextflow.Global
import nextflow.Session
import spock.lang.Specification
import spock.lang.Unroll
/**
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
class AzBatchOptsTest extends Specification {
@Unroll
def 'should fetch location' () {
expect:
new AzBatchOpts(endpoint: ENDPOINT, location: LOC, [:]).getLocation() == EXPECTED
where:
LOC | ENDPOINT | EXPECTED
null | null | null
'europe1' | null | 'europe1'
'europe1' | 'https://foo.xyz.batch.azure.com' | 'europe1'
null | 'https://foo.india2.batch.azure.com' | 'india2'
}
@Unroll
def 'should fetch account name' () {
expect:
new AzBatchOpts(endpoint: ENDPOINT, accountName: NAME, [:]).getAccountName() == EXPECTED
where:
NAME | ENDPOINT | EXPECTED
null | null | null
'foo' | null | 'foo'
'foo' | 'https://bar.eu1.batch.azure.com' | 'foo'
null | 'https://bar.eu2.batch.azure.com' | 'bar'
}
def 'should get account name & key'() {
when:
def opts1 = new AzBatchOpts([:], [:])
then:
opts1.accountName == null
opts1.accountKey == null
when:
def opts2 = new AzBatchOpts(
[accountName: 'xyz', accountKey: '123'],
[AZURE_BATCH_ACCOUNT_NAME: 'env-name', AZURE_BATCH_ACCOUNT_KEY:'env-key'])
then:
opts2.accountName == 'xyz'
opts2.accountKey == '123'
when:
def opts3 = new AzBatchOpts(
[:],
[AZURE_BATCH_ACCOUNT_NAME: 'env-name', AZURE_BATCH_ACCOUNT_KEY:'env-key'])
then:
opts3.accountName == 'env-name'
opts3.accountKey == 'env-key'
}
@Unroll
def 'should check azcopy install' () {
given:
Global.session = Mock(Session) { getConfig()>>[fusion:[enabled: FUSION]] }
AzBatchOpts opts = Spy(AzBatchOpts, constructorArgs: [ CONFIG,[:] ])
expect:
opts.getCopyToolInstallMode() == EXPECTED
where:
EXPECTED | CONFIG | FUSION
CopyToolInstallMode.task | [:] | false
CopyToolInstallMode.node | [allowPoolCreation: true] | false
CopyToolInstallMode.node | [autoPoolMode: true] | false
CopyToolInstallMode.node | [allowPoolCreation: true, copyToolInstallMode: 'node'] | false
CopyToolInstallMode.task | [allowPoolCreation: true, copyToolInstallMode: 'task'] | false
CopyToolInstallMode.task | [copyToolInstallMode: 'task'] | false
CopyToolInstallMode.node | [copyToolInstallMode: 'node'] | false
CopyToolInstallMode.off | [copyToolInstallMode: 'off'] | false
CopyToolInstallMode.off | [:] | true
}
def 'should set jobMaxWallClockTime' () {
when:
def opts1 = new AzBatchOpts([:], [:])
then:
opts1.jobMaxWallClockTime.toString() == '30d'
when:
def opts2 = new AzBatchOpts([jobMaxWallClockTime: '3d'], [:])
then:
opts2.jobMaxWallClockTime.toString() == '3d'
when:
def opts3 = new AzBatchOpts([jobMaxWallClockTime: '12h'], [:])
then:
opts3.jobMaxWallClockTime.toString() == '12h'
}
}

43
nextflow/plugins/nf-azure/src/test/nextflow/cloud/azure/config/AzCopyOptsTest.groovy Normal file
View File

@@ -0,0 +1,43 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.config
import spock.lang.Specification
/**
*
* @author Abhinav Sharma <abhi18av@outlook.com>
*/
class AzCopyOptsTest extends Specification {
def 'should get block size and blob tier'() {
when:
def opts1 = new AzCopyOpts([:])
then:
opts1.blobTier == 'None'
opts1.blockSize == '4'
when:
def opts2 = new AzCopyOpts(
[blobTier: 'Hot', blockSize: '100'])
then:
opts2.blobTier == 'Hot'
opts2.blockSize == '100'
}
}

87
nextflow/plugins/nf-azure/src/test/nextflow/cloud/azure/config/AzManagedIdentityOptsTest.groovy Normal file
View File

@@ -0,0 +1,87 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.config
import spock.lang.Specification
import spock.lang.Unroll
/**
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
class AzManagedIdentityOptsTest extends Specification {
def 'should create manage identity opts' () {
when:
def opts = new AzManagedIdentityOpts([:])
then:
opts.clientId == null
!opts.system
when:
opts = new AzManagedIdentityOpts(clientId: 'foo', system: false)
then:
opts.clientId == 'foo'
!opts.system
when:
opts = new AzManagedIdentityOpts(system: true)
then:
opts.clientId == null
opts.system
when:
opts = new AzManagedIdentityOpts(system: 'false')
then:
opts.clientId == null
!opts.system
when:
opts = new AzManagedIdentityOpts(system: 'true')
then:
opts.clientId == null
opts.system
}
@Unroll
def 'should get env' () {
when:
def opts = new AzManagedIdentityOpts(OPTS)
then:
opts.getEnv() == ENV
where:
OPTS | ENV
[:] | [AZURE_MANAGED_IDENTITY_USER: null, AZURE_MANAGED_IDENTITY_SYSTEM: false]
[clientId:'foo'] | [AZURE_MANAGED_IDENTITY_USER: 'foo', AZURE_MANAGED_IDENTITY_SYSTEM: false]
[system:'true'] | [AZURE_MANAGED_IDENTITY_USER: null, AZURE_MANAGED_IDENTITY_SYSTEM: true]
}
def 'should check is valid' (){
expect:
!new AzManagedIdentityOpts([:]).isConfigured()
and:
new AzManagedIdentityOpts([clientId: 'foo']).isConfigured()
new AzManagedIdentityOpts([system: true]).isConfigured()
when:
new AzManagedIdentityOpts([clientId: 'foo',system: true]).isConfigured()
then:
thrown(IllegalArgumentException)
}
}

101
nextflow/plugins/nf-azure/src/test/nextflow/cloud/azure/config/AzPoolOptsTest.groovy Normal file
View File

@@ -0,0 +1,101 @@
/*
* Copyright 2013-2026, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nextflow.cloud.azure.config
import nextflow.util.Duration
import spock.lang.Specification
/**
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
class AzPoolOptsTest extends Specification {
def 'should create pool options' () {
when:
def opts = new AzPoolOpts()
then:
!opts.runAs
!opts.privileged
opts.publisher == AzPoolOpts.DEFAULT_PUBLISHER
opts.offer == AzPoolOpts.DEFAULT_OFFER
opts.sku == AzPoolOpts.DEFAULT_SKU
opts.vmType == AzPoolOpts.DEFAULT_VM_TYPE
opts.fileShareRootPath == '/mnt/batch/tasks/fsmounts'
opts.vmCount == 1
!opts.autoScale
!opts.scaleFormula
!opts.schedulePolicy
opts.scaleInterval == AzPoolOpts.DEFAULT_SCALE_INTERVAL
opts.maxVmCount == opts.vmCount *3
!opts.registry
!opts.userName
!opts.password
!opts.virtualNetwork
!opts.lowPriority
!opts.startTask.script
!opts.startTask.privileged
}
def 'should create pool with custom options' () {
when:
def opts = new AzPoolOpts([
runAs:'foo',
privileged: true,
publisher: 'some-pub',
offer: 'some-offer',
sku: 'some-sku',
vmType: 'some-vmtype',
vmCount: 10,
autoScale: true,
scaleFormula: 'some-formula',
schedulePolicy: 'some-policy',
scaleInterval: Duration.of('10s'),
maxVmCount: 100,
registry: 'some-reg',
userName: 'some-user',
password: 'some-pwd',
virtualNetwork: 'some-vnet',
lowPriority: true,
startTask: [
script: 'echo hello-world',
privileged: true
]
])
then:
opts.runAs == 'foo'
opts.privileged
opts.publisher == 'some-pub'
opts.offer == 'some-offer'
opts.sku == 'some-sku'
opts.vmType == 'some-vmtype'
opts.fileShareRootPath == ''
opts.vmCount == 10
opts.autoScale
opts.scaleFormula == 'some-formula'
opts.schedulePolicy == 'some-policy'
opts.scaleInterval == Duration.of('10s')
opts.maxVmCount == 100
opts.registry == 'some-reg'
opts.userName == 'some-user'
opts.password == 'some-pwd'
opts.virtualNetwork == 'some-vnet'
opts.lowPriority
opts.startTask.script == 'echo hello-world'
opts.startTask.privileged
}
}

Some files were not shown because too many files have changed in this diff Show More