add nextflow d30e48d

nextflow/plugins/nf-cloudcache/README.md

@@ -0,0 +1,86 @@
+# Cloud cache plugin for Nextflow
+
+## Summary
+
+The Cloud cache plugin provides cloud-based caching support for Nextflow pipelines. It enables workflow resume capability when using cloud storage as the work directory.
+
+## Get Started
+
+To use this plugin, add it to your `nextflow.config`:
+
+```groovy
+plugins {
+    id 'nf-cloudcache'
+}
+```
+
+The plugin is automatically activated when using cloud storage (S3, GS, Azure Blob) as the work directory with resume enabled.
+
+```groovy
+workDir = 's3://my-bucket/work'
+```
+
+Run your pipeline with the `-resume` flag:
+
+```bash
+nextflow run main.nf -resume
+```
+
+## Examples
+
+### AWS S3 Cache
+
+```groovy
+plugins {
+    id 'nf-amazon'
+    id 'nf-cloudcache'
+}
+
+workDir = 's3://my-bucket/work'
+
+aws {
+    region = 'us-east-1'
+}
+```
+
+### Google Cloud Storage Cache
+
+```groovy
+plugins {
+    id 'nf-google'
+    id 'nf-cloudcache'
+}
+
+workDir = 'gs://my-bucket/work'
+
+google {
+    project = 'my-project'
+    location = 'us-central1'
+}
+```
+
+### Azure Blob Storage Cache
+
+```groovy
+plugins {
+    id 'nf-azure'
+    id 'nf-cloudcache'
+}
+
+workDir = 'az://my-container/work'
+
+azure {
+    storage {
+        accountName = 'mystorageaccount'
+        accountKey = System.getenv('AZURE_STORAGE_KEY')
+    }
+}
+```
+
+## Resources
+
+- [Nextflow Cache and Resume](https://nextflow.io/docs/latest/cache-and-resume.html)
+
+## License
+
+[Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0)

nextflow/plugins/nf-cloudcache/VERSION

@@ -0,0 +1 @@
+0.6.0

nextflow/plugins/nf-cloudcache/build.gradle

@@ -0,0 +1,58 @@
+/*
+ * Copyright 2013-2026, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+plugins {
+    id 'io.nextflow.nextflow-plugin' version "${nextflowPluginVersion}"
+    id 'java-test-fixtures'
+}
+
+nextflowPlugin {
+    nextflowVersion = '25.08.0-edge'
+
+    provider = "${nextflowPluginProvider}"
+    description = 'Implements cloud-based caching system to optimize workflow performance by reducing redundant computations and data transfers'
+    className = 'nextflow.CloudCachePlugin'
+    useDefaultDependencies = false
+    generateSpec = false
+    extensionPoints = [
+        'nextflow.cache.CloudCacheConfig',
+        'nextflow.cache.CloudCacheFactory'
+    ]
+}
+
+sourceSets {
+    main.java.srcDirs = []
+    main.groovy.srcDirs = ['src/main']
+    main.resources.srcDirs = ['src/resources']
+    test.groovy.srcDirs = ['src/test']
+    test.java.srcDirs = []
+    test.resources.srcDirs = []
+}
+
+configurations {
+    // see https://docs.gradle.org/4.1/userguide/dependency_management.html#sub:exclude_transitive_dependencies
+    runtimeClasspath.exclude group: 'org.slf4j', module: 'slf4j-api'
+}
+
+dependencies {
+    compileOnly project(':nextflow')
+    compileOnly 'org.slf4j:slf4j-api:2.0.17'
+    compileOnly 'org.pf4j:pf4j:3.14.1'
+
+    testImplementation(testFixtures(project(":nextflow")))
+    testImplementation "org.apache.groovy:groovy:4.0.31"
+    testImplementation "org.apache.groovy:groovy-nio:4.0.31"
+}
+

nextflow/plugins/nf-cloudcache/changelog.txt

@@ -0,0 +1,35 @@
+nf-cloudcache changelog
+=======================
+0.6.0 - 8 Feb 2026
+- Add CloudCacheConfig to register cloudcache config options (#6774) [ea8fea470]
+
+0.4.3 - 20 Jan 2025
+- Bump logback 1.5.13 + slf4j 2.0.16 [cc0163ac]
+- Bump groovy 4.0.24 missing deps [40670f7e]
+
+0.4.2 - 5 Aug 2024
+- Bump pf4j to version 3.12.0 [96117b9a]
+
+0.4.1 - 10 Mar 2024
+- Fix typo in error message [ci fast] [a7f23305]
+- minor cli docstring fix (#4759) [ci skip] [ee4b4a25]
+- Bump groovy 4.0.19 [854dc1f0]
+
+0.4.0 - 5 Feb 2024
+- Bump Groovy 4 (#4443) [9d32503b]
+
+0.3.3-patch1 - 28 May 2024
+- Bump dependency with Nextflow 23.10.2
+
+0.3.1 - 24 Nov 2023
+- Fix security vulnerabilities (#4513) [a310c777]
+
+0.3.0 - 10 Oct 2023
+- Add -cloudcache CLI option (#4385) [73fda582]
+
+0.2.0 - 17 Aug 2023
+- Remove lock file from cloudcache (#4167) [6e6ea579]
+- Enable cloud cache based on environment variable (#4160) [a66b0e63]
+
+0.1.0 - 22 Jul 2023
+- Initial version

nextflow/plugins/nf-cloudcache/src/main/nextflow/CloudCachePlugin.groovy

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2013-2026, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import nextflow.plugin.BasePlugin
+import org.pf4j.PluginWrapper
+/**
+ * Nextflow cloud cache plugin
+ *
+ * @author Ben Sherman <bentshermann@gmail.com>
+ */
+@Slf4j
+@CompileStatic
+class CloudCachePlugin extends BasePlugin {
+
+    CloudCachePlugin(PluginWrapper wrapper) {
+        super(wrapper)
+    }
+
+}

nextflow/plugins/nf-cloudcache/src/main/nextflow/cache/CloudCacheConfig.groovy

@@ -0,0 +1,57 @@
+/*
+ * Copyright 2013-2026, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.cache
+
+import groovy.transform.CompileStatic
+import nextflow.config.spec.ConfigOption
+import nextflow.config.spec.ConfigScope
+import nextflow.config.spec.ScopeName
+import nextflow.script.dsl.Description
+
+/**
+ * Model cloud cache configuration
+ *
+ * @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
+ */
+@ScopeName("cloudcache")
+@Description("""
+    The `cloudcache` scope controls the use of object storage as cache storage for workflow execution metadata.
+""")
+@CompileStatic
+class CloudCacheConfig implements ConfigScope {
+
+    @ConfigOption
+    @Description("""
+        Enable the use of cloud cache (default: `false`).
+    """)
+    final boolean enabled
+
+    @ConfigOption
+    @Description("""
+        The path to the cloud storage bucket for cache metadata (e.g. `s3://bucket/cache`).
+    """)
+    final String path
+
+    /* required by extension point -- do not remove */
+    CloudCacheConfig() {}
+
+    CloudCacheConfig(Map opts) {
+        this.enabled = opts.enabled as boolean
+        this.path = opts.path as String
+    }
+
+}

nextflow/plugins/nf-cloudcache/src/main/nextflow/cache/CloudCacheFactory.groovy

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2013-2026, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.cache
+
+import java.nio.file.Path
+
+import groovy.transform.CompileStatic
+import nextflow.Global
+import nextflow.Session
+import nextflow.exception.AbortOperationException
+import nextflow.plugin.Priority
+/**
+ * Implements the cloud cache factory
+ *
+ * @see CloudCacheStore
+ *
+ * @author Ben Sherman <bentshermann@gmail.com>
+ */
+@CompileStatic
+@Priority(-10)
+class CloudCacheFactory extends CacheFactory {
+
+    @Override
+    protected CacheDB newInstance(UUID uniqueId, String runName, Path home) {
+        if( !uniqueId ) throw new AbortOperationException("Missing cache `uuid`")
+        if( !runName ) throw new AbortOperationException("Missing cache `runName`")
+        final path = (Global.session as Session).cloudCachePath
+        if( !path )
+            throw new IllegalArgumentException("Cloud-cache path not defined - use either -with-cloudcache run option or NXF_CLOUDCACHE_PATH environment variable")
+        final store = new CloudCacheStore(uniqueId, runName, path)
+        return new CacheDB(store)
+    }
+
+}

nextflow/plugins/nf-cloudcache/src/main/nextflow/cache/CloudCacheStore.groovy

@@ -0,0 +1,160 @@
+/*
+ * Copyright 2013-2026, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.cache
+
+import java.nio.file.Files
+import java.nio.file.NoSuchFileException
+import java.nio.file.Path
+
+import com.google.common.hash.HashCode
+import groovy.transform.CompileStatic
+import nextflow.exception.AbortOperationException
+import nextflow.extension.FilesEx
+import nextflow.util.CacheHelper
+/**
+ * Implements the cloud cache store
+ *
+ * @author Ben Sherman <bentshermann@gmail.com>
+ */
+@CompileStatic
+class CloudCacheStore implements CacheStore {
+
+    private final int KEY_SIZE
+
+    /** The session UUID */
+    private UUID uniqueId
+
+    /** The unique run name associated with this cache instance */
+    private String runName
+
+    /** The base path for the entire cache */
+    private Path basePath
+
+    /** The base path for this cache instance */
+    private Path dataPath
+
+    /** The path to the index file */
+    private Path indexPath
+
+    /** Index file input stream */
+    private InputStream indexReader
+
+    /** Index file output stream */
+    private OutputStream indexWriter
+
+    CloudCacheStore(UUID uniqueId, String runName, Path basePath) {
+        assert uniqueId, "Missing cloudcache 'uniqueId' argument"
+        assert runName, "Missing cloudcache 'runName' argument"
+        assert basePath, "Missing cloudcache 'basePath' argument"
+        this.KEY_SIZE = CacheHelper.hasher('x').hash().asBytes().size()
+        this.uniqueId = uniqueId
+        this.runName = runName
+        this.basePath = basePath
+        this.dataPath = this.basePath.resolve("$uniqueId")
+        this.indexPath = dataPath.resolve("index.$runName")
+    }
+
+    @Override
+    CloudCacheStore open() {
+        indexWriter = new BufferedOutputStream(Files.newOutputStream(indexPath))
+        return this
+    }
+
+    @Override
+    CloudCacheStore openForRead() {
+        if( !dataPath.exists() )
+            throw new AbortOperationException("Missing cache directory: $dataPath")
+        indexReader = Files.newInputStream(indexPath)
+        return this
+    }
+
+    @Override
+    void drop() {
+        dataPath.deleteDir()
+    }
+
+    @Override
+    void close() {
+        FilesEx.closeQuietly(indexWriter)
+    }
+
+    @Override
+    void writeIndex(HashCode key, boolean cached) {
+        indexWriter.write(key.asBytes())
+        indexWriter.write(cached ? 1 : 0)
+    }
+
+    @Override
+    void deleteIndex() {
+        indexPath.delete()
+    }
+
+    @Override
+    Iterator<Index> iterateIndex() {
+        return new Iterator<Index>() {
+            private Index next
+
+            {
+                next = fetch()
+            }
+
+            @Override
+            boolean hasNext() {
+                return next != null
+            }
+
+            @Override
+            Index next() {
+                final result = next
+                next = fetch()
+                return result
+            }
+
+            private Index fetch() {
+                byte[] key = new byte[KEY_SIZE]
+                if( indexReader.read(key) == -1 )
+                    return null
+                final cached = indexReader.read() == 1
+                return new Index(HashCode.fromBytes(key), cached)
+            }
+        }
+    }
+
+    @Override
+    byte[] getEntry(HashCode key) {
+        try {
+            return getCachePath(key).bytes
+        }
+        catch( NoSuchFileException e ) {
+            return null
+        }
+    }
+
+    @Override
+    void putEntry(HashCode key, byte[] value) {
+        getCachePath(key).bytes = value
+    }
+
+    @Override
+    void deleteEntry(HashCode key) {
+        getCachePath(key).delete()
+    }
+
+    private Path getCachePath(HashCode key) {
+        dataPath.resolve(key.toString())
+    }
+}

nextflow/plugins/nf-cloudcache/src/test/nextflow/cache/CloudCacheConfigTest.groovy

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2013-2026, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.cache
+
+import spock.lang.Specification
+
+/**
+ *
+ * @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
+ */
+class CloudCacheConfigTest extends Specification {
+
+    def 'should create empty config' () {
+        when:
+        def config = new CloudCacheConfig([:])
+        then:
+        !config.enabled
+        config.path == null
+    }
+
+    def 'should create config with all options' () {
+        when:
+        def config = new CloudCacheConfig([enabled: true, path: 's3://bucket/cache'])
+        then:
+        config.enabled
+        config.path == 's3://bucket/cache'
+    }
+
+}