add nextflow d30e48d
This commit is contained in:
292
nextflow/packing.gradle
Normal file
292
nextflow/packing.gradle
Normal file
@@ -0,0 +1,292 @@
|
||||
configurations {
|
||||
defaultCfg.extendsFrom api
|
||||
//provided
|
||||
console.extendsFrom defaultCfg
|
||||
google.extendsFrom defaultCfg
|
||||
amazon.extendsFrom defaultCfg
|
||||
azure.extendsFrom defaultCfg
|
||||
legacy.extendsFrom defaultCfg
|
||||
tower.extendsFrom defaultCfg
|
||||
wave.extendsFrom defaultCfg
|
||||
seqera.extendsFrom defaultCfg
|
||||
}
|
||||
|
||||
dependencies {
|
||||
api project(':nextflow')
|
||||
// include Ivy at runtime in order to have Grape @Grab work correctly
|
||||
defaultCfg "org.apache.ivy:ivy:2.5.2"
|
||||
// default cfg = runtime + httpfs + lineage
|
||||
defaultCfg project(':nf-httpfs')
|
||||
defaultCfg project(':nf-lineage')
|
||||
console project(':plugins:nf-console')
|
||||
google project(':plugins:nf-google')
|
||||
amazon project(':plugins:nf-amazon')
|
||||
azure project(':plugins:nf-azure')
|
||||
tower project(':plugins:nf-tower')
|
||||
wave project(':plugins:nf-wave')
|
||||
seqera project(':plugins:nf-seqera')
|
||||
}
|
||||
|
||||
|
||||
// ext.mainClassName removed - now using application.mainClass in Shadow plugin
|
||||
ext.homeDir = System.properties['user.home']
|
||||
ext.nextflowDir = "$homeDir/.nextflow/framework/$version"
|
||||
ext.releaseDir = "$buildDir/releases"
|
||||
ext.s3CmdOpts="--acl public-read --storage-class STANDARD --region eu-west-1"
|
||||
|
||||
protected error(String message) {
|
||||
logger.error message
|
||||
throw new StopExecutionException(message)
|
||||
}
|
||||
|
||||
protected checkVersionExits(String version) {
|
||||
if(version.endsWith('-SNAPSHOT'))
|
||||
return
|
||||
|
||||
def cmd = "AWS_ACCESS_KEY_ID=${System.env.NXF_AWS_ACCESS} AWS_SECRET_ACCESS_KEY=${System.env.NXF_AWS_SECRET} aws s3 ls s3://www2.nextflow.io/releases/v$version/nextflow"
|
||||
def status=['bash','-c', cmd].execute().waitFor()
|
||||
if( status == 0 )
|
||||
error("STOP!! Version $version already deployed!")
|
||||
}
|
||||
|
||||
protected resolveDeps( String configName, String... x ) {
|
||||
|
||||
final deps = [] as Set
|
||||
final config = configurations.getByName(configName)
|
||||
//final root = config.getIncoming().getResolutionResult().root.moduleVersion.toString()
|
||||
config.getResolvedConfiguration().getResolvedArtifacts().each{ deps << coordinates(it) }
|
||||
|
||||
if( x )
|
||||
deps.addAll(x as List)
|
||||
|
||||
logger.info ">> Dependencies for configuration: $configName"
|
||||
deps.sort().each { logger.info " - $it" }
|
||||
|
||||
// make sure there aren't any version conflict
|
||||
def conflicts = deps.countBy { it.tokenize(":")[0..1].join(':') }.findAll { k,v -> v>1 }
|
||||
if( conflicts ) {
|
||||
def err = "There are multiple versions for the following lib(s):"
|
||||
conflicts.each { name, count -> err += "\n- $name: " + deps.findAll{ it.startsWith(name) }.collect{ it.tokenize(':')[2] } }
|
||||
throw new IllegalStateException(err)
|
||||
}
|
||||
|
||||
//println "** $configName\n${deps.sort().collect{"- $it"}.join('\n')}\n"
|
||||
|
||||
//println ">> Config: $configName \n${deps.sort().join('\n')}\n\n"
|
||||
|
||||
return deps.collect{ "$it(*:*)" }.join(' ')
|
||||
}
|
||||
|
||||
protected coordinates( it ) {
|
||||
if( it instanceof Dependency )
|
||||
return "${it.group}:${it.name}:${it.version}".toString()
|
||||
|
||||
if( it instanceof ResolvedArtifact ) {
|
||||
def result = it.moduleVersion.id.toString()
|
||||
if( it.classifier ) result += ":${it.classifier}"
|
||||
return result
|
||||
}
|
||||
|
||||
throw new IllegalArgumentException("Not a valid dependency object [${it?.class?.name}]: $it")
|
||||
}
|
||||
|
||||
/*
|
||||
* Compile and pack all packages
|
||||
*/
|
||||
task packOne( dependsOn: [compile, ":nextflow:shadowJar"]) {
|
||||
doLast {
|
||||
def source = "modules/nextflow/build/libs/nextflow-${version}-one.jar"
|
||||
ant.copy(file: source, todir: releaseDir, overwrite: true)
|
||||
ant.copy(file: source, todir: nextflowDir, overwrite: true)
|
||||
println "\n+ Nextflow package `ONE` copied to: $releaseDir/nextflow-${version}-one.jar"
|
||||
}
|
||||
}
|
||||
|
||||
task packDist( dependsOn: [compile, ":nextflow:shadowJar"]) {
|
||||
|
||||
doLast {
|
||||
file(releaseDir).mkdirs()
|
||||
// cleanup
|
||||
def source = file("modules/nextflow/build/libs/nextflow-${version}-one.jar")
|
||||
def target = file("$releaseDir/nextflow-${version}-dist"); target.delete()
|
||||
// append the big jar
|
||||
target.withOutputStream {
|
||||
it << file('nextflow').text.replaceAll(/NXF_PACK\=.*/, 'NXF_PACK=dist')
|
||||
it << new FileInputStream(source)
|
||||
}
|
||||
// execute permission
|
||||
"chmod +x $target".execute()
|
||||
// done
|
||||
println "+ Nextflow package `ALL` copied to: $target\n"
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Compile and pack all packages
|
||||
*/
|
||||
task pack( dependsOn: [packOne, packDist]) {
|
||||
|
||||
}
|
||||
|
||||
task deploy( type: Exec, dependsOn: [clean, compile, pack, validatePluginVersions]) {
|
||||
|
||||
def temp = File.createTempFile('upload',null)
|
||||
temp.deleteOnExit()
|
||||
def files = []
|
||||
|
||||
doFirst {
|
||||
checkVersionExits(version)
|
||||
|
||||
def path = new File(releaseDir)
|
||||
if( !path.exists() ) error("Releases path does not exist: $path")
|
||||
path.eachFile {
|
||||
if( it.name.startsWith("nextflow-$version"))
|
||||
files << it
|
||||
}
|
||||
|
||||
if( !files ) error("Can't find any file to upload -- Check path: $path")
|
||||
files << file('nextflow').absoluteFile
|
||||
files << file('nextflow.sha1').absoluteFile
|
||||
files << file('nextflow.sha256').absoluteFile
|
||||
files << file('nextflow.md5').absoluteFile
|
||||
|
||||
println "Uploading artifacts: "
|
||||
files.each { println "- $it"}
|
||||
|
||||
def script = []
|
||||
script << "export AWS_ACCESS_KEY_ID=${System.env.NXF_AWS_ACCESS}"
|
||||
script << "export AWS_SECRET_ACCESS_KEY=${System.env.NXF_AWS_SECRET}"
|
||||
script.addAll( files.collect { "aws s3 cp ${it} s3://www2.nextflow.io/releases/v${version}/${it.name} ${s3CmdOpts}"})
|
||||
|
||||
temp.text = script.join('\n')
|
||||
}
|
||||
|
||||
commandLine 'bash', '-e', temp.absolutePath
|
||||
}
|
||||
|
||||
task installLauncher(type: Copy, dependsOn: ['pack']) {
|
||||
from "$releaseDir/nextflow-$version-one.jar"
|
||||
into "$homeDir/.nextflow/framework/$version/"
|
||||
}
|
||||
|
||||
task installScratch(type: Copy, dependsOn: ['pack']) {
|
||||
from "$releaseDir/nextflow-$version-one.jar"
|
||||
into "${rootProject.projectDir}/test-e2e/.nextflow/framework/$version/"
|
||||
}
|
||||
|
||||
/*
|
||||
* build, tag and publish a and new docker packaged nextflow release
|
||||
*/
|
||||
task dockerImage(type: Exec) {
|
||||
|
||||
def temp = File.createTempFile('upload',null)
|
||||
temp.deleteOnExit()
|
||||
temp.text = """\
|
||||
grep $version nextflow
|
||||
cp nextflow docker/nextflow
|
||||
cd docker
|
||||
make release version=$version
|
||||
""".stripIndent()
|
||||
|
||||
commandLine 'bash', '-e', temp.absolutePath
|
||||
}
|
||||
|
||||
/*
|
||||
* Build a docker container image with the current snapshot
|
||||
* DO NOT CALL IT DIRECTLY! Use the `make dockerPack` command instead
|
||||
*/
|
||||
task dockerPack(type: Exec, dependsOn: ['packOne']) {
|
||||
|
||||
def source = new File("$releaseDir/nextflow-$version-one.jar")
|
||||
def target = new File("$buildDir/docker/.nextflow/framework/$version/")
|
||||
def dockerBase = new File("$buildDir/docker")
|
||||
if( dockerBase.exists() ) dockerBase.deleteDir()
|
||||
dockerBase.mkdirs()
|
||||
def dockerFile = new File("$buildDir/docker/Dockerfile")
|
||||
dockerFile.text = """
|
||||
FROM amazoncorretto:17-alpine-jdk
|
||||
RUN apk update && apk add bash && apk add coreutils && apk add curl
|
||||
COPY .nextflow /.nextflow
|
||||
COPY nextflow /usr/local/bin/nextflow
|
||||
COPY entry.sh /usr/local/bin/entry.sh
|
||||
COPY dist/docker /usr/local/bin/docker
|
||||
ENV NXF_HOME=/.nextflow
|
||||
RUN chmod +x /usr/local/bin/nextflow /usr/local/bin/entry.sh
|
||||
RUN nextflow info
|
||||
ENTRYPOINT ["/usr/local/bin/entry.sh"]
|
||||
"""
|
||||
|
||||
def temp = File.createTempFile('upload',null)
|
||||
temp.deleteOnExit()
|
||||
temp.text = """\
|
||||
mkdir -p $target
|
||||
cp $source $target
|
||||
cp nextflow $buildDir/docker
|
||||
cp docker/entry.sh $buildDir/docker
|
||||
cd $buildDir/docker
|
||||
mkdir -p dist
|
||||
curl -fsSLO https://get.docker.com/builds/Linux/x86_64/docker-17.03.1-ce.tgz && tar --strip-components=1 -xvzf docker-17.03.1-ce.tgz -C dist
|
||||
NXF_HOME=\$PWD/.nextflow ./nextflow info
|
||||
docker build -t nextflow/nextflow:$version .
|
||||
""".stripIndent()
|
||||
|
||||
commandLine 'bash', '-e', temp.absolutePath
|
||||
}
|
||||
|
||||
/*
|
||||
* Tag and upload the release
|
||||
*
|
||||
* https://github.com/aktau/github-release/
|
||||
*/
|
||||
task release(type: Exec, dependsOn: [pack, dockerImage, validatePluginVersions]) {
|
||||
|
||||
def launcherFile = file('nextflow').absoluteFile
|
||||
def launcherSha1 = file('nextflow.sha1').absoluteFile
|
||||
def launcherSha256 = file('nextflow.sha256').absoluteFile
|
||||
def nextflowAllFile = file("$releaseDir/nextflow-${version}-dist")
|
||||
def versionFile = file('VERSION').absoluteFile
|
||||
|
||||
def snapshot = version ==~ /^.+(-RC\d+|-SNAPSHOT)$/
|
||||
def edge = version ==~ /^.+(-edge|-EDGE)$/
|
||||
def isLatest = !snapshot && !edge
|
||||
def cmd = """\
|
||||
# tag the release
|
||||
git push || exit \$?
|
||||
(git tag -a v$version -m 'Tagging version $version [release]' -f && git push origin v$version -f) || exit \$?
|
||||
sleep 1
|
||||
gh release create --repo nextflow-io/nextflow --prerelease --title "Version $version" v$version
|
||||
sleep 1
|
||||
gh release upload --repo nextflow-io/nextflow v$version ${launcherFile} ${nextflowAllFile}
|
||||
""".stripIndent()
|
||||
|
||||
if( edge )
|
||||
cmd += """
|
||||
# publish the script as the latest
|
||||
export AWS_ACCESS_KEY_ID=${System.env.NXF_AWS_ACCESS}
|
||||
export AWS_SECRET_ACCESS_KEY=${System.env.NXF_AWS_SECRET}
|
||||
aws s3 cp $launcherFile s3://www2.nextflow.io/releases/edge/nextflow $s3CmdOpts
|
||||
aws s3 cp $launcherSha1 s3://www2.nextflow.io/releases/edge/nextflow.sha1 $s3CmdOpts
|
||||
aws s3 cp $launcherSha256 s3://www2.nextflow.io/releases/edge/nextflow.sha256 $s3CmdOpts
|
||||
aws s3 cp $launcherSha256 s3://www2.nextflow.io/releases/edge/nextflow.md5 $s3CmdOpts
|
||||
aws s3 cp $versionFile s3://www2.nextflow.io/releases/edge/version $s3CmdOpts
|
||||
""".stripIndent()
|
||||
|
||||
else if( isLatest )
|
||||
cmd += """
|
||||
# publish the script as the latest
|
||||
export AWS_ACCESS_KEY_ID=${System.env.NXF_AWS_ACCESS}
|
||||
export AWS_SECRET_ACCESS_KEY=${System.env.NXF_AWS_SECRET}
|
||||
aws s3 cp $launcherFile s3://www2.nextflow.io/releases/latest/nextflow $s3CmdOpts
|
||||
aws s3 cp $launcherSha1 s3://www2.nextflow.io/releases/latest/nextflow.sha1 $s3CmdOpts
|
||||
aws s3 cp $launcherSha256 s3://www2.nextflow.io/releases/latest/nextflow.sha256 $s3CmdOpts
|
||||
aws s3 cp $launcherSha256 s3://www2.nextflow.io/releases/latest/nextflow.md5 $s3CmdOpts
|
||||
aws s3 cp $versionFile s3://www2.nextflow.io/releases/latest/version $s3CmdOpts
|
||||
""".stripIndent()
|
||||
|
||||
def temp = File.createTempFile('upload',null)
|
||||
temp.deleteOnExit()
|
||||
temp.text = cmd
|
||||
commandLine 'bash', '-e', temp.absolutePath
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user