From 8a4348939949ce5532179cc8067350c9c2c598ef Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sun, 20 Aug 2023 19:44:47 +0200 Subject: [PATCH] Add support for Wave native build for singularity Signed-off-by: Paolo Di Tommaso --- build.gradle | 2 + .../script/bundle/ResourcesBundle.groovy | 15 +- .../script/bundle/ResourcesBundleTest.groovy | 27 +- plugins/nf-wave/build.gradle | 1 + .../main/io/seqera/wave/config/CondaOpts.java | 41 -- .../main/io/seqera/wave/config/SpackOpts.java | 48 --- .../plugin/SubmitContainerTokenRequest.groovy | 5 + .../io/seqera/wave/plugin/WaveAssets.groovy | 9 +- .../io/seqera/wave/plugin/WaveClient.groovy | 91 +++-- .../resolver/WaveContainerResolver.groovy | 19 +- .../io/seqera/wave/util/DockerHelper.java | 256 ------------ .../io/seqera/wave/util/TemplateRenderer.java | 150 ------- .../templates/conda/dockerfile-conda-file.txt | 6 - .../conda/dockerfile-conda-packages.txt | 7 - .../templates/spack/dockerfile-spack-file.txt | 18 - .../seqera/wave/config/CondaOptsTest.groovy | 47 --- .../seqera/wave/config/SpackOptsTest.groovy | 48 --- .../seqera/wave/plugin/WaveClientTest.groovy | 174 +++++++- .../resolver/WaveContainerResolverTest.groovy | 25 +- .../seqera/wave/util/DockerHelperTest.groovy | 381 ------------------ .../wave/util/TemplateRendererTest.groovy | 202 ---------- 21 files changed, 296 insertions(+), 1276 deletions(-) delete mode 100644 plugins/nf-wave/src/main/io/seqera/wave/config/CondaOpts.java delete mode 100644 plugins/nf-wave/src/main/io/seqera/wave/config/SpackOpts.java delete mode 100644 plugins/nf-wave/src/main/io/seqera/wave/util/DockerHelper.java delete mode 100644 plugins/nf-wave/src/main/io/seqera/wave/util/TemplateRenderer.java delete mode 100644 plugins/nf-wave/src/resources/templates/conda/dockerfile-conda-file.txt delete mode 100644 plugins/nf-wave/src/resources/templates/conda/dockerfile-conda-packages.txt delete mode 100644 plugins/nf-wave/src/resources/templates/spack/dockerfile-spack-file.txt delete mode 100644 plugins/nf-wave/src/test/io/seqera/wave/config/CondaOptsTest.groovy delete mode 100644 plugins/nf-wave/src/test/io/seqera/wave/config/SpackOptsTest.groovy delete mode 100644 plugins/nf-wave/src/test/io/seqera/wave/util/DockerHelperTest.groovy delete mode 100644 plugins/nf-wave/src/test/io/seqera/wave/util/TemplateRendererTest.groovy diff --git a/build.gradle b/build.gradle index 82c8422e3d..2b8f83e687 100644 --- a/build.gradle +++ b/build.gradle @@ -82,6 +82,8 @@ allprojects { mavenCentral() maven { url 'https://repo.eclipse.org/content/groups/releases' } maven { url 'https://oss.sonatype.org/content/repositories/snapshots' } + maven { url = "https://s3-eu-west-1.amazonaws.com/maven.seqera.io/releases" } + maven { url = "https://s3-eu-west-1.amazonaws.com/maven.seqera.io/snapshots" } } configurations { diff --git a/modules/nextflow/src/main/groovy/nextflow/script/bundle/ResourcesBundle.groovy b/modules/nextflow/src/main/groovy/nextflow/script/bundle/ResourcesBundle.groovy index a0f17094b0..6aa1ef9622 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/bundle/ResourcesBundle.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/bundle/ResourcesBundle.groovy @@ -45,13 +45,15 @@ class ResourcesBundle { private Path root private LinkedHashMap content = new LinkedHashMap<>(100) private Path dockerfile + private Path singularityfile private MemoryUnit maxFileSize = MAX_FILE_SIZE private MemoryUnit maxBundleSize = MAX_BUNDLE_SIZE private String baseDirectory ResourcesBundle(Path root) { this.root = root - this.dockerfile = dockefile0(root.resolveSibling('Dockerfile')) + this.dockerfile = pathIfExists0(root.resolveSibling('Dockerfile')) + this.singularityfile = pathIfExists0(root.resolveSibling('Singularityfile')) } ResourcesBundle withMaxFileSize(MemoryUnit mem) { @@ -68,7 +70,7 @@ class ResourcesBundle { Map content() { content } - static private Path dockefile0(Path path) { + static private Path pathIfExists0(Path path) { return path?.exists() ? path : null } @@ -100,6 +102,10 @@ class ResourcesBundle { return dockerfile } + Path getSingularityfile() { + return singularityfile + } + Set getPaths() { return new HashSet(content.values()) } @@ -125,7 +131,7 @@ class ResourcesBundle { } boolean asBoolean() { - return content.size() || dockerfile + return content.size() || dockerfile || singularityfile } /** @@ -189,6 +195,9 @@ class ResourcesBundle { if( dockerfile ) { allMeta.add(fileMeta(dockerfile.name, dockerfile)) } + if( singularityfile ) { + allMeta.add(fileMeta(singularityfile.name, singularityfile)) + } return CacheHelper.hasher(allMeta).hash().toString() } diff --git a/modules/nextflow/src/test/groovy/nextflow/script/bundle/ResourcesBundleTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/bundle/ResourcesBundleTest.groovy index 655dd82cd3..2ec7a41688 100644 --- a/modules/nextflow/src/test/groovy/nextflow/script/bundle/ResourcesBundleTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/script/bundle/ResourcesBundleTest.groovy @@ -106,12 +106,31 @@ class ResourcesBundleTest extends Specification { then: bundle.fingerprint() == '7b2200ff24230f76cea22e5eb15b1701' + } + + def 'should get singularityfile' () { + given: + def singularPath = folder.resolve('Singularityfile'); singularPath.text = "I'm the main file" + def bundlePath = folder.resolve('bundle') + and: + singularPath.setLastModified(LAST_MODIFIED) + singularPath.setPermissions(6,4,4) when: - // changing the last modified time, change the fingerprint - dockerPath.setLastModified(LAST_MODIFIED +100) + def bundle = ResourcesBundle.scan(bundlePath) then: - bundle.fingerprint() == '7b2200ff24230f76cea22e5eb15b1701' - + bundle.getSingularityfile() == singularPath + and: + bundle + !bundle.hasEntries() + and: + bundle.fingerprint() == '6933e9238f3363c8e013a35715fa0540' + + when: + // changing file permissions, change the fingerprint + singularPath.setPermissions(6,0,0) + then: + bundle.fingerprint() == '3ffe7f16cd5ae17e6ba7485e01972b20' + } def 'should check max file size'() { diff --git a/plugins/nf-wave/build.gradle b/plugins/nf-wave/build.gradle index 6c37eba4b1..61255db8e3 100644 --- a/plugins/nf-wave/build.gradle +++ b/plugins/nf-wave/build.gradle @@ -36,6 +36,7 @@ dependencies { api 'org.apache.commons:commons-lang3:3.12.0' api 'com.google.code.gson:gson:2.10.1' api 'org.yaml:snakeyaml:2.0' + api 'io.seqera:wave-utils:0.6.2' testImplementation(testFixtures(project(":nextflow"))) testImplementation "org.codehaus.groovy:groovy:3.0.18" diff --git a/plugins/nf-wave/src/main/io/seqera/wave/config/CondaOpts.java b/plugins/nf-wave/src/main/io/seqera/wave/config/CondaOpts.java deleted file mode 100644 index fe36f8aa8c..0000000000 --- a/plugins/nf-wave/src/main/io/seqera/wave/config/CondaOpts.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2013-2023, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package io.seqera.wave.config; - -import java.util.List; -import java.util.Map; - -/** - * Conda build options - * - * @author Paolo Di Tommaso - */ -public class CondaOpts { - final public static String DEFAULT_MAMBA_IMAGE = "mambaorg/micromamba:1.4.9"; - - final public String mambaImage; - final public List commands; - final public String basePackages; - - public CondaOpts(Map opts) { - this.mambaImage = opts.containsKey("mambaImage") ? opts.get("mambaImage").toString(): DEFAULT_MAMBA_IMAGE; - this.commands = opts.containsKey("commands") ? (List)opts.get("commands") : null; - this.basePackages = (String)opts.get("basePackages"); - } - -} diff --git a/plugins/nf-wave/src/main/io/seqera/wave/config/SpackOpts.java b/plugins/nf-wave/src/main/io/seqera/wave/config/SpackOpts.java deleted file mode 100644 index 1ccabcf22e..0000000000 --- a/plugins/nf-wave/src/main/io/seqera/wave/config/SpackOpts.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2013-2023, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package io.seqera.wave.config; - -import java.util.List; -import java.util.Map; - -/** - * Spack build options - * - * @author Marco De La Pierre - */ -public class SpackOpts { - - /** - * Custom Dockerfile `RUN` commands that can be used to customise the target container - */ - public final List commands; - - /** - * Spack packages that should be added to any Spack environment requested via Wave - */ - public final String basePackages; - - public SpackOpts() { - this(Map.of()); - } - public SpackOpts(Map opts) { - this.commands = opts.containsKey("commands") ? (List)opts.get("commands") : null; - this.basePackages = opts.containsKey("basePackages") ? opts.get("basePackages").toString() : null; - } - -} diff --git a/plugins/nf-wave/src/main/io/seqera/wave/plugin/SubmitContainerTokenRequest.groovy b/plugins/nf-wave/src/main/io/seqera/wave/plugin/SubmitContainerTokenRequest.groovy index 875cdfe338..1a219ef79c 100644 --- a/plugins/nf-wave/src/main/io/seqera/wave/plugin/SubmitContainerTokenRequest.groovy +++ b/plugins/nf-wave/src/main/io/seqera/wave/plugin/SubmitContainerTokenRequest.groovy @@ -111,4 +111,9 @@ class SubmitContainerTokenRequest { */ boolean freeze + /** + * Specify the format of the container file + */ + String format + } diff --git a/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveAssets.groovy b/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveAssets.groovy index 4541bf72b0..8201cdafb9 100644 --- a/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveAssets.groovy +++ b/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveAssets.groovy @@ -36,10 +36,11 @@ class WaveAssets { final String containerPlatform final ResourcesBundle moduleResources final ContainerConfig containerConfig - final String dockerFileContent + final String containerFile final Path condaFile final Path spackFile final ResourcesBundle projectResources + final boolean singularity static fromImage(String containerImage,String containerPlatform=null) { new WaveAssets(containerImage, containerPlatform) @@ -50,8 +51,8 @@ class WaveAssets { } String dockerFileEncoded() { - return dockerFileContent - ? dockerFileContent.bytes.encodeBase64() + return containerFile + ? containerFile.bytes.encodeBase64() : null } @@ -73,7 +74,7 @@ class WaveAssets { allMeta.add( this.containerImage ) allMeta.add( this.moduleResources?.fingerprint() ) allMeta.add( this.containerConfig?.fingerprint() ) - allMeta.add( this.dockerFileContent ) + allMeta.add( this.containerFile ) allMeta.add( this.condaFile?.text ) allMeta.add( this.spackFile?.text ) allMeta.add( this.projectResources?.fingerprint() ) diff --git a/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveClient.groovy b/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveClient.groovy index d189704e86..bf87c02135 100644 --- a/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveClient.groovy +++ b/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveClient.groovy @@ -165,10 +165,10 @@ class WaveClient { containerConfig.prependLayer(makeLayer(assets.projectResources)) } - if( !assets.containerImage && !assets.dockerFileContent ) + if( !assets.containerImage && !assets.containerFile ) throw new IllegalArgumentException("Wave container request requires at least a image or container file to build") - if( assets.containerImage && assets.dockerFileContent ) + if( assets.containerImage && assets.containerFile ) throw new IllegalArgumentException("Wave container image and container file cannot be specified in the same request") return new SubmitContainerTokenRequest( @@ -182,7 +182,8 @@ class WaveClient { cacheRepository: config.cacheRepository(), timestamp: OffsetDateTime.now().toString(), fingerprint: assets.fingerprint(), - freeze: config.freezeMode() + freeze: config.freezeMode(), + format: assets.singularity ? 'sif' : null ) } @@ -311,24 +312,29 @@ class WaveClient { } protected void checkConflicts(Map attrs, String name) { - if( attrs.dockerfile && attrs.conda ) { - throw new IllegalArgumentException("Process '${name}' declares both a 'conda' directive and a module bundle dockerfile that conflict each other") - } - if( attrs.container && attrs.dockerfile ) { - throw new IllegalArgumentException("Process '${name}' declares both a 'container' directive and a module bundle dockerfile that conflict each other") - } if( attrs.container && attrs.conda ) { throw new IllegalArgumentException("Process '${name}' declares both 'container' and 'conda' directives that conflict each other") } - if( attrs.dockerfile && attrs.spack ) { - throw new IllegalArgumentException("Process '${name}' declares both a 'spack' directive and a module bundle dockerfile that conflict each other") - } if( attrs.container && attrs.spack ) { throw new IllegalArgumentException("Process '${name}' declares both 'container' and 'spack' directives that conflict each other") } if( attrs.spack && attrs.conda ) { throw new IllegalArgumentException("Process '${name}' declares both 'spack' and 'conda' directives that conflict each other") } + checkConflicts0(attrs, name, 'dockerfile') + checkConflicts0(attrs, name, 'singularityfile') + } + + protected void checkConflicts0(Map attrs, String name, String fileType) { + if( attrs.get(fileType) && attrs.conda ) { + throw new IllegalArgumentException("Process '${name}' declares both a 'conda' directive and a module bundle $fileType that conflict each other") + } + if( attrs.container && attrs.get(fileType) ) { + throw new IllegalArgumentException("Process '${name}' declares both a 'container' directive and a module bundle $fileType that conflict each other") + } + if( attrs.get(fileType) && attrs.spack ) { + throw new IllegalArgumentException("Process '${name}' declares both a 'spack' directive and a module bundle $fileType that conflict each other") + } } Map resolveConflicts(Map attrs, List strategy) { @@ -341,6 +347,21 @@ class WaveClient { return result } + protected List patchStrategy(List strategy, boolean singularity) { + if( !singularity ) + return strategy + // when singularity is enabled, replaces `dockerfile` with `singularityfile` + // in the strategy if not specified explicitly + final p = strategy.indexOf('dockerfile') + if( p!=-1 && !strategy.contains('singularityfile') ) { + final result = new ArrayList(strategy) + result.remove(p) + result.add(p, 'singularityfile') + return Collections.unmodifiableList(result) + } + return strategy + } + static Architecture defaultArch() { try { return new Architecture(SysHelper.getArch()) @@ -352,7 +373,7 @@ class WaveClient { } @Memoized - WaveAssets resolveAssets(TaskRun task, String containerImage) { + WaveAssets resolveAssets(TaskRun task, String containerImage, boolean singularity) { // get the bundle final bundle = task.getModuleBundle() // get the Spack architecture @@ -367,49 +388,60 @@ class WaveClient { if( bundle!=null && bundle.dockerfile ) { attrs.dockerfile = bundle.dockerfile.text } + if( bundle!=null && bundle.singularityfile ) { + attrs.singularityfile = bundle.singularityfile.text + } // validate request attributes - if( config().strategy() ) - attrs = resolveConflicts(attrs, config().strategy()) + final strategy = config().strategy() + if( strategy ) + attrs = resolveConflicts(attrs, patchStrategy(strategy, singularity)) else checkConflicts(attrs, task.lazyName()) // resolve the wave assets - return resolveAssets0(attrs, bundle, dockerArch, spackArch) + return resolveAssets0(attrs, bundle, singularity, dockerArch, spackArch) } - protected WaveAssets resolveAssets0(Map attrs, ResourcesBundle bundle, String dockerArch, String spackArch) { + protected WaveAssets resolveAssets0(Map attrs, ResourcesBundle bundle, boolean singularity, String dockerArch, String spackArch) { - String dockerScript = attrs.dockerfile + final scriptType = singularity ? 'singularityfile' : 'dockerfile' + String containerScript = attrs.get(scriptType) final containerImage = attrs.container /* - * If 'conda' directive is specified use it to create a Dockefile + * If 'conda' directive is specified use it to create a container file * to assemble the target container */ Path condaFile = null if( attrs.conda ) { - if( dockerScript ) - throw new IllegalArgumentException("Unexpected conda and dockerfile conflict while resolving wave container") + if( containerScript ) + throw new IllegalArgumentException("Unexpected conda and $scriptType conflict while resolving wave container") // map the recipe to a dockerfile if( isCondaLocalFile(attrs.conda) ) { condaFile = Path.of(attrs.conda) - dockerScript = condaFileToDockerFile(config.condaOpts()) + containerScript = singularity + ? condaFileToSingularityFile(config.condaOpts()) + : condaFileToDockerFile(config.condaOpts()) } // 'conda' attributes is resolved as the conda packages to be used else { - dockerScript = condaPackagesToDockerFile(attrs.conda, condaChannels, config.condaOpts()) + containerScript = singularity + ? condaPackagesToSingularityFile(attrs.conda, condaChannels, config.condaOpts()) + : condaPackagesToDockerFile(attrs.conda, condaChannels, config.condaOpts()) } } /* - * If 'spack' directive is specified use it to create a Dockefile + * If 'spack' directive is specified use it to create a container file * to assemble the target container */ Path spackFile = null if( attrs.spack ) { - if( dockerScript ) + if( singularity ) + throw new IllegalArgumentException("Wave containers do not support (yet) the resolution of Spack package with Singularity") + if( containerScript ) throw new IllegalArgumentException("Unexpected spack and dockerfile conflict while resolving wave container") if( isSpackFile(attrs.spack) ) { @@ -420,14 +452,14 @@ class WaveClient { // create a minimal spack file with package spec from user input spackFile = spackPackagesToSpackFile(attrs.spack, config.spackOpts()) } - dockerScript = spackFileToDockerFile(config.spackOpts()) + containerScript = spackFileToDockerFile(config.spackOpts()) } /* * The process should declare at least a container image name via 'container' directive * or a dockerfile file to build, otherwise there's no job to be done by wave */ - if( !dockerScript && !containerImage ) { + if( !containerScript && !containerImage ) { return null } @@ -451,10 +483,11 @@ class WaveClient { platform, bundle, containerConfig, - dockerScript, + containerScript, condaFile, spackFile, - projectRes) + projectRes, + singularity) } @Memoized diff --git a/plugins/nf-wave/src/main/io/seqera/wave/plugin/resolver/WaveContainerResolver.groovy b/plugins/nf-wave/src/main/io/seqera/wave/plugin/resolver/WaveContainerResolver.groovy index 643e8dc755..46b00c464c 100644 --- a/plugins/nf-wave/src/main/io/seqera/wave/plugin/resolver/WaveContainerResolver.groovy +++ b/plugins/nf-wave/src/main/io/seqera/wave/plugin/resolver/WaveContainerResolver.groovy @@ -59,17 +59,19 @@ class WaveContainerResolver implements ContainerResolver { if( !client().enabled() ) return defaultResolver.resolveImage(task, imageName) + final freeze = client().config().freezeMode() + final engine= task.getContainerConfig().getEngine() + final nativeSingularityBuild = freeze && engine in SINGULARITY_LIKE if( !imageName ) { // when no image name is provided the module bundle should include a // Dockerfile or a Conda recipe or a Spack recipe to build // an image on-fly with an automatically assigned name - return waveContainer(task, null) + return waveContainer(task, null, nativeSingularityBuild) } - final engine= task.getContainerConfig().getEngine() if( engine in DOCKER_LIKE ) { final image = defaultResolver.resolveImage(task, imageName) - return waveContainer(task, image.target) + return waveContainer(task, image.target, false) } else if( engine in SINGULARITY_LIKE ) { // remove any `docker://` prefix if any @@ -80,8 +82,11 @@ class WaveContainerResolver implements ContainerResolver { return defaultResolver.resolveImage(task, imageName) } // fetch the wave container name - final image = waveContainer(task, imageName) - // then adapt it to singularity format + final image = waveContainer(task, imageName, nativeSingularityBuild) + // oras prefixed container are served directly + if( image && image.target.startsWith("oras://") ) + return image + // otherwise adapt it to singularity format return defaultResolver.resolveImage(task, image.target) } else @@ -102,9 +107,9 @@ class WaveContainerResolver implements ContainerResolver { * The container image name returned by the Wave backend or {@code null} * when the task does not request any container or dockerfile to build */ - protected ContainerInfo waveContainer(TaskRun task, String container) { + protected ContainerInfo waveContainer(TaskRun task, String container, boolean singularity) { validateContainerRepo(container) - final assets = client().resolveAssets(task, container) + final assets = client().resolveAssets(task, container, singularity) if( assets ) { return client().fetchContainerImage(assets) } diff --git a/plugins/nf-wave/src/main/io/seqera/wave/util/DockerHelper.java b/plugins/nf-wave/src/main/io/seqera/wave/util/DockerHelper.java deleted file mode 100644 index 33b844a3c8..0000000000 --- a/plugins/nf-wave/src/main/io/seqera/wave/util/DockerHelper.java +++ /dev/null @@ -1,256 +0,0 @@ -/* - * Copyright 2013-2023, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package io.seqera.wave.util; - -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.io.IOException; -import java.io.InputStream; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.io.File; - -import io.seqera.wave.config.CondaOpts; -import io.seqera.wave.config.SpackOpts; -import org.apache.commons.lang3.StringUtils; -import org.yaml.snakeyaml.Yaml; - -/** - * Helper class to create Dockerfile for Conda and Spack package managers - * - * @author Paolo Di Tommaso - */ -public class DockerHelper { - - static public List spackPackagesToList(String packages) { - if( packages==null || packages.isEmpty() ) - return null; - final List entries = Arrays.asList(packages.split(" ")); - final List result = new ArrayList<>(); - List current = new ArrayList<>(); - for( String it : entries ) { - if( it==null || it.isEmpty() || it.isBlank() ) - continue; - if( !Character.isLetterOrDigit(it.charAt(0)) || it.contains("=") ) { - current.add(it); - } - else { - if( current.size()>0 ) - result.add(String.join(" ",current)); - current = new ArrayList<>(); - current.add(it); - } - } - // remaining entries - if( current.size()>0 ) - result.add(String.join(" ",current)); - return result; - } - - static public String spackPackagesToSpackYaml(String packages, SpackOpts opts) { - final List base = spackPackagesToList(opts.basePackages); - final List custom = spackPackagesToList(packages); - if( base==null && custom==null ) - return null; - - final List specs = new ArrayList<>(); - if( base!=null ) - specs.addAll(base); - if( custom!=null ) - specs.addAll(custom); - - final Map concretizer = new LinkedHashMap<>(); - concretizer.put("unify", true); - concretizer.put("reuse", false); - - final Map spack = new LinkedHashMap<>(); - spack.put("specs", specs); - spack.put("concretizer", concretizer); - - final Map root = new LinkedHashMap<>(); - root.put("spack", spack); - - return new Yaml().dump(root); - } - - static public Path spackPackagesToSpackFile(String packages, SpackOpts opts) { - final String yaml = spackPackagesToSpackYaml(packages, opts); - if( yaml==null || yaml.length()==0 ) - return null; - return toYamlFile(yaml); - } - - static private Path toYamlFile(String yaml) { - try { - final File tempFile = File.createTempFile("nf-spack", ".yaml"); - tempFile.deleteOnExit(); - final Path result = tempFile.toPath(); - Files.write(result, yaml.getBytes()); - return result; - } - catch (IOException e) { - throw new IllegalStateException("Unable to write temporary Spack environment file - Reason: " + e.getMessage(), e); - } - } - - static public String spackFileToDockerFile(SpackOpts opts) { - // create bindings - final Map binding = spackBinding(opts); - // final ignored variables - final List ignore = List.of("spack_runner_image"); - // return the template - return renderTemplate0("/templates/spack/dockerfile-spack-file.txt", binding, ignore); - } - - static private Map spackBinding(SpackOpts opts) { - final Map binding = new HashMap<>(); - binding.put("add_commands", joinCommands(opts.commands)); - return binding; - } - - static public String condaPackagesToDockerFile(String packages, List condaChannels, CondaOpts opts) { - final List channels0 = condaChannels!=null ? condaChannels : List.of(); - final String channelsOpts = channels0.stream().map(it -> "-c "+it).collect(Collectors.joining(" ")); - final String image = opts.mambaImage; - final String target = packages.startsWith("http://") || packages.startsWith("https://") - ? "-f " + packages - : packages; - final Map binding = new HashMap<>(); - binding.put("base_image", image); - binding.put("channel_opts", channelsOpts); - binding.put("target", target); - binding.put("base_packages", mambaInstallBasePackage0(opts.basePackages)); - - final String result = renderTemplate0("/templates/conda/dockerfile-conda-packages.txt", binding) ; - return addCommands(result, opts.commands); - } - - static public String condaFileToDockerFile(CondaOpts opts) { - // create the binding map - final Map binding = new HashMap<>(); - binding.put("base_image", opts.mambaImage); - binding.put("base_packages", mambaInstallBasePackage0(opts.basePackages)); - - final String result = renderTemplate0("/templates/conda/dockerfile-conda-file.txt", binding); - return addCommands(result, opts.commands); - } - - static private String renderTemplate0(String templatePath, Map binding) { - return renderTemplate0(templatePath, binding, List.of()); - } - - static private String renderTemplate0(String templatePath, Map binding, List ignore) { - final URL template = DockerHelper.class.getResource(templatePath); - if( template==null ) - throw new IllegalStateException(String.format("Unable to load template '%s' from classpath", templatePath)); - try { - final InputStream reader = template.openStream(); - return new TemplateRenderer() - .withIgnore(ignore) - .render(reader, binding); - } - catch (IOException e) { - throw new IllegalStateException(String.format("Unable to read classpath template '%s'", templatePath), e); - } - } - - private static String mambaInstallBasePackage0(String basePackages) { - return !StringUtils.isEmpty(basePackages) - ? String.format("&& micromamba install -y -n base %s \\", basePackages) - : null; - } - - static private String addCommands(String result, List commands) { - if( commands==null || commands.isEmpty() ) - return result; - for( String cmd : commands ) { - result += cmd + "\n"; - } - return result; - } - - static private String joinCommands(List commands) { - if( commands==null || commands.size()==0 ) - return null; - StringBuilder result = new StringBuilder(); - for( String cmd : commands ) { - if( result.length()>0 ) - result.append("\n"); - result.append(cmd); - } - return result.toString(); - } - - public static Path addPackagesToSpackFile(String spackFile, SpackOpts opts) { - // Case A - both empty, nothing to do - if( StringUtils.isEmpty(spackFile) && StringUtils.isEmpty(opts.basePackages) ) - return null; - - // Case B - the spack file is empty, but some base package are given - // create a spack file with those packages - if( StringUtils.isEmpty(spackFile) ) { - return spackPackagesToSpackFile(null, opts); - } - - final Path spackEnvPath = Path.of(spackFile); - - // make sure the file exists - if( !Files.exists(spackEnvPath) ) { - throw new IllegalArgumentException("The specific Spack environment file cannot be found: " + spackFile); - } - - // Case C - if not base packages are given just return the spack file as a path - if( StringUtils.isEmpty(opts.basePackages) ) { - return spackEnvPath; - } - - // Case D - last case, both spack file and base packages are specified - // => parse the spack file yaml, add the base packages to it - final Yaml yaml = new Yaml(); - try { - // 1. parse the file - Map data = yaml.load(new FileReader(spackFile)); - // 2. parse the base packages - final List base = spackPackagesToList(opts.basePackages); - // 3. append to the specs - Map spack = (Map) data.get("spack"); - if( spack==null ) { - throw new IllegalArgumentException("The specified Spack environment file does not contain a root entry 'spack:' - offending file path: " + spackFile); - } - List specs = (List)spack.get("specs"); - if( specs==null ) { - specs = new ArrayList<>(); - spack.put("specs", specs); - } - specs.addAll(base); - // 5. return it as a new temp file - return toYamlFile( yaml.dump(data) ); - } - catch (FileNotFoundException e) { - throw new IllegalArgumentException("The specific Spack environment file cannot be found: " + spackFile, e); - } - } -} diff --git a/plugins/nf-wave/src/main/io/seqera/wave/util/TemplateRenderer.java b/plugins/nf-wave/src/main/io/seqera/wave/util/TemplateRenderer.java deleted file mode 100644 index af9d4b8f95..0000000000 --- a/plugins/nf-wave/src/main/io/seqera/wave/util/TemplateRenderer.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Copyright 2013-2023, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package io.seqera.wave.util; - -import java.io.InputStream; -import java.util.List; -import java.util.Map; -import java.util.Scanner; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Template rendering helper - * - * @author Paolo Di Tommaso - */ -public class TemplateRenderer { - - private static final Pattern PATTERN = Pattern.compile("\\{\\{([^}]+)}}"); - - private static final Pattern VAR1 = Pattern.compile("(\\s*)\\{\\{([\\d\\w_-]+)}}(\\s*$)"); - private static final Pattern VAR2 = Pattern.compile("(? ignoreNames = List.of(); - - public TemplateRenderer withIgnore(String... names) { - return withIgnore(List.of(names)); - } - - public TemplateRenderer withIgnore(List names) { - if( names!=null ) { - ignoreNames = List.copyOf(names); - } - return this; - } - - public String render(InputStream template, Map binding) { - String str = new Scanner(template).useDelimiter("\\A").next(); - return render(str, binding); - } - - public String render(String template, Map binding) { - final String[] lines = template.split("(?<=\n)"); - final StringBuilder result = new StringBuilder(); - for( String it : lines ) { - if( it==null || it.startsWith("##")) - continue; - final String resolved = replace0(it, binding); - if( resolved!=null ) - result.append(resolved); - } - return result.toString(); - } - - /** - * Simple template helper class replacing all variable enclosed by {{..}} - * with the corresponding variable specified in a map object - * - * @param template The template string - * @param binding The binding {@link Map} - * @return The templated having the variables replaced with the corresponding value - */ - String replace1(CharSequence template, Map binding) { - Matcher matcher = PATTERN.matcher(template); - - // Loop through each matched variable placeholder - StringBuilder builder = new StringBuilder(); - boolean isNull=false; - while (matcher.find()) { - String variable = matcher.group(1); - - // Check if the variable exists in the values map - if (binding.containsKey(variable)) { - Object value = binding.get(variable); - String str = value!=null ? value.toString() : ""; - isNull |= value==null; - matcher.appendReplacement(builder, str); - } - else if( !ignoreNames.contains(variable) ) { - throw new IllegalArgumentException(String.format("Unable to resolve template variable: {{%s}}", variable)); - } - } - matcher.appendTail(builder); - - final String result = builder.toString(); - return !isNull || !result.isBlank() ? result : null; - } - - String replace0(String line, Map binding) { - if( line==null || line.length()==0 ) - return line; - - Matcher matcher = VAR1.matcher(line); - if( matcher.matches() ) { - final String name = matcher.group(2); - if( ignoreNames.contains(name) ) - return line; - if( !binding.containsKey(name) ) - throw new IllegalArgumentException("Missing template key: "+name); - final String prefix = matcher.group(1); - final String value = binding.get(name); - if( value==null ) - return null; // <-- return null to skip this line - - final StringBuilder result = new StringBuilder(); - final String[] multi = value.split("(?<=\n)"); - for (String s : multi) { - result.append(prefix); - result.append(s); - } - result.append( matcher.group(3) ); - return result.toString(); - } - - final StringBuilder result = new StringBuilder(); - while( (matcher=VAR2.matcher(line)).find() ) { - final String name = matcher.group(1); - if( !binding.containsKey(name) && !ignoreNames.contains(name)) { - throw new IllegalArgumentException("Missing template key: "+name); - } - final String value = !ignoreNames.contains(name) - ? (binding.get(name)!=null ? binding.get(name) : "") - : "{{"+name+"}}"; - final int p = matcher.start(1); - final int q = matcher.end(1); - - result.append(line.substring(0,p-2)); - result.append(value); - line = line.substring(q+2); - } - result.append(line); - return result.toString(); - } - -} diff --git a/plugins/nf-wave/src/resources/templates/conda/dockerfile-conda-file.txt b/plugins/nf-wave/src/resources/templates/conda/dockerfile-conda-file.txt deleted file mode 100644 index d9be54e326..0000000000 --- a/plugins/nf-wave/src/resources/templates/conda/dockerfile-conda-file.txt +++ /dev/null @@ -1,6 +0,0 @@ -FROM {{base_image}} -COPY --chown=$MAMBA_USER:$MAMBA_USER conda.yml /tmp/conda.yml -RUN micromamba install -y -n base -f /tmp/conda.yml \ - {{base_packages}} - && micromamba clean -a -y -USER root diff --git a/plugins/nf-wave/src/resources/templates/conda/dockerfile-conda-packages.txt b/plugins/nf-wave/src/resources/templates/conda/dockerfile-conda-packages.txt deleted file mode 100644 index 67e01b4378..0000000000 --- a/plugins/nf-wave/src/resources/templates/conda/dockerfile-conda-packages.txt +++ /dev/null @@ -1,7 +0,0 @@ -FROM {{base_image}} -RUN \ - micromamba install -y -n base {{channel_opts}} \ - {{target}} \ - {{base_packages}} - && micromamba clean -a -y -USER root diff --git a/plugins/nf-wave/src/resources/templates/spack/dockerfile-spack-file.txt b/plugins/nf-wave/src/resources/templates/spack/dockerfile-spack-file.txt deleted file mode 100644 index bf6e436b15..0000000000 --- a/plugins/nf-wave/src/resources/templates/spack/dockerfile-spack-file.txt +++ /dev/null @@ -1,18 +0,0 @@ -# Runner image -FROM {{spack_runner_image}} - -COPY --from=builder /opt/spack-env /opt/spack-env -COPY --from=builder /opt/software /opt/software -COPY --from=builder /opt/._view /opt/._view - -# Entrypoint for Singularity -RUN mkdir -p /.singularity.d/env && \ - cp -p /opt/spack-env/z10_spack_environment.sh /.singularity.d/env/91-environment.sh -# Entrypoint for Docker -RUN echo "#!/usr/bin/env bash\n\nset -ef -o pipefail\nsource /opt/spack-env/z10_spack_environment.sh\nexec \"\$@\"" \ - >/opt/spack-env/spack_docker_entrypoint.sh && chmod a+x /opt/spack-env/spack_docker_entrypoint.sh - -{{add_commands}} - -ENTRYPOINT [ "/opt/spack-env/spack_docker_entrypoint.sh" ] -CMD [ "/bin/bash" ] diff --git a/plugins/nf-wave/src/test/io/seqera/wave/config/CondaOptsTest.groovy b/plugins/nf-wave/src/test/io/seqera/wave/config/CondaOptsTest.groovy deleted file mode 100644 index 055259888a..0000000000 --- a/plugins/nf-wave/src/test/io/seqera/wave/config/CondaOptsTest.groovy +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2013-2023, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package io.seqera.wave.config - -import spock.lang.Specification - -/** - * - * @author Paolo Di Tommaso - */ -class CondaOptsTest extends Specification { - - def 'check conda options' () { - when: - def opts = new CondaOpts([:]) - then: - opts.mambaImage == CondaOpts.DEFAULT_MAMBA_IMAGE - !opts.basePackages - !opts.commands - - when: - opts = new CondaOpts([ - mambaImage:'foo:latest', - commands: ['this','that'], - basePackages: 'some::more-package' - ]) - then: - opts.mambaImage == 'foo:latest' - opts.basePackages == 'some::more-package' - opts.commands == ['this','that'] - } -} diff --git a/plugins/nf-wave/src/test/io/seqera/wave/config/SpackOptsTest.groovy b/plugins/nf-wave/src/test/io/seqera/wave/config/SpackOptsTest.groovy deleted file mode 100644 index f3e5f5a955..0000000000 --- a/plugins/nf-wave/src/test/io/seqera/wave/config/SpackOptsTest.groovy +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2013-2023, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package io.seqera.wave.config - -import spock.lang.Specification - -/** - * - * @author Paolo Di Tommaso - */ -class SpackOptsTest extends Specification { - - def 'check spack default options' () { - given: - def opts = new SpackOpts() - expect: - opts.commands == null - opts.basePackages == null - } - - def 'check spack custom opts' () { - given: - def opts = new SpackOpts([ - basePackages: 'foo bar', - commands: ['run','--this','--that'] - ]) - - expect: - opts.commands == ['run','--this','--that'] - and: - opts.basePackages == 'foo bar' - } -} diff --git a/plugins/nf-wave/src/test/io/seqera/wave/plugin/WaveClientTest.groovy b/plugins/nf-wave/src/test/io/seqera/wave/plugin/WaveClientTest.groovy index 8e5bcb31df..6b6446cc0a 100644 --- a/plugins/nf-wave/src/test/io/seqera/wave/plugin/WaveClientTest.groovy +++ b/plugins/nf-wave/src/test/io/seqera/wave/plugin/WaveClientTest.groovy @@ -231,6 +231,33 @@ class WaveClientTest extends Specification { !req.containerConfig.layers } + def 'should create request object with singularityfile' () { + given: + def session = Mock(Session) { getConfig() >> [:]} + def SINGULARITY_FILE = 'From foo:latest' + def wave = new WaveClient(session) + and: + def assets = new WaveAssets(null, + 'linux/amd64', + null, + null, + SINGULARITY_FILE, + null, + null, + null, + true) + when: + def req = wave.makeRequest(assets) + then: + !req.containerImage + new String(req.containerFile.decodeBase64()) == SINGULARITY_FILE + !req.condaFile + !req.spackFile + !req.containerConfig.layers + and: + req.format == 'sif' + } + def 'should create request object with build and cache repos' () { given: def session = Mock(Session) { getConfig() >> [wave:[build:[repository:'some/repo',cacheRepository:'some/cache']]]} @@ -347,11 +374,11 @@ class WaveClientTest extends Specification { def client = new WaveClient(session) when: - def assets = client.resolveAssets(task, IMAGE) + def assets = client.resolveAssets(task, IMAGE, false) then: assets.containerImage == IMAGE !assets.moduleResources - !assets.dockerFileContent + !assets.containerFile !assets.containerConfig !assets.condaFile !assets.spackFile @@ -369,12 +396,12 @@ class WaveClientTest extends Specification { def client = new WaveClient(session) when: - def assets = client.resolveAssets(task, IMAGE) + def assets = client.resolveAssets(task, IMAGE, false) then: assets.containerImage == IMAGE assets.containerPlatform == 'linux/arm64' !assets.moduleResources - !assets.dockerFileContent + !assets.containerFile !assets.containerConfig !assets.condaFile !assets.spackFile @@ -392,11 +419,11 @@ class WaveClientTest extends Specification { def client = new WaveClient(session) when: - def assets = client.resolveAssets(task, IMAGE) + def assets = client.resolveAssets(task, IMAGE, false) then: assets.containerImage == IMAGE assets.moduleResources == BUNDLE - !assets.dockerFileContent + !assets.containerFile !assets.containerConfig !assets.condaFile !assets.spackFile @@ -416,7 +443,7 @@ class WaveClientTest extends Specification { WaveClient client = Spy(WaveClient, constructorArgs:[session]) when: - def assets = client.resolveAssets(task, IMAGE) + def assets = client.resolveAssets(task, IMAGE, false) then: client.resolveContainerConfig(ARCH) >> CONTAINER_CONFIG and: @@ -424,7 +451,7 @@ class WaveClientTest extends Specification { assets.moduleResources == BUNDLE assets.containerConfig == CONTAINER_CONFIG and: - !assets.dockerFileContent + !assets.containerFile !assets.condaFile !assets.spackFile !assets.projectResources @@ -445,9 +472,9 @@ class WaveClientTest extends Specification { def client = new WaveClient(session) when: - def assets = client.resolveAssets(task, null) + def assets = client.resolveAssets(task, null, false) then: - assets.dockerFileContent == 'FROM foo\nRUN this/that' + assets.containerFile == 'FROM foo\nRUN this/that' assets.moduleResources == BUNDLE !assets.containerImage !assets.containerConfig @@ -468,9 +495,9 @@ class WaveClientTest extends Specification { def client = new WaveClient(session) when: - def assets = client.resolveAssets(task, null) + def assets = client.resolveAssets(task, null, false) then: - assets.dockerFileContent == '''\ + assets.containerFile == '''\ FROM mambaorg/micromamba:1.4.9 RUN \\ micromamba install -y -n base -c conda-forge -c defaults \\ @@ -496,9 +523,9 @@ class WaveClientTest extends Specification { def client = new WaveClient(session) when: - def assets = client.resolveAssets(task, null) + def assets = client.resolveAssets(task, null, false) then: - assets.dockerFileContent == '''\ + assets.containerFile == '''\ # Runner image FROM {{spack_runner_image}} @@ -538,9 +565,9 @@ class WaveClientTest extends Specification { def client = new WaveClient(session) when: - def assets = client.resolveAssets(task, null) + def assets = client.resolveAssets(task, null, false) then: - assets.dockerFileContent == '''\ + assets.containerFile == '''\ FROM mambaorg/micromamba:1.4.9 COPY --chown=$MAMBA_USER:$MAMBA_USER conda.yml /tmp/conda.yml RUN micromamba install -y -n base -f /tmp/conda.yml \\ @@ -571,9 +598,9 @@ class WaveClientTest extends Specification { def client = new WaveClient(session) when: - def assets = client.resolveAssets(task, null) + def assets = client.resolveAssets(task, null, false) then: - assets.dockerFileContent == '''\ + assets.containerFile == '''\ # Runner image FROM {{spack_runner_image}} @@ -605,6 +632,78 @@ class WaveClientTest extends Specification { folder?.deleteDir() } + // ==== singularity native build + conda ==== + + def 'should create asset with conda recipe and singularity native build' () { + given: + def session = Mock(Session) { getConfig() >> [:]} + and: + def task = Mock(TaskRun) {getConfig() >> [conda:'salmon=1.2.3'] } + and: + def client = new WaveClient(session) + + when: + def assets = client.resolveAssets(task, null, true) + then: + assets.containerFile == '''\ + BootStrap: docker + From: mambaorg/micromamba:1.4.9 + %post + micromamba install -y -n base -c conda-forge -c defaults \\ + salmon=1.2.3 \\ + && micromamba clean -a -y + %environment + export PATH="$MAMBA_ROOT_PREFIX/bin:$PATH" + '''.stripIndent() + and: + assets.singularity + and: + !assets.moduleResources + !assets.containerImage + !assets.containerConfig + !assets.condaFile + !assets.spackFile + !assets.projectResources + } + + def 'should create asset with conda file and singularity native build' () { + given: + def folder = Files.createTempDirectory('test') + def condaFile = folder.resolve('conda.yml'); condaFile.text = 'the-conda-recipe-here' + and: + def session = Mock(Session) { getConfig() >> [:]} + def task = Mock(TaskRun) {getConfig() >> [conda:condaFile.toString()] } + and: + def client = new WaveClient(session) + + when: + def assets = client.resolveAssets(task, null, true) + then: + assets.containerFile == '''\ + BootStrap: docker + From: mambaorg/micromamba:1.4.9 + %files + {{wave_context_dir}}/conda.yml /tmp/conda.yml + %post + micromamba install -y -n base -f /tmp/conda.yml \\ + && micromamba clean -a -y + %environment + export PATH="$MAMBA_ROOT_PREFIX/bin:$PATH" + '''.stripIndent() + and: + assets.condaFile == condaFile + assets.singularity + and: + !assets.moduleResources + !assets.containerImage + !assets.containerConfig + !assets.spackFile + !assets.projectResources + + cleanup: + folder?.deleteDir() + } + def 'should create assets with project resources' () { given: def MODULE_RES = Mock(ResourcesBundle) @@ -623,7 +722,7 @@ class WaveClientTest extends Specification { WaveClient wave = Spy(WaveClient, constructorArgs: [session]) when: - def assets = wave.resolveAssets(task, 'image:latest') + def assets = wave.resolveAssets(task, 'image:latest', false) then: 1 * wave.projectResources(BIN_DIR) >> PROJECT_RES and: @@ -666,6 +765,23 @@ class WaveClientTest extends Specification { result == [spack:'x'] } + def 'should patch strategy for singularity' () { + given: + def session = Mock(Session) { getConfig() >> [:]} + and: + def client = new WaveClient(session) + + expect: + client.patchStrategy(Collections.unmodifiableList(STRATEGY), SING) == EXPECTED + + where: + STRATEGY | SING | EXPECTED + ['conda','dockerfile', 'spack'] | false | ['conda','dockerfile', 'spack'] + ['conda','dockerfile', 'spack'] | true | ['conda','singularityfile', 'spack'] + ['conda','dockerfile', 'spack'] | true | ['conda','singularityfile', 'spack'] + ['conda','singularityfile','dockerfile', 'spack'] | true | ['conda','singularityfile','dockerfile', 'spack'] + } + def 'should check conflicts' () { given: def session = Mock(Session) { getConfig() >> [:]} @@ -713,6 +829,25 @@ class WaveClientTest extends Specification { e = thrown(IllegalArgumentException) e.message == "Process 'foo' declares both 'spack' and 'conda' directives that conflict each other" + // singularity file checks + when: + client.checkConflicts([conda:'this', singularityfile:'that'], 'foo') + then: + e = thrown(IllegalArgumentException) + e.message == "Process 'foo' declares both a 'conda' directive and a module bundle singularityfile that conflict each other" + + when: + client.checkConflicts([container:'this', singularityfile:'that'], 'foo') + then: + e = thrown(IllegalArgumentException) + e.message == "Process 'foo' declares both a 'container' directive and a module bundle singularityfile that conflict each other" + + when: + client.checkConflicts([spack:'this', singularityfile:'that'], 'foo') + then: + e = thrown(IllegalArgumentException) + e.message == "Process 'foo' declares both a 'spack' directive and a module bundle singularityfile that conflict each other" + } def 'should get project resource bundle' () { @@ -762,7 +897,6 @@ class WaveClientTest extends Specification { assert (it[0] as SubmitContainerTokenRequest).towerWorkspaceId == 123 assert (it[0] as SubmitContainerTokenRequest).towerEndpoint == 'http://foo.com' } - } def 'should send request with tower access token and refresh token' () { diff --git a/plugins/nf-wave/src/test/io/seqera/wave/plugin/resolver/WaveContainerResolverTest.groovy b/plugins/nf-wave/src/test/io/seqera/wave/plugin/resolver/WaveContainerResolverTest.groovy index 7825dd0540..da190d728e 100644 --- a/plugins/nf-wave/src/test/io/seqera/wave/plugin/resolver/WaveContainerResolverTest.groovy +++ b/plugins/nf-wave/src/test/io/seqera/wave/plugin/resolver/WaveContainerResolverTest.groovy @@ -18,6 +18,7 @@ package io.seqera.wave.plugin.resolver import io.seqera.wave.plugin.WaveClient +import io.seqera.wave.plugin.config.WaveConfig import nextflow.container.ContainerConfig import nextflow.container.resolver.ContainerInfo import nextflow.container.resolver.DefaultContainerResolver @@ -35,6 +36,7 @@ class WaveContainerResolverTest extends Specification { given: def CONTAINER_NAME = "ubuntu:latest" def WAVE_CONTAINER = new ContainerInfo(CONTAINER_NAME, "wave.io/ubuntu:latest", "12345") + def ORAS_CONTAINER = new ContainerInfo(CONTAINER_NAME, "oras://wave.io/ubuntu:latest", "12345") def SINGULARITY_CONTAINER = new ContainerInfo('ubuntu:latest', '/some/singularity/ubuntu.img') and: def defaultResolver = Spy(DefaultContainerResolver) @@ -46,27 +48,40 @@ class WaveContainerResolverTest extends Specification { } } + // docker images when: def result = resolver.resolveImage(task, CONTAINER_NAME) then: - resolver.client() >> Mock(WaveClient) { enabled()>>true } + resolver.client() >> Mock(WaveClient) { enabled()>>true; config()>>Mock(WaveConfig) } _ * task.getContainerConfig() >> Mock(ContainerConfig) { getEngine()>>'docker' } and: - 1 * resolver.waveContainer(task, CONTAINER_NAME) >> WAVE_CONTAINER + 1 * resolver.waveContainer(task, CONTAINER_NAME, false) >> WAVE_CONTAINER and: result == WAVE_CONTAINER - + // singularity images when: result = resolver.resolveImage(task, CONTAINER_NAME) then: - resolver.client() >> Mock(WaveClient) { enabled()>>true } + resolver.client() >> Mock(WaveClient) { enabled()>>true; config()>>Mock(WaveConfig) } _ * task.getContainerConfig() >> Mock(ContainerConfig) { getEngine()>>'singularity' } and: - 1 * resolver.waveContainer(task, CONTAINER_NAME) >> WAVE_CONTAINER + 1 * resolver.waveContainer(task, CONTAINER_NAME, false) >> WAVE_CONTAINER 1 * defaultResolver.resolveImage(task, WAVE_CONTAINER.target) >> SINGULARITY_CONTAINER and: result == SINGULARITY_CONTAINER + + // singularity images + oras protocol + when: + result = resolver.resolveImage(task, CONTAINER_NAME) + then: + resolver.client() >> Mock(WaveClient) { enabled()>>true; config()>>Mock(WaveConfig) { freezeMode()>>true } } + _ * task.getContainerConfig() >> Mock(ContainerConfig) { getEngine()>>'singularity' } + and: + 1 * resolver.waveContainer(task, CONTAINER_NAME, true) >> ORAS_CONTAINER + 0 * defaultResolver.resolveImage(task, WAVE_CONTAINER.target) >> null + and: + result == ORAS_CONTAINER } def 'should validate container name' () { diff --git a/plugins/nf-wave/src/test/io/seqera/wave/util/DockerHelperTest.groovy b/plugins/nf-wave/src/test/io/seqera/wave/util/DockerHelperTest.groovy deleted file mode 100644 index 4112c96900..0000000000 --- a/plugins/nf-wave/src/test/io/seqera/wave/util/DockerHelperTest.groovy +++ /dev/null @@ -1,381 +0,0 @@ -/* - * Copyright 2013-2023, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package io.seqera.wave.util - -import java.nio.file.Files - -import spock.lang.Specification - -import io.seqera.wave.config.CondaOpts -import io.seqera.wave.config.SpackOpts - -/** - * - * @author Paolo Di Tommaso - */ -class DockerHelperTest extends Specification { - - def 'should create dockerfile content from conda file' () { - given: - def CONDA_OPTS = new CondaOpts([basePackages: 'conda-forge::procps-ng']) - - expect: - DockerHelper.condaFileToDockerFile(CONDA_OPTS)== '''\ - FROM mambaorg/micromamba:1.4.9 - COPY --chown=$MAMBA_USER:$MAMBA_USER conda.yml /tmp/conda.yml - RUN micromamba install -y -n base -f /tmp/conda.yml \\ - && micromamba install -y -n base conda-forge::procps-ng \\ - && micromamba clean -a -y - USER root - '''.stripIndent() - } - - def 'should create dockerfile content from conda file and base packages' () { - - expect: - DockerHelper.condaFileToDockerFile(new CondaOpts([:]))== '''\ - FROM mambaorg/micromamba:1.4.9 - COPY --chown=$MAMBA_USER:$MAMBA_USER conda.yml /tmp/conda.yml - RUN micromamba install -y -n base -f /tmp/conda.yml \\ - && micromamba clean -a -y - USER root - '''.stripIndent() - } - - - def 'should create dockerfile content from conda package' () { - given: - def PACKAGES = 'bwa=0.7.15 salmon=1.1.1' - def CHANNELS = ['conda-forge', 'defaults'] - expect: - DockerHelper.condaPackagesToDockerFile(PACKAGES, CHANNELS, new CondaOpts([:])) == '''\ - FROM mambaorg/micromamba:1.4.9 - RUN \\ - micromamba install -y -n base -c conda-forge -c defaults \\ - bwa=0.7.15 salmon=1.1.1 \\ - && micromamba clean -a -y - USER root - '''.stripIndent() - } - - def 'should create dockerfile with base packages' () { - given: - def CHANNELS = ['conda-forge', 'defaults'] - def CONDA_OPTS = new CondaOpts([basePackages: 'foo::one bar::two']) - def PACKAGES = 'bwa=0.7.15 salmon=1.1.1' - - expect: - DockerHelper.condaPackagesToDockerFile(PACKAGES, CHANNELS, CONDA_OPTS) == '''\ - FROM mambaorg/micromamba:1.4.9 - RUN \\ - micromamba install -y -n base -c conda-forge -c defaults \\ - bwa=0.7.15 salmon=1.1.1 \\ - && micromamba install -y -n base foo::one bar::two \\ - && micromamba clean -a -y - USER root - '''.stripIndent() - } - - def 'should create dockerfile content with custom channels' () { - given: - def CHANNELS = 'foo,bar'.tokenize(',') - def PACKAGES = 'bwa=0.7.15 salmon=1.1.1' - - expect: - DockerHelper.condaPackagesToDockerFile(PACKAGES, CHANNELS, new CondaOpts([:])) == '''\ - FROM mambaorg/micromamba:1.4.9 - RUN \\ - micromamba install -y -n base -c foo -c bar \\ - bwa=0.7.15 salmon=1.1.1 \\ - && micromamba clean -a -y - USER root - '''.stripIndent() - } - - def 'should create dockerfile content with custom conda config' () { - given: - def CHANNELS = ['conda-forge', 'defaults'] - def CONDA_OPTS = [mambaImage:'my-base:123', commands: ['USER my-user', 'RUN apt-get update -y && apt-get install -y nano']] - def PACKAGES = 'bwa=0.7.15 salmon=1.1.1' - - expect: - DockerHelper.condaPackagesToDockerFile(PACKAGES, CHANNELS, new CondaOpts(CONDA_OPTS)) == '''\ - FROM my-base:123 - RUN \\ - micromamba install -y -n base -c conda-forge -c defaults \\ - bwa=0.7.15 salmon=1.1.1 \\ - && micromamba clean -a -y - USER root - USER my-user - RUN apt-get update -y && apt-get install -y nano - '''.stripIndent() - } - - - def 'should create dockerfile content with remote conda lock' () { - given: - def CHANNELS = ['conda-forge', 'defaults'] - def OPTS = [mambaImage:'my-base:123', commands: ['USER my-user', 'RUN apt-get update -y && apt-get install -y procps']] - def PACKAGES = 'https://foo.com/some/conda-lock.yml' - - expect: - DockerHelper.condaPackagesToDockerFile(PACKAGES, CHANNELS, new CondaOpts(OPTS)) == '''\ - FROM my-base:123 - RUN \\ - micromamba install -y -n base -c conda-forge -c defaults \\ - -f https://foo.com/some/conda-lock.yml \\ - && micromamba clean -a -y - USER root - USER my-user - RUN apt-get update -y && apt-get install -y procps - '''.stripIndent() - } - - - def 'should create dockerfile content from spack package' () { - given: - def PACKAGES = 'bwa@0.7.15 salmon@1.1.1' - - expect: - DockerHelper.spackPackagesToSpackFile(PACKAGES, Mock(SpackOpts)).text == '''\ - spack: - specs: [bwa@0.7.15, salmon@1.1.1] - concretizer: {unify: true, reuse: false} - '''.stripIndent(true) - - DockerHelper.spackFileToDockerFile(new SpackOpts())== '''\ - # Runner image - FROM {{spack_runner_image}} - - COPY --from=builder /opt/spack-env /opt/spack-env - COPY --from=builder /opt/software /opt/software - COPY --from=builder /opt/._view /opt/._view - - # Entrypoint for Singularity - RUN mkdir -p /.singularity.d/env && \\ - cp -p /opt/spack-env/z10_spack_environment.sh /.singularity.d/env/91-environment.sh - # Entrypoint for Docker - RUN echo "#!/usr/bin/env bash\\n\\nset -ef -o pipefail\\nsource /opt/spack-env/z10_spack_environment.sh\\nexec \\"\\\$@\\"" \\ - >/opt/spack-env/spack_docker_entrypoint.sh && chmod a+x /opt/spack-env/spack_docker_entrypoint.sh - - - ENTRYPOINT [ "/opt/spack-env/spack_docker_entrypoint.sh" ] - CMD [ "/bin/bash" ] - '''.stripIndent() - } - - def 'should create dockerfile content with custom spack config' () { - given: - def SPACK_OPTS = [ commands:['USER hola'] ] - def PACKAGES = 'bwa@0.7.15 salmon@1.1.1' - - expect: - DockerHelper.spackPackagesToSpackFile(PACKAGES, Mock(SpackOpts)).text == '''\ - spack: - specs: [bwa@0.7.15, salmon@1.1.1] - concretizer: {unify: true, reuse: false} - '''.stripIndent(true) - - DockerHelper.spackFileToDockerFile(new SpackOpts(SPACK_OPTS))== '''\ - # Runner image - FROM {{spack_runner_image}} - - COPY --from=builder /opt/spack-env /opt/spack-env - COPY --from=builder /opt/software /opt/software - COPY --from=builder /opt/._view /opt/._view - - # Entrypoint for Singularity - RUN mkdir -p /.singularity.d/env && \\ - cp -p /opt/spack-env/z10_spack_environment.sh /.singularity.d/env/91-environment.sh - # Entrypoint for Docker - RUN echo "#!/usr/bin/env bash\\n\\nset -ef -o pipefail\\nsource /opt/spack-env/z10_spack_environment.sh\\nexec \\"\\\$@\\"" \\ - >/opt/spack-env/spack_docker_entrypoint.sh && chmod a+x /opt/spack-env/spack_docker_entrypoint.sh - - USER hola - - ENTRYPOINT [ "/opt/spack-env/spack_docker_entrypoint.sh" ] - CMD [ "/bin/bash" ] - '''.stripIndent() - } - - - def 'should create dockerfile content from spack file' () { - expect: - DockerHelper.spackFileToDockerFile(new SpackOpts())== '''\ - # Runner image - FROM {{spack_runner_image}} - - COPY --from=builder /opt/spack-env /opt/spack-env - COPY --from=builder /opt/software /opt/software - COPY --from=builder /opt/._view /opt/._view - - # Entrypoint for Singularity - RUN mkdir -p /.singularity.d/env && \\ - cp -p /opt/spack-env/z10_spack_environment.sh /.singularity.d/env/91-environment.sh - # Entrypoint for Docker - RUN echo "#!/usr/bin/env bash\\n\\nset -ef -o pipefail\\nsource /opt/spack-env/z10_spack_environment.sh\\nexec \\"\\\$@\\"" \\ - >/opt/spack-env/spack_docker_entrypoint.sh && chmod a+x /opt/spack-env/spack_docker_entrypoint.sh - - - ENTRYPOINT [ "/opt/spack-env/spack_docker_entrypoint.sh" ] - CMD [ "/bin/bash" ] - '''.stripIndent() - } - - def 'should return empty packages' () { - when: - def result = DockerHelper.spackPackagesToSpackYaml(null, new SpackOpts()) - then: - result == null - } - - def 'should convert a list of packages to a spack yaml' () { - when: - def result = DockerHelper.spackPackagesToSpackYaml('foo@1.2.3 x=one bar @2', new SpackOpts()) - then: - result == '''\ - spack: - specs: [foo@1.2.3 x=one, bar @2] - concretizer: {unify: true, reuse: false} - '''.stripIndent(true) - } - - - def 'should add base packages' () { - when: - def result = DockerHelper.spackPackagesToSpackYaml(null, new SpackOpts(basePackages: 'foo bar')) - then: - result == '''\ - spack: - specs: [foo, bar] - concretizer: {unify: true, reuse: false} - '''.stripIndent(true) - - when: - result = DockerHelper.spackPackagesToSpackYaml('this that @2', new SpackOpts(basePackages: 'foo bar @1')) - then: - result == '''\ - spack: - specs: [foo, bar @1, this, that @2] - concretizer: {unify: true, reuse: false} - '''.stripIndent(true) - } - - def 'should convert a list of packages to a spack file' () { - when: - def result = DockerHelper.spackPackagesToSpackFile('foo@1.2.3 x=one bar @2', new SpackOpts()) - then: - result.text == '''\ - spack: - specs: [foo@1.2.3 x=one, bar @2] - concretizer: {unify: true, reuse: false} - '''.stripIndent(true) - } - - def 'should parse a spack packages string' () { - expect: - DockerHelper.spackPackagesToList(PACKAGES) == EXPECTED - - where: - PACKAGES | EXPECTED - null | null - 'alpha' | ['alpha'] - 'alpha delta' | ['alpha', 'delta'] - 'alpha delta gamma' | ['alpha', 'delta', 'gamma'] - 'alpha 1aa' | ['alpha', '1aa'] - and: - 'alpha x=1' | ['alpha x=1'] - 'alpha x=1 delta' | ['alpha x=1', 'delta'] - 'alpha ^foo delta' | ['alpha ^foo', 'delta'] - and: - '^alpha ~beta foo' | ['^alpha ~beta', 'foo'] // <-- this should not be valid - - } - - def 'should merge spack file and base package' () { - given: - def folder = Files.createTempDirectory('test') - and: - def SPACK_FILE1 = folder.resolve('spack1.yaml') - SPACK_FILE1.text = '''\ - spack: - specs: [foo@1.2.3 x=one, bar @2] - concretizer: {unify: true, reuse: false} - '''.stripIndent(true) - and: - def SPACK_FILE2 = folder.resolve('spack2.yaml') - SPACK_FILE2.text = '''\ - spack: - concretizer: {unify: true, reuse: false} - '''.stripIndent(true) - and: - def SPACK_FILE3 = folder.resolve('spack3.yaml') - SPACK_FILE3.text = '''\ - foo: - concretizer: {unify: true, reuse: false} - '''.stripIndent(true) - - when: - def result = DockerHelper.addPackagesToSpackFile(null, new SpackOpts()) - then: - result == null - - when: - result = DockerHelper.addPackagesToSpackFile(SPACK_FILE1.toString(), new SpackOpts()) - then: - result.toString() == SPACK_FILE1.toString() - - when: - result = DockerHelper.addPackagesToSpackFile(SPACK_FILE1.toString(), new SpackOpts(basePackages: 'alpha delta')) - then: - result.text == '''\ - spack: - specs: [foo@1.2.3 x=one, bar @2, alpha, delta] - concretizer: {unify: true, reuse: false} - '''.stripIndent(true) - - - when: - result = DockerHelper.addPackagesToSpackFile(SPACK_FILE2.toString(), new SpackOpts(basePackages: 'alpha delta')) - then: - result.text == '''\ - spack: - concretizer: {unify: true, reuse: false} - specs: [alpha, delta] - '''.stripIndent(true) - - when: - DockerHelper.addPackagesToSpackFile(SPACK_FILE3.toString(), new SpackOpts(basePackages: 'foo')) - then: - thrown(IllegalArgumentException) - - when: - DockerHelper.addPackagesToSpackFile('missing file', new SpackOpts(basePackages: 'foo')) - then: - thrown(IllegalArgumentException) - - when: - DockerHelper.addPackagesToSpackFile('missing file', new SpackOpts()) - then: - thrown(IllegalArgumentException) - - cleanup: - folder?.deleteDir() - } - -} diff --git a/plugins/nf-wave/src/test/io/seqera/wave/util/TemplateRendererTest.groovy b/plugins/nf-wave/src/test/io/seqera/wave/util/TemplateRendererTest.groovy deleted file mode 100644 index 392893e16c..0000000000 --- a/plugins/nf-wave/src/test/io/seqera/wave/util/TemplateRendererTest.groovy +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Copyright 2013-2023, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package io.seqera.wave.util - -import spock.lang.Specification - -/** - * - * @author Paolo Di Tommaso - */ -class TemplateRendererTest extends Specification { - - def 'should replace vars' () { - given: - def binding = [foo: 'Hello', bar: 'world'] - def render = new TemplateRenderer() - expect: - render.replace0('{{foo}}', binding) == 'Hello' - render.replace0('{{foo}} ', binding) == 'Hello ' - render.replace0('{{foo}}\n', binding) == 'Hello\n' - render.replace0(' {{foo}}', binding) == ' Hello' - render.replace0(' {{foo}}\n', binding) == ' Hello\n' - render.replace0(' ${foo}', binding) == ' ${foo}' - render.replace0(' ${{foo}}', binding) == ' ${{foo}}' - render.replace0('{{foo}}', [foo:'']) == '' - render.replace0('{{foo}}', [foo:null]) == null - render.replace0(' {{foo}}\n', [foo:null]) == null - render.replace0('', binding) == '' - render.replace0(null, binding) == null - - render.replace0('{{foo}} {{bar}}!', binding) == 'Hello world!' - render.replace0('abc {{foo}} pq {{bar}} xyz', binding) == 'abc Hello pq world xyz' - render.replace0('{{foo}} 123 {{bar}} xyz {{foo}}', binding) == 'Hello 123 world xyz Hello' - render.replace0('1{{foo}}2{{foo}}3', [foo:'']) == '123' - render.replace0('1{{foo}}2{{foo}}3', [foo:null]) == '123' - } - - def 'should throw an exception when missing variables' () { - when: - new TemplateRenderer().replace0('{{x1}}', [:]) - then: - def e = thrown(IllegalArgumentException) - e.message == 'Missing template key: x1' - - when: - new TemplateRenderer().replace0('{{foo}} {{x2}}', [foo:'ciao']) - then: - e = thrown(IllegalArgumentException) - e.message == 'Missing template key: x2' - } - - def 'should not throw an exception when missing variables' () { - when: - def result = new TemplateRenderer().withIgnore("x1").replace0('{{x1}}', [x1:'one']) - then: - result == '{{x1}}' - - when: - result = new TemplateRenderer().withIgnore('x1','x2').replace0('{{x1}} {{x2}}', [x1:'one']) - then: - result == '{{x1}} {{x2}}' - } - - def 'should render template' () { - given: - def template = "Hello, {{name}}!\n" + - "Today is {{day}} and the weather is {{weather}}."; - and: - def binding = new HashMap(); - binding.put("name", "John"); - binding.put("day", "Monday"); - binding.put("weather", "sunny"); - - when: - def renderer = new TemplateRenderer() - and: - def result = renderer.render(template, binding); - - then: - result == 'Hello, John!\nToday is Monday and the weather is sunny.' - } - - def 'should render a template with comment'() { - given: - def template = """\ - ## remove this comment - 1: {{alpha}} - 2: {{delta}} {{delta}} - 3: {{gamma}} {{gamma}} {{gamma}} - 4: end - """.stripIndent() - and: - def binding = new HashMap(); - binding.put("alpha", "one"); - binding.put("delta", "two"); - binding.put("gamma", "three"); - - when: - def renderer = new TemplateRenderer() - and: - def result = renderer.render(new ByteArrayInputStream(template.bytes), binding); - - then: - result == """\ - 1: one - 2: two two - 3: three three three - 4: end - """.stripIndent() - } - - - def 'should render a template using an input stream'() { - given: - def template = """\ - {{one}} - {{two}} - xxx - {{three}} - zzz - """.stripIndent() - and: - def binding = [ - one: '1', // this is rendered - two:null, // a line containing a null variable is not rendered - three:'' // empty value is considered ok - ] - - when: - def renderer = new TemplateRenderer() - and: - def result = renderer.render(new ByteArrayInputStream(template.bytes), binding); - - then: - result == """\ - 1 - xxx - - zzz - """.stripIndent() - } - - def 'should render template with indentations' () { - given: - def binding = [foo: 'Hello', bar: 'world'] - - when: - def renderer = new TemplateRenderer() - and: - def result = renderer.render('{{foo}}\n{{bar}}', binding) - then: - result == 'Hello\nworld' - - when: - def template = '''\ - {{foo}} - {{bar}} - '''.stripIndent() - result = renderer.render(template, [foo:'11\n22\n33', bar:'Hello world']) - then: - result == '''\ - 11 - 22 - 33 - Hello world - '''.stripIndent() - - - when: - template = '''\ - {{x1}} - {{x2}} - {{x3}} - '''.stripIndent() - result = renderer.render(template, [x1:'aa\nbb\n', x2:null, x3:'pp\nqq']) - then: - result == '''\ - aa - bb - - pp - qq - '''.stripIndent() - - } - -}