diff --git a/plugins/nf-wave/build.gradle b/plugins/nf-wave/build.gradle index 19d449908a..a59f4aa1c3 100644 --- a/plugins/nf-wave/build.gradle +++ b/plugins/nf-wave/build.gradle @@ -33,6 +33,7 @@ dependencies { compileOnly 'org.slf4j:slf4j-api:2.0.7' compileOnly 'org.pf4j:pf4j:3.4.1' api 'org.apache.commons:commons-compress:1.21' + api 'org.apache.commons:commons-lang3:3.12.0' api 'com.google.code.gson:gson:2.10.1' testImplementation(testFixtures(project(":nextflow"))) diff --git a/plugins/nf-wave/src/main/io/seqera/wave/plugin/config/CondaOpts.groovy b/plugins/nf-wave/src/main/io/seqera/wave/config/CondaOpts.java similarity index 53% rename from plugins/nf-wave/src/main/io/seqera/wave/plugin/config/CondaOpts.groovy rename to plugins/nf-wave/src/main/io/seqera/wave/config/CondaOpts.java index 39ea81b94b..9d4fe2c952 100644 --- a/plugins/nf-wave/src/main/io/seqera/wave/plugin/config/CondaOpts.groovy +++ b/plugins/nf-wave/src/main/io/seqera/wave/config/CondaOpts.java @@ -15,25 +15,27 @@ * */ -package io.seqera.wave.plugin.config +package io.seqera.wave.config; + +import java.util.List; +import java.util.Map; /** * Conda build options * * @author Paolo Di Tommaso */ -class CondaOpts { - - final public static String DEFAULT_MAMBA_IMAGE = 'mambaorg/micromamba:1.4.2' +public class CondaOpts { + final public static String DEFAULT_MAMBA_IMAGE = "mambaorg/micromamba:1.4.2"; - final String mambaImage - final List commands - final String basePackages + final public String mambaImage; + final public List commands; + final public String basePackages; - CondaOpts(Map opts) { - this.mambaImage = opts.mambaImage ?: DEFAULT_MAMBA_IMAGE - this.commands = opts.commands as List - this.basePackages = opts.basePackages + public CondaOpts(Map opts) { + this.mambaImage = opts.containsKey("mambaImage") ? opts.get("mambaImage").toString(): DEFAULT_MAMBA_IMAGE; + this.commands = opts.containsKey("commands") ? (List)opts.get("commands") : null; + this.basePackages = (String)opts.get("basePackages"); } } diff --git a/plugins/nf-wave/src/main/io/seqera/wave/config/SpackOpts.java b/plugins/nf-wave/src/main/io/seqera/wave/config/SpackOpts.java new file mode 100644 index 0000000000..8cfcd9d602 --- /dev/null +++ b/plugins/nf-wave/src/main/io/seqera/wave/config/SpackOpts.java @@ -0,0 +1,58 @@ +/* + * Copyright 2013-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.seqera.wave.config; + +import java.util.List; +import java.util.Map; + +/** + * Spack build options + * + * @author Marco De La Pierre + */ +public class SpackOpts { + + final static public String DEFAULT_SPACK_BUILDER_IMAGE = "spack/ubuntu-jammy:v0.19.2"; + final static public String DEFAULT_SPACK_RUNNER_IMAGE = "ubuntu:22.04"; + final static public String DEFAULT_SPACK_OSPACKAGES = ""; + final static public String DEFAULT_SPACK_FLAGS = "-O3"; + + public final Boolean checksum; + public final String builderImage; + public final String runnerImage; + public final String osPackages; + public final String cFlags; + public final String cxxFlags; + public final String fFlags; + public final List commands; + + public SpackOpts() { + this(Map.of()); + } + public SpackOpts(Map opts) { + this.checksum = opts.get("checksum") == null || Boolean.parseBoolean(opts.get("checksum").toString()); + this.builderImage = opts.containsKey("builderImage") ? opts.get("builderImage").toString() : DEFAULT_SPACK_BUILDER_IMAGE; + this.runnerImage = opts.containsKey("runnerImage") ? opts.get("runnerImage").toString() : DEFAULT_SPACK_RUNNER_IMAGE; + this.osPackages = opts.containsKey("osPackages") ? opts.get("osPackages").toString() : DEFAULT_SPACK_OSPACKAGES; + this.cFlags = opts.containsKey("cFlags") ? opts.get("cFlags").toString() : DEFAULT_SPACK_FLAGS; + this.cxxFlags = opts.containsKey("cxxFlags") ? opts.get("cxxFlags").toString() : DEFAULT_SPACK_FLAGS; + this.fFlags = opts.containsKey("fFlags") ? opts.get("fFlags").toString() : DEFAULT_SPACK_FLAGS; + this.commands = opts.containsKey("commands") ? (List)opts.get("commands") : null; + } + +} diff --git a/plugins/nf-wave/src/main/io/seqera/wave/plugin/DescribeContainerResponse.groovy b/plugins/nf-wave/src/main/io/seqera/wave/plugin/DescribeContainerResponse.groovy index b1535e1911..1e788f3399 100644 --- a/plugins/nf-wave/src/main/io/seqera/wave/plugin/DescribeContainerResponse.groovy +++ b/plugins/nf-wave/src/main/io/seqera/wave/plugin/DescribeContainerResponse.groovy @@ -1,3 +1,20 @@ +/* + * Copyright 2013-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package io.seqera.wave.plugin import java.time.Instant diff --git a/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveClient.groovy b/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveClient.groovy index 9ceb3fe6be..9a2f8c5a83 100644 --- a/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveClient.groovy +++ b/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveClient.groovy @@ -17,6 +17,7 @@ package io.seqera.wave.plugin +import static io.seqera.wave.util.DockerHelper.* import java.net.http.HttpClient import java.net.http.HttpRequest @@ -54,12 +55,10 @@ import io.seqera.wave.plugin.packer.Packer import nextflow.Session import nextflow.SysEnv import nextflow.container.resolver.ContainerInfo -import nextflow.executor.BashTemplateEngine import nextflow.fusion.FusionConfig import nextflow.processor.Architecture import nextflow.processor.TaskRun import nextflow.script.bundle.ResourcesBundle -import nextflow.util.MustacheTemplateEngine import nextflow.util.SysHelper import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -380,10 +379,11 @@ class WaveClient { // map the recipe to a dockerfile if( isCondaLocalFile(attrs.conda) ) { condaFile = Path.of(attrs.conda) - dockerScript = condaFileToDockerFile() + dockerScript = condaFileToDockerFile(config.condaOpts()) } + // 'conda' attributes is resolved as the conda packages to be used else { - dockerScript = condaRecipeToDockerFile(attrs.conda) + dockerScript = condaPackagesToDockerFile(attrs.conda, condaChannels, config.condaOpts()) } } @@ -399,10 +399,10 @@ class WaveClient { // map the recipe to a dockerfile if( isSpackFile(attrs.spack) ) { spackFile = Path.of(attrs.spack) - dockerScript = spackFileToDockerFile(spackArch) + dockerScript = spackFileToDockerFile(spackArch, config.spackOpts()) } else { - dockerScript = spackRecipeToDockerFile(attrs.spack, spackArch) + dockerScript = spackPackagesToDockerFile(attrs.spack, spackArch, config.spackOpts()) } } @@ -464,105 +464,7 @@ class WaveClient { } } - protected String condaFileToDockerFile() { - final template = """\ - FROM {{base_image}} - COPY --chown=\$MAMBA_USER:\$MAMBA_USER conda.yml /tmp/conda.yml - RUN micromamba install -y -n base -f /tmp/conda.yml && \\ - {{base_packages}} - micromamba clean -a -y - """.stripIndent(true) - final image = config.condaOpts().mambaImage - - final basePackage = config.condaOpts().basePackages ? "micromamba install -y -n base ${config.condaOpts().basePackages} && \\".toString() : null - final binding = ['base_image': image, 'base_packages': basePackage] - final result = new MustacheTemplateEngine().render(template, binding) - - return addCommands(result) - } - - // Dockerfile template adpated from the Spack package manager - // https://github.com/spack/spack/blob/develop/share/spack/templates/container/Dockerfile - // LICENSE APACHE 2.0 - protected String spackFileToDockerFile(String spackArch) { - - String cmd_template = '' - final binding = [ - 'builder_image': config.spackOpts().builderImage, - 'c_flags': config.spackOpts().cFlags, - 'cxx_flags': config.spackOpts().cxxFlags, - 'f_flags': config.spackOpts().fFlags, - 'spack_arch': spackArch, - 'checksum_string': config.spackOpts().checksum ? '' : '-n ', - 'runner_image': config.spackOpts().runnerImage, - 'os_packages': config.spackOpts().osPackages, - 'add_commands': addCommands(cmd_template), - ] - final template = WaveClient.class.getResource('/templates/spack/dockerfile-spack-file.txt') - try(final reader = template.newReader()) { - final result = new BashTemplateEngine().render(reader, binding) - return result - } - } - protected String addCommands(String result) { - if( config.condaOpts().commands ) - for( String cmd : config.condaOpts().commands ) { - result += cmd + "\n" - } - if( config.spackOpts().commands ) - for( String cmd : config.spackOpts().commands ) { - result += cmd + "\n" - } - return result - } - - protected String condaRecipeToDockerFile(String recipe) { - final template = """\ - FROM {{base_image}} - RUN \\ - micromamba install -y -n base {{channel_opts}} \\ - {{target}} \\ - {{base_packages}} - && micromamba clean -a -y - """.stripIndent(true) - - final channelsOpts = condaChannels.collect(it -> "-c $it").join(' ') - final image = config.condaOpts().mambaImage - final target = recipe.startsWith('http://') || recipe.startsWith('https://') - ? "-f $recipe".toString() - : recipe - final basePackage = config.condaOpts().basePackages ? "&& micromamba install -y -n base ${config.condaOpts().basePackages} \\".toString() : null - final binding = [base_image: image, channel_opts: channelsOpts, target:target, base_packages: basePackage] - final result = new MustacheTemplateEngine().render(template, binding) - return addCommands(result) - } - - // Dockerfile template adpated from the Spack package manager - // https://github.com/spack/spack/blob/develop/share/spack/templates/container/Dockerfile - // LICENSE APACHE 2.0 - protected String spackRecipeToDockerFile(String recipe, String spackArch) { - - String cmd_template = '' - final binding = [ - 'recipe': recipe, - 'builder_image': config.spackOpts().builderImage, - 'c_flags': config.spackOpts().cFlags, - 'cxx_flags': config.spackOpts().cxxFlags, - 'f_flags': config.spackOpts().fFlags, - 'spack_arch': spackArch, - 'checksum_string': config.spackOpts().checksum ? '' : '-n ', - 'runner_image': config.spackOpts().runnerImage, - 'os_packages': config.spackOpts().osPackages, - 'add_commands': addCommands(cmd_template), - ] - final template = WaveClient.class.getResource('/templates/spack/dockerfile-spack-recipe.txt') - - try(final reader = template.newReader()) { - final result = new BashTemplateEngine().render(reader, binding) - return result - } - } static protected boolean isCondaLocalFile(String value) { if( value.contains('\n') ) diff --git a/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveObserver.groovy b/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveObserver.groovy index f48a67bc62..0dbdf863bf 100644 --- a/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveObserver.groovy +++ b/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveObserver.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022, Seqera Labs + * Copyright 2013-2023, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/plugins/nf-wave/src/main/io/seqera/wave/plugin/adapter/InstantAdapter.groovy b/plugins/nf-wave/src/main/io/seqera/wave/plugin/adapter/InstantAdapter.groovy index 3e369418a5..8ab42ab4be 100644 --- a/plugins/nf-wave/src/main/io/seqera/wave/plugin/adapter/InstantAdapter.groovy +++ b/plugins/nf-wave/src/main/io/seqera/wave/plugin/adapter/InstantAdapter.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022, Seqera Labs + * Copyright 2013-2023, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/plugins/nf-wave/src/main/io/seqera/wave/plugin/config/ReportOpts.groovy b/plugins/nf-wave/src/main/io/seqera/wave/plugin/config/ReportOpts.groovy index 08fd147cab..f8db921ce1 100644 --- a/plugins/nf-wave/src/main/io/seqera/wave/plugin/config/ReportOpts.groovy +++ b/plugins/nf-wave/src/main/io/seqera/wave/plugin/config/ReportOpts.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022, Seqera Labs + * Copyright 2013-2023, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/plugins/nf-wave/src/main/io/seqera/wave/plugin/config/RetryOpts.groovy b/plugins/nf-wave/src/main/io/seqera/wave/plugin/config/RetryOpts.groovy index bda00e3122..540b51801f 100644 --- a/plugins/nf-wave/src/main/io/seqera/wave/plugin/config/RetryOpts.groovy +++ b/plugins/nf-wave/src/main/io/seqera/wave/plugin/config/RetryOpts.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022, Seqera Labs + * Copyright 2013-2023, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/plugins/nf-wave/src/main/io/seqera/wave/plugin/config/SpackOpts.groovy b/plugins/nf-wave/src/main/io/seqera/wave/plugin/config/SpackOpts.groovy deleted file mode 100644 index 884444714e..0000000000 --- a/plugins/nf-wave/src/main/io/seqera/wave/plugin/config/SpackOpts.groovy +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2020-2022, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package io.seqera.wave.plugin.config - -/** - * Spack build options - * - * @author Marco De La Pierre - */ -class SpackOpts { - - final public String DEFAULT_SPACK_BUILDER_IMAGE = 'spack/ubuntu-jammy:v0.19.2' - final public String DEFAULT_SPACK_RUNNER_IMAGE = 'ubuntu:22.04' - final public String DEFAULT_SPACK_OSPACKAGES = '' - final public String DEFAULT_SPACK_FLAGS = '-O3' - - final Boolean checksum - final String builderImage - final String runnerImage - final String osPackages - final String cFlags - final String cxxFlags - final String fFlags - final List commands - - SpackOpts(Map opts) { - this.checksum = ( opts.checksum != null ) ? opts.checksum : true - this.builderImage = opts.builderImage ?: DEFAULT_SPACK_BUILDER_IMAGE - this.runnerImage = opts.runnerImage ?: DEFAULT_SPACK_RUNNER_IMAGE - this.osPackages = opts.osPackages ?: DEFAULT_SPACK_OSPACKAGES - this.cFlags = opts.cFlags ?: DEFAULT_SPACK_FLAGS - this.cxxFlags = opts.cxxFlags ?: DEFAULT_SPACK_FLAGS - this.fFlags = opts.fFlags ?: DEFAULT_SPACK_FLAGS - this.commands = opts.commands as List - } - -} diff --git a/plugins/nf-wave/src/main/io/seqera/wave/plugin/config/WaveConfig.groovy b/plugins/nf-wave/src/main/io/seqera/wave/plugin/config/WaveConfig.groovy index 45a185f77e..2a91c210b2 100644 --- a/plugins/nf-wave/src/main/io/seqera/wave/plugin/config/WaveConfig.groovy +++ b/plugins/nf-wave/src/main/io/seqera/wave/plugin/config/WaveConfig.groovy @@ -19,6 +19,8 @@ package io.seqera.wave.plugin.config import groovy.transform.CompileStatic import groovy.util.logging.Slf4j +import io.seqera.wave.config.CondaOpts +import io.seqera.wave.config.SpackOpts import nextflow.util.Duration /** * Model Wave client configuration diff --git a/plugins/nf-wave/src/main/io/seqera/wave/util/DockerHelper.java b/plugins/nf-wave/src/main/io/seqera/wave/util/DockerHelper.java new file mode 100644 index 0000000000..add03c7db6 --- /dev/null +++ b/plugins/nf-wave/src/main/io/seqera/wave/util/DockerHelper.java @@ -0,0 +1,134 @@ +/* + * Copyright 2013-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.seqera.wave.util; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import io.seqera.wave.config.CondaOpts; +import io.seqera.wave.config.SpackOpts; +import org.apache.commons.lang3.StringUtils; + +/** + * Helper class to create Dockerfile for Conda and Spack package managers + * + * @author Paolo Di Tommaso + */ +public class DockerHelper { + + static public String spackPackagesToDockerFile(String packages, String spackArch, SpackOpts opts) { + // create bindings + final Map binding = spackBinding(spackArch, opts); + binding.put("packages", packages); + // render the template + return renderTemplate0("/templates/spack/dockerfile-spack-packages.txt", binding); + } + + static public String spackFileToDockerFile(String spackArch, SpackOpts opts) { + // create bindings + final Map binding = spackBinding(spackArch, opts); + // return the template + return renderTemplate0("/templates/spack/dockerfile-spack-file.txt", binding); + } + + static private Map spackBinding(String spackArch, SpackOpts opts) { + final Map binding = new HashMap<>(); + binding.put("builder_image", opts.builderImage); + binding.put("f_flags", opts.fFlags); + binding.put("c_flags", opts.cFlags); + binding.put("cxx_flags", opts.cxxFlags); + binding.put("spack_arch", spackArch); + binding.put("checksum_string", opts.checksum ? "" : "-n "); + binding.put("runner_image", opts.runnerImage); + binding.put("os_packages", opts.osPackages); + binding.put("add_commands", joinCommands(opts.commands)); + return binding; + } + + static public String condaPackagesToDockerFile(String packages, List condaChannels, CondaOpts opts) { + final List channels0 = condaChannels!=null ? condaChannels : List.of(); + final String channelsOpts = channels0.stream().map(it -> "-c "+it).collect(Collectors.joining(" ")); + final String image = opts.mambaImage; + final String target = packages.startsWith("http://") || packages.startsWith("https://") + ? "-f " + packages + : packages; + final Map binding = new HashMap<>(); + binding.put("base_image", image); + binding.put("channel_opts", channelsOpts); + binding.put("target", target); + binding.put("base_packages", mambaInstallBasePackage0(opts.basePackages)); + + final String result = renderTemplate0("/templates/conda/dockerfile-conda-packages.txt", binding) ; + return addCommands(result, opts.commands); + } + + static public String condaFileToDockerFile(CondaOpts opts) { + // create the binding map + final Map binding = new HashMap<>(); + binding.put("base_image", opts.mambaImage); + binding.put("base_packages", mambaInstallBasePackage0(opts.basePackages)); + + final String result = renderTemplate0("/templates/conda/dockerfile-conda-file.txt", binding); + return addCommands(result, opts.commands); + } + + static private String renderTemplate0(String templatePath, Map binding) { + final URL template = DockerHelper.class.getResource(templatePath); + if( template==null ) + throw new IllegalStateException(String.format("Unable to load template '%s' from classpath", templatePath)); + try { + final InputStream reader = template.openStream(); + return TemplateRenderer.render(reader, binding); + } + catch (IOException e) { + throw new IllegalStateException(String.format("Unable to read classpath template '%s'", templatePath), e); + } + } + + private static String mambaInstallBasePackage0(String basePackages) { + return !StringUtils.isEmpty(basePackages) + ? String.format("&& micromamba install -y -n base %s \\", basePackages) + : null; + } + + static private String addCommands(String result, List commands) { + if( commands==null || commands.isEmpty() ) + return result; + for( String cmd : commands ) { + result += cmd + "\n"; + } + return result; + } + + static private String joinCommands(List commands) { + if( commands==null || commands.size()==0 ) + return null; + StringBuilder result = new StringBuilder(); + for( String cmd : commands ) { + if( result.length()>0 ) + result.append("\n"); + result.append(cmd); + } + return result.toString(); + } +} diff --git a/plugins/nf-wave/src/main/io/seqera/wave/util/TemplateRenderer.java b/plugins/nf-wave/src/main/io/seqera/wave/util/TemplateRenderer.java new file mode 100644 index 0000000000..ab45c37dda --- /dev/null +++ b/plugins/nf-wave/src/main/io/seqera/wave/util/TemplateRenderer.java @@ -0,0 +1,132 @@ +/* + * Copyright 2013-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.seqera.wave.util; + +import java.io.InputStream; +import java.util.Map; +import java.util.Scanner; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Template rendering helper + * + * @author Paolo Di Tommaso + */ +public class TemplateRenderer { + + private static final Pattern PATTERN = Pattern.compile("\\{\\{([^}]+)}}"); + + private static final Pattern VAR1 = Pattern.compile("(\\s*)\\{\\{([\\d\\w_-]+)}}(\\s*$)"); + private static final Pattern VAR2 = Pattern.compile("(? binding) { + String str = new Scanner(template).useDelimiter("\\A").next(); + return render(str, binding); + } + + public static String render(String template, Map binding) { + final String[] lines = template.split("(?<=\n)"); + final StringBuilder result = new StringBuilder(); + for( String it : lines ) { + if( it==null || it.startsWith("##")) + continue; + final String resolved = replace0(it, binding); + if( resolved!=null ) + result.append(resolved); + } + return result.toString(); + } + + /** + * Simple template helper class replacing all variable enclosed by {{..}} + * with the corresponding variable specified in a map object + * + * @param template The template string + * @param binding The binding {@link Map} + * @return The templated having the variables replaced with the corresponding value + */ + static String replace1(CharSequence template, Map binding) { + Matcher matcher = PATTERN.matcher(template); + + // Loop through each matched variable placeholder + StringBuilder builder = new StringBuilder(); + boolean isNull=false; + while (matcher.find()) { + String variable = matcher.group(1); + + // Check if the variable exists in the values map + if (binding.containsKey(variable)) { + Object value = binding.get(variable); + String str = value!=null ? value.toString() : ""; + isNull |= value==null; + matcher.appendReplacement(builder, str); + } + else { + throw new IllegalArgumentException(String.format("Unable to resolve template variable: {{%s}}", variable)); + } + } + matcher.appendTail(builder); + + final String result = builder.toString(); + return !isNull || !result.isBlank() ? result : null; + } + + static String replace0(String line, Map binding) { + if( line==null || line.length()==0 ) + return line; + + Matcher matcher = VAR1.matcher(line); + if( matcher.matches() ) { + final String name = matcher.group(2); + if( !binding.containsKey(name) ) + throw new IllegalArgumentException("Missing template key: "+name); + final String prefix = matcher.group(1); + final String value = binding.get(name); + if( value==null ) + return null; // <-- return null to skip this line + + final StringBuilder result = new StringBuilder(); + final String[] multi = value.split("(?<=\n)"); + for (String s : multi) { + result.append(prefix); + result.append(s); + } + result.append( matcher.group(3) ); + return result.toString(); + } + + final StringBuilder result = new StringBuilder(); + while( (matcher=VAR2.matcher(line)).find() ) { + String name = matcher.group(1); + if( !binding.containsKey(name)) + throw new IllegalArgumentException("Missing template key: "+name); + final String value = binding.get(name)!=null ? binding.get(name) : ""; + final int p = matcher.start(1); + final int q = matcher.end(1); + + result.append(line.substring(0,p-2)); + result.append(value); + line = line.substring(q+2); + } + result.append(line); + return result.toString(); + } + +} diff --git a/plugins/nf-wave/src/resources/templates/conda/dockerfile-conda-file.txt b/plugins/nf-wave/src/resources/templates/conda/dockerfile-conda-file.txt new file mode 100644 index 0000000000..a7cc7806e8 --- /dev/null +++ b/plugins/nf-wave/src/resources/templates/conda/dockerfile-conda-file.txt @@ -0,0 +1,5 @@ +FROM {{base_image}} +COPY --chown=$MAMBA_USER:$MAMBA_USER conda.yml /tmp/conda.yml +RUN micromamba install -y -n base -f /tmp/conda.yml \ + {{base_packages}} + && micromamba clean -a -y diff --git a/plugins/nf-wave/src/resources/templates/conda/dockerfile-conda-packages.txt b/plugins/nf-wave/src/resources/templates/conda/dockerfile-conda-packages.txt new file mode 100644 index 0000000000..6885c84ea0 --- /dev/null +++ b/plugins/nf-wave/src/resources/templates/conda/dockerfile-conda-packages.txt @@ -0,0 +1,6 @@ +FROM {{base_image}} +RUN \ + micromamba install -y -n base {{channel_opts}} \ + {{target}} \ + {{base_packages}} + && micromamba clean -a -y diff --git a/plugins/nf-wave/src/resources/templates/spack/dockerfile-spack-recipe.txt b/plugins/nf-wave/src/resources/templates/spack/dockerfile-spack-packages.txt similarity index 99% rename from plugins/nf-wave/src/resources/templates/spack/dockerfile-spack-recipe.txt rename to plugins/nf-wave/src/resources/templates/spack/dockerfile-spack-packages.txt index fbc0d5d9da..9d1e98e16e 100644 --- a/plugins/nf-wave/src/resources/templates/spack/dockerfile-spack-recipe.txt +++ b/plugins/nf-wave/src/resources/templates/spack/dockerfile-spack-packages.txt @@ -12,7 +12,7 @@ RUN mkdir -p /opt/spack-env \ /root/.spack/linux/compilers.yaml > /opt/spack-env/compilers.yaml \ && sed -i '/^spack:/a\ include: [/opt/spack-env/compilers.yaml]' /opt/spack-env/spack.yaml \ && cd /opt/spack-env && spack env activate . \ -&& spack add {{recipe}} \ +&& spack add {{packages}} \ && spack config add config:install_tree:/opt/software \ && spack config add concretizer:unify:true \ && spack config add concretizer:reuse:false \ diff --git a/plugins/nf-wave/src/test/io/seqera/wave/plugin/config/CondaOptsTest.groovy b/plugins/nf-wave/src/test/io/seqera/wave/config/CondaOptsTest.groovy similarity index 97% rename from plugins/nf-wave/src/test/io/seqera/wave/plugin/config/CondaOptsTest.groovy rename to plugins/nf-wave/src/test/io/seqera/wave/config/CondaOptsTest.groovy index 284250a845..055259888a 100644 --- a/plugins/nf-wave/src/test/io/seqera/wave/plugin/config/CondaOptsTest.groovy +++ b/plugins/nf-wave/src/test/io/seqera/wave/config/CondaOptsTest.groovy @@ -15,7 +15,7 @@ * */ -package io.seqera.wave.plugin.config +package io.seqera.wave.config import spock.lang.Specification diff --git a/plugins/nf-wave/src/test/io/seqera/wave/config/SpackOptsTest.groovy b/plugins/nf-wave/src/test/io/seqera/wave/config/SpackOptsTest.groovy new file mode 100644 index 0000000000..0bed7ff9e5 --- /dev/null +++ b/plugins/nf-wave/src/test/io/seqera/wave/config/SpackOptsTest.groovy @@ -0,0 +1,67 @@ +/* + * Copyright 2013-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.seqera.wave.config + +import spock.lang.Specification + +/** + * + * @author Paolo Di Tommaso + */ +class SpackOptsTest extends Specification { + + def 'check spack default options' () { + given: + def opts = new SpackOpts() + expect: + opts.checksum + opts.builderImage == SpackOpts.DEFAULT_SPACK_BUILDER_IMAGE + opts.runnerImage == SpackOpts.DEFAULT_SPACK_RUNNER_IMAGE + opts.osPackages == SpackOpts.DEFAULT_SPACK_OSPACKAGES + opts.cFlags == SpackOpts.DEFAULT_SPACK_FLAGS + opts.cxxFlags == SpackOpts.DEFAULT_SPACK_FLAGS + opts.fFlags == SpackOpts.DEFAULT_SPACK_FLAGS + opts.commands == null + } + + def 'check spack custom opts' () { + given: + def opts = new SpackOpts([ + checksum:false, + builderImage: 'my/builder:image', + runnerImage: 'my/runner:image', + osPackages: 'my-os-packages', + cFlags: "--my-c-flags", + cxxFlags: '--my-cxx-flags', + fFlags: '--my-f-flags', + commands: ['run','--this','--that'] + ]) + + expect: + !opts.checksum + and: + opts.builderImage == 'my/builder:image' + opts.runnerImage == 'my/runner:image' + opts.osPackages == 'my-os-packages' + and: + opts.cFlags == '--my-c-flags' + opts.cxxFlags == '--my-cxx-flags' + opts.fFlags == '--my-f-flags' + opts.commands == ['run','--this','--that'] + } +} diff --git a/plugins/nf-wave/src/test/io/seqera/wave/plugin/WaveClientTest.groovy b/plugins/nf-wave/src/test/io/seqera/wave/plugin/WaveClientTest.groovy index ff249a293b..a91f1923bd 100644 --- a/plugins/nf-wave/src/test/io/seqera/wave/plugin/WaveClientTest.groovy +++ b/plugins/nf-wave/src/test/io/seqera/wave/plugin/WaveClientTest.groovy @@ -31,7 +31,6 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.Session import nextflow.SysEnv -import nextflow.conda.CondaConfig import nextflow.extension.FilesEx import nextflow.file.FileHelper import nextflow.processor.TaskRun @@ -315,344 +314,6 @@ class WaveClientTest extends Specification { req.containerConfig.layers[1] == MODULE_LAYER } - def 'should create dockerfile content from conda recipe' () { - given: - def session = Mock(Session) { getConfig() >> [:]} - def RECIPE = 'bwa=0.7.15 salmon=1.1.1' - when: - def client = new WaveClient(session) - then: - client.condaRecipeToDockerFile(RECIPE) == '''\ - FROM mambaorg/micromamba:1.4.2 - RUN \\ - micromamba install -y -n base -c conda-forge -c defaults \\ - bwa=0.7.15 salmon=1.1.1 \\ - && micromamba clean -a -y - '''.stripIndent() - } - - def 'should create dockerfile with base packages' () { - given: - def CONDA_OPTS = [basePackages: 'foo::one bar::two'] - def session = Mock(Session) { getConfig() >> [wave:[build:[conda:CONDA_OPTS]]]} - def RECIPE = 'bwa=0.7.15 salmon=1.1.1' - when: - def client = new WaveClient(session) - then: - client.condaRecipeToDockerFile(RECIPE) == '''\ - FROM mambaorg/micromamba:1.4.2 - RUN \\ - micromamba install -y -n base -c conda-forge -c defaults \\ - bwa=0.7.15 salmon=1.1.1 \\ - && micromamba install -y -n base foo::one bar::two \\ - && micromamba clean -a -y - '''.stripIndent() - } - - def 'should create dockerfile content from spack recipe' () { - given: - def session = Mock(Session) { getConfig() >> [:]} - def RECIPE = 'bwa@0.7.15 salmon@1.1.1' - def ARCH = 'x86_64' - when: - def client = new WaveClient(session) - then: - client.spackRecipeToDockerFile(RECIPE, ARCH) == '''\ -# Builder image -FROM spack/ubuntu-jammy:v0.19.2 as builder - -RUN mkdir -p /opt/spack-env \\ -&& spack env create -d /opt/spack-env \\ -&& sed -e 's;compilers:;compilers::;' \\ - -e 's;^ *flags: *{}; flags:\\n cflags: -O3\\n cxxflags: -O3\\n fflags: -O3;' \\ - /root/.spack/linux/compilers.yaml > /opt/spack-env/compilers.yaml \\ -&& sed -i '/^spack:/a\\ include: [/opt/spack-env/compilers.yaml]' /opt/spack-env/spack.yaml \\ -&& cd /opt/spack-env && spack env activate . \\ -&& spack add bwa@0.7.15 salmon@1.1.1 \\ -&& spack config add config:install_tree:/opt/software \\ -&& spack config add concretizer:unify:true \\ -&& spack config add concretizer:reuse:false \\ -&& spack config add packages:all:target:[x86_64] \\ -&& echo -e "\\ - view: /opt/view \\n\\ -" >> /opt/spack-env/spack.yaml - -# Install packages, clean afterwards, finally strip binaries -RUN cd /opt/spack-env && spack env activate . \\ -&& spack concretize -f \\ -&& spack install --fail-fast && spack gc -y \\ -&& find -L /opt/._view/* -type f -exec readlink -f '{}' \\; | \\ - xargs file -i | \\ - grep 'charset=binary' | \\ - grep 'x-executable\\|x-archive\\|x-sharedlib' | \\ - awk -F: '{print \$1}' | xargs strip -s - -RUN cd /opt/spack-env && \\ - spack env activate --sh -d . >> /opt/spack-env/z10_spack_environment.sh && \\ - original_view=\$( cd /opt ; ls -1d ._view/* ) && \\ - sed -i "s;/view/;/\$original_view/;" /opt/spack-env/z10_spack_environment.sh && \\ - echo "# Needed for Perl applications" >>/opt/spack-env/z10_spack_environment.sh && \\ - echo "export PERL5LIB=\$(eval ls -d /opt/._view/*/lib/5.*):\$PERL5LIB" >>/opt/spack-env/z10_spack_environment.sh && \\ - rm -rf /opt/view - -# Runner image -FROM ubuntu:22.04 - -COPY --from=builder /opt/spack-env /opt/spack-env -COPY --from=builder /opt/software /opt/software -COPY --from=builder /opt/._view /opt/._view - -# Near OS-agnostic package addition -RUN ( apt update -y && apt install -y procps libgomp1 && rm -rf /var/lib/apt/lists/* ) || \\ - ( yum install -y procps libgomp && yum clean all && rm -rf /var/cache/yum ) || \\ - ( zypper ref && zypper install -y procps libgomp1 && zypper clean -a ) || \\ - ( apk update && apk add --no-cache procps libgomp bash && rm -rf /var/cache/apk ) - -# Entrypoint for Singularity -RUN mkdir -p /.singularity.d/env && \\ - cp -p /opt/spack-env/z10_spack_environment.sh /.singularity.d/env/91-environment.sh -# Entrypoint for Docker -RUN echo "#!/usr/bin/env bash\\n\\nset -ef -o pipefail\\nsource /opt/spack-env/z10_spack_environment.sh\\nexec \\"\\\$@\\"" \\ - >/opt/spack-env/spack_docker_entrypoint.sh && chmod a+x /opt/spack-env/spack_docker_entrypoint.sh - - -ENTRYPOINT [ "/opt/spack-env/spack_docker_entrypoint.sh" ] -CMD [ "/bin/bash" ] -'''//.stripIndent() - } - - def 'should create dockerfile content with custom channels' () { - given: - def session = Mock(Session) { - getConfig() >> [:] - getCondaConfig() >> new CondaConfig([channels:'foo,bar'], [:]) - } - def RECIPE = 'bwa=0.7.15 salmon=1.1.1' - when: - def client = new WaveClient(session) - then: - client.condaRecipeToDockerFile(RECIPE) == '''\ - FROM mambaorg/micromamba:1.4.2 - RUN \\ - micromamba install -y -n base -c foo -c bar \\ - bwa=0.7.15 salmon=1.1.1 \\ - && micromamba clean -a -y - '''.stripIndent() - } - - def 'should create dockerfile content with custom conda config' () { - given: - def CONDA_OPTS = [mambaImage:'my-base:123', commands: ['USER my-user', 'RUN apt-get update -y && apt-get install -y nano']] - def session = Mock(Session) { getConfig() >> [wave:[build:[conda:CONDA_OPTS]]]} - def RECIPE = 'bwa=0.7.15 salmon=1.1.1' - when: - def client = new WaveClient(session) - then: - client.condaRecipeToDockerFile(RECIPE) == '''\ - FROM my-base:123 - RUN \\ - micromamba install -y -n base -c conda-forge -c defaults \\ - bwa=0.7.15 salmon=1.1.1 \\ - && micromamba clean -a -y - USER my-user - RUN apt-get update -y && apt-get install -y nano - '''.stripIndent() - } - - - def 'should create dockerfile content with remote conda lock' () { - given: - def CONDA_OPTS = [mambaImage:'my-base:123', commands: ['USER my-user', 'RUN apt-get update -y && apt-get install -y procps']] - def session = Mock(Session) { getConfig() >> [wave:[build:[conda:CONDA_OPTS]]]} - def RECIPE = 'https://foo.com/some/conda-lock.yml' - when: - def client = new WaveClient(session) - then: - client.condaRecipeToDockerFile(RECIPE) == '''\ - FROM my-base:123 - RUN \\ - micromamba install -y -n base -c conda-forge -c defaults \\ - -f https://foo.com/some/conda-lock.yml \\ - && micromamba clean -a -y - USER my-user - RUN apt-get update -y && apt-get install -y procps - '''.stripIndent() - } - - def 'should create dockerfile content with custom spack config' () { - given: - def SPACK_OPTS = [ checksum:false, builderImage:'spack/foo:1', runnerImage:'ubuntu/foo', osPackages:'libfoo', cFlags:'-foo', cxxFlags:'-foo2', fFlags:'-foo3', commands:['USER hola'] ] - def session = Mock(Session) { getConfig() >> [wave:[build:[spack:SPACK_OPTS]]]} - def RECIPE = 'bwa@0.7.15 salmon@1.1.1' - def ARCH = 'nextcpu' - when: - def client = new WaveClient(session) - then: - client.spackRecipeToDockerFile(RECIPE, ARCH) == '''\ -# Builder image -FROM spack/foo:1 as builder - -RUN mkdir -p /opt/spack-env \\ -&& spack env create -d /opt/spack-env \\ -&& sed -e 's;compilers:;compilers::;' \\ - -e 's;^ *flags: *{}; flags:\\n cflags: -foo\\n cxxflags: -foo2\\n fflags: -foo3;' \\ - /root/.spack/linux/compilers.yaml > /opt/spack-env/compilers.yaml \\ -&& sed -i '/^spack:/a\\ include: [/opt/spack-env/compilers.yaml]' /opt/spack-env/spack.yaml \\ -&& cd /opt/spack-env && spack env activate . \\ -&& spack add bwa@0.7.15 salmon@1.1.1 \\ -&& spack config add config:install_tree:/opt/software \\ -&& spack config add concretizer:unify:true \\ -&& spack config add concretizer:reuse:false \\ -&& spack config add packages:all:target:[nextcpu] \\ -&& echo -e "\\ - view: /opt/view \\n\\ -" >> /opt/spack-env/spack.yaml - -# Install packages, clean afterwards, finally strip binaries -RUN cd /opt/spack-env && spack env activate . \\ -&& spack concretize -f \\ -&& spack install --fail-fast -n && spack gc -y \\ -&& find -L /opt/._view/* -type f -exec readlink -f '{}' \\; | \\ - xargs file -i | \\ - grep 'charset=binary' | \\ - grep 'x-executable\\|x-archive\\|x-sharedlib' | \\ - awk -F: '{print \$1}' | xargs strip -s - -RUN cd /opt/spack-env && \\ - spack env activate --sh -d . >> /opt/spack-env/z10_spack_environment.sh && \\ - original_view=\$( cd /opt ; ls -1d ._view/* ) && \\ - sed -i "s;/view/;/\$original_view/;" /opt/spack-env/z10_spack_environment.sh && \\ - echo "# Needed for Perl applications" >>/opt/spack-env/z10_spack_environment.sh && \\ - echo "export PERL5LIB=\$(eval ls -d /opt/._view/*/lib/5.*):\$PERL5LIB" >>/opt/spack-env/z10_spack_environment.sh && \\ - rm -rf /opt/view - -# Runner image -FROM ubuntu/foo - -COPY --from=builder /opt/spack-env /opt/spack-env -COPY --from=builder /opt/software /opt/software -COPY --from=builder /opt/._view /opt/._view - -# Near OS-agnostic package addition -RUN ( apt update -y && apt install -y procps libgomp1 libfoo && rm -rf /var/lib/apt/lists/* ) || \\ - ( yum install -y procps libgomp libfoo && yum clean all && rm -rf /var/cache/yum ) || \\ - ( zypper ref && zypper install -y procps libgomp1 libfoo && zypper clean -a ) || \\ - ( apk update && apk add --no-cache procps libgomp bash libfoo && rm -rf /var/cache/apk ) - -# Entrypoint for Singularity -RUN mkdir -p /.singularity.d/env && \\ - cp -p /opt/spack-env/z10_spack_environment.sh /.singularity.d/env/91-environment.sh -# Entrypoint for Docker -RUN echo "#!/usr/bin/env bash\\n\\nset -ef -o pipefail\\nsource /opt/spack-env/z10_spack_environment.sh\\nexec \\"\\\$@\\"" \\ - >/opt/spack-env/spack_docker_entrypoint.sh && chmod a+x /opt/spack-env/spack_docker_entrypoint.sh - -USER hola - -ENTRYPOINT [ "/opt/spack-env/spack_docker_entrypoint.sh" ] -CMD [ "/bin/bash" ] -'''//.stripIndent() - } - - def 'should create dockerfile content from conda file' () { - given: - def CONDA_OPTS = [basePackages: 'conda-forge::procps-ng'] - def session = Mock(Session) { getConfig() >> [wave:[build:[conda:CONDA_OPTS]]]} - when: - def client = new WaveClient(session) - then: - client.condaFileToDockerFile()== '''\ - FROM mambaorg/micromamba:1.4.2 - COPY --chown=$MAMBA_USER:$MAMBA_USER conda.yml /tmp/conda.yml - RUN micromamba install -y -n base -f /tmp/conda.yml && \\ - micromamba install -y -n base conda-forge::procps-ng && \\ - micromamba clean -a -y - '''.stripIndent() - } - - def 'should create dockerfile content from conda file and base packages' () { - given: - def session = Mock(Session) { getConfig() >> [:]} - when: - def client = new WaveClient(session) - then: - client.condaFileToDockerFile()== '''\ - FROM mambaorg/micromamba:1.4.2 - COPY --chown=$MAMBA_USER:$MAMBA_USER conda.yml /tmp/conda.yml - RUN micromamba install -y -n base -f /tmp/conda.yml && \\ - micromamba clean -a -y - '''.stripIndent() - } - - def 'should create dockerfile content from spack file' () { - given: - def session = Mock(Session) { getConfig() >> [:]} - def ARCH = 'x86_64' - when: - def client = new WaveClient(session) - then: - client.spackFileToDockerFile(ARCH)== '''\ -# Builder image -FROM spack/ubuntu-jammy:v0.19.2 as builder -COPY spack.yaml /tmp/spack.yaml - -RUN mkdir -p /opt/spack-env \\ -&& sed -e 's;compilers:;compilers::;' \\ - -e 's;^ *flags: *{}; flags:\\n cflags: -O3\\n cxxflags: -O3\\n fflags: -O3;' \\ - /root/.spack/linux/compilers.yaml > /opt/spack-env/compilers.yaml \\ -&& sed '/^spack:/a\\ include: [/opt/spack-env/compilers.yaml]' /tmp/spack.yaml > /opt/spack-env/spack.yaml \\ -&& cd /opt/spack-env && spack env activate . \\ -&& spack config add config:install_tree:/opt/software \\ -&& spack config add concretizer:unify:true \\ -&& spack config add concretizer:reuse:false \\ -&& spack config add packages:all:target:[x86_64] \\ -&& echo -e "\\ - view: /opt/view \\n\\ -" >> /opt/spack-env/spack.yaml - -# Install packages, clean afterwards, finally strip binaries -RUN cd /opt/spack-env && spack env activate . \\ -&& spack concretize -f \\ -&& spack install --fail-fast && spack gc -y \\ -&& find -L /opt/._view/* -type f -exec readlink -f '{}' \\; | \\ - xargs file -i | \\ - grep 'charset=binary' | \\ - grep 'x-executable\\|x-archive\\|x-sharedlib' | \\ - awk -F: '{print \$1}' | xargs strip -s - -RUN cd /opt/spack-env && \\ - spack env activate --sh -d . >> /opt/spack-env/z10_spack_environment.sh && \\ - original_view=\$( cd /opt ; ls -1d ._view/* ) && \\ - sed -i "s;/view/;/\$original_view/;" /opt/spack-env/z10_spack_environment.sh && \\ - echo "# Needed for Perl applications" >>/opt/spack-env/z10_spack_environment.sh && \\ - echo "export PERL5LIB=\$(eval ls -d /opt/._view/*/lib/5.*):\$PERL5LIB" >>/opt/spack-env/z10_spack_environment.sh && \\ - rm -rf /opt/view - -# Runner image -FROM ubuntu:22.04 - -COPY --from=builder /opt/spack-env /opt/spack-env -COPY --from=builder /opt/software /opt/software -COPY --from=builder /opt/._view /opt/._view - -# Near OS-agnostic package addition -RUN ( apt update -y && apt install -y procps libgomp1 && rm -rf /var/lib/apt/lists/* ) || \\ - ( yum install -y procps libgomp && yum clean all && rm -rf /var/cache/yum ) || \\ - ( zypper ref && zypper install -y procps libgomp1 && zypper clean -a ) || \\ - ( apk update && apk add --no-cache procps libgomp bash && rm -rf /var/cache/apk ) - -# Entrypoint for Singularity -RUN mkdir -p /.singularity.d/env && \\ - cp -p /opt/spack-env/z10_spack_environment.sh /.singularity.d/env/91-environment.sh -# Entrypoint for Docker -RUN echo "#!/usr/bin/env bash\\n\\nset -ef -o pipefail\\nsource /opt/spack-env/z10_spack_environment.sh\\nexec \\"\\\$@\\"" \\ - >/opt/spack-env/spack_docker_entrypoint.sh && chmod a+x /opt/spack-env/spack_docker_entrypoint.sh - - -ENTRYPOINT [ "/opt/spack-env/spack_docker_entrypoint.sh" ] -CMD [ "/bin/bash" ] -'''//.stripIndent() - - } def 'should create asset with image' () { given: @@ -900,8 +561,8 @@ CMD [ "/bin/bash" ] assets.dockerFileContent == '''\ FROM mambaorg/micromamba:1.4.2 COPY --chown=$MAMBA_USER:$MAMBA_USER conda.yml /tmp/conda.yml - RUN micromamba install -y -n base -f /tmp/conda.yml && \\ - micromamba clean -a -y + RUN micromamba install -y -n base -f /tmp/conda.yml \\ + && micromamba clean -a -y '''.stripIndent() and: assets.condaFile == condaFile diff --git a/plugins/nf-wave/src/test/io/seqera/wave/plugin/WaveObserverTest.groovy b/plugins/nf-wave/src/test/io/seqera/wave/plugin/WaveObserverTest.groovy index 7957e3f77e..69be66d69e 100644 --- a/plugins/nf-wave/src/test/io/seqera/wave/plugin/WaveObserverTest.groovy +++ b/plugins/nf-wave/src/test/io/seqera/wave/plugin/WaveObserverTest.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022, Seqera Labs + * Copyright 2013-2023, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/plugins/nf-wave/src/test/io/seqera/wave/plugin/config/RetryOptsTest.groovy b/plugins/nf-wave/src/test/io/seqera/wave/plugin/config/RetryOptsTest.groovy index 395b11ee62..0c53c8cbbb 100644 --- a/plugins/nf-wave/src/test/io/seqera/wave/plugin/config/RetryOptsTest.groovy +++ b/plugins/nf-wave/src/test/io/seqera/wave/plugin/config/RetryOptsTest.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022, Seqera Labs + * Copyright 2013-2023, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/plugins/nf-wave/src/test/io/seqera/wave/util/DockerHelperTest.groovy b/plugins/nf-wave/src/test/io/seqera/wave/util/DockerHelperTest.groovy new file mode 100644 index 0000000000..99175c2c5b --- /dev/null +++ b/plugins/nf-wave/src/test/io/seqera/wave/util/DockerHelperTest.groovy @@ -0,0 +1,353 @@ +/* + * Copyright 2013-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.seqera.wave.util + +import spock.lang.Specification + +import io.seqera.wave.config.CondaOpts +import io.seqera.wave.config.SpackOpts + +/** + * + * @author Paolo Di Tommaso + */ +class DockerHelperTest extends Specification { + + def 'should create dockerfile content from conda file' () { + given: + def CONDA_OPTS = new CondaOpts([basePackages: 'conda-forge::procps-ng']) + + expect: + DockerHelper.condaFileToDockerFile(CONDA_OPTS)== '''\ + FROM mambaorg/micromamba:1.4.2 + COPY --chown=$MAMBA_USER:$MAMBA_USER conda.yml /tmp/conda.yml + RUN micromamba install -y -n base -f /tmp/conda.yml \\ + && micromamba install -y -n base conda-forge::procps-ng \\ + && micromamba clean -a -y + '''.stripIndent() + } + + def 'should create dockerfile content from conda file and base packages' () { + + expect: + DockerHelper.condaFileToDockerFile(new CondaOpts([:]))== '''\ + FROM mambaorg/micromamba:1.4.2 + COPY --chown=$MAMBA_USER:$MAMBA_USER conda.yml /tmp/conda.yml + RUN micromamba install -y -n base -f /tmp/conda.yml \\ + && micromamba clean -a -y + '''.stripIndent() + } + + + def 'should create dockerfile content from conda package' () { + given: + def PACKAGES = 'bwa=0.7.15 salmon=1.1.1' + def CHANNELS = ['conda-forge', 'defaults'] + expect: + DockerHelper.condaPackagesToDockerFile(PACKAGES, CHANNELS, new CondaOpts([:])) == '''\ + FROM mambaorg/micromamba:1.4.2 + RUN \\ + micromamba install -y -n base -c conda-forge -c defaults \\ + bwa=0.7.15 salmon=1.1.1 \\ + && micromamba clean -a -y + '''.stripIndent() + } + + def 'should create dockerfile with base packages' () { + given: + def CHANNELS = ['conda-forge', 'defaults'] + def CONDA_OPTS = new CondaOpts([basePackages: 'foo::one bar::two']) + def PACKAGES = 'bwa=0.7.15 salmon=1.1.1' + + expect: + DockerHelper.condaPackagesToDockerFile(PACKAGES, CHANNELS, CONDA_OPTS) == '''\ + FROM mambaorg/micromamba:1.4.2 + RUN \\ + micromamba install -y -n base -c conda-forge -c defaults \\ + bwa=0.7.15 salmon=1.1.1 \\ + && micromamba install -y -n base foo::one bar::two \\ + && micromamba clean -a -y + '''.stripIndent() + } + + def 'should create dockerfile content with custom channels' () { + given: + def CHANNELS = 'foo,bar'.tokenize(',') + def PACKAGES = 'bwa=0.7.15 salmon=1.1.1' + + expect: + DockerHelper.condaPackagesToDockerFile(PACKAGES, CHANNELS, new CondaOpts([:])) == '''\ + FROM mambaorg/micromamba:1.4.2 + RUN \\ + micromamba install -y -n base -c foo -c bar \\ + bwa=0.7.15 salmon=1.1.1 \\ + && micromamba clean -a -y + '''.stripIndent() + } + + def 'should create dockerfile content with custom conda config' () { + given: + def CHANNELS = ['conda-forge', 'defaults'] + def CONDA_OPTS = [mambaImage:'my-base:123', commands: ['USER my-user', 'RUN apt-get update -y && apt-get install -y nano']] + def PACKAGES = 'bwa=0.7.15 salmon=1.1.1' + + expect: + DockerHelper.condaPackagesToDockerFile(PACKAGES, CHANNELS, new CondaOpts(CONDA_OPTS)) == '''\ + FROM my-base:123 + RUN \\ + micromamba install -y -n base -c conda-forge -c defaults \\ + bwa=0.7.15 salmon=1.1.1 \\ + && micromamba clean -a -y + USER my-user + RUN apt-get update -y && apt-get install -y nano + '''.stripIndent() + } + + + def 'should create dockerfile content with remote conda lock' () { + given: + def CHANNELS = ['conda-forge', 'defaults'] + def OPTS = [mambaImage:'my-base:123', commands: ['USER my-user', 'RUN apt-get update -y && apt-get install -y procps']] + def PACKAGES = 'https://foo.com/some/conda-lock.yml' + + expect: + DockerHelper.condaPackagesToDockerFile(PACKAGES, CHANNELS, new CondaOpts(OPTS)) == '''\ + FROM my-base:123 + RUN \\ + micromamba install -y -n base -c conda-forge -c defaults \\ + -f https://foo.com/some/conda-lock.yml \\ + && micromamba clean -a -y + USER my-user + RUN apt-get update -y && apt-get install -y procps + '''.stripIndent() + } + + + def 'should create dockerfile content from spack package' () { + given: + def PACKAGES = 'bwa@0.7.15 salmon@1.1.1' + def ARCH = 'x86_64' + + expect: + DockerHelper.spackPackagesToDockerFile(PACKAGES, ARCH, new SpackOpts([:])) == '''\ +# Builder image +FROM spack/ubuntu-jammy:v0.19.2 as builder + +RUN mkdir -p /opt/spack-env \\ +&& spack env create -d /opt/spack-env \\ +&& sed -e 's;compilers:;compilers::;' \\ + -e 's;^ *flags: *{}; flags:\\n cflags: -O3\\n cxxflags: -O3\\n fflags: -O3;' \\ + /root/.spack/linux/compilers.yaml > /opt/spack-env/compilers.yaml \\ +&& sed -i '/^spack:/a\\ include: [/opt/spack-env/compilers.yaml]' /opt/spack-env/spack.yaml \\ +&& cd /opt/spack-env && spack env activate . \\ +&& spack add bwa@0.7.15 salmon@1.1.1 \\ +&& spack config add config:install_tree:/opt/software \\ +&& spack config add concretizer:unify:true \\ +&& spack config add concretizer:reuse:false \\ +&& spack config add packages:all:target:[x86_64] \\ +&& echo -e "\\ + view: /opt/view \\n\\ +" >> /opt/spack-env/spack.yaml + +# Install packages, clean afterwards, finally strip binaries +RUN cd /opt/spack-env && spack env activate . \\ +&& spack concretize -f \\ +&& spack install --fail-fast && spack gc -y \\ +&& find -L /opt/._view/* -type f -exec readlink -f '{}' \\; | \\ + xargs file -i | \\ + grep 'charset=binary' | \\ + grep 'x-executable\\|x-archive\\|x-sharedlib' | \\ + awk -F: '{print \$1}' | xargs strip -s + +RUN cd /opt/spack-env && \\ + spack env activate --sh -d . >> /opt/spack-env/z10_spack_environment.sh && \\ + original_view=\$( cd /opt ; ls -1d ._view/* ) && \\ + sed -i "s;/view/;/\$original_view/;" /opt/spack-env/z10_spack_environment.sh && \\ + echo "# Needed for Perl applications" >>/opt/spack-env/z10_spack_environment.sh && \\ + echo "export PERL5LIB=\$(eval ls -d /opt/._view/*/lib/5.*):\$PERL5LIB" >>/opt/spack-env/z10_spack_environment.sh && \\ + rm -rf /opt/view + +# Runner image +FROM ubuntu:22.04 + +COPY --from=builder /opt/spack-env /opt/spack-env +COPY --from=builder /opt/software /opt/software +COPY --from=builder /opt/._view /opt/._view + +# Near OS-agnostic package addition +RUN ( apt update -y && apt install -y procps libgomp1 && rm -rf /var/lib/apt/lists/* ) || \\ + ( yum install -y procps libgomp && yum clean all && rm -rf /var/cache/yum ) || \\ + ( zypper ref && zypper install -y procps libgomp1 && zypper clean -a ) || \\ + ( apk update && apk add --no-cache procps libgomp bash && rm -rf /var/cache/apk ) + +# Entrypoint for Singularity +RUN mkdir -p /.singularity.d/env && \\ + cp -p /opt/spack-env/z10_spack_environment.sh /.singularity.d/env/91-environment.sh +# Entrypoint for Docker +RUN echo "#!/usr/bin/env bash\\n\\nset -ef -o pipefail\\nsource /opt/spack-env/z10_spack_environment.sh\\nexec \\"\\\$@\\"" \\ + >/opt/spack-env/spack_docker_entrypoint.sh && chmod a+x /opt/spack-env/spack_docker_entrypoint.sh + + +ENTRYPOINT [ "/opt/spack-env/spack_docker_entrypoint.sh" ] +CMD [ "/bin/bash" ] +'''//.stripIndent() + } + + def 'should create dockerfile content with custom spack config' () { + given: + def SPACK_OPTS = [ checksum:false, builderImage:'spack/foo:1', runnerImage:'ubuntu/foo', osPackages:'libfoo', cFlags:'-foo', cxxFlags:'-foo2', fFlags:'-foo3', commands:['USER hola'] ] + def PACKAGES = 'bwa@0.7.15 salmon@1.1.1' + def ARCH = 'nextcpu' + + expect: + DockerHelper.spackPackagesToDockerFile(PACKAGES, ARCH, new SpackOpts(SPACK_OPTS)) == '''\ +# Builder image +FROM spack/foo:1 as builder + +RUN mkdir -p /opt/spack-env \\ +&& spack env create -d /opt/spack-env \\ +&& sed -e 's;compilers:;compilers::;' \\ + -e 's;^ *flags: *{}; flags:\\n cflags: -foo\\n cxxflags: -foo2\\n fflags: -foo3;' \\ + /root/.spack/linux/compilers.yaml > /opt/spack-env/compilers.yaml \\ +&& sed -i '/^spack:/a\\ include: [/opt/spack-env/compilers.yaml]' /opt/spack-env/spack.yaml \\ +&& cd /opt/spack-env && spack env activate . \\ +&& spack add bwa@0.7.15 salmon@1.1.1 \\ +&& spack config add config:install_tree:/opt/software \\ +&& spack config add concretizer:unify:true \\ +&& spack config add concretizer:reuse:false \\ +&& spack config add packages:all:target:[nextcpu] \\ +&& echo -e "\\ + view: /opt/view \\n\\ +" >> /opt/spack-env/spack.yaml + +# Install packages, clean afterwards, finally strip binaries +RUN cd /opt/spack-env && spack env activate . \\ +&& spack concretize -f \\ +&& spack install --fail-fast -n && spack gc -y \\ +&& find -L /opt/._view/* -type f -exec readlink -f '{}' \\; | \\ + xargs file -i | \\ + grep 'charset=binary' | \\ + grep 'x-executable\\|x-archive\\|x-sharedlib' | \\ + awk -F: '{print \$1}' | xargs strip -s + +RUN cd /opt/spack-env && \\ + spack env activate --sh -d . >> /opt/spack-env/z10_spack_environment.sh && \\ + original_view=\$( cd /opt ; ls -1d ._view/* ) && \\ + sed -i "s;/view/;/\$original_view/;" /opt/spack-env/z10_spack_environment.sh && \\ + echo "# Needed for Perl applications" >>/opt/spack-env/z10_spack_environment.sh && \\ + echo "export PERL5LIB=\$(eval ls -d /opt/._view/*/lib/5.*):\$PERL5LIB" >>/opt/spack-env/z10_spack_environment.sh && \\ + rm -rf /opt/view + +# Runner image +FROM ubuntu/foo + +COPY --from=builder /opt/spack-env /opt/spack-env +COPY --from=builder /opt/software /opt/software +COPY --from=builder /opt/._view /opt/._view + +# Near OS-agnostic package addition +RUN ( apt update -y && apt install -y procps libgomp1 libfoo && rm -rf /var/lib/apt/lists/* ) || \\ + ( yum install -y procps libgomp libfoo && yum clean all && rm -rf /var/cache/yum ) || \\ + ( zypper ref && zypper install -y procps libgomp1 libfoo && zypper clean -a ) || \\ + ( apk update && apk add --no-cache procps libgomp bash libfoo && rm -rf /var/cache/apk ) + +# Entrypoint for Singularity +RUN mkdir -p /.singularity.d/env && \\ + cp -p /opt/spack-env/z10_spack_environment.sh /.singularity.d/env/91-environment.sh +# Entrypoint for Docker +RUN echo "#!/usr/bin/env bash\\n\\nset -ef -o pipefail\\nsource /opt/spack-env/z10_spack_environment.sh\\nexec \\"\\\$@\\"" \\ + >/opt/spack-env/spack_docker_entrypoint.sh && chmod a+x /opt/spack-env/spack_docker_entrypoint.sh + +USER hola + +ENTRYPOINT [ "/opt/spack-env/spack_docker_entrypoint.sh" ] +CMD [ "/bin/bash" ] +'''//.stripIndent() + } + + + def 'should create dockerfile content from spack file' () { + given: + def ARCH = 'x86_64' + + expect: + DockerHelper.spackFileToDockerFile(ARCH, new SpackOpts())== '''\ +# Builder image +FROM spack/ubuntu-jammy:v0.19.2 as builder +COPY spack.yaml /tmp/spack.yaml + +RUN mkdir -p /opt/spack-env \\ +&& sed -e 's;compilers:;compilers::;' \\ + -e 's;^ *flags: *{}; flags:\\n cflags: -O3\\n cxxflags: -O3\\n fflags: -O3;' \\ + /root/.spack/linux/compilers.yaml > /opt/spack-env/compilers.yaml \\ +&& sed '/^spack:/a\\ include: [/opt/spack-env/compilers.yaml]' /tmp/spack.yaml > /opt/spack-env/spack.yaml \\ +&& cd /opt/spack-env && spack env activate . \\ +&& spack config add config:install_tree:/opt/software \\ +&& spack config add concretizer:unify:true \\ +&& spack config add concretizer:reuse:false \\ +&& spack config add packages:all:target:[x86_64] \\ +&& echo -e "\\ + view: /opt/view \\n\\ +" >> /opt/spack-env/spack.yaml + +# Install packages, clean afterwards, finally strip binaries +RUN cd /opt/spack-env && spack env activate . \\ +&& spack concretize -f \\ +&& spack install --fail-fast && spack gc -y \\ +&& find -L /opt/._view/* -type f -exec readlink -f '{}' \\; | \\ + xargs file -i | \\ + grep 'charset=binary' | \\ + grep 'x-executable\\|x-archive\\|x-sharedlib' | \\ + awk -F: '{print \$1}' | xargs strip -s + +RUN cd /opt/spack-env && \\ + spack env activate --sh -d . >> /opt/spack-env/z10_spack_environment.sh && \\ + original_view=\$( cd /opt ; ls -1d ._view/* ) && \\ + sed -i "s;/view/;/\$original_view/;" /opt/spack-env/z10_spack_environment.sh && \\ + echo "# Needed for Perl applications" >>/opt/spack-env/z10_spack_environment.sh && \\ + echo "export PERL5LIB=\$(eval ls -d /opt/._view/*/lib/5.*):\$PERL5LIB" >>/opt/spack-env/z10_spack_environment.sh && \\ + rm -rf /opt/view + +# Runner image +FROM ubuntu:22.04 + +COPY --from=builder /opt/spack-env /opt/spack-env +COPY --from=builder /opt/software /opt/software +COPY --from=builder /opt/._view /opt/._view + +# Near OS-agnostic package addition +RUN ( apt update -y && apt install -y procps libgomp1 && rm -rf /var/lib/apt/lists/* ) || \\ + ( yum install -y procps libgomp && yum clean all && rm -rf /var/cache/yum ) || \\ + ( zypper ref && zypper install -y procps libgomp1 && zypper clean -a ) || \\ + ( apk update && apk add --no-cache procps libgomp bash && rm -rf /var/cache/apk ) + +# Entrypoint for Singularity +RUN mkdir -p /.singularity.d/env && \\ + cp -p /opt/spack-env/z10_spack_environment.sh /.singularity.d/env/91-environment.sh +# Entrypoint for Docker +RUN echo "#!/usr/bin/env bash\\n\\nset -ef -o pipefail\\nsource /opt/spack-env/z10_spack_environment.sh\\nexec \\"\\\$@\\"" \\ + >/opt/spack-env/spack_docker_entrypoint.sh && chmod a+x /opt/spack-env/spack_docker_entrypoint.sh + + +ENTRYPOINT [ "/opt/spack-env/spack_docker_entrypoint.sh" ] +CMD [ "/bin/bash" ] +'''//.stripIndent() + + } + +} diff --git a/plugins/nf-wave/src/test/io/seqera/wave/util/TemplateRendererTest.groovy b/plugins/nf-wave/src/test/io/seqera/wave/util/TemplateRendererTest.groovy new file mode 100644 index 0000000000..b043594168 --- /dev/null +++ b/plugins/nf-wave/src/test/io/seqera/wave/util/TemplateRendererTest.groovy @@ -0,0 +1,181 @@ +/* + * Copyright 2013-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.seqera.wave.util + +import spock.lang.Specification + +/** + * + * @author Paolo Di Tommaso + */ +class TemplateHelperTest extends Specification { + + def 'should replace vars' () { + given: + def binding = [foo: 'Hello', bar: 'world'] + + expect: + TemplateHelper.replace0('{{foo}}', binding) == 'Hello' + TemplateHelper.replace0('{{foo}} ', binding) == 'Hello ' + TemplateHelper.replace0('{{foo}}\n', binding) == 'Hello\n' + TemplateHelper.replace0(' {{foo}}', binding) == ' Hello' + TemplateHelper.replace0(' {{foo}}\n', binding) == ' Hello\n' + TemplateHelper.replace0(' ${foo}', binding) == ' ${foo}' + TemplateHelper.replace0(' ${{foo}}', binding) == ' ${{foo}}' + TemplateHelper.replace0('{{foo}}', [foo:'']) == '' + TemplateHelper.replace0('{{foo}}', [foo:null]) == null + TemplateHelper.replace0(' {{foo}}\n', [foo:null]) == null + TemplateHelper.replace0('', binding) == '' + TemplateHelper.replace0(null, binding) == null + + TemplateHelper.replace0('{{foo}} {{bar}}!', binding) == 'Hello world!' + TemplateHelper.replace0('abc {{foo}} pq {{bar}} xyz', binding) == 'abc Hello pq world xyz' + TemplateHelper.replace0('{{foo}} 123 {{bar}} xyz {{foo}}', binding) == 'Hello 123 world xyz Hello' + TemplateHelper.replace0('1{{foo}}2{{foo}}3', [foo:'']) == '123' + TemplateHelper.replace0('1{{foo}}2{{foo}}3', [foo:null]) == '123' + + when: + TemplateHelper.replace0('{{x1}}', binding) + then: + def e = thrown(IllegalArgumentException) + e.message == 'Missing template key: x1' + + when: + TemplateHelper.replace0('{{foo}} {{x2}}', binding) + then: + e = thrown(IllegalArgumentException) + e.message == 'Missing template key: x2' + + } + + def 'should render template' () { + given: + def template = "Hello, {{name}}!\n" + + "Today is {{day}} and the weather is {{weather}}."; + and: + def binding = new HashMap(); + binding.put("name", "John"); + binding.put("day", "Monday"); + binding.put("weather", "sunny"); + + when: + def result = TemplateHelper.render(template, binding); + + then: + result == 'Hello, John!\nToday is Monday and the weather is sunny.' + } + + def 'should render a template with comment'() { + given: + def template = """\ + ## remove this comment + 1: {{alpha}} + 2: {{delta}} {{delta}} + 3: {{gamma}} {{gamma}} {{gamma}} + 4: end + """.stripIndent() + and: + def binding = new HashMap(); + binding.put("alpha", "one"); + binding.put("delta", "two"); + binding.put("gamma", "three"); + + when: + def result = TemplateHelper.render(new ByteArrayInputStream(template.bytes), binding); + + then: + result == """\ + 1: one + 2: two two + 3: three three three + 4: end + """.stripIndent() + } + + + def 'should render a template using an input stream'() { + given: + def template = """\ + {{one}} + {{two}} + xxx + {{three}} + zzz + """.stripIndent() + and: + def binding = [ + one: '1', // this is rendered + two:null, // a line containing a null variable is not rendered + three:'' // empty value is considered ok + ] + + when: + def result = TemplateHelper.render(new ByteArrayInputStream(template.bytes), binding); + + then: + result == """\ + 1 + xxx + + zzz + """.stripIndent() + } + + def 'should render template with indentations' () { + given: + def binding = [foo: 'Hello', bar: 'world'] + + when: + def result = TemplateHelper.render('{{foo}}\n{{bar}}', binding) + then: + result == 'Hello\nworld' + + when: + def template = '''\ + {{foo}} + {{bar}} + '''.stripIndent() + result = TemplateHelper.render(template, [foo:'11\n22\n33', bar:'Hello world']) + then: + result == '''\ + 11 + 22 + 33 + Hello world + '''.stripIndent() + + + when: + template = '''\ + {{x1}} + {{x2}} + {{x3}} + '''.stripIndent() + result = TemplateHelper.render(template, [x1:'aa\nbb\n', x2:null, x3:'pp\nqq']) + then: + result == '''\ + aa + bb + + pp + qq + '''.stripIndent() + + } + +}