diff --git a/modules/nf-core/quarto/partial/environment.yml b/modules/nf-core/quarto/partial/environment.yml new file mode 100644 index 000000000000..baffa16207d2 --- /dev/null +++ b/modules/nf-core/quarto/partial/environment.yml @@ -0,0 +1,12 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +channels: + - conda-forge + - bioconda +dependencies: + # renovate: datasource=conda depName=conda-forge/quarto + - conda-forge::jupyter=1.1.1 + - conda-forge::matplotlib=3.10.3 + - conda-forge::papermill=2.6.0 + - conda-forge::quarto=1.7.31 + - conda-forge::r-rmarkdown=2.29 diff --git a/modules/nf-core/quarto/partial/main.nf b/modules/nf-core/quarto/partial/main.nf new file mode 100644 index 000000000000..51013749d67b --- /dev/null +++ b/modules/nf-core/quarto/partial/main.nf @@ -0,0 +1,102 @@ +// NB: You'll likely want to override this with a container containing all +// required dependencies for your analyses, or use wave to build the container +// for you from the environment.yml. You'll at least need Quarto itself, +// Papermill and whatever language you are running your analyses on; you can see +// an example in this module's environment file. +process QUARTO_PARTIAL { + tag "${prefix}" + label 'process_low' + conda "${moduleDir}/environment.yml" + container "${workflow.containerEngine in ['singularity', 'apptainer'] && !task.ext.singularity_pull_docker_container + ? 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/28/28717ccd9ce22dbfc219f3db088d5a1fc2ca1f575b5c65621218596dcdbaac95/data' + : 'community.wave.seqera.io/library/jupyter_matplotlib_papermill_quarto_r-rmarkdown:6d15193ce3dfc665'}" + + input: + tuple val(meta), path(notebook) + val parameters + path input_files + + output: + tuple val(meta), path("${prefix}{.md,_files}") , emit: partial + tuple val(meta), path(notebook) , emit: notebook + tuple val(meta), path("params.yml") , emit: params_yaml + tuple val(meta), path("${notebook_parameters.artifact_dir}/*") , emit: artifacts, optional: true + tuple val("${task.process}"), val('quarto') , eval('quarto -v') , emit: versions_quarto , topic: versions + tuple val("${task.process}"), val('papermill'), eval('papermill --version | cut -f1 -d" "'), emit: versions_papermill, topic: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + // Partial is meant to be run once (not per sample), hence the naming scheme + prefix = task.ext.prefix ?: "${notebook.baseName}" + // Implicit parameters can be overwritten by supplying a value with parameters + notebook_parameters = [ + meta: meta, + cpus: task.cpus, + artifact_dir: "artifacts", + ] + (parameters ?: [:]) + // Parse parameters through a YAML file, which is better than CLI because: + // - No issue with escaping + // - Allows passing nested maps instead of just single values + // - Allows running with the language-agnostic `--execute-params` + def yamlBuilder = new groovy.yaml.YamlBuilder() + yamlBuilder.call(notebook_parameters) + def yaml_content = yamlBuilder.toString().tokenize('\n').join("\n ") + """ + # Dump parameters to yaml file + cat <<- END_YAML_PARAMS > params.yml + ${yaml_content} + END_YAML_PARAMS + + # Create output directory + mkdir "${notebook_parameters.artifact_dir}" + + # Set environment variables needed for Quarto rendering + export XDG_CACHE_HOME="./.xdg_cache_home" + export XDG_DATA_HOME="./.xdg_data_home" + + # Fix Quarto for Apptainer (see https://community.seqera.io/t/confusion-over-why-a-tool-works-in-docker-but-fails-in-singularity-when-the-installation-doesnt-differ-i-e-using-wave-micromamba/1244) + ENV_QUARTO=/opt/conda/etc/conda/activate.d/quarto.sh + set +u + if [ -z "\${QUARTO_DENO}" ] && [ -f "\${ENV_QUARTO}" ]; then + source "\${ENV_QUARTO}" + fi + set -u + + # Set parallelism for BLAS/MKL etc. to avoid over-booking of resources + export MKL_NUM_THREADS="${task.cpus}" + export OPENBLAS_NUM_THREADS="${task.cpus}" + export OMP_NUM_THREADS="${task.cpus}" + export NUMBA_NUM_THREADS="${task.cpus}" + + # Render partial + quarto render \\ + ${notebook} \\ + ${args} \\ + --to markdown \\ + --execute-params params.yml \\ + --output ${prefix}.md + """ + + stub: + prefix = task.ext.prefix ?: "${notebook.baseName}" + notebook_parameters = [ + meta: meta, + cpus: task.cpus, + artifact_dir: "artifacts", + ] + (parameters ?: [:]) + """ + # Note: The fix is also needed in the stub for `quarto -v` to work. + ENV_QUARTO=/opt/conda/etc/conda/activate.d/quarto.sh + set +u + if [ -z "\${QUARTO_DENO}" ] && [ -f "\${ENV_QUARTO}" ]; then + source "\${ENV_QUARTO}" + fi + set -u + + touch ${prefix}.md + touch params.yml + """ +} diff --git a/modules/nf-core/quarto/partial/meta.yml b/modules/nf-core/quarto/partial/meta.yml new file mode 100644 index 000000000000..d4c30db9d8dc --- /dev/null +++ b/modules/nf-core/quarto/partial/meta.yml @@ -0,0 +1,132 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json +name: "quarto_partial" +description: Render a Quarto partial, including parametrization +keywords: + - quarto + - notebook + - reports + - python + - r +tools: + - "quarto": + description: An open-source scientific and technical publishing system. + homepage: https://quarto.org/ + documentation: https://quarto.org/docs/reference/ + tool_dev_url: https://github.com/quarto-dev/quarto-cli + licence: + - "MIT" + identifier: "" + - papermill: + description: Parameterize, execute, and analyze notebooks + homepage: https://github.com/nteract/papermill + documentation: http://papermill.readthedocs.io/en/latest/ + tool_dev_url: https://github.com/nteract/papermill + licence: + - "BSD 3-clause" + identifier: "" +input: + - - meta: + type: map + description: | + Groovy Map containing sample information, e.g. `[ id:'sample1' ]`. + - notebook: + type: file + description: The Quarto notebook to be rendered into a partial. + pattern: "*.qmd" + ontologies: [] + - parameters: + type: map + description: | + Groovy map with notebook parameters which will be passed to Quarto to generate parametrized partials. + - input_files: + type: file + description: One or multiple files serving as input data for the partial. + pattern: "*" + ontologies: [] +output: + partial: + - - meta: + type: map + description: | + Groovy Map containing sample information, e.g. `[ id:'sample1' ]`. + - ${prefix}{.md,_files}: + type: file + description: Partial (a markdown file) and partial files (a directory) + generated by Quarto. + pattern: "${prefix}{.md,_files}.md" + ontologies: [] + notebook: + - - meta: + type: map + description: | + Groovy Map containing sample information, e.g. `[ id:'sample1' ]`. + - notebook: + type: file + description: The original Quarto notebook that was rendered into a + partial; allows the user to continue working on the notebook. + pattern: "*.qmd" + ontologies: [] + params_yaml: + - - meta: + type: map + description: | + Groovy Map containing sample information, e.g. `[ id:'sample1' ]`. + - params.yml: + type: file + description: Parameters used during rendering. + pattern: "*" + ontologies: [] + artifacts: + - - meta: + type: map + description: | + Groovy Map containing sample information, e.g. `[ id:'sample1' ]`. + - ${notebook_parameters.artifact_dir}/*: + type: file + description: Artifacts generated during rendering. + pattern: "*" + ontologies: [] + versions_quarto: + - - ${task.process}: + type: string + description: The name of the process + - quarto: + type: string + description: The name of the tool + - quarto -v: + type: eval + description: The expression to obtain the version of the tool + versions_papermill: + - - ${task.process}: + type: string + description: The name of the process + - papermill: + type: string + description: The name of the tool + - papermill --version | cut -f1 -d" ": + type: eval + description: The expression to obtain the version of the tool +topics: + versions: + - - ${task.process}: + type: string + description: The name of the process + - quarto: + type: string + description: The name of the tool + - quarto -v: + type: eval + description: The expression to obtain the version of the tool + - - ${task.process}: + type: string + description: The name of the process + - papermill: + type: string + description: The name of the tool + - papermill --version | cut -f1 -d" ": + type: eval + description: The expression to obtain the version of the tool +authors: + - "@fasterius" +maintainers: + - "@fasterius" diff --git a/modules/nf-core/quarto/partial/tests/main.nf.test b/modules/nf-core/quarto/partial/tests/main.nf.test new file mode 100644 index 000000000000..d0a835b7b2a0 --- /dev/null +++ b/modules/nf-core/quarto/partial/tests/main.nf.test @@ -0,0 +1,179 @@ +nextflow_process { + + name "Test Process QUARTO_PARTIAL" + script "../main.nf" + process "QUARTO_PARTIAL" + + tag "modules" + tag "modules_nfcore" + tag "quarto" + tag "quarto/partial" + + test("test partial - r") { + + when { + process { + """ + input[0] = [ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'generic/notebooks/quarto/quarto_r.qmd', checkIfExists: true) // Notebook + ] + input[1] = [:] // Parameters + input[2] = [] // Input files + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot( + process.out.findAll { key, val -> key.startsWith('versions') }, + process.out.artifacts, + process.out.params_yaml, + process.out.partial[0][1].find { it.toString().endsWith('.md') } + ).match() }, + // Explicitly check that output contains executed code. The + // `partial` output contains both a markdown file (the partial) + // and any associated files in a `_files/` directory, but only + // if the partial contains figures or other required resources, + // hence the explicit `endsWith` statement. + { + def md = process.out.partial[0][1].find { it.toString().endsWith('.md') } + assert path(md).readLines().any { it.contains('Hello World 1') } + }, + { assert process.out.partial[0][1].find { it.toString().endsWith('_files') } != null } + ) + } + + } + + test("test partial - python") { + + when { + process { + """ + input[0] = [ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'generic/notebooks/quarto/quarto_python.qmd', checkIfExists: true) // Notebook + ] + input[1] = [:] // Parameters + input[2] = [] // Input files + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot( + process.out.findAll { key, val -> key.startsWith('versions') }, + process.out.artifacts, + process.out.params_yaml, + process.out.partial[0][1].find { it.toString().endsWith('.md') } + ).match() }, + { + def md = process.out.partial[0][1].find { it.toString().endsWith('.md') } + assert path(md).readLines().any { it.contains('Hello World 1') } + }, + { assert process.out.partial[0][1].find { it.toString().endsWith('_files') } != null } + ) + } + + } + + test("test partial - parametrized - r") { + + when { + process { + """ + input[0] = [ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'generic/notebooks/quarto/quarto_r.qmd', checkIfExists: true) // Notebook + ] + input[1] = [input_filename: "hello.txt", n_iter: 12] // Parameters + input[2] = file(params.modules_testdata_base_path + 'generic/txt/hello.txt', checkIfExists: true) // Input files + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot( + process.out.findAll { key, val -> key.startsWith('versions') }, + process.out.artifacts, + process.out.params_yaml, + process.out.partial[0][1].find { it.toString().endsWith('.md') } + ).match() }, + { + def md = process.out.partial[0][1].find { it.toString().endsWith('.md') } + assert path(md).readLines().any { it.contains('Hello World 12') } + }, + { assert process.out.partial[0][1].find { it.toString().endsWith('_files') } != null } + ) + } + + } + + test("test partial - parametrized - python") { + + when { + process { + """ + input[0] = [ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'generic/notebooks/quarto/quarto_python.qmd', checkIfExists: true) // Notebook + ] + input[1] = [input_filename: "hello.txt", n_iter: 12] // Parameters + input[2] = file(params.modules_testdata_base_path + 'generic/txt/hello.txt', checkIfExists: true) // Input files + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot( + process.out.findAll { key, val -> key.startsWith('versions') }, + process.out.artifacts, + process.out.params_yaml, + process.out.partial[0][1].find { it.toString().endsWith('.md') } + ).match() }, + { + def md = process.out.partial[0][1].find { it.toString().endsWith('.md') } + assert path(md).readLines().any { it.contains('Hello World 12') } + }, + { assert process.out.partial[0][1].find { it.toString().endsWith('_files') } != null } + ) + } + + } + + test("test partial - stub - r") { + + options "-stub" + + when { + process { + """ + input[0] = [ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'generic/notebooks/quarto/quarto_r.qmd', checkIfExists: true) // Notebook + ] + input[1] = [:] // Parameters + input[2] = [] // Input files + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() }, + ) + } + + } + +} diff --git a/modules/nf-core/quarto/partial/tests/main.nf.test.snap b/modules/nf-core/quarto/partial/tests/main.nf.test.snap new file mode 100644 index 000000000000..f55f20db59e0 --- /dev/null +++ b/modules/nf-core/quarto/partial/tests/main.nf.test.snap @@ -0,0 +1,263 @@ +{ + "test partial - python": { + "content": [ + { + "versions_papermill": [ + [ + "QUARTO_PARTIAL", + "papermill", + "2.6.0" + ] + ], + "versions_quarto": [ + [ + "QUARTO_PARTIAL", + "quarto", + "1.7.31" + ] + ] + }, + [ + [ + { + "id": "test" + }, + "artifact.txt:md5,8ddd8be4b179a529afa5f2ffae4b9858" + ] + ], + [ + [ + { + "id": "test" + }, + "params.yml:md5,8b9e438c0eb4850e20035a1222cd151e" + ] + ], + "quarto_python.md:md5,2e2dc1e7dcfa0e8f5ac8471e5b090315" + ], + "timestamp": "2026-06-29T10:37:37.061321", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.4" + } + }, + "test partial - r": { + "content": [ + { + "versions_papermill": [ + [ + "QUARTO_PARTIAL", + "papermill", + "2.6.0" + ] + ], + "versions_quarto": [ + [ + "QUARTO_PARTIAL", + "quarto", + "1.7.31" + ] + ] + }, + [ + [ + { + "id": "test" + }, + "artifact.txt:md5,b10a8db164e0754105b7a99be72e3fe5" + ] + ], + [ + [ + { + "id": "test" + }, + "params.yml:md5,8b9e438c0eb4850e20035a1222cd151e" + ] + ], + "quarto_r.md:md5,406ad0baf6c27a7a8287d997b885eb78" + ], + "timestamp": "2026-06-29T10:37:27.14031", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.4" + } + }, + "test partial - parametrized - python": { + "content": [ + { + "versions_papermill": [ + [ + "QUARTO_PARTIAL", + "papermill", + "2.6.0" + ] + ], + "versions_quarto": [ + [ + "QUARTO_PARTIAL", + "quarto", + "1.7.31" + ] + ] + }, + [ + [ + { + "id": "test" + }, + "artifact.txt:md5,8ddd8be4b179a529afa5f2ffae4b9858" + ] + ], + [ + [ + { + "id": "test" + }, + "params.yml:md5,0dbcd58276179f0700e144e45d8d3c76" + ] + ], + "quarto_python.md:md5,1b9b10928f1ed3ddd048f232c0b38b30" + ], + "timestamp": "2026-06-29T10:37:55.445441", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.4" + } + }, + "test partial - stub - r": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "quarto_r.md:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + { + "id": "test" + }, + "quarto_r.qmd:md5,b3fa8b456efae62495c0b278a4f7694c" + ] + ], + "2": [ + [ + { + "id": "test" + }, + "params.yml:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "3": [ + + ], + "4": [ + [ + "QUARTO_PARTIAL", + "quarto", + "1.7.31" + ] + ], + "5": [ + [ + "QUARTO_PARTIAL", + "papermill", + "2.6.0" + ] + ], + "artifacts": [ + + ], + "notebook": [ + [ + { + "id": "test" + }, + "quarto_r.qmd:md5,b3fa8b456efae62495c0b278a4f7694c" + ] + ], + "params_yaml": [ + [ + { + "id": "test" + }, + "params.yml:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "partial": [ + [ + { + "id": "test" + }, + "quarto_r.md:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions_papermill": [ + [ + "QUARTO_PARTIAL", + "papermill", + "2.6.0" + ] + ], + "versions_quarto": [ + [ + "QUARTO_PARTIAL", + "quarto", + "1.7.31" + ] + ] + } + ], + "timestamp": "2026-06-26T20:14:01.188355", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.4" + } + }, + "test partial - parametrized - r": { + "content": [ + { + "versions_papermill": [ + [ + "QUARTO_PARTIAL", + "papermill", + "2.6.0" + ] + ], + "versions_quarto": [ + [ + "QUARTO_PARTIAL", + "quarto", + "1.7.31" + ] + ] + }, + [ + [ + { + "id": "test" + }, + "artifact.txt:md5,b10a8db164e0754105b7a99be72e3fe5" + ] + ], + [ + [ + { + "id": "test" + }, + "params.yml:md5,0dbcd58276179f0700e144e45d8d3c76" + ] + ], + "quarto_r.md:md5,f23da813661ebb77d10914c155002507" + ], + "timestamp": "2026-06-29T10:37:45.572784", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.4" + } + } +}