diff --git a/_data/contributors.yml b/_data/contributors.yml new file mode 100644 index 0000000..1335d45 --- /dev/null +++ b/_data/contributors.yml @@ -0,0 +1,14 @@ +- name: Nils Kleinbölting + ORCID: https://orcid.org/0000-0001-9124-5203 + +- name: Dilfuza Djamalova + ORCID: https://orcid.org/0009-0004-7782-2894 + +- name: Sebastian Jünemann + ORCID: https://orcid.org/0000-0002-3714-0328 + +- name: Alexander Sczyrba + ORCID: https://orcid.org/0000-0002-4405-3847 + +- name: Abhijeet Shah + ORCID: https://orcid.org/0000-0002-0467-4884 \ No newline at end of file diff --git a/_data/tutorial_groups.yml b/_data/tutorial_groups.yml index 51b20dc..93813b0 100644 --- a/_data/tutorial_groups.yml +++ b/_data/tutorial_groups.yml @@ -28,6 +28,8 @@ - basicsAmpliconQC - nanopore - mgtk_short + - genome-annotation + - genome-assembly - title: Data Management on the Cloud description: Tutorials focused on data management best practices, solutions on the Cloud, including how to adapt a solution to the cloud (Kubernetes and S3 storage) diff --git a/_includes/attributions.html b/_includes/attributions.html index 3182c8c..da9c4a0 100644 --- a/_includes/attributions.html +++ b/_includes/attributions.html @@ -23,12 +23,34 @@
Contributions
- {% if authors %} -

Author(s): {{ authors | join: ", " }}

- {% endif %} + {% if authors %} +

+ Author(s): + {% for author in authors %} + {% assign author_data = site.data.contributors | where: "name", author | first %} + {% if author_data and author_data.ORCID %} + {{ author }} + {% else %} + {{ author }} + {% endif %} + {% unless forloop.last %}, {% endunless %} + {% endfor %} +

+ {% endif %} {% if editors %} -

Editor(s): {{ editors | join: ", " }}

+

+ Editor(s): + {% for editor in editors %} + {% assign editor_data = site.data.contributors | where: "name", editor | first %} + {% if editor_data and editor_data.ORCID %} + {{ editor }} + {% else %} + {{ editor }} + {% endif %} + {% unless forloop.last %}, {% endunless %} + {% endfor %} +

{% endif %}
diff --git a/_includes/pathway_lifecycle.html b/_includes/pathway_lifecycle.html new file mode 100644 index 0000000..688cb4d --- /dev/null +++ b/_includes/pathway_lifecycle.html @@ -0,0 +1,29 @@ +{% comment %} + Display learning path's lifecycle. + Possible values: "under development", "active", "archived". + If not set, the learning path is considered "under development". +{% endcomment %} + +{% if page.life_cycle == nil %} +{% assign life_cycle = include.life_cycle %} +{% else %} +{% assign life_cycle = page.life_cycle %} +{% endif %} + +{% if life_cycle == "under development" %} +
+ Under development: This learning path is in an early stage of development and may change significantly. +
+{% elsif life_cycle == "active" %} +
+ Active: This learning path is actively maintained and updated. +
+{% elsif life_cycle == "archived" %} +
+ Archived: This learning path is no longer actively maintained. +
+{% else %} +
+ Unknown: The lifecycle status of this learning path is not defined. Please contact us for more information. +
+{% endif %} \ No newline at end of file diff --git a/_includes/tutorial_feedback.html b/_includes/tutorial_feedback.html index ed9899e..2c09091 100644 --- a/_includes/tutorial_feedback.html +++ b/_includes/tutorial_feedback.html @@ -3,8 +3,8 @@ {% endcomment %}
- 💬 Feedback: Found something unclear or want to suggest an improvement? Open a feedback issue. + 💬 Feedback: Found something unclear or want to suggest an improvement? Open a feedback issue.

👥 Contribution: We also welcome contributions when you spot an opportunity to improve the training materials. - Please review the contribution page first. Then, edit this material on GitHub to suggest your improvements. + Please review the contribution page first. Then, edit this material on GitHub to suggest your improvements.
\ No newline at end of file diff --git a/_layouts/pathway.html b/_layouts/pathway.html index df5ee17..bf8d505 100644 --- a/_layouts/pathway.html +++ b/_layouts/pathway.html @@ -12,6 +12,7 @@ + @@ -131,6 +132,7 @@

{{ page.title }}

{{ page.description }}

+ {% include _includes/pathway_lifecycle.html %}
@@ -143,14 +145,16 @@

{{ section.section }}

- + + Tutorial Slides Description - Learning Objectives + Learning Objectives + Estimated Time @@ -191,6 +195,7 @@

{{ section.section }}

{% endfor %} + {{ tutorial.time_estimation }} {% else %} @@ -201,21 +206,9 @@

{{ section.section }}

{% endfor %} - -

This material is reviewed by our Editorial Board:

-
    - {% for editor in page.editorial_board %} - {% if editor.orcid %} - {{ editor.name }} - {% else %} - {{ editor.name }} - {% endif %} - {% endfor %} -
- - {% include _includes/tutorial_feedback.html %} - -
+ {% include _includes/attributions.html %} + {% include _includes/tutorial_feedback.html %} +
diff --git a/_layouts/tutorial.html b/_layouts/tutorial.html deleted file mode 100644 index 90b3fdc..0000000 --- a/_layouts/tutorial.html +++ /dev/null @@ -1,9 +0,0 @@ ---- -layout: tutorial_hands_on ---- -
-{% include _includes/tutorial_overview.html %} - {{content}} -{% include _includes/tutorial_keypoints.html %} -{% include _includes/tutorial_feedback.html %} -
\ No newline at end of file diff --git a/_pathways/mgworkshop.md b/_pathways/mgworkshop.md deleted file mode 100644 index d28301c..0000000 --- a/_pathways/mgworkshop.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -layout: pathway -title: "Metagenomics workshop" -description: "Introduction to assembly based metagenomics" -tags: [shell, metagenomics, assembly, binning, taxonomy] -pathway: - - section: "Module 1: Introduction to basic Unix commands" - description: "Learn the basic Unix shell commands used in cloud environments." - tutorials: - - name: unix-course - version: main - - - section: "Module 2: Assembly" - description: "Learn how to run metagenome assemblies." - tutorials: - - name: mgworkshop_assembly - version: main -editorial_board: - - name: - orcid: ---- - diff --git a/_pathways/nanopore-workshop.md b/_pathways/nanopore-workshop.md new file mode 100644 index 0000000..76d3f49 --- /dev/null +++ b/_pathways/nanopore-workshop.md @@ -0,0 +1,50 @@ +--- +layout: pathway +title: "ONT Sequencing: From Reads to Annotation" +description: "This learning path introduces the complete workflow for analyzing isolate genome sequencing data generated with Oxford Nanopore Technologies (ONT) and Illumina platforms. Starting from raw ONT signal data, you will learn how to perform basecalling, quality control, genome assembly, polishing, hybrid assembly, genome annotation, and downstream analysis of long-read metagenomic data." +keywords: [ONT, Nanopore, Basecalling, dorado, flye, SPAdes, QUAST, polypolish, hybrid assembly, binning, prokka, bakta, EDGAR] +level: intermediate +life_cycle: under development + +pathway: + - section: "Module 1: Introduction to basic Unix commands" + description: "This module introduces essential Unix shell commands and concepts required for working in computational environments. You will learn how to navigate file systems, manipulate files, and execute basic commands commonly used in bioinformatics workflows." + tutorials: + - name: unix-course + version: main + + - section: "Module 2: Basecalling ONT data" + description: "This module introduces the preprocessing workflow for Oxford Nanopore Technologies (ONT) sequencing data, starting from raw signal files. You will learn how to perform basecalling, assess read quality, and prepare high-quality sequencing reads for downstream genome analysis." + tutorials: + - name: nanopore + version: main + + - section: "Module 3: Metagenomics assembly" + description: "This module introduces genome assembly approaches for long-read and short-read sequencing data. You will learn how to assemble prokaryotic genomes from ONT and Illumina reads, evaluate assembly quality, improve assemblies through polishing and hybrid assembly strategies, and assess the final assembly results. The module also provides an introduction to metagenome assembly workflows using dedicated bioinformatics tools." + tutorials: + - name: mgworkshop_assembly + version: main + + - name: genome-assembly + version: main + + - section: "Module 4: Genome annotation" + description: "This module covers the functional annotation of bacterial genomes. You will learn how to identify genomic features, predict genes, assign functional information, and interpret genome annotations using commonly used annotation tools." + tutorials: + - name: genome-annotation + version: main + + - section: "Module 5: Long-Read Metagenomics using the Metagenomics-Toolkit" + description: "This module introduces long-read metagenomics analysis using the Metagenomics-Toolkit. You will learn how to prepare ONT sequencing data, run the initial analysis workflow, and explore the first steps of metagenomic data processing and interpretation." + tutorials: + - name: mgtk_short + version: main + +contributions: + authorship: + - Nils Kleinbölting + editing: + - Dilfuza Djamalova + funding: +--- + diff --git a/_tutorials/genome-annotation/main/tutorial.md b/_tutorials/genome-annotation/main/tutorial.md new file mode 100644 index 0000000..93af7a6 --- /dev/null +++ b/_tutorials/genome-annotation/main/tutorial.md @@ -0,0 +1,153 @@ +--- +layout: tutorial_hands_on +title: Prokaryotic Genome Annotation (hands-on) +description: "Learn the steps of prokaryotic genome annotation by generating and comparing annotations with Prokka and Bakta. The tutorial also introduces EDGAR for comparative genome analysis, including core-genome, pan-genome and ortholog identification." +time_estimation: 2H +level: beginner +keywords: prokka, bakta, EDGAR +questions: + - "How do I functionally annotate a bacterial genome using Prokka and Bakta, and what role does EDGAR play in comparative genomics?" +objectives: + - "Generate comprehensive genome annotations with Prokka and Bakta, and interpret differences in functional naming and hypothetical protein rates." + - "Explain how comparative genomics frameworks like EDGAR identify core genomes, pan-genomes, and unique singleton genes." +key_points: + - "Modern annotation systems like Bakta significantly reduce the rate of uncharacterized 'hypothetical proteins' by integrating curated, up-to-date RefSeq and UniProt cross-references." + - "Comparative workflows like EDGAR utilize BLAST Score Ratios (BSR) across multiple annotated genomes to instantly segregate the core genome from unique singleton genes." +version: main +life_cycle: under development +contributions: + authorship: + - Nils Kleinbölting + editing: + - Dilfuza Djamalova + funding: +--- + +This tutorial demonstrates how to annotate a polished prokaryotic genome assembly using Prokka and Bakta, compare their annotation outputs, and briefly explore comparative genomics with EDGAR. It uses the genome assembly generated in the previous tutorial [Assembly and assembly evaluation (hands-on)]({{ site.url }}{{ site.baseurl }}/tutorials/genome-assembly/main/tutorial/) as the input for all analyses. + +>Prerequisites +> - Please complete the [Unix/Linux introduction tutorial]({{ site.url }}{{ site.baseurl }}/tutorials/unix-course/main/tutorial/) before this tutorial. +> - We assume you have successfully connected to an instance in the de.NBI cloud with the software pre-installed. Otherwise you will need to install the required tools on your own and make sure you have sufficient resources available. +> - Throughout the course we assume you are working on data downloaded to a volume under `/vol/longread/`, we create a link `~/workdir/` to that folder, if you are working somewhere else, adjust the `~/workdir` link to that location and all commands should work as outlined in the course. +> - We also assume that you have a machine with **28 cores** available, if not - adjust the commands that specify a certain number of threads / cores accordingly. +{: .details} + +## Introduction to Prokaryotic Genome Annotation + +Once you have successfully assembled and polished a bacterial chromosome, it consists simply of a long, uncharacterized string of nucleotides (A, C, G, T). To make this data useful for biological research, you must perform **genome annotation**. This process involves identifying the structural features of the genome—such as protein-coding sequences (CDS), transfer RNAs (tRNAs), and ribosomal RNAs (rRNAs)—and assigning functional biological identities to them based on sequence similarity to known databases. + +In this module, we will compare two popular tools used for this task: + +* **Prokka:** For nearly a decade, Prokka has been the legacy workhorse tool for rapid prokaryotic genome annotation. It coordinates an ensemble of open-source tools (like Prodigal for CDS finding and Aragorn for tRNAs) to generate comprehensive annotation suites in minutes. However, because its internal reference databases are no longer actively maintained, it often over-assigns generic functional names or labels proteins as "hypothetical protein". + +> Optional: How to install Prokka +It's quite complicated to install without conda/docker/singularity. Check out the [github repository](https://github.com/tseemann/prokka) and use one of those methods. +{: .tip} + +* **Bakta:** A modern, next-generation annotation platform designed specifically for microbial genomes. Bakta addresses Prokka's database stagnation by utilizing a thoroughly curated, regularly updated SQLite database synchronized with NCBI RefSeq, UniProt, and specialized feature resources. It provides highly accurate protein names, precise cross-reference tags (DBXrefs), and native tracking of non-coding RNAs (ncRNAs), pseudogenes, and antimicrobial resistance (AMR) gene identifiers. + +> Optional: How to install Bakta +It's quite complicated to install without conda/docker/singularity. Check out the [github repository](https://github.com/oschwenders/bakta) and use one of those methods. +{: .tip} +--- + +## Hands-on: Annotating Your Assembly + +We will run both annotators on our polished long-read assembly (`flye_polished.fasta`) and evaluate how their structural findings and functional naming conventions differ. + +>Step 1: Running Prokka +> Execute Prokka by specifying an output directory and a custom file prefix: +> +>```bash +>prokka --cpus 28 --outdir ~/workdir/prokka_output --prefix prokka_ont ~/workdir/polypolish/flye_polished.fasta +>``` +{: .hands-on} + +Don't worry about the `Could not run command: tbl2asn` message if it appears. We don't need the `asn` file. + +>Step 2: Running Bakta +> +> Unlike Prokka, Bakta relies on a separate, heavy database containing millions of curated proteins. For this workshop, this database has been pre-staged for you. Run Bakta using the following command: +> +>```bash +>conda activate bakta +>bakta --threads 28 --db ~/bakta_db/db-light --output ~/workdir/bakta_output ~/workdir/polypolish/flye_polished.fasta +>conda deactivate +>source ~/longread/bin/activate +>``` +{: .hands-on} + +--- + +## Hands-on: Comparing Annotation Profiles + +Both tools generate various standardized outputs, including GFF3, GenBank, and FASTA files. To quickly benchmark their structural predictions, we can review the text-based summary logs (`.txt`) produced by each pipeline. + +>Step 3: Inspecting Summary Outputs +> +>Use `cat` to print out both overview profiles in your terminal: +> +>```bash +># View the Prokka summary report +>cat ~/workdir/prokka_output/prokka_ont.txt +> +># View the Bakta summary report +>cat ~/workdir/bakta_output/bakta_ont.txt +>``` +{: .hands-on} + +> Analyzing Annotation Discrepancies +> Look closely at the total counts of Coding Sequences (CDS), tRNAs, and rRNAs in both outputs. Are the numbers identical? If they differ, what could cause one tool to predict more genes than the other? +> +> > Solution +> > Even though they use the same underlying software for core gene finding (Prodigal), the total counts often differ slightly. Bakta uses stricter structural filters and a much larger database, allowing it to accurately split overlapping reading frames, filter out false positive predictions, and identify specialized elements like pseudogenes or small non-coding RNAs that Prokka completely misses. +> {: .solution} +{: .question} + +>Step 4: Comparing Functional Descriptions +> +>A major difference lies in how specifically proteins are named. Let's use `grep` to check how many genes were left uncharacterized as "hypothetical protein" in both annotation suites: +> +>```bash +># Count hypothetical proteins in Prokka's GFF output +>grep -c "hypothetical protein" ~/workdir/prokka_output/prokka_ont.gff +> +># Count hypothetical proteins in Bakta's GFF output +>grep -c "hypothetical protein" ~/workdir/bakta_output/bakta_ont.gff +>``` +{: .hands-on} + +> Interpreting Naming Quality +> You will notice that Bakta significantly reduces the fraction of `hypothetical protein` labels compared to Prokka. Thanks to its modern reference integration with UniProt and RefSeq, Bakta can assign definitive, functional gene names to sequences where Prokka could only find vague, outdated family matches. +{: .comment} + +--- + +## Comparative Genomics with EDGAR + +Once individual genomes are annotated, the next logical milestone is to explore how multiple strains or species relate to one another. For this downstream phase, we shift from localized command-line annotation to web-based comparative genomics using **EDGAR** (Efficient Database framework for comparative Genome Analyses). + +* **Official Server Link:** [http://edgar3.computational.bio](http://edgar3.computational.bio) + +### How EDGAR Works: +EDGAR is a fully automated high-throughput platform tailored for the deep comparative analysis of prokaryotic genomes. Users upload their fully annotated genome files (such as the `.gff` or GenBank files generated by Bakta) into public or password-protected private projects. + +The underlying pipeline performs intensive all-versus-all sequence alignments across all selected strains. By evaluating **BLAST Score Ratios (BSR)**, EDGAR accurately determines orthology relational paths to delineate specific genomic subsets: +1. **The Core Genome:** The conserved set of genes shared identically across *all* analyzed organisms, often used to build highly precise core-genome phylogenetic trees. +2. **The Pan-Genome:** The complete global pool of all unique genes present across the entire group. +3. **Singleton Genes:** Unique genes present in only *one* specific strain, which are crucial for identifying specific downstream traits like pathogenicity islands or unique metabolic capabilities. + +Furthermore, EDGAR calculates average nucleotide identity (ANI) metrics and renders publication-ready visualizations, including Venn diagrams, UpSet plots, and synteny maps mapping gene order conservation across syntenic chromosomal layouts. + +--- + +## APPENDIX: References for tools used within the tutorial +* **Prokka (Rapid Prokaryotic Genome Annotation):** + * **GitHub:** [https://github.com/tseemann/prokka](https://github.com/tseemann/prokka) + * **Publication:** *Seemann, T. (2014). Prokka: rapid prokaryotic genome annotation. Bioinformatics.* +* **Bakta (Next-generation Microbial Genome Annotation):** + * **GitHub:** [https://github.com/oschwenders/bakta](https://github.com/oschwenders/bakta) + * **Publication:** *Schwengers, O. et al. (2021). Bakta: rapid and standardized annotation of bacterial genomes and plasmids. Microbial Genomics.* +* **EDGAR (Comparative Genomics Framework):** + * **Webserver Platform:** [http://edgar3.computational.bio](http://edgar3.computational.bio) + * **Publication:** *Dieckmann, M. A. et al. (2021). EDGAR 3.0: comparative genomics and phylogenomics on a scalable infrastructure. Nucleic Acids Research.* \ No newline at end of file diff --git a/_tutorials/genome-assembly/main/tutorial.md b/_tutorials/genome-assembly/main/tutorial.md new file mode 100644 index 0000000..aab18c8 --- /dev/null +++ b/_tutorials/genome-assembly/main/tutorial.md @@ -0,0 +1,347 @@ +--- +layout: tutorial_hands_on +title: Assembly and assembly evaluation (hands-on) +description: "This tutorial demonstrates how to assemble a prokaryotic genome using Oxford Nanopore and Illumina sequencing data, evaluate assembly quality, improve assemblies through polishing and hybrid assembly approaches, and assess the final results." +time_estimation: 2H +level: beginner +keywords: flye, SPAdes, QUAST, polypolish, hybrid assembly +questions: + - "What are the core differences between De Bruijn Graph (DBG) and Overlap-Layout-Consensus (OLC) assembly approaches?" + - "How do I perform a long-read assembly with Flye and a short-read assembly with SPAdes?" + - "How can we improve a long-read assembly via short-read polishing with Polypolish?" + - "What is a hybrid assembly approach, and how does SPAdes execute it?" + - "How do I evaluate and compare multiple assemblies using QUAST and Bandage?" +objectives: + - "Reconstruct a bacterial genome using Flye and SPAdes, and visually evaluate assembly graph topologies in Bandage." + - "Execute the multi-step Polypolish workflow (all-mapping, filtering by insert size, and polishing) to correct homopolymer indels." +key_points: + - "Long reads effortlessly resolve repetitive genomic structures via the OLC paradigm, allowing Flye to assemble fully closed circular bacterial chromosomes." + - "Polypolish guards against false corrections in repeat regions by only proposing sequence fixes if all alternative short-read alignments agree on the mismatch." + - "Hybrid co-assembly leverages highly accurate short-read graphs while using long reads as structural scaffolds to resolve complex repeat branches." +version: main +life_cycle: under development +contributions: + authorship: + - Nils Kleinbölting + editing: + - Dilfuza Djamalova + funding: +--- + +This tutorial demonstrates how to assemble a prokaryotic genome using Oxford Nanopore and Illumina sequencing data, evaluate assembly quality, improve assemblies through polishing and hybrid assembly approaches, and assess the final results. The sequencing reads generated in the previous tutorial, [Basecalling and QC of ONT data]({{ site.url }}{{ site.baseurl }}/tutorials/nanopore/main/tutorial/), will be used as input for all analyses. The resulting polished genome assembly will serve as the starting point for the subsequent [Prokaryotic Genome Annotation (hands-on)]({{ site.url }}{{ site.baseurl }}/tutorials/genome-annotation/main/tutorial/) tutorial. + +>Prerequisites +> - Please complete the [Unix/Linux introduction tutorial]({{ site.url }}{{ site.baseurl }}/tutorials/unix-course/main/tutorial/) before this tutorial. +> - We assume you have successfully connected to an instance in the de.NBI cloud with the software pre-installed. Otherwise you will need to install the required tools on your own and make sure you have sufficient resources available. +> - Throughout the course we assume you are working on data downloaded to a volume under `/vol/longread/`, we create a link `~/workdir/` to that folder, if you are working somewhere else, adjust the `~/workdir` link to that location and all commands should work as outlined in the course. +> - We also assume that you have a machine with **28 cores** available, if not - adjust the commands that specify a certain number of threads / cores accordingly. +{: .details} + + +## Assembly and assembly evaluation + +### Introduction to De Novo Genome Assembly + +Genome assembly is the process of piecing together massive amounts of short or long DNA fragments (reads) to reconstruct the original underlying chromosome. Because we do not use a reference genome during *de novo* assembly, the algorithms rely strictly on sequence overlaps. Two main algorithmic paradigms dominate the field: + +* **De Bruijn Graph (DBG):** Primarily used for short reads (e.g., Illumina). Reads are broken down into smaller fixed-length strings called **$$k$$-mers**. Overlaps are tracked by constructing a network where nodes or edges represent shared $$k$$-mers. DBG is computationally efficient for processing hundreds of millions of short reads and highly accurate, but it struggles enormously with genomic repeats because the short $$k$$-mer contexts cannot resolve long duplicate regions. +* **Overlap-Layout-Consensus (OLC):** Primarily used for long reads (e.g., ONT, PacBio). The algorithm calculates all-versus-all alignments between full reads (**Overlap**), constructs an alignment graph to simplify paths and resolve structures (**Layout**), and finally determines the most accurate sequence across overlapping reads (**Consensus**). Long reads easily span across genomic repeats, allowing OLC-based pipelines to assemble completely closed chromosomes. This approach is usually not feasible for short reads due to the massive amount of alignments that have to be computed. + +--- + +### Understanding the Assembly and Assembly Evaluation Tools + +#### 1. Flye +Flye is a specialized *de novo* assembler designed for long, error-prone reads. Instead of building a classic OLC overlap graph (which scales poorly with high read depths), Flye constructs an unpolished **repeat graph**. It collapses complex genomic repeats into single edges, and then utilizes the long span of individual read paths to accurately untangle and separate those repeat copies. + +> Optional: How to install Flye +Run this (or follow instructions in github): +> ```bash +> pip install setuptools +> git clone https://github.com/fenderglass/Flye +> cd Flye +> python setup.py install +> ``` +{: .tip} + +#### 2. SPAdes +SPAdes (St. Petersburg Genome Assembler) is the gold standard for bacterial short-read assemblies. It relies on multi-sized De Bruijn Graphs (combining multiple $k$-mer lengths) to simultaneously optimize specificity and sensitivity, providing robust performance across single-isolate cultures and single-cell sequencing. + +> Optional: How to install Flye +Run this (or follow instructions in github): +> ```bash +> wget https://github.com/ablab/spades/releases/download/v4.3.0/SPAdes-4.3.0-Linux.tar.gz +> tar -xzvf SPAdes-4.3.0-Linux.tar.gz +> export PATH=PATH:$(pwd)/SPAdes-4.3.0-Linux/bin/ +> ``` +{: .tip} + +#### 3. QUAST +QUAST (Quality Assessment Tool) is an evaluation utility that calculates structural metrics (like contig counts, N50 value, and total length) and identifies misassemblies by aligning your assembled contigs back against a trusted reference genome. + +> Optional: How to install Quast +Run this (or follow instructions in github): +> ```bash +> wget https://github.com/ablab/quast/releases/download/quast_5.3.0/quast-5.3.0.tar.gz +> tar -xzvf quast-5.3.0.tar.gz +> cd quast-5.3.0 +> ./setup.py install +> ``` +{: .tip} + +#### 4. Bandage +Bandage (Bioinformatics Application for Navigating De Novo Assembly Graphs Easily) is a graphical interface utility that reads Graphical Assembly Graph (`.gfa`) files. It allows you to see how contigs connect to one another, helping you determine whether your bacterial genome successfully assembled into a single closed circular chromosome. + +> Optional: How to install Bandage +Run this (or follow instructions in github): +> ```bash +> wget https://github.com/rrwick/Bandage/releases/download/v0.8.1/Bandage_Ubuntu_dynamic_v0_8_1.zip +> unzip Bandage_Ubuntu_dynamic_v0_8_1.zip +> sudo mv Bandage /usr/local/bin/ +> #might be necessary: +> sudo apt install libqt5svg5 +> ``` +{: .tip} + +--- + +### Hands-on: Building and Evaluating Assemblies + +In this section, we will run separate long-read and short-read assembly pipelines, statistically benchmark their outputs against our reference, inspect their connectivity graphs, and align the draft contigs visually. + +#### Step 1: Long-Read Assembly with Flye + +Because modern Dorado basecalled data achieves exceptional accuracy (entering the Q20 standard), we use Flye's high-fidelity option (`--nano-hq`) to generate our draft genome: + +```bash +flye --nano-hq ~/workdir/coursedata/ont.fastq.gz --out-dir ~/workdir/flye_output --threads 28 +``` + +#### Step 2: Short-Read Assembly with SPAdes + +Next, we generate a corresponding short-read assembly utilizing our paired-end Illumina datasets: + +```bash +spades.py -1 ~/workdir/coursedata/illumina/Barcode11_TSLF_S10_L001_R1_001.fastq.gz \ + -2 ~/workdir/coursedata/illumina/Barcode11_TSLF_S10_L001_R2_001.fastq.gz \ + -o ~/workdir/spades_output --threads 28 +``` + +#### Step 3: Benchmarking Assemblies with QUAST + +We can now run a direct comparative evaluation between both assembly results using our known genome sequence as a reference: + +```bash +quast.py ~/workdir/flye_output/assembly.fasta \ + ~/workdir/spades_output/contigs.fasta \ + -r ~/workdir/coursedata/reference.fasta \ + -o ~/workdir/quast_output +``` + +Open the interactive QUAST HTML summary document in your browser to view the benchmark comparison: + +```bash +firefox ~/workdir/quast_output/report.html +``` + +> Analyzing Assembly Metrics +> Look at the metric comparisons in the QUAST report. Which assembly contains fewer total contigs? Which possesses a higher N50 score? What does this tell you about the power of long reads? +> +> > Solution +> > Typically, the Flye long-read assembly will result in significantly fewer contigs (often a single continuous contig for a closed bacterial chromosome) and a drastically higher N50 score approaching the true size of the genome. The SPAdes short-read assembly is usually split across multiple fragments because short fragments cannot resolve genomic repeats. +> {: .solution} +{: .question} + +--- + +#### Step 4: Visualizing Graphs in Bandage + +Statistical metrics only tell half the story. We need to look at the assembly graphs to see the structure of our contigs. + +1. Launch the **Bandage** GUI application via your terminal: +```bash + Bandage + ``` +2. In the Bandage menu, navigate to **File** -> **Load graph**. +3. First, load the Flye assembly graph file located at `~/workdir/flye_output/assembly_graph.gfa` and click **Draw graph**. +4. Next, clear the screen and load the SPAdes assembly graph file found at `~/workdir/spades_output/assembly_graph_with_scaffolds.gfa` and click **Draw graph**. + +> Interpreting Graph Topologies +> In the Flye window, you should see a single, beautiful, interconnected closed loop representing the intact circular bacterial chromosome. In contrast, the SPAdes graph will likely display a highly fragmented web of disjointed paths and isolated nodes, highlighting where the short-read assembly broke down at repeat boundaries. +{: .comment} + +--- + +#### Step 5: Aligning Contigs to Reference for IGV + +Finally, we want to align our assembled fasta contigs back against the reference genome to visually spot missing structural parts or mismatches in IGV. We use `minimap2` with the `-ax asm5` preset, which is optimized for aligning highly accurate genome assemblies. + +```bash +# Map the Flye assembly contigs +minimap2 -t 28 -ax asm5 ~/workdir/coursedata/reference.fasta ~/workdir/flye_output/assembly.fasta > ~/workdir/mappings/flye_vs_ref.sam +samtools view -S -b ~/workdir/mappings/flye_vs_ref.sam | samtools sort -o ~/workdir/mappings/flye_vs_ref_sorted.bam +samtools index ~/workdir/mappings/flye_vs_ref_sorted.bam + +# Map the SPAdes assembly contigs +minimap2 -t 28 -ax asm5 ~/workdir/coursedata/reference.fasta ~/workdir/spades_output/scaffolds.fasta > ~/workdir/mappings/spades_vs_ref.sam +samtools view -S -b ~/workdir/mappings/spades_vs_ref.sam | samtools sort -o ~/workdir/mappings/spades_vs_ref_sorted.bam +samtools index ~/workdir/mappings/spades_vs_ref_sorted.bam +``` + +#### Verification in IGV: +1. Open **IGV**, and make sure your reference genome (`~/workdir/coursedata/reference.fasta`) is actively loaded. +2. Load both new alignment files via **File** -> **Load from File...**: + * `~/workdir/mappings/flye_vs_ref_sorted.bam` + * `~/workdir/mappings/spades_vs_ref_sorted.bam` +3. Inspect the alignment tracks to identify gaps or fragmentation points where the short-read assembly failed to recover structural elements. + +--- + +## Improving the flye assembly and trying a hybrid assembly approach + +### Short-Read Polishing with Polypolish + +Even though modern ONT R10.4.1 chemistry combined with Dorado pushes raw read accuracy into the Q20 (>99%) range, long-read assemblies can still retain minor systematic errors. These errors are most frequently found in homopolymer runs (e.g., long stretches of AAAA), manifesting as small insertions or deletions (indels). To fix these remaining micro-errors, we can perform a process called **polishing** using highly accurate Illumina short reads. + +We will use **Polypolish**, a short-read polishing tool designed specifically for long-read assemblies. + +> How Polypolish Avoids False Corrections +> Traditional polishers take all short-read alignments and use a consensus to alter the assembly. However, in repetitive genomic regions, short reads frequently misalign to the wrong repeat copy, leading the polisher to introduce errors rather than fix them. +> +> Polypolish solves this by examining the alternative alignments for each short read. If a read can map to multiple places in the assembly, Polypolish will only propose a correction if *all* possible target sites agree on the mismatch. If the mapping is ambiguous, it leaves the sequence untouched, preventing false corrections in repeat boundaries. +{: .comment} + +To ensure Polypolish operates effectively, we must execute a specific multi-step pipeline: +1. Map short reads **separately** (R1 and R2 independently) using `bwa mem` with the `-a` option. This option forces the aligner to output *all* possible alignment locations for a read, not just the single best hit. +2. Run `polypolish filter` to calculate the expected insert size of read pairs and filter out low-confidence alignments. +3. Run `polypolish polish` to correct the assembly using the filtered pileups. + +> Optional: How to install Polypolish +Run this (or follow instructions in github): +> ```bash +> wget https://github.com/rrwick/Polypolish/releases/download/v0.6.1/polypolish-linux-x86_64-musl-v0.6.1.tar.gz +> tar -xzvf polypolish-linux-x86_64-musl-v0.6.1.tar.gz +> sudo mv polypolish /usr/local/bin/ +> ``` +{: .tip} + +--- + +### Hands-on: Polishing the Long-Read Assembly + +#### Step 1: Mapping Short Reads with All Alignments Enabled + +First, let's build the BWA index of our long-read genome assembly and map both Illumina forward and reverse files completely independently using the required `-a` flag: + +```bash +# Index the Flye draft genome +bwa index ~/workdir/flye_output/assembly.fasta + +# Create directory for polypolish files +mkdir polypolish +# Map R1 and R2 forward/reverse reads completely independently with the -a flag +bwa mem -t 28 -a ~/workdir/flye_output/assembly.fasta ~/workdir/coursedata/illumina/Barcode11_TSLF_S10_L001_R1_001.fastq.gz > ~/workdir/polypolish/polypolish_r1.sam +bwa mem -t 28 -a ~/workdir/flye_output/assembly.fasta ~/workdir/coursedata/illumina/Barcode11_TSLF_S10_L001_R2_001.fastq.gz > ~/workdir/polypolish/polypolish_r2.sam +``` + +#### Step 2: Filtering Alignments by Insert Size + +Next, we pass our independent raw SAM files into Polypolish's filtering subcommand. This evaluates read pairing distances to clear away non-specific background mappings: + +```bash +polypolish filter --in1 ~/workdir/polypolish/polypolish_r1.sam --in2 ~/workdir/polypolish/polypolish_r2.sam --out1 ~/workdir/polypolish/filtered_r1.sam --out2 ~/workdir/polypolish/filtered_r2.sam +``` + +#### Step 3: Executing the Final Consensus Polish + +Now, we provide the original unpolished Flye assembly along with both freshly filtered alignment tracks to create our refined fasta file: + +```bash +polypolish polish ~/workdir/flye_output/assembly.fasta ~/workdir/polypolish/filtered_r1.sam ~/workdir/polypolish/filtered_r2.sam > ~/workdir/polypolish/flye_polished.fasta +``` + +--- + +### Hybrid Assembly with SPAdes + +Instead of assembling long reads first and polishing them later, a **hybrid assembly** combines both data types simultaneously into a single algorithmic workflow. + +We will use the hybrid mode of **SPAdes**. The SPAdes hybrid approach works as follows: +1. It builds a high-accuracy, highly-resolved **De Bruijn Graph** using only the pristine Illumina short reads. +2. It then maps the ONT long reads onto this graph. The long reads act as structural templates or "scaffolds" to bridge across repeat-induced gaps and resolve complex branches within the graph structure. + +This approach combines the single-nucleotide accuracy of short reads with the structural spanning power of long reads seamlessly. + +#### Step 4: Running Hybrid SPAdes + +Execute the hybrid SPAdes pipeline by supplying both your paired-end short reads and your combined long-read datasets: + +```bash +spades.py -1 ~/workdir/coursedata/illumina/Barcode11_TSLF_S10_L001_R1_001.fastq.gz \ + -2 ~/workdir/coursedata/illumina/Barcode11_TSLF_S10_L001_R2_001.fastq.gz \ + --nanopore ~/workdir/coursedata/ont.fastq.gz \ + -o ~/workdir/spades_hybrid_output --threads 28 +``` + +--- + +### Hands-on: Comprehensive Assembly Evaluation + +We now have four distinct assembly variants tracking our target genome. Let's run a final comparative evaluation with QUAST to see how polishing and hybrid strategies alter genome completeness and accuracy metrics. + +The 4 assembly variants to evaluate are: +1. `flye_output/assembly.fasta` (ONT Long-Reads Only) +2. `spades_output/scaffolds.fasta` (Illumina Short-Reads Only) +3. `flye_polished.fasta` (ONT Long-Reads Polished with Short-Reads) +4. `spades_hybrid_output/contigs.fasta` (Hybrid Co-Assembly) + +#### Step 5: Comparing all Four Frameworks in QUAST + +Run QUAST with all four assembly files against the true reference genome: + +```bash +quast.py ~/workdir/flye_output/assembly.fasta \ + ~/workdir/spades_output/contigs.fasta \ + ~/workdir/polypolish/flye_polished.fasta \ + ~/workdir/spades_hybrid_output/contigs.fasta \ + -l "flye,spades,polypolish,hybrid_spades" \ + -r ~/workdir/coursedata/reference.fasta \ + -t 28 \ + -o ~/workdir/quast_final_output +``` + +Open the resulting dashboard summary report in your browser: + +```bash +firefox ~/workdir/quast_final_output/report.html +``` + +> Evaluating the Impact of Polishing and Hybridization +> Compare the column profiles of the unpolished Flye assembly vs. the polished Flye assembly. Look at metrics like "mismatches per 100 kbp" or "indels per 100 kbp". What changes do you observe? How does the Hybrid assembly compare in contig count? +> +> > Solution +> > Polishing with Polypolish typically causes a significant drop in the number of indels per 100 kbp compared to raw Flye contigs, which often restores disrupted open reading frames and increases the total number of fully recovered genes. The Hybrid SPAdes assembly often improves in contiguity, but depending on repeat complexity, it may still contain a few more contig fragments than Flye's completely closed loop structure. +> {: .solution} +{: .question} + +--- + +## APPENDIX: References for tools used within the tutorial +* **Flye** + * **GitHub:** [https://github.com/fenderglass/Flye](https://github.com/fenderglass/Flye) + * **Publication:** *Kolmogorov, M. et al. (2019). Assembly of long, error-prone reads using repeat graphs. Nature Biotechnology.* +* **SPAdes** + * **GitHub:** [https://github.com/ablab/spades](https://github.com/ablab/spades) + * **Publication:** *Bankevich, A. et al. (2012). SPAdes: a new genome assembly algorithm and its applications to single-cell sequencing. Journal of Computational Biology.* +* **QUAST** + * **GitHub:** [https://github.com/ablab/quast](https://github.com/ablab/quast) + * **Publication:** *Gurevich, A. et al. (2013). QUAST: quality assessment tool for genome assemblies. Bioinformatics.* +* **Bandage** + * **GitHub:** [https://github.com/rrwick/Bandage](https://github.com/rrwick/Bandage) + * **Publication:** *Wick, R. R. et al. (2015). Bandage: interactive visualization of de novo genome assembly graphs. Bioinformatics.* +* **Polypolish (Short-read Polisher for Long-read Assemblies):** + * **GitHub:** [https://github.com/rrwick/Polypolish](https://github.com/rrwick/Polypolish) + * **Publication:** *Wick, R. R. & Holt, K. E. (2022). Polypolish: Short-read polishing of long-read bacterial genome assemblies. PLoS Computational Biology.* +* **SPAdes (Hybrid Assembly Mode Support):** + * **GitHub:** [https://github.com/ablab/spades](https://github.com/ablab/spades) + * **Publication:** *Antipov, D. et al. (2016). hybridSPAdes: an algorithm for genome assembly from microbial long and short reads. Bioinformatics.* \ No newline at end of file diff --git a/_tutorials/mgtk_short/main/tutorial.md b/_tutorials/mgtk_short/main/tutorial.md index b10b047..0abdf86 100644 --- a/_tutorials/mgtk_short/main/tutorial.md +++ b/_tutorials/mgtk_short/main/tutorial.md @@ -1,7 +1,7 @@ --- layout: tutorial_hands_on title: "Introduction to Long-Read Metagenomics using the Metagenomics-Toolkit" -description: "This tutorial will guide you through the first steps to run the Metagenmoics-Toolkit on ONT data" +description: "This tutorial will guide you through the first steps to run the Metagenomics-Toolkit on ONT data" time_estimation: 1H level: beginner keywords: ONT, Nanopore, Metagenomics, Assembly, Classification, Binning, Annotation, Workflow, Nextflow @@ -24,11 +24,12 @@ contributions: authorship: - Nils Kleinbölting editing: + - Dilfuza Djamalova funding: --- This tutorial is a very short introduction to the Metagenomics-Toolkit which shows the main steps in analysing Nanopore long-read metagenomics data using the Metagenomics-Toolkit. -A more detailed introduction and Tutorials can be found [here](https://metagenomics.github.io/metagenomics-tk/latest/). +A more detailed introduction and tutorials can be found [here](https://metagenomics.github.io/metagenomics-tk/latest/). In this part you will learn how to configure and run the Toolkit and what the output of a Toolkit run looks like. ## Tutorial Scope and Requirements @@ -39,21 +40,15 @@ sample separately (*per-sample*), and the second part runs a combined downstream While there are several optimizations for running the Toolkit on a cloud-based setup, during this workshop you will run the Toolkit on a single machine. -### Requirements - -* Basic Linux command-line usage - -* This tutorial has been tested on a machine with 28 CPUs and 64 GB of RAM with Ubuntu installed on it. - -* Docker: Install Docker by following the official Docker installation [instructions](https://docs.docker.com/engine/install/ubuntu/). - -* Java: In order to run Nextflow, you need to install Java on your machine, which can be achieved via `sudo apt install default-jre`. - -* Nextflow should be installed. Please check the official Nextflow [instructions](https://www.nextflow.io/docs/latest/install.html#install-nextflow). - -* Throughout the course we assume you are working on data downloaded to a volume under `/vol/longread/`, we create a link `~/workdir/` to that folder, if you are working somewhere else, adjust the `~/workdir` link to that location and all commands should work as outlined in the course. - -* We also assume that you have a machine with **28 cores** and **64GB of RAM** available, if not - adjust the configuration that specifies a certain number of cores/memory accordingly. +>Requirements +>* Please complete the [Unix/Linux introduction tutorial]({{ site.url }}{{ site.baseurl }}/tutorials/unix-course/main/tutorial/) before this tutorial. +>* This tutorial has been tested on a machine with 28 CPUs and 64 GB of RAM with Ubuntu installed on it. +>* Docker: Install Docker by following the official Docker installation [instructions](https://docs.docker.com/engine/install/ubuntu/). +>* Java: In order to run Nextflow, you need to install Java on your machine, which can be achieved via `sudo apt install default-jre`. +>* Nextflow should be installed. Please check the official Nextflow [instructions](https://www.nextflow.io/docs/latest/install.html#install-nextflow). +>* Throughout the course we assume you are working on data downloaded to a volume under `/vol/longread/`, we create a link `~/workdir/` to that folder, if you are working somewhere else, adjust the `~/workdir` link to that location and all commands should work as outlined in the course. +>* We also assume that you have a machine with **28 cores** and **64GB of RAM** available, if not - adjust the configuration that specifies a certain number of cores/memory accordingly. +{: .details} ### **Download the data and preparations** @@ -63,14 +58,13 @@ First **(if not already done)**, create a link to `/vol/longread` (or the folder ln -s /vol/longread/ ~/workdir cd ~/workdir ``` + You might need to change the permissions of `/vol/longread`, for example (in the cloud setup we use for the on-site course) with: ```bash +# Adjust accordingly to your setup sudo chown ubuntu:ubuntu /vol/longread/ ``` -(Adjust accordingly to your setup) - ---- Next, we download our tutorial dataset (into a data folder): @@ -79,7 +73,6 @@ cd ~/workdir mkdir mgtk_data cd ~/workdir/mgtk_data/ wget https://s3.bi.denbi.de/cmg/mgcourses/mgtk_short/sample0_5p.fastq.gz -cd ~/workdir ``` ## Metagenomics-Toolkit Introduction @@ -157,7 +150,7 @@ scratch: false > Computational Resources -Please note that computational resources are also global parameters and will be handled in the third part of this configuration section. +> Please note that computational resources are also global parameters and will be handled in the third part of this configuration section. {: .tip} @@ -268,24 +261,24 @@ The Toolkit output fulfills the following schema: SAMPLE_NAME/RUN_ID/MODULE/MODULE_VERSION/TOOL ``` -* **RUN_ID:** The run ID will be part of the output path and allows to distinguish between different pipeline configurations that were used for the same dataset. +* `RUN_ID`: The run ID will be part of the output path and allows to distinguish between different pipeline configurations that were used for the same dataset. -* **MODULE** is the analysis that is executed by the Toolkit (e.g. qc, assembly, etc.). +* `MODULE` is the analysis that is executed by the Toolkit (e.g., qc, assembly, etc.). -* **MODULE_VERSION** is the version number of the module. +* `MODULE_VERSION` is the version number of the module. -* **TOOL** is the tool or method that is executed by the Toolkit. +* `TOOL` is the tool or method that is executed by the Toolkit. Below you can see an example output structure configured for long-read data. Every output folder includes four log files: -* **.command.err:** Contains the standard error. +* `.command.err`: Contains the standard error. -* **.command.out:** Contains the standard output. +* `.command.out`: Contains the standard output. -* **.command.log:** Contains the combined standard error and standard output. +* `.command.log`: Contains the combined standard error and standard output. -* **.command.sh:** Contains the command that was executed. +* `.command.sh`: Contains the command that was executed. ```bash output/ @@ -336,7 +329,8 @@ cd ~/workdir gedit samples.tsv ``` -Then add the following content to the file: +Then add the following content to the file. + **Important:** Make sure there are actually tabs in between the columns! And adjust the path if your is not `/vol/longread/`. @@ -442,7 +436,7 @@ NXF_VER=25.10.4 nextflow run main.nf \ Then check the results in the `output` folder. Check the tutorials in the Metagenomics-Toolkit documentation for further information. ---- + ### APPENDIX: Reference Links for Metagenomics-Toolkit Tools * **Nextflow (Workflow Orchestration Engine):** diff --git a/_tutorials/mgworkshop_assembly/main/tutorial.md b/_tutorials/mgworkshop_assembly/main/tutorial.md index dba1919..b6a7dff 100644 --- a/_tutorials/mgworkshop_assembly/main/tutorial.md +++ b/_tutorials/mgworkshop_assembly/main/tutorial.md @@ -1,17 +1,20 @@ --- layout: tutorial_hands_on -title: Metagenomic Assembly -description: "This tutorial will guide you through the typical steps of metagenome assembly. " +title: "Metagenome Assembly: concepts and hands-on comparison of assemblers" +description: "This tutorial introduces the principles and challenges of metagenome assembly, including de Bruijn graph-based approaches, k-mer selection, and metagenome-specific assembly strategies. You will apply several assemblers, including Velvet, MEGAHIT, metaSPAdes, IDBA-UD, and Ray, and compare their assembly performance." time_estimation: 6H level: beginner -keywords: [FIXME] +keywords: [metagenomics, genome assembly, OLC, de Bruijn graph, Velvet, MEGAHIT, metaSPAdes, IDBA-UD, Ray] questions: - - "How do I run a metagenome assembly?" - - "How do I compare metagenome assemblies?" + - "What are the main challenges of reconstructing genomes from metagenomic sequencing data?" + - "How do assembly algorithms differ?" + - "How do k-mer size, sequencing errors, repeats, and uneven coverage influence assembly quality?" + - "Which assembly strategies and tools are commonly used for complex metagenomic datasets?" objectives: - - + - "Recognize common sources of assembly ambiguity, such as repeats, sequencing errors, and uneven species abundance." + - "Run multiple metagenome assemblers on sequencing reads and generate assembly statistics." key_points: - - + - "No single assembler performs best for all datasets; tool selection depends on dataset complexity, accuracy requirements, and available computational resources." version: - main life_cycle: under development @@ -21,12 +24,12 @@ contributions: - Alexander Sczyrba - Sebastian Jünemann editing: + - Dilfuza Djamalova funding: --- >Prerequisites -> -> - Please do the linux introduction before this tutorial. +> - Please complete the [Unix/Linux introduction tutorial]({{ site.url }}{{ site.baseurl }}/tutorials/unix-course/main/tutorial/) before this tutorial. > - We assume you have successfully connected to an instance in the de.NBI cloud with the software pre-installed. Otherwise you will need to install the required tools on your own and make sure you have sufficient resources available. {: .details} @@ -54,7 +57,9 @@ Metagenome assembly resembles the process of solving a large jigsaw puzzle witho ### Classical Genome Sequencing Strategies -Before short-read next-generation sequencing became dominant, genome assembly commonly relied on two main strategies: ordered shotgun sequencing and whole genome shotgun sequencing. In ordered shotgun sequencing, DNA fragments were cloned into vectors such as BACs and arranged into a physical map. This map provided positional information, making it possible to sequence clones individually and then piece them together with high confidence. The physical map served as a scaffold that guaranteed correct global arrangement, though generating it required significant laboratory effort. +Before short-read next-generation sequencing became dominant, genome assembly commonly relied on two main strategies: ordered shotgun sequencing and whole genome shotgun sequencing. + +In ordered shotgun sequencing, DNA fragments were cloned into vectors such as BACs and arranged into a physical map. This map provided positional information, making it possible to sequence clones individually and then piece them together with high confidence. The physical map served as a scaffold that guaranteed correct global arrangement, though generating it required significant laboratory effort. **Figure 4.** *Hierarchical (ordered) shotgun sequencing schematic adapted from Venter et al., Nature 2001 * ![Hierarchical shotgun sequencing]({{ "/tutorials/mgworkshop_assembly/images/hierarchical_shotgun.png" | relative_url }}){: .responsive-img } @@ -218,11 +223,11 @@ Metagenome assembly reconstructs genomic sequences from mixtures of organisms, r --- -## **Tutorial** +## **Hands-on tutorial** We are going to use different assemblers and compare the results. -### **Data download** +### **Download data** We have prepared a small toy data set for this tutorial. It's simulated data, so there is actually no need for quality control. @@ -469,7 +474,7 @@ contigs are located in `ray_51/Contigs.fasta` (and {: .hands_on} >Step 4: getN50 -> Now that you have run assemblies using Velvet, MEGAHIT, metaSPAdes, IDBA-UD and Ray, let's have a > quick look at the assembly statistics of all of them:: +> Now that you have run assemblies using Velvet, MEGAHIT, metaSPAdes, IDBA-UD and Ray, let's have a quick look at the assembly statistics of all of them: >>Code-in >> ```bash >> cd /mnt/WGS-data diff --git a/_tutorials/nanopore/main/data-preparation.md b/_tutorials/nanopore/main/data-preparation.md new file mode 100644 index 0000000..eb29c20 --- /dev/null +++ b/_tutorials/nanopore/main/data-preparation.md @@ -0,0 +1,42 @@ +## **Download the data and preparations** + +First, create a link to `/vol/longread` (or the folder in which you want to work during the course) and switch to that directory: + +```bash +ln -s /vol/longread/ ~/workdir +cd ~/workdir +``` +You might need to change the permissions of `/vol/longread`, for example (in the cloud setup we use for the on-site course) with: + +```bash +sudo chown ubuntu:ubuntu /vol/longread/ +``` +(Adjust accordingly to your setup) + +>IMPORTANT +>Some software is installed within a python virtual environment, you need to activate it with: +> +>```bash +>source ~/longread/bin/activate +>``` +>If some tool cannot be executed during this tutorial - make sure the environment is active! Indicated byt `(longread)` in your commandline. +{: .details} + +Next, we download our tutorial dataset and extract it: + +```bash +cd ~/workdir +wget https://s3.bi.denbi.de/cmg/mgcourses/longread2026/coursedata.tar.gz +tar -xzvf coursedata.tar.gz +``` + +Have a quick look at the content of the `coursedata` folder: + +```bash +ls -l ~/workdir/coursedata/ +``` +It contains the following components: +1. **illumina/** A folder containing fastq files with Illumina reads +2. **ont/** A folder containing fastq files with Nanopore reads +3. **raw/** A folder containing a file with raw Nanopore data which we will inspect in the next section +4. **reference.fasta** A reference fasta for the strain we are going to analyze diff --git a/_tutorials/nanopore/main/tutorial.md b/_tutorials/nanopore/main/tutorial.md index 74559e9..2898cf8 100644 --- a/_tutorials/nanopore/main/tutorial.md +++ b/_tutorials/nanopore/main/tutorial.md @@ -1,33 +1,29 @@ --- layout: tutorial_hands_on -title: Nanopore Workshop -description: "This tutorial will guide you through the typical steps of analyzing read data from an isolate genome using ONT sequencing data and Illumina. " -time_estimation: 6H +title: Basecalling and QC of ONT data +description: "This tutorial introduces the preprocessing workflow for Oxford Nanopore Technologies (ONT) sequencing data, from raw signal files to quality-controlled reads for downstream analysis." +time_estimation: 2H level: beginner -keywords: ONT, Nanopore, Assembly, Polishing, Genomics, Annotation +keywords: ONT, Nanopore, Basecalling, dorado questions: - - "How do I basecall ONT data?" - - "What is the difference between FAST5 and POD5 files?" - - "How can we inspect and convert raw Nanopore data formats?" - - "How do I assemble ONT data" - - "How do I map ONT data?" - - "How do I polish my ONT assembly using Illumina data?" - - "How do run a hybrid assembly?" - - "How do I compare assemblies?" - - "How do I annotate my genome?" + - "How do I basecall and extract native base modifications from ONT data?" + - "What is the structural difference between FAST5 and POD5 files, and how do I convert them?" + - "How can we perform long-read quality control and filtering using fastplong, FastQC, and NanoPlot?" objectives: - - "Understand the physical and electrical principles of Nanopore sequencing." - - "Explain the transition from early HMM-based basecallers to modern Transformer-based models." - - "Interpret Phred quality scores and differentiate between Fast, HAC, and SUP models." - - "Compare the structural differences between HDF5-based FAST5 and Arrow-based POD5." - - "Use command-line tools to inspect legacy raw data." - - "Convert legacy FAST5 files into the modern POD5 standard." + - "Understand the physical and electrical principles of Nanopore sequencing and signal recording." + - "Explain the transition from early HMM-based basecallers to modern Transformer-based models like Dorado." + - "Differentiate between Fast, HAC, and SUP basecalling models and interpret Phred quality scores." + - "Compare the data architecture of HDF5-based FAST5 and Apache Arrow-based POD5." + - "Inspect raw metadata and convert legacy FAST5 datasets into the production-standard POD5 format." + - "Apply fastplong to trim adapters and filter long reads by length and quality." + - "Align long and short reads to a reference genome using minimap2 and BWA, and process files via samtools for IGV visualization." key_points: - - "Basecalling translates raw ionic current disruptions ('squiggles') into nucleotide sequences." - - "The evolution from Albacore to Guppy and Dorado represents a major shift from CPUs to GPU-accelerated deep learning." - - "Modern R10.4.1 chemistry paired with Dorado pushes accuracies past Q20 (>99%), matching or exceeding short-read standards." - - "FAST5 uses a hierarchical structure that causes I/O bottlenecks during high-throughput basecalling." - - "POD5 uses a flat, columnar format (Apache Arrow) that offers faster multi-threaded access and smaller file sizes." + - "Basecalling translates raw ionic current disruptions (i.e., squiggles) into discrete nucleotide sequences." + - "The evolution from Albacore to Guppy and Dorado represents a major architectural shift from CPU-driven statistical models to GPU-accelerated deep learning." + - "Modern R10.4.1 chemistry paired with Dorado pushes raw read accuracy past Q20 (>99.0%) and duplex reads past Q30 (>99.9%)." + - "POD5 replaces the heavy, hierarchical, bottleneck-prone FAST5 format with a flat, columnar layout (Apache Arrow) optimized for fast multi-threaded I/O." + - "Native chemical base modifications (like 5mC methylation) alter the ionic current signature and can be detected in real-time without bisulfite treatment using specialized Dorado modification models." + - "Standard short-read QC tools like FastQC often misinterpret long-read length variations; dedicated tools like NanoPlot and fastplong provide highly tailored long-read summary metrics." version: - main life_cycle: under development @@ -35,59 +31,19 @@ contributions: authorship: - Nils Kleinbölting editing: + - Dilfuza Djamalova funding: --- >Prerequisites -> -> - Please do the linux introduction before this tutorial. +> - Please complete the [Unix/Linux introduction tutorial]({{ site.url }}{{ site.baseurl }}/tutorials/unix-course/main/tutorial/) before this tutorial. > - Basic understanding of the ONT and Illumina sequencing technology. > - We assume you have successfully connected to an instance in the de.NBI cloud with the software pre-installed. Otherwise you will need to install the required tools on your own and make sure you have sufficient resources available. > - Throughout the course we assume you are working on data downloaded to a volume under `/vol/longread/`, we create a link `~/workdir/` to that folder, if you are working somewhere else, adjust the `~/workdir` link to that location and all commands should work as outlined in the course. > - We also assume that you have a machine with **28 cores** available, if not - adjust the commands that specify a certain number of threads / cores accordingly. {: .details} -## **Download the data and preparations** - -First, create a link to `/vol/longread` (or the folder in which you want to work during the course) and switch to that directory: - -```bash -ln -s /vol/longread/ ~/workdir -cd ~/workdir -``` -You might need to change the permissions of `/vol/longread`, for example (in the cloud setup we use for the on-site course) with: - -```bash -sudo chown ubuntu:ubuntu /vol/longread/ -``` -(Adjust accordingly to your setup) - -**IMPORTANT**: Some software is installed within a python virtual environment, you need to activate it with: - -```bash -source ~/longread/bin/activate -``` -If some tool cannot be executed during this tutorial - make sure the environment is active! Indicated byt `(longread)` in your commandline. - - -Next, we download our tutorial dataset and extract it: - -```bash -cd ~/workdir -wget https://s3.bi.denbi.de/cmg/mgcourses/longread2026/coursedata.tar.gz -tar -xzvf coursedata.tar.gz -``` - -Have a quick look at the content of the `coursedata` folder: - -```bash -ls -l ~/workdir/coursedata/ -``` -It contains the following components: -1. **illumina/** A folder containing fastq files with Illumina reads -2. **ont/** A folder containing fastq files with Nanopore reads -3. **raw/** A folder containing a file with raw Nanopore data which we will inspect in the next section -4. **reference.fasta** A reference fasta for the strain we are going to analyze +{% include _tutorials/nanopore/main/data-preparation.md %} ## Basecalling ONT data @@ -634,405 +590,10 @@ IGV > Browse through the alignment tracks. What is the main difference in terms of sequencing errors? What specific problem is caused by the ONT errors? > > > Solution -> > Illumina has less errors and they are usullay substitutions, ONT has a high amount of insertions. Which causes a problem for gene prediction and annotation due to frameshifts. +> > Illumina has less errors and they are usually substitutions, ONT has a high amount of insertions. Which causes a problem for gene prediction and annotation due to frameshifts. > {: .solution} {: .question} -## Assembly and assembly evaluation - -### Introduction to De Novo Genome Assembly - -Genome assembly is the process of piecing together massive amounts of short or long DNA fragments (reads) to reconstruct the original underlying chromosome. Because we do not use a reference genome during *de novo* assembly, the algorithms rely strictly on sequence overlaps. Two main algorithmic paradigms dominate the field: - -* **De Bruijn Graph (DBG):** Primarily used for short reads (e.g., Illumina). Reads are broken down into smaller fixed-length strings called **$$k$$-mers**. Overlaps are tracked by constructing a network where nodes or edges represent shared $$k$$-mers. DBG is computationally efficient for processing hundreds of millions of short reads and highly accurate, but it struggles enormously with genomic repeats because the short $$k$$-mer contexts cannot resolve long duplicate regions. -* **Overlap-Layout-Consensus (OLC):** Primarily used for long reads (e.g., ONT, PacBio). The algorithm calculates all-versus-all alignments between full reads (**Overlap**), constructs an alignment graph to simplify paths and resolve structures (**Layout**), and finally determines the most accurate sequence across overlapping reads (**Consensus**). Long reads easily span across genomic repeats, allowing OLC-based pipelines to assemble completely closed chromosomes. This approach is usually not feasible for short reads due to the massive amount of alignments that have to be computed. - ---- - -### Understanding the Assembly and Assembly Evaluation Tools - -#### 1. Flye -Flye is a specialized *de novo* assembler designed for long, error-prone reads. Instead of building a classic OLC overlap graph (which scales poorly with high read depths), Flye constructs an unpolished **repeat graph**. It collapses complex genomic repeats into single edges, and then utilizes the long span of individual read paths to accurately untangle and separate those repeat copies. - -> Optional: How to install Flye -Run this (or follow instructions in github): -> ```bash -> pip install setuptools -> git clone https://github.com/fenderglass/Flye -> cd Flye -> python setup.py install -> ``` -{: .tip} - -#### 2. SPAdes -SPAdes (St. Petersburg Genome Assembler) is the gold standard for bacterial short-read assemblies. It relies on multi-sized De Bruijn Graphs (combining multiple $k$-mer lengths) to simultaneously optimize specificity and sensitivity, providing robust performance across single-isolate cultures and single-cell sequencing. - -> Optional: How to install Flye -Run this (or follow instructions in github): -> ```bash -> wget https://github.com/ablab/spades/releases/download/v4.3.0/SPAdes-4.3.0-Linux.tar.gz -> tar -xzvf SPAdes-4.3.0-Linux.tar.gz -> export PATH=PATH:$(pwd)/SPAdes-4.3.0-Linux/bin/ -> ``` -{: .tip} - -#### 3. QUAST -QUAST (Quality Assessment Tool) is an evaluation utility that calculates structural metrics (like contig counts, N50 value, and total length) and identifies misassemblies by aligning your assembled contigs back against a trusted reference genome. - -> Optional: How to install Quast -Run this (or follow instructions in github): -> ```bash -> wget https://github.com/ablab/quast/releases/download/quast_5.3.0/quast-5.3.0.tar.gz -> tar -xzvf quast-5.3.0.tar.gz -> cd quast-5.3.0 -> ./setup.py install -> ``` -{: .tip} - -#### 4. Bandage -Bandage (Bioinformatics Application for Navigating De Novo Assembly Graphs Easily) is a graphical interface utility that reads Graphical Assembly Graph (`.gfa`) files. It allows you to see how contigs connect to one another, helping you determine whether your bacterial genome successfully assembled into a single closed circular chromosome. - -> Optional: How to install Bandage -Run this (or follow instructions in github): -> ```bash -> wget https://github.com/rrwick/Bandage/releases/download/v0.8.1/Bandage_Ubuntu_dynamic_v0_8_1.zip -> unzip Bandage_Ubuntu_dynamic_v0_8_1.zip -> sudo mv Bandage /usr/local/bin/ -> #might be necessary: -> sudo apt install libqt5svg5 -> ``` -{: .tip} - ---- - -### Hands-on: Building and Evaluating Assemblies - -In this section, we will run separate long-read and short-read assembly pipelines, statistically benchmark their outputs against our reference, inspect their connectivity graphs, and align the draft contigs visually. - -#### Step 1: Long-Read Assembly with Flye - -Because modern Dorado basecalled data achieves exceptional accuracy (entering the Q20 standard), we use Flye's high-fidelity option (`--nano-hq`) to generate our draft genome: - -```bash -flye --nano-hq ~/workdir/coursedata/ont.fastq.gz --out-dir ~/workdir/flye_output --threads 28 -``` - -#### Step 2: Short-Read Assembly with SPAdes - -Next, we generate a corresponding short-read assembly utilizing our paired-end Illumina datasets: - -```bash -spades.py -1 ~/workdir/coursedata/illumina/Barcode11_TSLF_S10_L001_R1_001.fastq.gz \ - -2 ~/workdir/coursedata/illumina/Barcode11_TSLF_S10_L001_R2_001.fastq.gz \ - -o ~/workdir/spades_output --threads 28 -``` - -#### Step 3: Benchmarking Assemblies with QUAST - -We can now run a direct comparative evaluation between both assembly results using our known genome sequence as a reference: - -```bash -quast.py ~/workdir/flye_output/assembly.fasta \ - ~/workdir/spades_output/contigs.fasta \ - -r ~/workdir/coursedata/reference.fasta \ - -o ~/workdir/quast_output -``` - -Open the interactive QUAST HTML summary document in your browser to view the benchmark comparison: - -```bash -firefox ~/workdir/quast_output/report.html -``` - -> Analyzing Assembly Metrics -> Look at the metric comparisons in the QUAST report. Which assembly contains fewer total contigs? Which possesses a higher N50 score? What does this tell you about the power of long reads? -> -> > Solution -> > Typically, the Flye long-read assembly will result in significantly fewer contigs (often a single continuous contig for a closed bacterial chromosome) and a drastically higher N50 score approaching the true size of the genome. The SPAdes short-read assembly is usually split across multiple fragments because short fragments cannot resolve genomic repeats. -> {: .solution} -{: .question} - ---- - -#### Step 4: Visualizing Graphs in Bandage - -Statistical metrics only tell half the story. We need to look at the assembly graphs to see the structure of our contigs. - -1. Launch the **Bandage** GUI application via your terminal: -```bash - Bandage - ``` -2. In the Bandage menu, navigate to **File** -> **Load graph**. -3. First, load the Flye assembly graph file located at `~/workdir/flye_output/assembly_graph.gfa` and click **Draw graph**. -4. Next, clear the screen and load the SPAdes assembly graph file found at `~/workdir/spades_output/assembly_graph_with_scaffolds.gfa` and click **Draw graph**. - -> Interpreting Graph Topologies -> In the Flye window, you should see a single, beautiful, interconnected closed loop representing the intact circular bacterial chromosome. In contrast, the SPAdes graph will likely display a highly fragmented web of disjointed paths and isolated nodes, highlighting where the short-read assembly broke down at repeat boundaries. -{: .comment} - ---- - -#### Step 5: Aligning Contigs to Reference for IGV - -Finally, we want to align our assembled fasta contigs back against the reference genome to visually spot missing structural parts or mismatches in IGV. We use `minimap2` with the `-ax asm5` preset, which is optimized for aligning highly accurate genome assemblies. - -```bash -# Map the Flye assembly contigs -minimap2 -t 28 -ax asm5 ~/workdir/coursedata/reference.fasta ~/workdir/flye_output/assembly.fasta > ~/workdir/mappings/flye_vs_ref.sam -samtools view -S -b ~/workdir/mappings/flye_vs_ref.sam | samtools sort -o ~/workdir/mappings/flye_vs_ref_sorted.bam -samtools index ~/workdir/mappings/flye_vs_ref_sorted.bam - -# Map the SPAdes assembly contigs -minimap2 -t 28 -ax asm5 ~/workdir/coursedata/reference.fasta ~/workdir/spades_output/scaffolds.fasta > ~/workdir/mappings/spades_vs_ref.sam -samtools view -S -b ~/workdir/mappings/spades_vs_ref.sam | samtools sort -o ~/workdir/mappings/spades_vs_ref_sorted.bam -samtools index ~/workdir/mappings/spades_vs_ref_sorted.bam -``` - -#### Verification in IGV: -1. Open **IGV**, and make sure your reference genome (`~/workdir/coursedata/reference.fasta`) is actively loaded. -2. Load both new alignment files via **File** -> **Load from File...**: - * `~/workdir/mappings/flye_vs_ref_sorted.bam` - * `~/workdir/mappings/spades_vs_ref_sorted.bam` -3. Inspect the alignment tracks to identify gaps or fragmentation points where the short-read assembly failed to recover structural elements. - ---- - -## Improving the flye assembly and trying a hybrid assembly approach - -### Short-Read Polishing with Polypolish - -Even though modern ONT R10.4.1 chemistry combined with Dorado pushes raw read accuracy into the Q20 (>99%) range, long-read assemblies can still retain minor systematic errors. These errors are most frequently found in homopolymer runs (e.g., long stretches of AAAA), manifesting as small insertions or deletions (indels). To fix these remaining micro-errors, we can perform a process called **polishing** using highly accurate Illumina short reads. - -We will use **Polypolish**, a short-read polishing tool designed specifically for long-read assemblies. - -> How Polypolish Avoids False Corrections -> Traditional polishers take all short-read alignments and use a consensus to alter the assembly. However, in repetitive genomic regions, short reads frequently misalign to the wrong repeat copy, leading the polisher to introduce errors rather than fix them. -> -> Polypolish solves this by examining the alternative alignments for each short read. If a read can map to multiple places in the assembly, Polypolish will only propose a correction if *all* possible target sites agree on the mismatch. If the mapping is ambiguous, it leaves the sequence untouched, preventing false corrections in repeat boundaries. -{: .comment} - -To ensure Polypolish operates effectively, we must execute a specific multi-step pipeline: -1. Map short reads **separately** (R1 and R2 independently) using `bwa mem` with the `-a` option. This option forces the aligner to output *all* possible alignment locations for a read, not just the single best hit. -2. Run `polypolish filter` to calculate the expected insert size of read pairs and filter out low-confidence alignments. -3. Run `polypolish polish` to correct the assembly using the filtered pileups. - -> Optional: How to install Polypolish -Run this (or follow instructions in github): -> ```bash -> wget https://github.com/rrwick/Polypolish/releases/download/v0.6.1/polypolish-linux-x86_64-musl-v0.6.1.tar.gz -> tar -xzvf polypolish-linux-x86_64-musl-v0.6.1.tar.gz -> sudo mv polypolish /usr/local/bin/ -> ``` -{: .tip} - ---- - -### Hands-on: Polishing the Long-Read Assembly - -#### Step 1: Mapping Short Reads with All Alignments Enabled - -First, let's build the BWA index of our long-read genome assembly and map both Illumina forward and reverse files completely independently using the required `-a` flag: - -```bash -# Index the Flye draft genome -bwa index ~/workdir/flye_output/assembly.fasta - -# Create directory for polypolish files -mkdir polypolish -# Map R1 and R2 forward/reverse reads completely independently with the -a flag -bwa mem -t 28 -a ~/workdir/flye_output/assembly.fasta ~/workdir/coursedata/illumina/Barcode11_TSLF_S10_L001_R1_001.fastq.gz > ~/workdir/polypolish/polypolish_r1.sam -bwa mem -t 28 -a ~/workdir/flye_output/assembly.fasta ~/workdir/coursedata/illumina/Barcode11_TSLF_S10_L001_R2_001.fastq.gz > ~/workdir/polypolish/polypolish_r2.sam -``` - -#### Step 2: Filtering Alignments by Insert Size - -Next, we pass our independent raw SAM files into Polypolish's filtering subcommand. This evaluates read pairing distances to clear away non-specific background mappings: - -```bash -polypolish filter --in1 ~/workdir/polypolish/polypolish_r1.sam --in2 ~/workdir/polypolish/polypolish_r2.sam --out1 ~/workdir/polypolish/filtered_r1.sam --out2 ~/workdir/polypolish/filtered_r2.sam -``` - -#### Step 3: Executing the Final Consensus Polish - -Now, we provide the original unpolished Flye assembly along with both freshly filtered alignment tracks to create our refined fasta file: - -```bash -polypolish polish ~/workdir/flye_output/assembly.fasta ~/workdir/polypolish/filtered_r1.sam ~/workdir/polypolish/filtered_r2.sam > ~/workdir/polypolish/flye_polished.fasta -``` - ---- - -### Hybrid Assembly with SPAdes - -Instead of assembling long reads first and polishing them later, a **hybrid assembly** combines both data types simultaneously into a single algorithmic workflow. - -We will use the hybrid mode of **SPAdes**. The SPAdes hybrid approach works as follows: -1. It builds a high-accuracy, highly-resolved **De Bruijn Graph** using only the pristine Illumina short reads. -2. It then maps the ONT long reads onto this graph. The long reads act as structural templates or "scaffolds" to bridge across repeat-induced gaps and resolve complex branches within the graph structure. - -This approach combines the single-nucleotide accuracy of short reads with the structural spanning power of long reads seamlessly. - -#### Step 4: Running Hybrid SPAdes - -Execute the hybrid SPAdes pipeline by supplying both your paired-end short reads and your combined long-read datasets: - -```bash -spades.py -1 ~/workdir/coursedata/illumina/Barcode11_TSLF_S10_L001_R1_001.fastq.gz \ - -2 ~/workdir/coursedata/illumina/Barcode11_TSLF_S10_L001_R2_001.fastq.gz \ - --nanopore ~/workdir/coursedata/ont.fastq.gz \ - -o ~/workdir/spades_hybrid_output --threads 28 -``` - ---- - -### Hands-on: Comprehensive Assembly Evaluation - -We now have four distinct assembly variants tracking our target genome. Let's run a final comparative evaluation with QUAST to see how polishing and hybrid strategies alter genome completeness and accuracy metrics. - -The 4 assembly variants to evaluate are: -1. `flye_output/assembly.fasta` (ONT Long-Reads Only) -2. `spades_output/scaffolds.fasta` (Illumina Short-Reads Only) -3. `flye_polished.fasta` (ONT Long-Reads Polished with Short-Reads) -4. `spades_hybrid_output/contigs.fasta` (Hybrid Co-Assembly) - -#### Step 5: Comparing all Four Frameworks in QUAST - -Run QUAST with all four assembly files against the true reference genome: - -```bash -quast.py ~/workdir/flye_output/assembly.fasta \ - ~/workdir/spades_output/contigs.fasta \ - ~/workdir/polypolish/flye_polished.fasta \ - ~/workdir/spades_hybrid_output/contigs.fasta \ - -l "flye,spades,polypolish,hybrid_spades" \ - -r ~/workdir/coursedata/reference.fasta \ - -t 28 \ - -o ~/workdir/quast_final_output -``` - -Open the resulting dashboard summary report in your browser: - -```bash -firefox ~/workdir/quast_final_output/report.html -``` - -> Evaluating the Impact of Polishing and Hybridization -> Compare the column profiles of the unpolished Flye assembly vs. the polished Flye assembly. Look at metrics like "mismatches per 100 kbp" or "indels per 100 kbp". What changes do you observe? How does the Hybrid assembly compare in contig count? -> -> > Solution -> > Polishing with Polypolish typically causes a significant drop in the number of indels per 100 kbp compared to raw Flye contigs, which often restores disrupted open reading frames and increases the total number of fully recovered genes. The Hybrid SPAdes assembly often improves in contiguity, but depending on repeat complexity, it may still contain a few more contig fragments than Flye's completely closed loop structure. -> {: .solution} -{: .question} - ---- - -## Genome Annotation - -### Introduction to Prokaryotic Genome Annotation - -Once you have successfully assembled and polished a bacterial chromosome, it consists simply of a long, uncharacterized string of nucleotides (A, C, G, T). To make this data useful for biological research, you must perform **genome annotation**. This process involves identifying the structural features of the genome—such as protein-coding sequences (CDS), transfer RNAs (tRNAs), and ribosomal RNAs (rRNAs)—and assigning functional biological identities to them based on sequence similarity to known databases. - -In this module, we will compare two popular tools used for this task: - -* **Prokka:** For nearly a decade, Prokka has been the legacy workhorse tool for rapid prokaryotic genome annotation. It coordinates an ensemble of open-source tools (like Prodigal for CDS finding and Aragorn for tRNAs) to generate comprehensive annotation suites in minutes. However, because its internal reference databases are no longer actively maintained, it often over-assigns generic functional names or labels proteins as "hypothetical protein". - -> Optional: How to install Prokka -It's quite complicated to install without conda/docker/singularity. Check out the github and use one of those methods. -{: .tip} - -* **Bakta:** A modern, next-generation annotation platform designed specifically for microbial genomes. Bakta addresses Prokka's database stagnation by utilizing a thoroughly curated, regularly updated SQLite database synchronized with NCBI RefSeq, UniProt, and specialized feature resources. It provides highly accurate protein names, precise cross-reference tags (DBXrefs), and native tracking of non-coding RNAs (ncRNAs), pseudogenes, and antimicrobial resistance (AMR) gene identifiers. - -> Optional: How to install Bakta -It's quite complicated to install without conda/docker/singularity. Check out the github and use one of those methods. -{: .tip} ---- - -### Hands-on: Annotating Your Assembly - -We will run both annotators on our polished long-read assembly (`flye_polished.fasta`) and evaluate how their structural findings and functional naming conventions differ. - -#### Step 1: Running Prokka - -Execute Prokka by specifying an output directory and a custom file prefix: - -```bash -prokka --cpus 28 --outdir ~/workdir/prokka_output --prefix prokka_ont ~/workdir/polypolish/flye_polished.fasta -``` - -Don;t worry about the `Could not run command: tbl2asn` message if it appears. We don't need the `asn` file. - -#### Step 2: Running Bakta - -Unlike Prokka, Bakta relies on a separate, heavy database containing millions of curated proteins. For this workshop, this database has been pre-staged for you. Run Bakta using the following command: - -```bash -conda activate bakta -bakta --threads 28 --db ~/bakta_db/db-light --output ~/workdir/bakta_output ~/workdir/polypolish/flye_polished.fasta -conda deactivate -source ~/longread/bin/activate -``` - ---- - -### Hands-on: Comparing Annotation Profiles - -Both tools generate various standardized outputs, including GFF3, GenBank, and FASTA files. To quickly benchmark their structural predictions, we can review the text-based summary logs (`.txt`) produced by each pipeline. - -#### Step 3: Inspecting Summary Outputs - -Use `cat` to print out both overview profiles in your terminal: - -```bash -# View the Prokka summary report -cat ~/workdir/prokka_output/prokka_ont.txt - -# View the Bakta summary report -cat ~/workdir/bakta_output/bakta_ont.txt -``` - -> Analyzing Annotation Discrepancies -> Look closely at the total counts of Coding Sequences (CDS), tRNAs, and rRNAs in both outputs. Are the numbers identical? If they differ, what could cause one tool to predict more genes than the other? -> -> > Solution -> > Even though they use the same underlying software for core gene finding (Prodigal), the total counts often differ slightly. Bakta uses stricter structural filters and a much larger database, allowing it to accurately split overlapping reading frames, filter out false positive predictions, and identify specialized elements like pseudogenes or small non-coding RNAs that Prokka completely misses. -> {: .solution} -{: .question} - -#### Step 4: Comparing Functional Descriptions - -A major difference lies in how specifically proteins are named. Let's use `grep` to check how many genes were left uncharacterized as "hypothetical protein" in both annotation suites: - -```bash -# Count hypothetical proteins in Prokka's GFF output -grep -c "hypothetical protein" ~/workdir/prokka_output/prokka_ont.gff - -# Count hypothetical proteins in Bakta's GFF output -grep -c "hypothetical protein" ~/workdir/bakta_output/bakta_ont.gff -``` - -> Interpreting Naming Quality -> You will notice that Bakta significantly reduces the fraction of "hypothetical protein" labels compared to Prokka. Thanks to its modern reference integration with UniProt and RefSeq, Bakta can assign definitive, functional gene names to sequences where Prokka could only find vague, outdated family matches. -{: .comment} - ---- - -### Comparative Genomics with EDGAR - -Once individual genomes are annotated, the next logical milestone is to explore how multiple strains or species relate to one another. For this downstream phase, we shift from localized command-line annotation to web-based comparative genomics using **EDGAR** (Efficient Database framework for comparative Genome Analyses). - -* **Official Server Link:** [http://edgar3.computational.bio](http://edgar3.computational.bio) - -#### How EDGAR Works: -EDGAR is a fully automated high-throughput platform tailored for the deep comparative analysis of prokaryotic genomes. Users upload their fully annotated genome files (such as the `.gff` or GenBank files generated by Bakta) into public or password-protected private projects. - -The underlying pipeline performs intensive all-versus-all sequence alignments across all selected strains. By evaluating **BLAST Score Ratios (BSR)**, EDGAR accurately determines orthology relational paths to delineate specific genomic subsets: -1. **The Core Genome:** The conserved set of genes shared identically across *all* analyzed organisms, often used to build highly precise core-genome phylogenetic trees. -2. **The Pan-Genome:** The complete global pool of all unique genes present across the entire group. -3. **Singleton Genes:** Unique genes present in only *one* specific strain, which are crucial for identifying specific downstream traits like pathogenicity islands or unique metabolic capabilities. - -Furthermore, EDGAR calculates average nucleotide identity (ANI) metrics and renders publication-ready visualizations, including Venn diagrams, UpSet plots, and synteny maps mapping gene order conservation across syntenic chromosomal layouts. - ---- ## APPENDIX: References for tools used within the tutorial @@ -1060,30 +621,5 @@ Furthermore, EDGAR calculates average nucleotide identity (ANI) metrics and rend * **IGV (Integrative Genomics Viewer):** * **Homepage:** [https://software.broadinstitute.org/software/igv/](https://software.broadinstitute.org/software/igv/) * **Publication:** *Robinson, J. T. et al. (2011). Integrative Genomics Viewer. Nature Biotechnology.* -* **Flye** - * **GitHub:** [https://github.com/fenderglass/Flye](https://github.com/fenderglass/Flye) - * **Publication:** *Kolmogorov, M. et al. (2019). Assembly of long, error-prone reads using repeat graphs. Nature Biotechnology.* -* **SPAdes** - * **GitHub:** [https://github.com/ablab/spades](https://github.com/ablab/spades) - * **Publication:** *Bankevich, A. et al. (2012). SPAdes: a new genome assembly algorithm and its applications to single-cell sequencing. Journal of Computational Biology.* -* **QUAST** - * **GitHub:** [https://github.com/ablab/quast](https://github.com/ablab/quast) - * **Publication:** *Gurevich, A. et al. (2013). QUAST: quality assessment tool for genome assemblies. Bioinformatics.* -* **Bandage** - * **GitHub:** [https://github.com/rrwick/Bandage](https://github.com/rrwick/Bandage) - * **Publication:** *Wick, R. R. et al. (2015). Bandage: interactive visualization of de novo genome assembly graphs. Bioinformatics.* -* **Polypolish (Short-read Polisher for Long-read Assemblies):** - * **GitHub:** [https://github.com/rrwick/Polypolish](https://github.com/rrwick/Polypolish) - * **Publication:** *Wick, R. R. & Holt, K. E. (2022). Polypolish: Short-read polishing of long-read bacterial genome assemblies. PLoS Computational Biology.* -* **SPAdes (Hybrid Assembly Mode Support):** - * **GitHub:** [https://github.com/ablab/spades](https://github.com/ablab/spades) - * **Publication:** *Antipov, D. et al. (2016). hybridSPAdes: an algorithm for genome assembly from microbial long and short reads. Bioinformatics.* -* **Prokka (Rapid Prokaryotic Genome Annotation):** - * **GitHub:** [https://github.com/tseemann/prokka](https://github.com/tseemann/prokka) - * **Publication:** *Seemann, T. (2014). Prokka: rapid prokaryotic genome annotation. Bioinformatics.* -* **Bakta (Next-generation Microbial Genome Annotation):** - * **GitHub:** [https://github.com/oschwenders/bakta](https://github.com/oschwenders/bakta) - * **Publication:** *Schwengers, O. et al. (2021). Bakta: rapid and standardized annotation of bacterial genomes and plasmids. Microbial Genomics.* -* **EDGAR (Comparative Genomics Framework):** - * **Webserver Platform:** [http://edgar3.computational.bio](http://edgar3.computational.bio) - * **Publication:** *Dieckmann, M. A. et al. (2021). EDGAR 3.0: comparative genomics and phylogenomics on a scalable infrastructure. Nucleic Acids Research.* + + diff --git a/assets/css/dekcd_tutorial.scss b/assets/css/dekcd_tutorial.scss index adfb31a..2e698c8 100644 --- a/assets/css/dekcd_tutorial.scss +++ b/assets/css/dekcd_tutorial.scss @@ -198,4 +198,5 @@ mjx-container[jax="CHTML"] { line-height: 1 !important; padding: 0 !important; margin: 0 !important; -} \ No newline at end of file +} + diff --git a/assets/js/lesson.js b/assets/js/lesson.js index 1e55800..f40052e 100644 --- a/assets/js/lesson.js +++ b/assets/js/lesson.js @@ -2,7 +2,7 @@ $("table").addClass("table table-striped"); -// Handle foldable boxes (on click and at start) +// Handle foldable boxes (on title click and at start) $(document).ready(function() { // Container selectors for foldable blocks @@ -16,19 +16,23 @@ $(document).ready(function() { // Hide all children except the title element $(">*:not(" + titleSelector + ")", container).hide(); - // Add fold/unfold icon to the title - $(titleSelector + ":first", container).append("");}); - //$(titleSelector + ":first", container).append("");}); - - // Toggle on click (whole box fold/unfold) - $(foldableSelector).on("click", function(event) { - // Do not toggle when clicking inside a nested box or external links/buttons - if (!$(event.target).closest(foldableSelector).is(this)) { - return; + // Add fold/unfold icon to the title (avoid duplicates) + var title = container.children(titleSelector).first(); + if (title.find(".fold-unfold").length === 0) { + title.append(""); } + }); - var container = $(this); - var title = container.children(titleSelector).first(); + // Remove previous handlers + $(foldableSelector).off("click"); + + // Toggle only when clicking the title + $(document).on("click", titleSelector, function(event) { + event.preventDefault(); + event.stopPropagation(); + + var title = $(this); + var container = title.closest(foldableSelector); var body = container.children(":not(" + titleSelector + ")"); var icon = title.children("i.fold-unfold"); @@ -36,12 +40,15 @@ $(document).ready(function() { body.toggle(400); // Toggle the icon class - //icon.toggleClass("glyphicon-collapse-down glyphicon-collapse-up"); icon.toggleClass("bi-chevron-expand bi-chevron-contract"); }); -}); + // Prevent clicks inside the content from toggling + $(document).on("click", foldableSelector + " > :not(" + titleSelector + ")", function(event) { + event.stopPropagation(); + }); +}); // Handle searches. // Relies on document having 'meta' element with name 'search-domain'. diff --git a/pathways.md b/pathways.md index 3a1a366..95d6d70 100644 --- a/pathways.md +++ b/pathways.md @@ -19,8 +19,8 @@ permalink: /pathways/

{{ pathway.description }}

{% if pathway.tags %}
- {% for tag in pathway.tags %} - {{ tag }} + {% for keyword in pathway.keywords %} + {{ keyword }} {% endfor %}
{% endif %}