From f2e2de557fca13eac683ae4d5624ee4d39534add Mon Sep 17 00:00:00 2001 From: Michael Harper Date: Fri, 5 Jan 2024 12:55:41 +1100 Subject: [PATCH 1/8] adding analyst tutorial markdown and jupyter notebook --- .../Final_analyst_tutorial.ipynb | 441 ++++++++++++++++++ .../analyst_tutorial/analyst_tutorial.md | 122 +++++ 2 files changed, 563 insertions(+) create mode 100644 onboarding_documentation/technical_documentation/analyst_tutorial/Final_analyst_tutorial.ipynb create mode 100644 onboarding_documentation/technical_documentation/analyst_tutorial/analyst_tutorial.md diff --git a/onboarding_documentation/technical_documentation/analyst_tutorial/Final_analyst_tutorial.ipynb b/onboarding_documentation/technical_documentation/analyst_tutorial/Final_analyst_tutorial.ipynb new file mode 100644 index 0000000..71c0d54 --- /dev/null +++ b/onboarding_documentation/technical_documentation/analyst_tutorial/Final_analyst_tutorial.ipynb @@ -0,0 +1,441 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "0d6ccf14-c2b7-4d1c-8c1f-642d2ed2aba5", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + " \n", + "
\n", + " \n", + " Loading BokehJS ...\n", + "
\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": "(function(root) {\n function now() {\n return new Date();\n }\n\n const force = true;\n\n if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\nconst JS_MIME_TYPE = 'application/javascript';\n const HTML_MIME_TYPE = 'text/html';\n const EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n const CLASS_NAME = 'output_bokeh rendered_html';\n\n /**\n * Render data to the DOM node\n */\n function render(props, node) {\n const script = document.createElement(\"script\");\n node.appendChild(script);\n }\n\n /**\n * Handle when an output is cleared or removed\n */\n function handleClearOutput(event, handle) {\n function drop(id) {\n const view = Bokeh.index.get_by_id(id)\n if (view != null) {\n view.model.document.clear()\n Bokeh.index.delete(view)\n }\n }\n\n const cell = handle.cell;\n\n const id = cell.output_area._bokeh_element_id;\n const server_id = cell.output_area._bokeh_server_id;\n\n // Clean up Bokeh references\n if (id != null) {\n drop(id)\n }\n\n if (server_id !== undefined) {\n // Clean up Bokeh references\n const cmd_clean = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n cell.notebook.kernel.execute(cmd_clean, {\n iopub: {\n output: function(msg) {\n const id = msg.content.text.trim()\n drop(id)\n }\n }\n });\n // Destroy server and session\n const cmd_destroy = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n cell.notebook.kernel.execute(cmd_destroy);\n }\n }\n\n /**\n * Handle when a new output is added\n */\n function handleAddOutput(event, handle) {\n const output_area = handle.output_area;\n const output = handle.output;\n\n // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n if ((output.output_type != \"display_data\") || (!Object.prototype.hasOwnProperty.call(output.data, EXEC_MIME_TYPE))) {\n return\n }\n\n const toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n\n if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n // store reference to embed id on output_area\n output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n }\n if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n const bk_div = document.createElement(\"div\");\n bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n const script_attrs = bk_div.children[0].attributes;\n for (let i = 0; i < script_attrs.length; i++) {\n toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n toinsert[toinsert.length - 1].firstChild.textContent = bk_div.children[0].textContent\n }\n // store reference to server id on output_area\n output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n }\n }\n\n function register_renderer(events, OutputArea) {\n\n function append_mime(data, metadata, element) {\n // create a DOM node to render to\n const toinsert = this.create_output_subarea(\n metadata,\n CLASS_NAME,\n EXEC_MIME_TYPE\n );\n this.keyboard_manager.register_events(toinsert);\n // Render to node\n const props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n render(props, toinsert[toinsert.length - 1]);\n element.append(toinsert);\n return toinsert\n }\n\n /* Handle when an output is cleared or removed */\n events.on('clear_output.CodeCell', handleClearOutput);\n events.on('delete.Cell', handleClearOutput);\n\n /* Handle when a new output is added */\n events.on('output_added.OutputArea', handleAddOutput);\n\n /**\n * Register the mime type and append_mime function with output_area\n */\n OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n /* Is output safe? */\n safe: true,\n /* Index of renderer in `output_area.display_order` */\n index: 0\n });\n }\n\n // register the mime type if in Jupyter Notebook environment and previously unregistered\n if (root.Jupyter !== undefined) {\n const events = require('base/js/events');\n const OutputArea = require('notebook/js/outputarea').OutputArea;\n\n if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n register_renderer(events, OutputArea);\n }\n }\n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n const NB_LOAD_WARNING = {'data': {'text/html':\n \"
\\n\"+\n \"

\\n\"+\n \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n \"

\\n\"+\n \"\\n\"+\n \"\\n\"+\n \"from bokeh.resources import INLINE\\n\"+\n \"output_notebook(resources=INLINE)\\n\"+\n \"\\n\"+\n \"
\"}};\n\n function display_loaded() {\n const el = document.getElementById(\"cc7e9275-9ef1-4dfd-8d37-4a188441122d\");\n if (el != null) {\n el.textContent = \"BokehJS is loading...\";\n }\n if (root.Bokeh !== undefined) {\n if (el != null) {\n el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(display_loaded, 100)\n }\n }\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls == null || js_urls.length === 0) {\n run_callbacks();\n return null;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n\n function on_error(url) {\n console.error(\"failed to load \" + url);\n }\n\n for (let i = 0; i < css_urls.length; i++) {\n const url = css_urls[i];\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error.bind(null, url);\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n }\n\n for (let i = 0; i < js_urls.length; i++) {\n const url = js_urls[i];\n const element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error.bind(null, url);\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n const js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.3.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.3.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.3.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.3.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-3.3.2.min.js\"];\n const css_urls = [];\n\n const inline_js = [ function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\nfunction(Bokeh) {\n }\n ];\n\n function run_inline_js() {\n if (root.Bokeh !== undefined || force === true) {\n for (let i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }\nif (force === true) {\n display_loaded();\n }} else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n } else if (force !== true) {\n const cell = $(document.getElementById(\"cc7e9275-9ef1-4dfd-8d37-4a188441122d\")).parents('.cell').data().cell;\n cell.output_area.append_execute_result(NB_LOAD_WARNING)\n }\n }\n\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(css_urls, js_urls, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));", + "application/vnd.bokehjs_load.v0+json": "" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + " \n", + "
\n", + " \n", + " Loading BokehJS ...\n", + "
\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": "(function(root) {\n function now() {\n return new Date();\n }\n\n const force = true;\n\n if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\nconst JS_MIME_TYPE = 'application/javascript';\n const HTML_MIME_TYPE = 'text/html';\n const EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n const CLASS_NAME = 'output_bokeh rendered_html';\n\n /**\n * Render data to the DOM node\n */\n function render(props, node) {\n const script = document.createElement(\"script\");\n node.appendChild(script);\n }\n\n /**\n * Handle when an output is cleared or removed\n */\n function handleClearOutput(event, handle) {\n function drop(id) {\n const view = Bokeh.index.get_by_id(id)\n if (view != null) {\n view.model.document.clear()\n Bokeh.index.delete(view)\n }\n }\n\n const cell = handle.cell;\n\n const id = cell.output_area._bokeh_element_id;\n const server_id = cell.output_area._bokeh_server_id;\n\n // Clean up Bokeh references\n if (id != null) {\n drop(id)\n }\n\n if (server_id !== undefined) {\n // Clean up Bokeh references\n const cmd_clean = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n cell.notebook.kernel.execute(cmd_clean, {\n iopub: {\n output: function(msg) {\n const id = msg.content.text.trim()\n drop(id)\n }\n }\n });\n // Destroy server and session\n const cmd_destroy = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n cell.notebook.kernel.execute(cmd_destroy);\n }\n }\n\n /**\n * Handle when a new output is added\n */\n function handleAddOutput(event, handle) {\n const output_area = handle.output_area;\n const output = handle.output;\n\n // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n if ((output.output_type != \"display_data\") || (!Object.prototype.hasOwnProperty.call(output.data, EXEC_MIME_TYPE))) {\n return\n }\n\n const toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n\n if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n // store reference to embed id on output_area\n output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n }\n if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n const bk_div = document.createElement(\"div\");\n bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n const script_attrs = bk_div.children[0].attributes;\n for (let i = 0; i < script_attrs.length; i++) {\n toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n toinsert[toinsert.length - 1].firstChild.textContent = bk_div.children[0].textContent\n }\n // store reference to server id on output_area\n output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n }\n }\n\n function register_renderer(events, OutputArea) {\n\n function append_mime(data, metadata, element) {\n // create a DOM node to render to\n const toinsert = this.create_output_subarea(\n metadata,\n CLASS_NAME,\n EXEC_MIME_TYPE\n );\n this.keyboard_manager.register_events(toinsert);\n // Render to node\n const props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n render(props, toinsert[toinsert.length - 1]);\n element.append(toinsert);\n return toinsert\n }\n\n /* Handle when an output is cleared or removed */\n events.on('clear_output.CodeCell', handleClearOutput);\n events.on('delete.Cell', handleClearOutput);\n\n /* Handle when a new output is added */\n events.on('output_added.OutputArea', handleAddOutput);\n\n /**\n * Register the mime type and append_mime function with output_area\n */\n OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n /* Is output safe? */\n safe: true,\n /* Index of renderer in `output_area.display_order` */\n index: 0\n });\n }\n\n // register the mime type if in Jupyter Notebook environment and previously unregistered\n if (root.Jupyter !== undefined) {\n const events = require('base/js/events');\n const OutputArea = require('notebook/js/outputarea').OutputArea;\n\n if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n register_renderer(events, OutputArea);\n }\n }\n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n const NB_LOAD_WARNING = {'data': {'text/html':\n \"
\\n\"+\n \"

\\n\"+\n \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n \"

\\n\"+\n \"\\n\"+\n \"\\n\"+\n \"from bokeh.resources import INLINE\\n\"+\n \"output_notebook(resources=INLINE)\\n\"+\n \"\\n\"+\n \"
\"}};\n\n function display_loaded() {\n const el = document.getElementById(\"f4f32713-7290-47db-90d0-9f7f5246a434\");\n if (el != null) {\n el.textContent = \"BokehJS is loading...\";\n }\n if (root.Bokeh !== undefined) {\n if (el != null) {\n el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(display_loaded, 100)\n }\n }\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls == null || js_urls.length === 0) {\n run_callbacks();\n return null;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n\n function on_error(url) {\n console.error(\"failed to load \" + url);\n }\n\n for (let i = 0; i < css_urls.length; i++) {\n const url = css_urls[i];\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error.bind(null, url);\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n }\n\n for (let i = 0; i < js_urls.length; i++) {\n const url = js_urls[i];\n const element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error.bind(null, url);\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n const js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.3.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.3.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.3.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.3.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-3.3.2.min.js\"];\n const css_urls = [];\n\n const inline_js = [ function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\nfunction(Bokeh) {\n }\n ];\n\n function run_inline_js() {\n if (root.Bokeh !== undefined || force === true) {\n for (let i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }\nif (force === true) {\n display_loaded();\n }} else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n } else if (force !== true) {\n const cell = $(document.getElementById(\"f4f32713-7290-47db-90d0-9f7f5246a434\")).parents('.cell').data().cell;\n cell.output_area.append_execute_result(NB_LOAD_WARNING)\n }\n }\n\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(css_urls, js_urls, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));", + "application/vnd.bokehjs_load.v0+json": "" + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import hail as hl\n", + "import pprint\n", + "from hail.plot import show\n", + "hl.plot.output_notebook()" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "fe84e1cc-0b8d-473b-a239-24e7ae14187b", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Initializing Hail with default parameters...\n", + "SLF4J: No SLF4J providers were found.\n", + "SLF4J: Defaulting to no-operation (NOP) logger implementation\n", + "SLF4J: See https://www.slf4j.org/codes.html#noProviders for further details.\n", + "SLF4J: Class path contains SLF4J bindings targeting slf4j-api versions 1.7.x or earlier.\n", + "SLF4J: Ignoring binding found at [jar:file:/opt/conda/envs/python310/lib/python3.10/site-packages/pyspark/jars/log4j-slf4j-impl-2.17.2.jar!/org/slf4j/impl/StaticLoggerBinder.class]\n", + "SLF4J: See https://www.slf4j.org/codes.html#ignoredBindings for an explanation.\n", + "Setting default log level to \"WARN\".\n", + "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", + "Running on Apache Spark version 3.3.4\n", + "SparkUI available at http://dd7b7b83e829:4040\n", + "Welcome to\n", + " __ __ <>__\n", + " / /_/ /__ __/ /\n", + " / __ / _ `/ / /\n", + " /_/ /_/\\_,_/_/_/ version 0.2.126-ee77707f4fab\n", + "LOGGING: writing to /home/jupyter/hail-20240104-2101-0.2.126-ee77707f4fab.log\n" + ] + } + ], + "source": [ + "vds = hl.vds.read_vds('gs://cpg-bioheart-test/vds/5-0.vds/')\n", + "sample_qc_ht = hl.read_table('gs://cpg-bioheart-test/large_cohort/5-0/sample_qc.ht/')\n", + "relateds_to_drop_ht = hl.read_table('gs://cpg-bioheart-test/large_cohort/5-0/relateds_to_drop.ht/')\n", + "pop_ht = hl.read_table('gs://cpg-bioheart-test/large_cohort/5-0/ancestry/inferred_pop.ht/')\n", + "vqsr_ht = hl.read_table('gs://cpg-bioheart-test/large_cohort/5-0/vqsr.ht/')\n", + "freq_ht = hl.read_table('gs://cpg-bioheart-test/large_cohort/5-0/frequencies.ht/')\n", + "dense_mt = hl.read_matrix_table('gs://cpg-bioheart-test/large_cohort/5-0/dense_subset.mt/')" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "4289f74d-60a4-4dfb-b17b-e198df09acff", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Row-level tables require a split+dense matrix table:\n", + "vds = hl.vds.split_multi(vds, filter_changed_loci=True)\n", + "mt = hl.vds.to_dense_mt(vds)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "1e74f5e6-1c91-445e-af95-fd5ff131e912", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Hard-filtering samples and variants:\n", + "mt = mt.filter_cols(hl.len(sample_qc_ht[mt.col_key].filters) > 0, keep=False)\n", + "mt = mt.filter_cols(hl.is_defined(relateds_to_drop_ht[mt.col_key]), keep=False)\n", + "mt = mt.filter_rows(hl.len(vqsr_ht[mt.row_key].filters) > 0, keep=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "510813e0-e22b-4be2-9d7b-c6f4e715be31", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Annotating samples and variants:\n", + "mt = mt.annotate_cols(**sample_qc_ht[mt.col_key])\n", + "mt = mt.rename({'filters' : 'sample_qc_filters'}) # to avoid conflict with vqsr column labels\n", + "mt = mt.annotate_cols(**pop_ht[mt.col_key])\n", + "mt = mt.annotate_rows(**vqsr_ht[mt.row_key])\n", + "mt = mt.annotate_rows(**freq_ht[mt.row_key])" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "b6ae5753-b90a-48d5-a435-9d76929cb758", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-01-04 21:02:29.088 Hail: INFO: Reading table to impute column types\n", + "2024-01-04 21:02:30.624 Hail: INFO: Finished type imputation\n", + " Loading field 'Sample' as type str (imputed)\n", + " Loading field 'population' as type str (imputed)\n", + " Loading field 'superpopulation' as type str (imputed)\n" + ] + } + ], + "source": [ + "# import annotations table with ancestry information\n", + "# even though we have an 'ancestry' output from the pipeline, we won't use it\n", + "table = (hl.import_table('annotations.txt', impute=True).key_by('Sample'))" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "049c9540-1e6d-470e-877c-f17db3a215df", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# annotate matrix table columns with ancestry information\n", + "mt = mt.annotate_cols(superpopulation=table[mt.s].superpopulation, \n", + " population=table[mt.s].population)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "2f286ecc-0784-478c-b84e-76254f36aac2", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-01-04 21:02:55.654 Hail: WARN: aggregate_cols(): Aggregates over cols ordered by 'col_key'.\n", + " To preserve matrix table column order, first unkey columns with 'key_cols_by()'\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + "
\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": "(function(root) {\n function embed_document(root) {\n const docs_json = {\"73a1dfc6-515f-48ef-98b1-6f59889e5062\":{\"version\":\"3.3.2\",\"title\":\"Bokeh Application\",\"roots\":[{\"type\":\"object\",\"name\":\"Figure\",\"id\":\"p1001\",\"attributes\":{\"width\":800,\"height\":800,\"x_range\":{\"type\":\"object\",\"name\":\"DataRange1d\",\"id\":\"p1002\"},\"y_range\":{\"type\":\"object\",\"name\":\"DataRange1d\",\"id\":\"p1003\"},\"x_scale\":{\"type\":\"object\",\"name\":\"LinearScale\",\"id\":\"p1011\"},\"y_scale\":{\"type\":\"object\",\"name\":\"LinearScale\",\"id\":\"p1012\"},\"title\":{\"type\":\"object\",\"name\":\"Title\",\"id\":\"p1004\",\"attributes\":{\"text\":\"PCA\"}},\"renderers\":[{\"type\":\"object\",\"name\":\"GlyphRenderer\",\"id\":\"p1049\",\"attributes\":{\"data_source\":{\"type\":\"object\",\"name\":\"ColumnDataSource\",\"id\":\"p1035\",\"attributes\":{\"selected\":{\"type\":\"object\",\"name\":\"Selection\",\"id\":\"p1036\",\"attributes\":{\"indices\":[],\"line_indices\":[]}},\"selection_policy\":{\"type\":\"object\",\"name\":\"UnionRenderers\",\"id\":\"p1037\"},\"data\":{\"type\":\"map\",\"entries\":[[\"index\",{\"type\":\"ndarray\",\"array\":{\"type\":\"bytes\",\"data\":\"AAAAAAEAAAACAAAAAwAAAAQAAAA=\"},\"shape\":[5],\"dtype\":\"int32\",\"order\":\"little\"}],[\"x\",{\"type\":\"ndarray\",\"array\":{\"type\":\"bytes\",\"data\":\"/5bzoNJX1L/ocxozGmrQv1sAadQxh8y/7L6DYfwsx78zNYL3AU7vPw==\"},\"shape\":[5],\"dtype\":\"float64\",\"order\":\"little\"}],[\"y\",{\"type\":\"ndarray\",\"array\":{\"type\":\"bytes\",\"data\":\"DRAK+CIZ4b+lpfkOFV3oP2frKWnzkdy/owBruc3ezz94/qsWelWevw==\"},\"shape\":[5],\"dtype\":\"float64\",\"order\":\"little\"}],[\"label\",{\"type\":\"ndarray\",\"array\":[\"AMR\",\"EUR\",\"AMR\",\"EAS\",\"EUR\"],\"shape\":[5],\"dtype\":\"object\",\"order\":\"little\"}]]}}},\"view\":{\"type\":\"object\",\"name\":\"CDSView\",\"id\":\"p1041\",\"attributes\":{\"filter\":{\"type\":\"object\",\"name\":\"IntersectionFilter\",\"id\":\"p1040\",\"attributes\":{\"operands\":[{\"type\":\"object\",\"name\":\"GroupFilter\",\"id\":\"p1039\",\"attributes\":{\"column_name\":\"label\",\"group\":\"AMR\"}}]}}}},\"glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1046\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"type\":\"object\",\"name\":\"CategoricalColorMapper\",\"id\":\"p1038\",\"attributes\":{\"palette\":[\"#1f77b4\",\"#ff7f0e\",\"#2ca02c\",\"#d62728\",\"#9467bd\",\"#8c564b\",\"#e377c2\",\"#7f7f7f\",\"#bcbd22\",\"#17becf\"],\"factors\":[\"EUR\",\"AMR\",\"EAS\"]}}},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}}}},\"nonselection_glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1047\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"line_alpha\":{\"type\":\"value\",\"value\":0.1},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"fill_alpha\":{\"type\":\"value\",\"value\":0.1},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.1}}},\"muted_glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1048\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"line_alpha\":{\"type\":\"value\",\"value\":0.2},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"fill_alpha\":{\"type\":\"value\",\"value\":0.2},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.2}}}}},{\"type\":\"object\",\"name\":\"GlyphRenderer\",\"id\":\"p1062\",\"attributes\":{\"data_source\":{\"id\":\"p1035\"},\"view\":{\"type\":\"object\",\"name\":\"CDSView\",\"id\":\"p1054\",\"attributes\":{\"filter\":{\"type\":\"object\",\"name\":\"IntersectionFilter\",\"id\":\"p1053\",\"attributes\":{\"operands\":[{\"type\":\"object\",\"name\":\"GroupFilter\",\"id\":\"p1052\",\"attributes\":{\"column_name\":\"label\",\"group\":\"EAS\"}}]}}}},\"glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1059\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}}}},\"nonselection_glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1060\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"line_alpha\":{\"type\":\"value\",\"value\":0.1},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"fill_alpha\":{\"type\":\"value\",\"value\":0.1},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.1}}},\"muted_glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1061\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"line_alpha\":{\"type\":\"value\",\"value\":0.2},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"fill_alpha\":{\"type\":\"value\",\"value\":0.2},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.2}}}}},{\"type\":\"object\",\"name\":\"GlyphRenderer\",\"id\":\"p1075\",\"attributes\":{\"data_source\":{\"id\":\"p1035\"},\"view\":{\"type\":\"object\",\"name\":\"CDSView\",\"id\":\"p1067\",\"attributes\":{\"filter\":{\"type\":\"object\",\"name\":\"IntersectionFilter\",\"id\":\"p1066\",\"attributes\":{\"operands\":[{\"type\":\"object\",\"name\":\"GroupFilter\",\"id\":\"p1065\",\"attributes\":{\"column_name\":\"label\",\"group\":\"EUR\"}}]}}}},\"glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1072\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}}}},\"nonselection_glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1073\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"line_alpha\":{\"type\":\"value\",\"value\":0.1},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"fill_alpha\":{\"type\":\"value\",\"value\":0.1},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.1}}},\"muted_glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1074\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"line_alpha\":{\"type\":\"value\",\"value\":0.2},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"fill_alpha\":{\"type\":\"value\",\"value\":0.2},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.2}}}}}],\"toolbar\":{\"type\":\"object\",\"name\":\"Toolbar\",\"id\":\"p1010\",\"attributes\":{\"tools\":[{\"type\":\"object\",\"name\":\"PanTool\",\"id\":\"p1023\"},{\"type\":\"object\",\"name\":\"WheelZoomTool\",\"id\":\"p1024\",\"attributes\":{\"renderers\":\"auto\"}},{\"type\":\"object\",\"name\":\"BoxZoomTool\",\"id\":\"p1025\",\"attributes\":{\"overlay\":{\"type\":\"object\",\"name\":\"BoxAnnotation\",\"id\":\"p1026\",\"attributes\":{\"syncable\":false,\"level\":\"overlay\",\"visible\":false,\"left\":{\"type\":\"number\",\"value\":\"nan\"},\"right\":{\"type\":\"number\",\"value\":\"nan\"},\"top\":{\"type\":\"number\",\"value\":\"nan\"},\"bottom\":{\"type\":\"number\",\"value\":\"nan\"},\"left_units\":\"canvas\",\"right_units\":\"canvas\",\"top_units\":\"canvas\",\"bottom_units\":\"canvas\",\"line_color\":\"black\",\"line_alpha\":1.0,\"line_width\":2,\"line_dash\":[4,4],\"fill_color\":\"lightgrey\",\"fill_alpha\":0.5}}}},{\"type\":\"object\",\"name\":\"SaveTool\",\"id\":\"p1031\"},{\"type\":\"object\",\"name\":\"ResetTool\",\"id\":\"p1032\"},{\"type\":\"object\",\"name\":\"HelpTool\",\"id\":\"p1033\"},{\"type\":\"object\",\"name\":\"HoverTool\",\"id\":\"p1034\",\"attributes\":{\"renderers\":\"auto\",\"tooltips\":[[\"x\",\"@x\"],[\"y\",\"@y\"]]}}]}},\"left\":[{\"type\":\"object\",\"name\":\"LinearAxis\",\"id\":\"p1018\",\"attributes\":{\"ticker\":{\"type\":\"object\",\"name\":\"BasicTicker\",\"id\":\"p1019\",\"attributes\":{\"mantissas\":[1,2,5]}},\"formatter\":{\"type\":\"object\",\"name\":\"BasicTickFormatter\",\"id\":\"p1020\"},\"axis_label\":\"PC2\",\"major_label_policy\":{\"type\":\"object\",\"name\":\"AllLabels\",\"id\":\"p1021\"}}},{\"type\":\"object\",\"name\":\"Legend\",\"id\":\"p1081\",\"attributes\":{\"click_policy\":\"hide\",\"items\":[{\"type\":\"object\",\"name\":\"LegendItem\",\"id\":\"p1078\",\"attributes\":{\"label\":{\"type\":\"value\",\"value\":\"AMR\"},\"renderers\":[{\"id\":\"p1049\"}]}},{\"type\":\"object\",\"name\":\"LegendItem\",\"id\":\"p1079\",\"attributes\":{\"label\":{\"type\":\"value\",\"value\":\"EAS\"},\"renderers\":[{\"id\":\"p1062\"}]}},{\"type\":\"object\",\"name\":\"LegendItem\",\"id\":\"p1080\",\"attributes\":{\"label\":{\"type\":\"value\",\"value\":\"EUR\"},\"renderers\":[{\"id\":\"p1075\"}]}}]}},{\"type\":\"object\",\"name\":\"ColorBar\",\"id\":\"p1082\",\"attributes\":{\"visible\":false,\"major_label_policy\":{\"type\":\"object\",\"name\":\"NoOverlap\",\"id\":\"p1083\"},\"color_mapper\":{\"id\":\"p1038\"}}}],\"below\":[{\"type\":\"object\",\"name\":\"LinearAxis\",\"id\":\"p1013\",\"attributes\":{\"ticker\":{\"type\":\"object\",\"name\":\"BasicTicker\",\"id\":\"p1014\",\"attributes\":{\"mantissas\":[1,2,5]}},\"formatter\":{\"type\":\"object\",\"name\":\"BasicTickFormatter\",\"id\":\"p1015\"},\"axis_label\":\"PC1\",\"major_label_policy\":{\"type\":\"object\",\"name\":\"AllLabels\",\"id\":\"p1016\"}}}],\"center\":[{\"type\":\"object\",\"name\":\"Grid\",\"id\":\"p1017\",\"attributes\":{\"axis\":{\"id\":\"p1013\"}}},{\"type\":\"object\",\"name\":\"Grid\",\"id\":\"p1022\",\"attributes\":{\"dimension\":1,\"axis\":{\"id\":\"p1018\"}}}]}}]}};\n const render_items = [{\"docid\":\"73a1dfc6-515f-48ef-98b1-6f59889e5062\",\"roots\":{\"p1001\":\"f725dce6-7696-42f2-bb79-81f286875e0b\"},\"root_ids\":[\"p1001\"]}];\n root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n }\n if (root.Bokeh !== undefined) {\n embed_document(root);\n } else {\n let attempts = 0;\n const timer = setInterval(function(root) {\n if (root.Bokeh !== undefined) {\n clearInterval(timer);\n embed_document(root);\n } else {\n attempts++;\n if (attempts > 100) {\n clearInterval(timer);\n console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n }\n }\n }, 10, root)\n }\n})(window);", + "application/vnd.bokehjs_exec.v0+json": "" + }, + "metadata": { + "application/vnd.bokehjs_exec.v0+json": { + "id": "p1001" + } + }, + "output_type": "display_data" + } + ], + "source": [ + "# plot ancestry\n", + "p = hl.plot.scatter(mt.pca_scores[0],\n", + " mt.pca_scores[1],\n", + " label=mt.superpopulation,\n", + " title='PCA', xlabel='PC1', ylabel='PC2')\n", + "show(p)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "f86783b0-340c-4e23-a7c7-fd1b47127ad5", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-01-04 21:14:29.256 Hail: INFO: downloading 1KG VCF ...\n", + " Source: https://storage.googleapis.com/hail-tutorial/1kg.vcf.bgz\n", + "2024-01-04 21:14:31.451 Hail: INFO: importing VCF and writing to matrix table...\n", + "2024-01-04 21:14:32.133 Hail: INFO: scanning VCF for sortedness...\n", + "2024-01-04 21:14:35.146 Hail: INFO: Coerced sorted VCF - no additional import work to do\n", + "2024-01-04 21:14:40.409 Hail: INFO: wrote matrix table with 10879 rows and 284 columns in 16 partitions to data/1kg.mt\n", + "2024-01-04 21:14:40.510 Hail: INFO: downloading 1KG annotations ...\n", + " Source: https://storage.googleapis.com/hail-tutorial/1kg_annotations.txt\n", + "2024-01-04 21:14:41.056 Hail: INFO: downloading Ensembl gene annotations ...\n", + " Source: https://storage.googleapis.com/hail-tutorial/ensembl_gene_annotations.txt\n", + "2024-01-04 21:14:42.298 Hail: INFO: Done!\n" + ] + } + ], + "source": [ + "hl.utils.get_1kg('data/')" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "d22599dd-007c-4fc4-b246-948ba059c680", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "mt_1kg = hl.read_matrix_table('data/1kg.mt')" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "ddd6d2ba-0609-4752-99dc-47c33b5b2d23", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-01-04 21:25:13.742 Hail: INFO: Reading table to impute column types\n", + "2024-01-04 21:25:14.248 Hail: INFO: Finished type imputation\n", + " Loading field 'Sample' as type str (imputed)\n", + " Loading field 'Population' as type str (imputed)\n", + " Loading field 'SuperPopulation' as type str (imputed)\n", + " Loading field 'isFemale' as type bool (imputed)\n", + " Loading field 'PurpleHair' as type bool (imputed)\n", + " Loading field 'CaffeineConsumption' as type int32 (imputed)\n" + ] + } + ], + "source": [ + "table_1kg = (hl.import_table('data/1kg_annotations.txt', impute=True)\n", + " .key_by('Sample'))\n", + "mt_1kg = mt_1kg.annotate_cols(pheno = table_1kg[mt_1kg.s])" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "72024335-3aa6-4039-8c00-0b3c212f8a27", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Filtering 3.60% entries out of downstream analysis.\n" + ] + } + ], + "source": [ + "# Quality control\n", + "mt_1kg = hl.sample_qc(mt_1kg)\n", + "mt_1kg = mt_1kg.filter_cols((mt_1kg.sample_qc.dp_stats.mean >= 4) & (mt_1kg.sample_qc.call_rate >= 0.97))\n", + "\n", + "ab = mt_1kg.AD[1] / hl.sum(mt_1kg.AD)\n", + "\n", + "filter_condition_ab = ((mt_1kg.GT.is_hom_ref() & (ab <= 0.1)) |\n", + " (mt_1kg.GT.is_het() & (ab >= 0.25) & (ab <= 0.75)) |\n", + " (mt_1kg.GT.is_hom_var() & (ab >= 0.9)))\n", + "\n", + "fraction_filtered = mt_1kg.aggregate_entries(hl.agg.fraction(~filter_condition_ab))\n", + "print(f'Filtering {fraction_filtered * 100:.2f}% entries out of downstream analysis.')\n", + "mt_1kg = mt_1kg.filter_entries(filter_condition_ab)\n", + "\n", + "mt_1kg = hl.variant_qc(mt_1kg)\n", + "\n", + "mt_1kg = mt_1kg.filter_rows(mt_1kg.variant_qc.AF[1] > 0.01)\n", + "mt_1kg = mt_1kg.filter_rows(mt_1kg.variant_qc.p_value_hwe > 1e-6)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c999853d-4819-4bd3-9608-2330cbebcd54", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Calculate PCA scores\n", + "eigenvalues, pcs, _ = hl.hwe_normalized_pca(mt_1kg.GT)\n", + "mt_1kg = mt_1kg.annotate_cols(scores = pcs[mt_1kg.s].scores)" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "61b2ad7e-517f-4a3b-afc5-5952c00d4952", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[Stage 221:==================================> (10 + 4) / 16]\r" + ] + }, + { + "data": { + "text/html": [ + "\n", + "
\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": "(function(root) {\n function embed_document(root) {\n const docs_json = {\"2c4bc5ec-810f-42fb-aa3c-79a3e0026f5f\":{\"version\":\"3.3.2\",\"title\":\"Bokeh Application\",\"roots\":[{\"type\":\"object\",\"name\":\"Figure\",\"id\":\"p1087\",\"attributes\":{\"width\":800,\"height\":800,\"x_range\":{\"type\":\"object\",\"name\":\"DataRange1d\",\"id\":\"p1088\"},\"y_range\":{\"type\":\"object\",\"name\":\"DataRange1d\",\"id\":\"p1089\"},\"x_scale\":{\"type\":\"object\",\"name\":\"LinearScale\",\"id\":\"p1097\"},\"y_scale\":{\"type\":\"object\",\"name\":\"LinearScale\",\"id\":\"p1098\"},\"title\":{\"type\":\"object\",\"name\":\"Title\",\"id\":\"p1090\",\"attributes\":{\"text\":\"PCA\"}},\"renderers\":[{\"type\":\"object\",\"name\":\"GlyphRenderer\",\"id\":\"p1135\",\"attributes\":{\"data_source\":{\"type\":\"object\",\"name\":\"ColumnDataSource\",\"id\":\"p1121\",\"attributes\":{\"selected\":{\"type\":\"object\",\"name\":\"Selection\",\"id\":\"p1122\",\"attributes\":{\"indices\":[],\"line_indices\":[]}},\"selection_policy\":{\"type\":\"object\",\"name\":\"UnionRenderers\",\"id\":\"p1123\"},\"data\":{\"type\":\"map\",\"entries\":[[\"index\",{\"type\":\"ndarray\",\"array\":{\"type\":\"bytes\",\"data\":\"AAAAAAEAAAACAAAAAwAAAAQAAAAFAAAABgAAAAcAAAAIAAAACQAAAAoAAAALAAAADAAAAA0AAAAOAAAADwAAABAAAAARAAAAEgAAABMAAAAUAAAAFQAAABYAAAAXAAAAGAAAABkAAAAaAAAAGwAAABwAAAAdAAAAHgAAAB8AAAAgAAAAIQAAACIAAAAjAAAAJAAAACUAAAAmAAAAJwAAACgAAAApAAAAKgAAACsAAAAsAAAALQAAAC4AAAAvAAAAMAAAADEAAAAyAAAAMwAAADQAAAA1AAAANgAAADcAAAA4AAAAOQAAADoAAAA7AAAAPAAAAD0AAAA+AAAAPwAAAEAAAABBAAAAQgAAAEMAAABEAAAARQAAAEYAAABHAAAASAAAAEkAAABKAAAASwAAAEwAAABNAAAATgAAAE8AAABQAAAAUQAAAFIAAABTAAAAVAAAAFUAAABWAAAAVwAAAFgAAABZAAAAWgAAAFsAAABcAAAAXQAAAF4AAABfAAAAYAAAAGEAAABiAAAAYwAAAGQAAABlAAAAZgAAAGcAAABoAAAAaQAAAGoAAABrAAAAbAAAAG0AAABuAAAAbwAAAHAAAABxAAAAcgAAAHMAAAB0AAAAdQAAAHYAAAB3AAAAeAAAAHkAAAB6AAAAewAAAHwAAAB9AAAAfgAAAH8AAACAAAAAgQAAAIIAAACDAAAAhAAAAIUAAACGAAAAhwAAAIgAAACJAAAAigAAAIsAAACMAAAAjQAAAI4AAACPAAAAkAAAAJEAAACSAAAAkwAAAJQAAACVAAAAlgAAAJcAAACYAAAAmQAAAJoAAACbAAAAnAAAAJ0AAACeAAAAnwAAAKAAAAChAAAAogAAAKMAAACkAAAApQAAAKYAAACnAAAAqAAAAKkAAACqAAAAqwAAAKwAAACtAAAArgAAAK8AAACwAAAAsQAAALIAAACzAAAAtAAAALUAAAC2AAAAtwAAALgAAAC5AAAAugAAALsAAAC8AAAAvQAAAL4AAAC/AAAAwAAAAMEAAADCAAAAwwAAAMQAAADFAAAAxgAAAMcAAADIAAAAyQAAAMoAAADLAAAAzAAAAM0AAADOAAAAzwAAANAAAADRAAAA0gAAANMAAADUAAAA1QAAANYAAADXAAAA2AAAANkAAADaAAAA2wAAANwAAADdAAAA3gAAAN8AAADgAAAA4QAAAOIAAADjAAAA5AAAAOUAAADmAAAA5wAAAOgAAADpAAAA6gAAAOsAAADsAAAA7QAAAO4AAADvAAAA8AAAAPEAAADyAAAA8wAAAPQAAAD1AAAA9gAAAPcAAAD4AAAA\"},\"shape\":[249],\"dtype\":\"int32\",\"order\":\"little\"}],[\"x\",{\"type\":\"ndarray\",\"array\":{\"type\":\"bytes\",\"data\":\"h4HgrWMY0L/KAy8D+fTPv7PtPLjOOs+/8GR6Muv2zr+riJ13YenOvw6wsN0y5c6/PANUDvTwzr84GfKVvfjOv8d+o6q/qs6/ObqiJjCnzr9JipmpNpPOv7uBQex5X86/7vmIMR9ozr/5WRR3d3bOv6PKCUmXbc6/CazYbJQ9zr/z2yn+TTfOv7vp3RGVOs6/JhbPIVn8zb9Z+OvPI/TNv820IsS9zM2/+/9/nBLDzb+H1AuX4bnNv7lfoTnJtM2/gakxveuNzb/2Yx4ZIarNv+Eh1q9Dq82/AFnBHZWEzb8FHAJL1YrNv3Yzyxa5Us2/QvfeZ1Efzb/anUh2u0PNv1qFvuRYD82/zZYPN0Edzb+d7/hQCtLMv5VtR7yg7cy/VKdn2sGfzL9pPOhjXKTMvzoqlhwojcy/W/llGsKXzL/FFPnHBjPMvxwliyR3U8y/Q0OcgSUQzL8uFIlxah7Mv0bt9uPK/su/ZmVRSAfry7/EMpW/AfTLv72eW6fLzcu/cySQom6hy78jKthYTZjLvzS24b6Hn8u/JClpVslty7/nNlaYJHTLv7g1pbWWbMu/Pw3MHQg9y790nZkZ2APLv2eSi6h//sq/Gn4VlGfwyr/WHdu+L93Jv78q7yMogcm/ekxp2rD1yL9f+289P+PIv9ycAs/PtMa/QbhuTTkSxr8SKwglXWDFv+IDQrjm6sS/22zvVk/HxL+JA5IP1FzEvxIGLr3fx8O/RJ6Z02hkw78l4rt1kyzDv6r5huhNS8O/x9qSJ+o0w7+q/AN+zujCvxpLxapF88K/txOtJ67Twr/NqenOEXvCv8lY4y5ZScK/VO0vX4dfwr+ua9Bg0BHCvzCwLK8K7sG/8EtNbEeBwb/Uq8C8N4jBv+BV25iMTsG/epbdLpBfwb90RDbsE0DBv7vWwzAdOcG/gpTxL80Jwb8X3qwuijHBv0vcXMcA+cC/J0mWcZHtwL+udOoYM+vAv8qayzUX48C/ekn8GvbBwL/OHB1QG73Av4xyTnZMqcC/r1Y7gNDUwL9EZ+NHrbrAv5IUJ3940MC/vFQkt2SzwL+qkR1rHITAv+K4Tzy7oMC/bmRiweSOwL+pQJpptZHAv+hcWFW9ccC/1CnP9mxJwL9m2+gGQWHAv+3KHoPlX8C/vsm4azYuwL8Asy24oznAv3FfpVpmK8C/MYcsgrAowL+o3KJFgzPAv0/Z0mkMFcC/N0On6K0RwL8sk51AgxHAv6MmJs/SAsC/0RM8613Yv7/GOHKVYLW/v0yy2l5rsL+/LC8id89ov7+aIzyFO2W/v12sP1CCeb+/AfpFLo1Yv7+WK/R6/Su/v5Hx57BqOb+/WzDD1ilfv7/FhfTFzqW+v75tGWvaq76/Json9nPAvr+jvPnHeFW+v1id4jk9Sr6/AsOdKOhOvr9vR2cp0tq9v6m17YbwI76/9XcByDcbvr+Z9oD1zjO+vxf0CpRlNb6/uOx6bcEnvr85ZpsV77K9vz1QkKwUmb2/aoNCuKjTvb+H0MyVj869vxjguEpAqL2/uMLqdwCivb+FPVRPki29v7ttTJ0zWL2/EdiVGVD+vL8P+ZJBnLq8v4hHKVscD72/tdj8tEGLvL+jYP7Gp2y8v+TF5iTc1Lu/MBB13TDPu788DoPgHVi7v+2iTrLKhru/sWMcKfZAu79Z77hosdS6v4+kPMQQ2bq/ojdei9cVu78ccGvGBw67v+Rt0zyaurq/Jq8349z4ub/aVFwH6X25v5ATeEAP/ri/Uv2oEWizuL/fIpvpbIK4v2DnsJpv0bi/6X8A3gVLuL8BhN3jqPy3vz+Eza0xfbe/LLBPMzqdt79l5MEnqZ+3vwGuQhiSSLe/r4IdbBO/tr9ehCn2QS+2v6V0PTdZzrW/dvW8m4Ijtr88UkKocTa0v/bViBH+D7O/9erqzPJjsb96GlN7/Ryov7ipLS3d66W/T0wZeXjQkr+uLsZ3ytvOP56hwBW4cNE/Btq3DBYQ0j+r6g+N5q/VPzaEjcDQzNY/xqHQ6HHV1z8t+XrAd2vYP03dBOztRdo/DKkxVCmW2j83YUltjcPaP7WqN0ruGts/c1MThkJm2z8hbcQBLmPbP9xzMGkIANw/VoifdFwv3D/VngLZ40vcP3aL0cgsPtw/x0U6FU1e3D+YxAk4aHncP2tI/PrGj9w/gq/QXsqU3D81F6EDlZvcP8CLNUhjvdw/P9bUDNvW3D9I+dugge7cP7pKyDRD/Nw/GyK+CL8L3T8ae8shVgDdP5bazZ7NC90/zcqHtoE93T8trIVEEFLdP4IsKMRmc90/Nih29EN13T8teBkCuHTdP4381JZ6hN0/yTNF69Wi3T/jFDbb6bvdP0gS/CeBsd0/SugQHPK43T9QevsBlK7dP8Fiy1M9w90/NSm7TSns3T//s1Os7PHdP9uL5PeM8N0/UHS3Janv3T9rJCE8rPrdPynXt8KoCN4/jQUFlqD/3T9ImqmabPzdPyp4slqv/N0/3Crk3VdL3j+dA9pO51TePyRMk1SxY94/6Nzpr5xa3j8PJEoYz5PeP6ZQFA2Mpt4/lagfr2XF3j/35+FT0cveP/aIOLaf094/MwrgSizu3j+lfXuC9wjfP8htghcRB98//15qSCM/3z8bLkllW1ffPzsMDG1M3t8/\"},\"shape\":[249],\"dtype\":\"float64\",\"order\":\"little\"}],[\"y\",{\"type\":\"ndarray\",\"array\":{\"type\":\"bytes\",\"data\":\"d7LZF8r40T9HJ5QTSLPSP/WDK6pT8tE/UPj8zADp0T8c70YGWFzSP2RNklOTv9I/+2/DjxsE0z9Iualfi6fTP73CXqbrrdI/gOlsih4d0j9X26AvOCfSP1O0dFEZRdE/JedfGTa10T8Y+63gi3fTP9rlXW/ujNQ/Cmrw6wtK0j8LCbfjztDSPzuq8jhzUtM/r2WJk1GV0T8mKSyehCXSP6q+iBAr99A/4KaqdFnJ0T/6XbqJ3vTRP2atlnpAJdI/rh/ZW15N0T8qC5qVeGfRP/bcbXCBgdE/WWSz7Jr40j81Ep3XmRPTP8T5f2yepdI/vxr+3kX00D+u1yy1533RP+P/j4F+EtM/fzFnY4N30z9gxsiqdCfSP6fbJyvR5dI/PnPLVL090T+zcrVI6/7RP+pE2b/ettI/Y2HhCg3y0j9SVqDCnw3SP5280Lz6B9M/f8TWjcE00j9+MhbXyVPSP4NNhnlJqtM/kT9oUgyEsT+nCGBYUZbNP6zC+kca/dE/XnqH665i0T/DB/q01KXSP1tRTo8wk9Q/CQQk2eYq0j9J/73FFvLSP1C5MGKzSdM/2LAIHQ/T0j/+D7r2c/XRP/IFenXxdNE/8tMPCofp0j/DBBngJV7TPyKaW2iRS9E/3+9XDtourT+u01KBWGjTPwm7xqzd35E/+OY1HZcjrb+0BmlQftarv08NYg0+y7O/1G8Xb2yRur81Of7T+fuyv0xBPFpQzau/nEqX1KjTvb+sxLj4yFe7v+U1+xVQAbi/59yoikFgsL9iUcNeAFq3v7xqxIbeQra/FBsn2CgDpL9N8gEkmHuiv9a4njSIfLu/9mJQAPP7q79NN4E3il3Av/25AbBAcsK/JAseQuQXx7/2ftsvmwG5v2zHi15xQ9C/jfA5FdO0v79LrBLBFUixvzokx2MgL9O/5UyzfX+K0b+oSONr6Q7Gv0nQ2rEX+cy/pR/Sv5S3wL+Ww5wc/Oi7v/pIB8jrVLu/TMtONmWs07+iK/Tl19bHv9chFXCyhMW/9wQiriXwvr/FXQOu0pOwv3RNmZTX1Ka/jd5x8CuIpb/Af79sYBLUv8y1tdQac8S/tuMS2hEKvr+en9qAfIy7v9LFI6MJKM+/PB3vIF3ewb9oq1jPrhnAv02WAklc97q/NbQ2yKYEz79vTRJ8CXjFvyt4sVVNTb2/RVJmDsrcvL+cSiS1MG+7v7b856Moube/qJYwfrDf0r9rG2fUumvSv2nq5Ib6lbq/s45g2VYok78zmwSv3cTSv/Wf/L3I49G/50e+7Mvmv78qfDZQML29v56UuiVqZLa/cyIazDb10b8uMXNRSezGvxRaMuxmsrq/ZPLE9lVYtL/6mO2nMELUv9SOnEWJWsC/2hA4A+yspb+aP3KQg17Rv1YT0vPFSLq/hfBNadQgq79BbEAR+vzTv7Hq64IeY9K/O9yu+yJJ0b8meaG41YC9v91wDBcW9ry/bPEV0ME3sr9xlwazGAHNvz/WVagiBMy/3lkhACNHxr9687Zy3DbEv9dpG6KZyKK/xaTJWrbJjb9NqkcUVoXSvxArM02EdNK/jhtyZ9Vx1L8BGH//HLjTv86T9NFdZbm/ZF74rz/jwb9sqPcRLGa8vz+AjorWptK/Ez8SJIfa0b+r+ojDwJ3TvzQHPta+IdO/np7VVw5Q0r8HF+qWBhvUvx9ZzE4RZ9K/33m/NDlE0r9GVu84Hsa+vxtk8T7596A/Gov2yW3U0r9AOj4uStzRv211dwkOWNG/spk+a8Ws0r9KVfR7lZDQv+/TigIV482/o/wYVKM1079Odke4+bfRvzpbgPvQotK/DmC9QUcSzr8eCWVWZN+/v8BzRasTh8e/b9xctQoMx78igE0+Q6vSv9GgZUnIkdK/Rj/TX4Id0b/tBhNO32rSv4AsoJPRAs6/a/fX3+y6yr/dUYk9/rHKvwjsW/4qhsC/2nMgDjGjw7+3rD2sN4mkv+SNYkAdh5G/sauOFI7sn7/mLwolJSR/P3lj5q/CQ0a/o1kYsHenjD+bw1gswaGQP34qYERkCK0/tDGMCuEylz9SxjgXAfSmP1XxjviwDbA/KPA11Y/umz/xD+2SpgimP7nazXh1Cqo/10O9GtpmqT9cjAwQ2HytP93vmkLxlq8/4dCkolhMqD+TZzCRPX6wP88eBgdqb7A/SXMz5RNfsT9JOZedenShP0brwN9FmbE/4HD5uZMfpD/hb+kkGqeeP5Dc8fuCSrA/BQW6fGcFqj/BzhWHjaOqP3SlT4FXIbE/H8X+UI88rD++A8WJnPiyP2ReHxAOwKc/r0z3fOBdqD8cQipemVi1P/M+hpgAPrA/B6byVa9ArD+NIihlIuGoP8ryss0OFK4/400j8VCtsD/iJPz3E0exPxTfbkSqsK8/Zt7JMTqKrT/W48uW6QWzPwoE0iEMW7M/iDiIAiD0uT8LmDSRUwGuP7dmt7J+q64/Z2/2N8mgsj+keN1p1LG0P2W51Z898bg/xknyU5yNrD8I1NEilQewP3n1eFLaSbE/hwHHAJ0ctT9OfGGsd2qvPxohF3PHk6s/naA1zS9PqD+aIw/YTXKqP9UO9u2ST7E/UhcyH2y2sj8BRYOKRXKpPzplKCR6BrA/LV8s+Wo/sT8aIxkUq22zP6XqGNGrnLI/\"},\"shape\":[249],\"dtype\":\"float64\",\"order\":\"little\"}],[\"label\",{\"type\":\"ndarray\",\"array\":[\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"AMR\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"AMR\",\"EAS\",\"AMR\",\"AMR\",\"AMR\",\"SAS\",\"SAS\",\"SAS\",\"SAS\",\"SAS\",\"SAS\",\"SAS\",\"SAS\",\"SAS\",\"AMR\",\"SAS\",\"SAS\",\"SAS\",\"SAS\",\"SAS\",\"SAS\",\"SAS\",\"SAS\",\"EUR\",\"SAS\",\"SAS\",\"EUR\",\"EUR\",\"AMR\",\"AMR\",\"SAS\",\"SAS\",\"SAS\",\"EUR\",\"SAS\",\"SAS\",\"SAS\",\"AMR\",\"SAS\",\"SAS\",\"EUR\",\"SAS\",\"SAS\",\"SAS\",\"EUR\",\"SAS\",\"SAS\",\"SAS\",\"EUR\",\"SAS\",\"SAS\",\"SAS\",\"SAS\",\"AMR\",\"EUR\",\"EUR\",\"SAS\",\"AMR\",\"EUR\",\"EUR\",\"SAS\",\"SAS\",\"SAS\",\"EUR\",\"SAS\",\"SAS\",\"SAS\",\"EUR\",\"SAS\",\"SAS\",\"EUR\",\"AMR\",\"AMR\",\"EUR\",\"EUR\",\"EUR\",\"SAS\",\"SAS\",\"AMR\",\"AMR\",\"AMR\",\"SAS\",\"SAS\",\"AMR\",\"AMR\",\"EUR\",\"EUR\",\"EUR\",\"EUR\",\"SAS\",\"SAS\",\"SAS\",\"EUR\",\"EUR\",\"EUR\",\"EUR\",\"EUR\",\"EUR\",\"EUR\",\"EUR\",\"SAS\",\"AMR\",\"EUR\",\"EUR\",\"EUR\",\"EUR\",\"AMR\",\"AMR\",\"EUR\",\"EUR\",\"EUR\",\"AMR\",\"AMR\",\"AMR\",\"AMR\",\"EUR\",\"EUR\",\"AMR\",\"EUR\",\"AMR\",\"AMR\",\"AMR\",\"AMR\",\"AMR\",\"AFR\",\"AMR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\"],\"shape\":[249],\"dtype\":\"object\",\"order\":\"little\"}]]}}},\"view\":{\"type\":\"object\",\"name\":\"CDSView\",\"id\":\"p1127\",\"attributes\":{\"filter\":{\"type\":\"object\",\"name\":\"IntersectionFilter\",\"id\":\"p1126\",\"attributes\":{\"operands\":[{\"type\":\"object\",\"name\":\"GroupFilter\",\"id\":\"p1125\",\"attributes\":{\"column_name\":\"label\",\"group\":\"AFR\"}}]}}}},\"glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1132\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"type\":\"object\",\"name\":\"CategoricalColorMapper\",\"id\":\"p1124\",\"attributes\":{\"palette\":[\"#1f77b4\",\"#ff7f0e\",\"#2ca02c\",\"#d62728\",\"#9467bd\",\"#8c564b\",\"#e377c2\",\"#7f7f7f\",\"#bcbd22\",\"#17becf\"],\"factors\":[\"EAS\",\"AMR\",\"EUR\",\"SAS\",\"AFR\"]}}},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}}}},\"nonselection_glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1133\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"line_alpha\":{\"type\":\"value\",\"value\":0.1},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"fill_alpha\":{\"type\":\"value\",\"value\":0.1},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.1}}},\"muted_glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1134\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"line_alpha\":{\"type\":\"value\",\"value\":0.2},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"fill_alpha\":{\"type\":\"value\",\"value\":0.2},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.2}}}}},{\"type\":\"object\",\"name\":\"GlyphRenderer\",\"id\":\"p1148\",\"attributes\":{\"data_source\":{\"id\":\"p1121\"},\"view\":{\"type\":\"object\",\"name\":\"CDSView\",\"id\":\"p1140\",\"attributes\":{\"filter\":{\"type\":\"object\",\"name\":\"IntersectionFilter\",\"id\":\"p1139\",\"attributes\":{\"operands\":[{\"type\":\"object\",\"name\":\"GroupFilter\",\"id\":\"p1138\",\"attributes\":{\"column_name\":\"label\",\"group\":\"AMR\"}}]}}}},\"glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1145\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}}}},\"nonselection_glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1146\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"line_alpha\":{\"type\":\"value\",\"value\":0.1},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"fill_alpha\":{\"type\":\"value\",\"value\":0.1},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.1}}},\"muted_glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1147\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"line_alpha\":{\"type\":\"value\",\"value\":0.2},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"fill_alpha\":{\"type\":\"value\",\"value\":0.2},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.2}}}}},{\"type\":\"object\",\"name\":\"GlyphRenderer\",\"id\":\"p1161\",\"attributes\":{\"data_source\":{\"id\":\"p1121\"},\"view\":{\"type\":\"object\",\"name\":\"CDSView\",\"id\":\"p1153\",\"attributes\":{\"filter\":{\"type\":\"object\",\"name\":\"IntersectionFilter\",\"id\":\"p1152\",\"attributes\":{\"operands\":[{\"type\":\"object\",\"name\":\"GroupFilter\",\"id\":\"p1151\",\"attributes\":{\"column_name\":\"label\",\"group\":\"EAS\"}}]}}}},\"glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1158\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}}}},\"nonselection_glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1159\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"line_alpha\":{\"type\":\"value\",\"value\":0.1},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"fill_alpha\":{\"type\":\"value\",\"value\":0.1},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.1}}},\"muted_glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1160\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"line_alpha\":{\"type\":\"value\",\"value\":0.2},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"fill_alpha\":{\"type\":\"value\",\"value\":0.2},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.2}}}}},{\"type\":\"object\",\"name\":\"GlyphRenderer\",\"id\":\"p1174\",\"attributes\":{\"data_source\":{\"id\":\"p1121\"},\"view\":{\"type\":\"object\",\"name\":\"CDSView\",\"id\":\"p1166\",\"attributes\":{\"filter\":{\"type\":\"object\",\"name\":\"IntersectionFilter\",\"id\":\"p1165\",\"attributes\":{\"operands\":[{\"type\":\"object\",\"name\":\"GroupFilter\",\"id\":\"p1164\",\"attributes\":{\"column_name\":\"label\",\"group\":\"EUR\"}}]}}}},\"glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1171\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}}}},\"nonselection_glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1172\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"line_alpha\":{\"type\":\"value\",\"value\":0.1},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"fill_alpha\":{\"type\":\"value\",\"value\":0.1},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.1}}},\"muted_glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1173\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"line_alpha\":{\"type\":\"value\",\"value\":0.2},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"fill_alpha\":{\"type\":\"value\",\"value\":0.2},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.2}}}}},{\"type\":\"object\",\"name\":\"GlyphRenderer\",\"id\":\"p1187\",\"attributes\":{\"data_source\":{\"id\":\"p1121\"},\"view\":{\"type\":\"object\",\"name\":\"CDSView\",\"id\":\"p1179\",\"attributes\":{\"filter\":{\"type\":\"object\",\"name\":\"IntersectionFilter\",\"id\":\"p1178\",\"attributes\":{\"operands\":[{\"type\":\"object\",\"name\":\"GroupFilter\",\"id\":\"p1177\",\"attributes\":{\"column_name\":\"label\",\"group\":\"SAS\"}}]}}}},\"glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1184\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}}}},\"nonselection_glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1185\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"line_alpha\":{\"type\":\"value\",\"value\":0.1},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"fill_alpha\":{\"type\":\"value\",\"value\":0.1},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.1}}},\"muted_glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1186\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"line_alpha\":{\"type\":\"value\",\"value\":0.2},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"fill_alpha\":{\"type\":\"value\",\"value\":0.2},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.2}}}}}],\"toolbar\":{\"type\":\"object\",\"name\":\"Toolbar\",\"id\":\"p1096\",\"attributes\":{\"tools\":[{\"type\":\"object\",\"name\":\"PanTool\",\"id\":\"p1109\"},{\"type\":\"object\",\"name\":\"WheelZoomTool\",\"id\":\"p1110\",\"attributes\":{\"renderers\":\"auto\"}},{\"type\":\"object\",\"name\":\"BoxZoomTool\",\"id\":\"p1111\",\"attributes\":{\"overlay\":{\"type\":\"object\",\"name\":\"BoxAnnotation\",\"id\":\"p1112\",\"attributes\":{\"syncable\":false,\"level\":\"overlay\",\"visible\":false,\"left\":{\"type\":\"number\",\"value\":\"nan\"},\"right\":{\"type\":\"number\",\"value\":\"nan\"},\"top\":{\"type\":\"number\",\"value\":\"nan\"},\"bottom\":{\"type\":\"number\",\"value\":\"nan\"},\"left_units\":\"canvas\",\"right_units\":\"canvas\",\"top_units\":\"canvas\",\"bottom_units\":\"canvas\",\"line_color\":\"black\",\"line_alpha\":1.0,\"line_width\":2,\"line_dash\":[4,4],\"fill_color\":\"lightgrey\",\"fill_alpha\":0.5}}}},{\"type\":\"object\",\"name\":\"SaveTool\",\"id\":\"p1117\"},{\"type\":\"object\",\"name\":\"ResetTool\",\"id\":\"p1118\"},{\"type\":\"object\",\"name\":\"HelpTool\",\"id\":\"p1119\"},{\"type\":\"object\",\"name\":\"HoverTool\",\"id\":\"p1120\",\"attributes\":{\"renderers\":\"auto\",\"tooltips\":[[\"x\",\"@x\"],[\"y\",\"@y\"]]}}]}},\"left\":[{\"type\":\"object\",\"name\":\"LinearAxis\",\"id\":\"p1104\",\"attributes\":{\"ticker\":{\"type\":\"object\",\"name\":\"BasicTicker\",\"id\":\"p1105\",\"attributes\":{\"mantissas\":[1,2,5]}},\"formatter\":{\"type\":\"object\",\"name\":\"BasicTickFormatter\",\"id\":\"p1106\"},\"axis_label\":\"PC2\",\"major_label_policy\":{\"type\":\"object\",\"name\":\"AllLabels\",\"id\":\"p1107\"}}},{\"type\":\"object\",\"name\":\"Legend\",\"id\":\"p1195\",\"attributes\":{\"click_policy\":\"hide\",\"items\":[{\"type\":\"object\",\"name\":\"LegendItem\",\"id\":\"p1190\",\"attributes\":{\"label\":{\"type\":\"value\",\"value\":\"AFR\"},\"renderers\":[{\"id\":\"p1135\"}]}},{\"type\":\"object\",\"name\":\"LegendItem\",\"id\":\"p1191\",\"attributes\":{\"label\":{\"type\":\"value\",\"value\":\"AMR\"},\"renderers\":[{\"id\":\"p1148\"}]}},{\"type\":\"object\",\"name\":\"LegendItem\",\"id\":\"p1192\",\"attributes\":{\"label\":{\"type\":\"value\",\"value\":\"EAS\"},\"renderers\":[{\"id\":\"p1161\"}]}},{\"type\":\"object\",\"name\":\"LegendItem\",\"id\":\"p1193\",\"attributes\":{\"label\":{\"type\":\"value\",\"value\":\"EUR\"},\"renderers\":[{\"id\":\"p1174\"}]}},{\"type\":\"object\",\"name\":\"LegendItem\",\"id\":\"p1194\",\"attributes\":{\"label\":{\"type\":\"value\",\"value\":\"SAS\"},\"renderers\":[{\"id\":\"p1187\"}]}}]}},{\"type\":\"object\",\"name\":\"ColorBar\",\"id\":\"p1196\",\"attributes\":{\"visible\":false,\"major_label_policy\":{\"type\":\"object\",\"name\":\"NoOverlap\",\"id\":\"p1197\"},\"color_mapper\":{\"id\":\"p1124\"}}}],\"below\":[{\"type\":\"object\",\"name\":\"LinearAxis\",\"id\":\"p1099\",\"attributes\":{\"ticker\":{\"type\":\"object\",\"name\":\"BasicTicker\",\"id\":\"p1100\",\"attributes\":{\"mantissas\":[1,2,5]}},\"formatter\":{\"type\":\"object\",\"name\":\"BasicTickFormatter\",\"id\":\"p1101\"},\"axis_label\":\"PC1\",\"major_label_policy\":{\"type\":\"object\",\"name\":\"AllLabels\",\"id\":\"p1102\"}}}],\"center\":[{\"type\":\"object\",\"name\":\"Grid\",\"id\":\"p1103\",\"attributes\":{\"axis\":{\"id\":\"p1099\"}}},{\"type\":\"object\",\"name\":\"Grid\",\"id\":\"p1108\",\"attributes\":{\"dimension\":1,\"axis\":{\"id\":\"p1104\"}}}]}}]}};\n const render_items = [{\"docid\":\"2c4bc5ec-810f-42fb-aa3c-79a3e0026f5f\",\"roots\":{\"p1087\":\"e436bca5-7b35-4012-bf85-0ce527df74ae\"},\"root_ids\":[\"p1087\"]}];\n root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n }\n if (root.Bokeh !== undefined) {\n embed_document(root);\n } else {\n let attempts = 0;\n const timer = setInterval(function(root) {\n if (root.Bokeh !== undefined) {\n clearInterval(timer);\n embed_document(root);\n } else {\n attempts++;\n if (attempts > 100) {\n clearInterval(timer);\n console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n }\n }\n }, 10, root)\n }\n})(window);", + "application/vnd.bokehjs_exec.v0+json": "" + }, + "metadata": { + "application/vnd.bokehjs_exec.v0+json": { + "id": "p1087" + } + }, + "output_type": "display_data" + } + ], + "source": [ + "p = hl.plot.scatter(mt_1kg.scores[0],\n", + " mt_1kg.scores[1],\n", + " label=mt_1kg.pheno.SuperPopulation,\n", + " title='PCA', xlabel='PC1', ylabel='PC2')\n", + "show(p)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [conda env:python310]", + "language": "python", + "name": "conda-env-python310-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/onboarding_documentation/technical_documentation/analyst_tutorial/analyst_tutorial.md b/onboarding_documentation/technical_documentation/analyst_tutorial/analyst_tutorial.md new file mode 100644 index 0000000..2400162 --- /dev/null +++ b/onboarding_documentation/technical_documentation/analyst_tutorial/analyst_tutorial.md @@ -0,0 +1,122 @@ +As an analyst you will be running analysis jobs via `analysis-runner`, producing output, and then analysing the output. This tutorial will walk you through the process of running an analysis job, and then analysing the output. + +In this tutorial, we will begin with raw fastq files and process them using the large-cohort pipeline. This pipeline performs several steps: + +1. Aligns the reads to the reference genome. +2. Conducts quality control checks. +3. Calls variants and combines these variant calls. +4. Performs an ancestry analysis. +5. Plots the results of the ancestry analysis. + +While the large-cohort pipeline includes additional stages, for the purpose of this tutorial, we will only execute up to and including the Ancestry stage, which carries out Principal Component Analysis (PCA). + +For a description of what each stage of the `large-cohort` pipeline does, please see the `large-cohort` pipeline documentation [here](https://github.com/populationgenomics/production-pipelines#large-cohort-workflow) + +## Table of Contents +- [1. Getting test data](#getting-test-data) +- [2. Writing a config file](#writing-a-config-file) +- [3. Submitting an analysis job](#submitting-an-analysis-job) +- [4. Analysing the output](#analysing-the-output) + +## 1. Getting test data +As an analyst it is vitally important we run our analysis jobs on test data before running them on real data. This is to ensure that the analysis job is working as expected, and that the output is as expected. This is also important to ensure that the analysis job is not going to take too long (and therefore cost too much!) to run on real data. As pipelines are constantly being updated, it is also important to run the analysis job on test data to ensure that the pipeline has not been broken by a recent update. + +For this tutorial we will be using the `fewgenomes` dataset, which is a collection of genomes from different sources. First things first, let's check if you have access to the `fewgenomes` dataset. To do this, run the following command: + +```bash +gsutil ls gs://cpg-fewgenomes-main/ +``` + +The output of the above should list a number of files ending in `.fastq.gz`. These are read files, which are the input to the pipeline. If you do not see these files, then you do not have access to the `fewgenomes` dataset, please contact your manager. + +### Subsetting the bucket +Our aim is to validate the pipeline's functionality and output by running it on a subset of the fewgenomes dataset. To do this we will use the script [create_test_subset.py](https://github.com/populationgenomics/metamist/blob/f6c226d08a8ee9875014d8c99cfe119742221efb/scripts/create_test_subset.py) in the metamist repository. This script selects a random set of samples from `fewgenomes` and copies their read files to the `fewgenomes-test` bucket. Execute the following command after navigating to the main branch of your metamist repository in the CLI. + +To run this script we must first be in the `metamist` repository. So navigate to your `metamist` repository, ensure you're on the `main` branch of `metamist` and then run the following command from CLI: + +**Note: We will need to set up more genomes in the actual bucket we're using** +```bash +analysis-runner \ +--dataset fewgenomes --description "populate fewgenomes test subset" --output-dir "fewgenomes-test" \ +--access-level full \ +scripts/create_test_subset.py --project fewgenomes --samples XPG280371 XPG280389 XPG280397 XPG280405 XPG280413 --skip-ped +``` +**FOR REFERENCE: The above was taken from [this](https://centrepopgen.slack.com/archives/C03FA2M1MR9/p1700020527448029?thread_ts=1699935103.776929&cid=C03FA2M1MR9) Slack thread** + + +## 2. Writing a config file + +Config files are used to specify the parameters of an analysis job. They are written in TOML format. There is already a great description of config files and how they're used in the `team-docs` repo [here](https://github.com/populationgenomics/team-docs/blob/13755bd51356b50ce11e6be78a76e53ed0a3ccb1/cpg_utils_config.md). + +In short; config files contain all the parameters needed to run the job we want such as images, references, specific stages of the pipeline to run, and the input data. They are required for the `analysis-runner` to start. + +When running stages in the `large cohort` pipeline there are default config files that are automatically chosen (see the default [here](https://github.com/populationgenomics/production-pipelines/blob/0bcf9775206f10ee91ac197c8c178f844ecad447/cpg_workflows/defaults.toml)) with subsequent user-defined parameters overriding these defaults as necessary. The default config file for the `large cohort` pipeline can be found [here](https://github.com/populationgenomics/production-pipelines/blob/main/configs/defaults/large_cohort.toml) and in it you can see a description of some of the parameters and what they're used for. + +#### Task: Write a config file +- If we are wanting to run the `large cohort` pipeline on the `fewgenomes-test` dataset, we will need to create a config file that is capable of doing this. +- Have a go at writing your own config file capable of running the `large cohort` pipeline on `fewgenomes-test` up until the `Combiner` stage. +- You can use the default config file as a starting point, and then override the necessary parameters to run the pipeline on `fewgenomes-test`. +- *Hint*: You will **not** need to change anything in the default `large cohort` config file. + +
+Click to reveal the config file + +```TOML +[workflow] +input_datasets = ['fewgenomes-test'] +sequencing_type = 'genome' +output_version = '1.0' # Do we want them to specify output_version? +only_sgs = [] # to be used to demonstrate how to run on a subset of samples +``` + +
+ +Once the config file has been written, save it with an appropriate name (e.g. `fewgenomes-test.toml`) in a directory that makes sense on your local machine. In the following step we will be using `analysis-runner` to submit the job and point it to the config file we have just written. + +## 3. Submitting an analysis job +Now that we have a config file, we can submit an analysis job. To do this we will use the `analysis-runner` tool. The `analysis-runner` tool is a command line tool that is used to submit analysis jobs to the cloud. + +Because we are running the `large cohort` pipeline we will need to be inside the `Production-pipelines` repository. So in a directory you want to keep `Production-pipelines` in, run the following command: + +```bash +git clone --recurse-submodules git@github.com:populationgenomics/production-pipelines.git +cd production-pipelines +``` + +Ensure you are on the `main` branch. + +For guidance on how to use the `analysis-runner` tool, please see the `analysis-runner` documentation [here](https://github.com/populationgenomics/analysis-runner). It will need to be `pip` installed before use. + +It is generally good practice to create a bash script that contains the `analysis-runner` command to submit the job. This is so that the command can be easily re-run if necessary. + +#### Task: Submit an analysis job +- Using the instructions in the `analysis-runner` repo linked above as well as the config file you have just written, write the command to submit analysis job to the cloud using the `analysis-runner` tool. + +
+Click to reveal the command + +```bash +analysis-runner \ +--dataset fewgenomes-test \ +--description "Running large cohort up to Combiner stage on fewgenomes" \ +--output-dir "fewgenomes-test-YOUR-NAME" \ +--access-level test \ +--config configs/genome.toml \ +--config configs/defaults/large_cohort.toml \ +--config path/to/you/config/config_filename.toml \ +--image australia-southeast1-docker.pkg.dev/cpg-common/images/cpg_workflows:latest \ +main.py large_cohort +``` +
+ + + +## 4. Analysing the output + +Now that we have some output to work with, lets begin analysing. + +Analysis is best done within a Jupyter Notebook. However, in order to interact with our data in the cloud using Hail and jupyter notebooks we need to do some set up first. + +Instructions on setting up a Jupyter Notebook in the cloud can be found [here](https://github.com/populationgenomics/team-docs/blob/main/notebooks.md) + +Please continue this tutorial once you have a Jupyter Notebook running in the cloud, good luck! From 355f02121a3862df4baaded8f09121e6637a8b47 Mon Sep 17 00:00:00 2001 From: Michael Harper Date: Fri, 5 Jan 2024 13:06:16 +1100 Subject: [PATCH 2/8] changed from fewgenomes to bioheart in markdown for consistency until there is an onboarding dataset --- .../analyst_tutorial/analyst_tutorial.md | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/onboarding_documentation/technical_documentation/analyst_tutorial/analyst_tutorial.md b/onboarding_documentation/technical_documentation/analyst_tutorial/analyst_tutorial.md index 2400162..cee9c83 100644 --- a/onboarding_documentation/technical_documentation/analyst_tutorial/analyst_tutorial.md +++ b/onboarding_documentation/technical_documentation/analyst_tutorial/analyst_tutorial.md @@ -21,25 +21,25 @@ For a description of what each stage of the `large-cohort` pipeline does, please ## 1. Getting test data As an analyst it is vitally important we run our analysis jobs on test data before running them on real data. This is to ensure that the analysis job is working as expected, and that the output is as expected. This is also important to ensure that the analysis job is not going to take too long (and therefore cost too much!) to run on real data. As pipelines are constantly being updated, it is also important to run the analysis job on test data to ensure that the pipeline has not been broken by a recent update. -For this tutorial we will be using the `fewgenomes` dataset, which is a collection of genomes from different sources. First things first, let's check if you have access to the `fewgenomes` dataset. To do this, run the following command: +For this tutorial we will be using the `bioheart` dataset, which is a collection of genomes from different sources. First things first, let's check if you have access to the `bioheart` dataset. To do this, run the following command: ```bash -gsutil ls gs://cpg-fewgenomes-main/ +gsutil ls gs://cpg-bioheart-main/ ``` -The output of the above should list a number of files ending in `.fastq.gz`. These are read files, which are the input to the pipeline. If you do not see these files, then you do not have access to the `fewgenomes` dataset, please contact your manager. +The output of the above should list a number of files ending in `.fastq.gz`. These are read files, which are the input to the pipeline. If you do not see these files, then you do not have access to the `bioheart` dataset, please contact your manager. ### Subsetting the bucket -Our aim is to validate the pipeline's functionality and output by running it on a subset of the fewgenomes dataset. To do this we will use the script [create_test_subset.py](https://github.com/populationgenomics/metamist/blob/f6c226d08a8ee9875014d8c99cfe119742221efb/scripts/create_test_subset.py) in the metamist repository. This script selects a random set of samples from `fewgenomes` and copies their read files to the `fewgenomes-test` bucket. Execute the following command after navigating to the main branch of your metamist repository in the CLI. +Our aim is to validate the pipeline's functionality and output by running it on a subset of the bioheart dataset. To do this we will use the script [create_test_subset.py](https://github.com/populationgenomics/metamist/blob/f6c226d08a8ee9875014d8c99cfe119742221efb/scripts/create_test_subset.py) in the metamist repository. This script selects a random set of samples from `bioheart` and copies their read files to the `bioheart-test` bucket. Execute the following command after navigating to the main branch of your metamist repository in the CLI. To run this script we must first be in the `metamist` repository. So navigate to your `metamist` repository, ensure you're on the `main` branch of `metamist` and then run the following command from CLI: **Note: We will need to set up more genomes in the actual bucket we're using** ```bash analysis-runner \ ---dataset fewgenomes --description "populate fewgenomes test subset" --output-dir "fewgenomes-test" \ +--dataset bioheart --description "populate bioheart test subset" --output-dir "bioheart-test" \ --access-level full \ -scripts/create_test_subset.py --project fewgenomes --samples XPG280371 XPG280389 XPG280397 XPG280405 XPG280413 --skip-ped +scripts/create_test_subset.py --project bioheart --samples XPG280371 XPG280389 XPG280397 XPG280405 XPG280413 --skip-ped ``` **FOR REFERENCE: The above was taken from [this](https://centrepopgen.slack.com/archives/C03FA2M1MR9/p1700020527448029?thread_ts=1699935103.776929&cid=C03FA2M1MR9) Slack thread** @@ -53,9 +53,9 @@ In short; config files contain all the parameters needed to run the job we want When running stages in the `large cohort` pipeline there are default config files that are automatically chosen (see the default [here](https://github.com/populationgenomics/production-pipelines/blob/0bcf9775206f10ee91ac197c8c178f844ecad447/cpg_workflows/defaults.toml)) with subsequent user-defined parameters overriding these defaults as necessary. The default config file for the `large cohort` pipeline can be found [here](https://github.com/populationgenomics/production-pipelines/blob/main/configs/defaults/large_cohort.toml) and in it you can see a description of some of the parameters and what they're used for. #### Task: Write a config file -- If we are wanting to run the `large cohort` pipeline on the `fewgenomes-test` dataset, we will need to create a config file that is capable of doing this. -- Have a go at writing your own config file capable of running the `large cohort` pipeline on `fewgenomes-test` up until the `Combiner` stage. -- You can use the default config file as a starting point, and then override the necessary parameters to run the pipeline on `fewgenomes-test`. +- If we are wanting to run the `large cohort` pipeline on the `bioheart-test` dataset, we will need to create a config file that is capable of doing this. +- Have a go at writing your own config file capable of running the `large cohort` pipeline on `bioheart-test` up until the `Combiner` stage. +- You can use the default config file as a starting point, and then override the necessary parameters to run the pipeline on `bioheart-test`. - *Hint*: You will **not** need to change anything in the default `large cohort` config file.
@@ -63,7 +63,7 @@ When running stages in the `large cohort` pipeline there are default config file ```TOML [workflow] -input_datasets = ['fewgenomes-test'] +input_datasets = ['bioheart-test'] sequencing_type = 'genome' output_version = '1.0' # Do we want them to specify output_version? only_sgs = [] # to be used to demonstrate how to run on a subset of samples @@ -71,7 +71,7 @@ only_sgs = [] # to be used to demonstrate how to ru
-Once the config file has been written, save it with an appropriate name (e.g. `fewgenomes-test.toml`) in a directory that makes sense on your local machine. In the following step we will be using `analysis-runner` to submit the job and point it to the config file we have just written. +Once the config file has been written, save it with an appropriate name (e.g. `bioheart-test.toml`) in a directory that makes sense on your local machine. In the following step we will be using `analysis-runner` to submit the job and point it to the config file we have just written. ## 3. Submitting an analysis job Now that we have a config file, we can submit an analysis job. To do this we will use the `analysis-runner` tool. The `analysis-runner` tool is a command line tool that is used to submit analysis jobs to the cloud. @@ -97,9 +97,9 @@ It is generally good practice to create a bash script that contains the `analysi ```bash analysis-runner \ ---dataset fewgenomes-test \ ---description "Running large cohort up to Combiner stage on fewgenomes" \ ---output-dir "fewgenomes-test-YOUR-NAME" \ +--dataset bioheart-test \ +--description "Running large cohort up to Combiner stage on bioheart" \ +--output-dir "bioheart-test-YOUR-NAME" \ --access-level test \ --config configs/genome.toml \ --config configs/defaults/large_cohort.toml \ From 6aee918d4de17ee426838454faa59c89661d1ff7 Mon Sep 17 00:00:00 2001 From: Michael Harper Date: Tue, 9 Jan 2024 13:07:57 +1100 Subject: [PATCH 3/8] removed task-based approach of tutorial in favour of guided approach. Also improved descriptions of tools and improved clarity --- .../analyst_tutorial/analyst_tutorial.md | 85 ++++++++++++------- 1 file changed, 54 insertions(+), 31 deletions(-) diff --git a/onboarding_documentation/technical_documentation/analyst_tutorial/analyst_tutorial.md b/onboarding_documentation/technical_documentation/analyst_tutorial/analyst_tutorial.md index cee9c83..aa755db 100644 --- a/onboarding_documentation/technical_documentation/analyst_tutorial/analyst_tutorial.md +++ b/onboarding_documentation/technical_documentation/analyst_tutorial/analyst_tutorial.md @@ -1,4 +1,4 @@ -As an analyst you will be running analysis jobs via `analysis-runner`, producing output, and then analysing the output. This tutorial will walk you through the process of running an analysis job, and then analysing the output. +As an analyst you will be running analysis jobs via the `analysis-runner`, producing output, and then analysing the output. This tutorial will walk you through the process of running an analysis job, and then analysing the output. In this tutorial, we will begin with raw fastq files and process them using the large-cohort pipeline. This pipeline performs several steps: @@ -19,22 +19,29 @@ For a description of what each stage of the `large-cohort` pipeline does, please - [4. Analysing the output](#analysing-the-output) ## 1. Getting test data -As an analyst it is vitally important we run our analysis jobs on test data before running them on real data. This is to ensure that the analysis job is working as expected, and that the output is as expected. This is also important to ensure that the analysis job is not going to take too long (and therefore cost too much!) to run on real data. As pipelines are constantly being updated, it is also important to run the analysis job on test data to ensure that the pipeline has not been broken by a recent update. +As an analyst, it's crucial to initially run our analysis jobs on test data before executing them on the actual data. The primary reason for this approach is cost efficiency. Large datasets can be incredibly expensive to process and debug. By using test data, we can estimate the time and cost of running the job on the full dataset, allowing us to optimise resources. Additionally, this process helps us ensure that the pipeline is functioning correctly and producing the expected output. While these checks can also be performed on the main dataset, using test data allows for quicker iterations and adjustments. It's also a good practice to run the analysis job on test data whenever the pipeline is updated, to verify that the updates haven't introduced any issues. -For this tutorial we will be using the `bioheart` dataset, which is a collection of genomes from different sources. First things first, let's check if you have access to the `bioheart` dataset. To do this, run the following command: +In this tutorial, we will be working with the bioheart project dataset. The BiohEART project aims to identify individuals at risk of atherosclerosis by detecting genetic markers in blood before the disease manifests. Our first step is to verify your access to the bioheart dataset. To do this, execute the following command: ```bash -gsutil ls gs://cpg-bioheart-main/ +gsutil ls gs://cpg-bioheart-main-upload/ ``` -The output of the above should list a number of files ending in `.fastq.gz`. These are read files, which are the input to the pipeline. If you do not see these files, then you do not have access to the `bioheart` dataset, please contact your manager. +The output of the above command should list a number of files ending in .fastq.gz. These are read files, which serve as the input to the pipeline. If instead you encounter an AccessDeniedException: 403, it means you do not have the necessary access to the bioheart dataset. The error message will look something like this: +``` +AccessDeniedException: 403 your.username@populationgenomics.org.au does not have storage.objects.list access to the Google Cloud Storage bucket. Permission 'storage.objects.list' denied on resource (or it may not exist). +``` ### Subsetting the bucket -Our aim is to validate the pipeline's functionality and output by running it on a subset of the bioheart dataset. To do this we will use the script [create_test_subset.py](https://github.com/populationgenomics/metamist/blob/f6c226d08a8ee9875014d8c99cfe119742221efb/scripts/create_test_subset.py) in the metamist repository. This script selects a random set of samples from `bioheart` and copies their read files to the `bioheart-test` bucket. Execute the following command after navigating to the main branch of your metamist repository in the CLI. +Our goal is to validate the pipeline's functionality and output by running it on a subset of a dataset. For this tutorial, we'll use a subset of the `bioheart` dataset. We'll employ the [create_test_subset.py](https://github.com/populationgenomics/metamist/blob/f6c226d08a8ee9875014d8c99cfe119742221efb/scripts/create_test_subset.py) script from the `metamist` repository to achieve this. It's important to note that this script is not specific to the bioheart project and can be used with any dataset. The script selects a random set of samples from the specified dataset and copies their read files to a test bucket. + +Before executing this script, ensure that you have installed all necessary dependencies, which include `gsutil` and `analysis-runner`. The `analysis-runner` is a command-line tool that facilitates the submission of analysis jobs to the cloud. It is instrumental in tracking analysis runs, guaranteeing the reproducibility of analyses, and simplifying the process of conducting analyses. In our setup, direct execution of scripts on production data is not permitted. Instead, we mandate that code be committed, reviewed, and subsequently executed. The `analysis-runner` is the tool that enables this process, acting as a bridge between the user and the cloud resources. It operates by allowing an analyst to submit a request for an analysis via the CLI. The `analysis-runner` server then processes this request, verifies the user's authorisation, creates a Batch pipeline from a specific commit in a GitHub repository, and initiates the analysis on behalf of the user. + +For more information on the analysis-runner please check the repo [here](https://github.com/populationgenomics/analysis-runner) and an exercise [here](https://github.com/populationgenomics/team-docs/tree/main/exercise-analysis-runner). -To run this script we must first be in the `metamist` repository. So navigate to your `metamist` repository, ensure you're on the `main` branch of `metamist` and then run the following command from CLI: +If you haven't done so already, clone the `metamist` repository to your local machine. Once you've done that, navigate to your `metamist` repository. Make sure you're on the `main` branch of `metamist`, then execute the following command from the CLI: -**Note: We will need to set up more genomes in the actual bucket we're using** +**TODO: We will need to set up more genomes in the actual bucket we're using** ```bash analysis-runner \ --dataset bioheart --description "populate bioheart test subset" --output-dir "bioheart-test" \ @@ -46,37 +53,37 @@ scripts/create_test_subset.py --project bioheart --samples XPG280371 XPG280389 X ## 2. Writing a config file -Config files are used to specify the parameters of an analysis job. They are written in TOML format. There is already a great description of config files and how they're used in the `team-docs` repo [here](https://github.com/populationgenomics/team-docs/blob/13755bd51356b50ce11e6be78a76e53ed0a3ccb1/cpg_utils_config.md). +Config files, written in TOML format, are used to specify the parameters of an analysis job for the large cohort pipeline. They contain all the necessary parameters to run the job, such as images, references, specific stages of the pipeline to run, and the input data. You can find a comprehensive description of config files and their usage in the `team-docs` repo [here](https://github.com/populationgenomics/team-docs/blob/13755bd51356b50ce11e6be78a76e53ed0a3ccb1/cpg_utils_config.md). -In short; config files contain all the parameters needed to run the job we want such as images, references, specific stages of the pipeline to run, and the input data. They are required for the `analysis-runner` to start. +Please note that while config files are essential for the large cohort pipeline, they are not required to start the `analysis-runner`, which is a wrapper that can be used independently. -When running stages in the `large cohort` pipeline there are default config files that are automatically chosen (see the default [here](https://github.com/populationgenomics/production-pipelines/blob/0bcf9775206f10ee91ac197c8c178f844ecad447/cpg_workflows/defaults.toml)) with subsequent user-defined parameters overriding these defaults as necessary. The default config file for the `large cohort` pipeline can be found [here](https://github.com/populationgenomics/production-pipelines/blob/main/configs/defaults/large_cohort.toml) and in it you can see a description of some of the parameters and what they're used for. +When running stages in the large cohort pipeline, there are set config files that are used, one of which is the defaults file you can see [here](https://github.com/populationgenomics/production-pipelines/blob/0bcf9775206f10ee91ac197c8c178f844ecad447/cpg_workflows/defaults.toml). It's important to understand that this defaults file does not cover all possible parameters. It provides a base set of parameters, but it may not include every parameter that could be relevant for your specific analysis. User-defined parameters in your specific config file can override these defaults as necessary. You can find the default config file for the large cohort pipeline [here](https://github.com/populationgenomics/production-pipelines/blob/main/configs/defaults/large_cohort.toml), where you can see a description of some of the parameters and their uses. -#### Task: Write a config file -- If we are wanting to run the `large cohort` pipeline on the `bioheart-test` dataset, we will need to create a config file that is capable of doing this. -- Have a go at writing your own config file capable of running the `large cohort` pipeline on `bioheart-test` up until the `Combiner` stage. -- You can use the default config file as a starting point, and then override the necessary parameters to run the pipeline on `bioheart-test`. -- *Hint*: You will **not** need to change anything in the default `large cohort` config file. +#### Guide: Write a config file +If we want to run the `large cohort `pipeline on the `bioheart-test` dataset, we need to create a config file that is capable of doing this. Here's a step-by-step guide on how to do it: -
-Click to reveal the config file +Since the default config file serves as our base, we only need to define the parameters that deviate from these defaults. This includes parameters that are left undefined in the default file due to their dataset-specific nature. In this case, we need to specify the input dataset, the sequencing type, and the output version. We also need to specify the sequencing groups we want to run on. We can do this by creating a new config file specifying the following paramters in the `[workflow]` of the `toml` file: ```TOML [workflow] input_datasets = ['bioheart-test'] sequencing_type = 'genome' -output_version = '1.0' # Do we want them to specify output_version? +output_version = '1.0' # TODO: Do we want them to specify output_version? only_sgs = [] # to be used to demonstrate how to run on a subset of samples ``` +- `input_datasets`: This parameter specifies the datasets to be loaded as inputs for the analysis. If this parameter is not provided, the datasets will be determined automatically based on the input of `analysis-runner`. It's an array that can contain multiple dataset names e.g. `['bioheart', 'tob-wgs']`. +- `only_sgs`: This parameter is used to limit the analysis to specific `sequencingGroups`. If you want to run the analysis on a subset of sequencing groups, you can specify their IDs in this array. +- `sequencing_type`: This parameter is used to limit the data to a specific sequencing type. The default value is 'genome', but it can be changed to other sequencing types like 'exome' depending on the analysis. +- `output_version`: This parameter is used to suffix the location of the workflow outputs (get_workflow().prefix) with a specific string. By default, the hash of all input paths will be used. This can be useful for versioning your outputs, especially when you run the same analysis multiple times with different parameters or input data. -
+- Please note that you do not need to change anything in the default `large cohort` config file. The changes you make should be in the new config file you're creating for the `bioheart-test` analysis. -Once the config file has been written, save it with an appropriate name (e.g. `bioheart-test.toml`) in a directory that makes sense on your local machine. In the following step we will be using `analysis-runner` to submit the job and point it to the config file we have just written. +Once the config file has been written, save it with an appropriate name (e.g. `bioheart-test.toml`). Instead of saving it on your local machine, push it to the `production-pipelines-configuration` repository, which is a private repository. In the following step, we will use the `analysis-runner` to submit the job and point it to the config file we have just pushed. ## 3. Submitting an analysis job Now that we have a config file, we can submit an analysis job. To do this we will use the `analysis-runner` tool. The `analysis-runner` tool is a command line tool that is used to submit analysis jobs to the cloud. -Because we are running the `large cohort` pipeline we will need to be inside the `Production-pipelines` repository. So in a directory you want to keep `Production-pipelines` in, run the following command: +Remember, we are using the `large cohort` pipeline to conduct an analysis of several genomes in the `Bioheart` dataset. As the `large cohort` pipeline is a component of the `Production-pipelines` repository, we need to be within this repository to run it. This means that our current working directory should be the `Production-pipelines` repository when executing the pipeline. So our next step is to clone the `Production-pipelines` repository to our local machine. To do this, execute the following command in a directory of your choosing: ```bash git clone --recurse-submodules git@github.com:populationgenomics/production-pipelines.git @@ -85,31 +92,47 @@ cd production-pipelines Ensure you are on the `main` branch. -For guidance on how to use the `analysis-runner` tool, please see the `analysis-runner` documentation [here](https://github.com/populationgenomics/analysis-runner). It will need to be `pip` installed before use. +#### Guide: Submit an analysis job +When working in any pipeline with many moving parts, it's important to be able to track the progress of your analysis. As mentioned previously, the `analysis-runner` is our tool of choice for this purpose. Therefore, in order to start our analysis, we point the `analysis-runner` to our config file with parameters specific to our analysis. -It is generally good practice to create a bash script that contains the `analysis-runner` command to submit the job. This is so that the command can be easily re-run if necessary. +The `analysis-runner` tool is a powerful utility for submitting analysis jobs. Detailed instructions on its usage can be found in the [official documentation](https://github.com/populationgenomics/analysis-runner) (also linked earlier). Before you can use it, ensure it's installed via `pip`. -#### Task: Submit an analysis job -- Using the instructions in the `analysis-runner` repo linked above as well as the config file you have just written, write the command to submit analysis job to the cloud using the `analysis-runner` tool. +You have the option to either encapsulate the `analysis-runner` command within a bash script or run it directly from the command line, based on your preference. -
-Click to reveal the command +Let's break down the structure of an `analysis-runner` command: +- `--dataset`: This parameter specifies the dataset to be loaded as input for the analysis. +- `--description`: This parameter is used to provide a description of the analysis job. +- `--access-level`: This parameter is used to specify the access level of the analysis job. The access level can be either `test` or `full`. The `test` access level is used for test data, while the `full` access level is used for production data. +- `--output-dir`: This parameter is used to specify the output directory for the analysis job. The output directory is where the output files will be stored in the bucket of `dataset` provided (in this case `bioheart-test`) + - Because we specified the `access-level` as `test`, the output directory will be in the `bioheart-test` bucket. Had we specified `full` as the `access-level`, the output directory would be in the `bioheart-main` bucket. +- `--config`: This parameter is used to specify the config files to be used for the analysis job. The config files are used to specify the parameters of the analysis job. You can specify multiple config files by repeating this parameter. + - **IMPORTANT:** *The order of the config files is important. The config files are read in the order they are specified. This means that if you specify the same parameter in multiple config files, the value of the parameter in the last config file will be used. This is why the config file we created is listed last as an input.* +- `--image`: This parameter is used to specify the docker image to be used for the analysis job. The docker image is used to specify the environment in which the analysis job will be run. + - TODO: what else should we say about this? +- `main.py`: This is the Python script that initiates the pipeline. In our scenario, the `large cohort` pipeline is started by this script. +- `large_cohort`: This argument identifies the specific pipeline to execute. Here, it's set to `large_cohort`, indicating that we're running the `large cohort` pipeline, which is initiated by the `main.py` script. + +Putting it all together, we get the following: ```bash analysis-runner \ ---dataset bioheart-test \ +--dataset bioheart \ --description "Running large cohort up to Combiner stage on bioheart" \ --output-dir "bioheart-test-YOUR-NAME" \ --access-level test \ ---config configs/genome.toml \ --config configs/defaults/large_cohort.toml \ --config path/to/you/config/config_filename.toml \ --image australia-southeast1-docker.pkg.dev/cpg-common/images/cpg_workflows:latest \ main.py large_cohort ``` -
+The output of the `analysis-runner` command is link to the Hail Batch driver job. Something like this: +``` +Submitting analysis-runner@ for dataset "bioheart" +Request submitted successfully: https://batch.hail.populationgenomics.org.au/batches/ +``` +This driver job is responsible for setting up all the subsequent jobs to run the analysis. When this driver job is complete, a link to another Hail Batch service will be provided. This is where all the individual jobs of the pipeline will be listed as well as their status. Once this secondary Batch is completed, the output will be available in `bioheart-test`. ## 4. Analysing the output From b467ce47e9ce56156e7c3bcf1d368855e81a0355 Mon Sep 17 00:00:00 2001 From: Michael Harper Date: Tue, 9 Jan 2024 13:18:51 +1100 Subject: [PATCH 4/8] clarify instructions on using accompanying notebook --- .../analyst_tutorial/analyst_tutorial.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onboarding_documentation/technical_documentation/analyst_tutorial/analyst_tutorial.md b/onboarding_documentation/technical_documentation/analyst_tutorial/analyst_tutorial.md index aa755db..1bd1c52 100644 --- a/onboarding_documentation/technical_documentation/analyst_tutorial/analyst_tutorial.md +++ b/onboarding_documentation/technical_documentation/analyst_tutorial/analyst_tutorial.md @@ -140,6 +140,6 @@ Now that we have some output to work with, lets begin analysing. Analysis is best done within a Jupyter Notebook. However, in order to interact with our data in the cloud using Hail and jupyter notebooks we need to do some set up first. -Instructions on setting up a Jupyter Notebook in the cloud can be found [here](https://github.com/populationgenomics/team-docs/blob/main/notebooks.md) +Instructions on setting up a Jupyter Notebook in the cloud can be found [here](https://github.com/populationgenomics/team-docs/blob/main/notebooks.md). -Please continue this tutorial once you have a Jupyter Notebook running in the cloud, good luck! +Once you have your notebook setup on GCP follow along with the instructions in this tutorials accompanying notebook, copying over code as required, good luck! From 05c1b3a99cb3137b10c66ed714c7e4f3593cdc6a Mon Sep 17 00:00:00 2001 From: Michael Harper Date: Tue, 9 Jan 2024 18:47:39 +1100 Subject: [PATCH 5/8] added descriptions and documentation to the jupyer notebook to improve clarity and ease of access to newcomers --- .../Final_analyst_tutorial.ipynb | 181 +++++++++++++++++- 1 file changed, 178 insertions(+), 3 deletions(-) diff --git a/onboarding_documentation/technical_documentation/analyst_tutorial/Final_analyst_tutorial.ipynb b/onboarding_documentation/technical_documentation/analyst_tutorial/Final_analyst_tutorial.ipynb index 71c0d54..496a3bf 100644 --- a/onboarding_documentation/technical_documentation/analyst_tutorial/Final_analyst_tutorial.ipynb +++ b/onboarding_documentation/technical_documentation/analyst_tutorial/Final_analyst_tutorial.ipynb @@ -1,5 +1,89 @@ { "cells": [ + { + "cell_type": "markdown", + "id": "e87b9d2a", + "metadata": {}, + "source": [ + "## Analyst Tutorial: Ancestry Comparisong using PCA" + ] + }, + { + "cell_type": "markdown", + "id": "0f95cb81", + "metadata": {}, + "source": [ + "This Jupyter Notebook documents the analysis of ancestry comparison using Principal Component Analysis (PCA). The analysis is performed on a dataset generated by the Large Cohort Pipeline, which processed raw FASTQ files and generated PCA values. Additionally, we will compare the ancestry of our dataset to a public dataset by calculating PCA values for the public dataset.\n", + "\n", + "### Background\n", + "The Large Cohort Pipeline is a powerful tool for processing genomic data and extracting valuable insights. By submitting raw FASTQ files to the pipeline, we obtained PCA values that provide information about the genetic ancestry of our dataset. To gain further insights and compare our dataset with a public dataset, we will calculate PCA values for the public dataset as well.\n", + "\n", + "### Objectives\n", + "The main objectives of this analysis are as follows:\n", + "1. Compare the genetic ancestry of our dataset with the public dataset using PCA.\n", + "2. Visualize the PCA results to identify patterns and similarities/differences between the datasets.\n", + "3. Gain insights into the genetic diversity and population structure of the datasets.\n", + "4. Gain familiarity with using Hail to conduct analyses of large datasets\n", + "\n", + "### Methodology\n", + "We will the Hail for genomic data analysis, including PCA calculation and visualisation. The notebook consists of multiple cells, each containing code snippets and markdown cells for documentation.\n", + "\n", + "### Conclusion\n", + "By comparing the PCA values of our dataset with the public dataset, we aim to gain insights into the genetic ancestry and population structure of both datasets. The analysis will provide valuable information for understanding the genetic diversity and relationships between different populations.\n", + "\n", + "Let's proceed with the analysis and explore the fascinating world of genetic ancestry comparison using PCA!" + ] + }, + { + "cell_type": "markdown", + "id": "e642d8ed", + "metadata": {}, + "source": [ + "### Hail Background\n", + "**Hail**: Hail is an open-source, scalable framework for exploring and analyzing genomic data. It's built to handle large-scale genomic data and provides tools for data manipulation, statistical analysis, and visualisation. Hail is particularly powerful for genome-wide association studies (GWAS) and other genetic analyses.\n", + "\n", + "**Hail Tables**: Hail Tables are one of the two primary data structures in Hail. They are similar to SQL tables, R data.frames, or pandas DataFrames. A Hail Table is a distributed dataset, that can be used to hold various types of data, including numerical, categorical, and genomic data. Hail Tables support a variety of operations like filtering, aggregation, and joins.\n", + "\n", + "**Hail MatrixTables**: MatrixTables are the second primary data structure in Hail and are designed to hold large-scale, distributed genomic data. A MatrixTable consists of a two-dimensional matrix of entries, indexed by row keys (representing variant data) and column keys (representing sample data). Each cell in the matrix represents the genetic data for a specific variant and sample. Along with the matrix, there are also row, column, and global fields for storing related data. MatrixTables support a variety of operations like filtering, aggregation, linear algebra operations, and joins." + ] + }, + { + "cell_type": "markdown", + "id": "20e64b58", + "metadata": {}, + "source": [ + "Hail provides tutorials specifically designed to introduce individuals to working with Tables and MatrixTables:\n", + "- [Table tutorial](https://hail.is/docs/0.2/tutorials/03-tables.html)\n", + "- [MatrixTable tutorial](https://hail.is/docs/0.2/tutorials/07-matrixtable.html)\n", + "\n", + "Additionally, [here](https://hail.is/docs/0.2/tutorials-landing.html) is Hail's complete list of tutorials that provide users with the opportunity to explore more manipulations using their data structures" + ] + }, + { + "cell_type": "markdown", + "id": "ee7ab28a", + "metadata": {}, + "source": [ + "![Hail Matrix Table Diagram](Hail_matrix_table_diagram.png)\n", + "Source: https://hail.is/docs/0.2/_static/cheatsheets/hail_matrix_tables_cheat_sheet.pdf" + ] + }, + { + "cell_type": "markdown", + "id": "4cc981f2", + "metadata": {}, + "source": [ + "## Imports and Reading in Data from GCP" + ] + }, + { + "cell_type": "markdown", + "id": "7f93b1b4", + "metadata": {}, + "source": [ + "Before starting, we need to import the necessary packages. If you have followed the notebook setup properly, Hail should already be installed along with Python and some basic packages, and can be imported directly." + ] + }, { "cell_type": "code", "execution_count": 1, @@ -72,6 +156,37 @@ "hl.plot.output_notebook()" ] }, + { + "cell_type": "markdown", + "id": "da901394", + "metadata": {}, + "source": [ + "This notebook should be set up inside the bucket that contains the data we are wanting to analyse, this way we have permissions to access the data. In the case of this tutorial it is the `bioheart` bucket. While some buckets/projects are capable of reading the contents of other buckets, not all of them are, so it is best to work from a notebook that is within the bucket containing our data." + ] + }, + { + "cell_type": "markdown", + "id": "25ca7847", + "metadata": {}, + "source": [ + "#### Reading in Data\n", + "We will now be reading in our data. If you remember in our config file we set the parameter `output_version` to be `5.0`. This has created a directory within the `large_cohort` directory of the `cpg-bioheart-test` bucket.\n", + "While we have mentioned `Tables` (`.ht`) and `MatrixTables` (`.mt`) above, we will also be working with `Variant Dataset` (`.vds`) files. \n", + "- `vds` or `Variant Dataset` ([documentation](https://hail.is/docs/0.2/vds/index.html))\n", + " - Contains variant information for every sample in the cohort\n", + " - Briefly; because we are dealing with cohort-level data there will be regions of genome that, for every sample in the cohort, is homozygous reference, as well as information where sample(s) are non-reference. To maintain both reference and non-reference information in a scalable manner, a `.vds` file is made up of two `MatrixTables`, the `reference_data` and `variant_data` `MatrixTable`'s.\n", + "- `sample_qc_ht`: this Hail Table is generated by the `SampleQC` stage of the `large cohort` pipeline. It contains the sex of each sample in the cohort and PCA values for each sample to be used in ancestry comparison as well as quality metrics.\n", + "- `relateds_to_drop_ht`: this Hail Table is generated by the `Relatedness` stage of the `large cohort` pipeline. It contains information about related samples in the cohort and which samples, if related, should be dropped from the cohort. For samples that are related, they are ranked according to their sample QC metrics and the sample with the worst metrics is dropped from the cohort so that there is only one sample left per family. See code [here](https://github.com/populationgenomics/production-pipelines/blob/824019c6eb0387d10aff047145e92583cd3e701c/cpg_workflows/large_cohort/relatedness.py#L78)\n", + "- `pop_ht`: a Hail Table containing the assigned global population labels. Hail uses a Random Forest machine learning model based on PCA scores to assign global population labels to samples. See code [here](https://github.com/populationgenomics/production-pipelines/blob/824019c6eb0387d10aff047145e92583cd3e701c/cpg_workflows/large_cohort/ancestry_pca.py#L215). Population label is assigned based on previously known ancestry of the HGDP and 1KG datasets (calculated from gnomAD).\n", + "- `vqsr.ht`: Hail Table resulting from the Variant Quality Score Recalibration process. This table contains information about the variants that were filtered out during the VQSR process.\n", + "- `freq_ht`: Hail Table containing information about the allele frequency of each variant in the cohort. This table is generated by the `VariantQC` stage of the `large cohort` pipeline. Generates frequency annotations of AF, AC, AN, InbreedingCoeff\n", + " - AF (Allele Frequency): This is the proportion of all alleles at a given location in a population that are of a particular type\n", + " - AC (Allele Count): This is the total number of times a specific allele appears in a population.\n", + " - AN (Allele Number): This is the total number of alleles at a given location in a population.\n", + " - InbreedingCoeff: This is a measure of the likelihood of genetic traits in a population being inherited due to the parents being closely related.\n", + "- `dense_mt`: Hail Matrix Table containing the genotype information for each sample in the cohort. This table is generated by the `Genotype` and subsequent `GenotypeQC` stages of the `large cohort` pipeline. This table contains the genotype information for a set of pre-determined QC sites (according to `gs://cpg-common-main/references/gnomad/v0/sample_qc/pre_ld_pruning_qc_variants.ht`) for each sample in the cohort. The table is dense, meaning that it contains all variants for all samples in the cohort." + ] + }, { "cell_type": "code", "execution_count": 2, @@ -114,6 +229,24 @@ "dense_mt = hl.read_matrix_table('gs://cpg-bioheart-test/large_cohort/5-0/dense_subset.mt/')" ] }, + { + "cell_type": "markdown", + "id": "3197c3d7", + "metadata": {}, + "source": [ + "### Pre-processing and Filtering" + ] + }, + { + "cell_type": "markdown", + "id": "d522cce8", + "metadata": {}, + "source": [ + "As stated previously, a `hl.vds` object is a combination of two `MatrixTables`; `vds.reference_data` and `vds.variant_data`, which are both sparse representations of the data. `vds.reference_data` contains reference data as 'blocks' of information, meaning that reference data is not explicitly stored at each variant site but is represented in intervals ('blocks') where it is necessary. `vds.variant_data` is a sparse matrix of non-reference calls. Our analysis requires a dense representation of the data, meaning each position in the genome is represented in the data i.e. reference data is explicitly stored at each position and not in 'blocks', as well as non-reference calls being present. To achieve this, we will use the `to_dense_mt()` function to convert our `vds` into a `MatrixTable` with a dense representation of the data.\n", + "\n", + "Additionally, we use the `split_multi()` to split multi-allelic variants into bi-allelic variants. After splitting, each row represents a single variant." + ] + }, { "cell_type": "code", "execution_count": 3, @@ -128,6 +261,18 @@ "mt = hl.vds.to_dense_mt(vds)" ] }, + { + "cell_type": "markdown", + "id": "94860935", + "metadata": {}, + "source": [ + "#### Sample vs Row level filtering\n", + "We will now filter our `MatrixTable` according to the outputs of the pipeline. \n", + "Any filtering that works using the column key is sample-level filtering and thus will be filtering on sample-level metrics. \n", + "For example: any samples that did not pass the `SampleQC` stage of the pipeline will be removed from the `MatrixTable` if they are not in the `sample_qc_ht` table. This is sample-level filtering because we are filtering on the sample-level metrics in the `sample_qc_ht` table.\n", + "`vqsr.ht` contains information about the variants that were filtered out during the VQSR process. We will use this table to filter out variants that did not pass the VQSR process. This is row-level filtering because we are filtering on the variant-level metrics in the `vqsr.ht` table." + ] + }, { "cell_type": "code", "execution_count": 4, @@ -138,9 +283,23 @@ "outputs": [], "source": [ "# Hard-filtering samples and variants:\n", - "mt = mt.filter_cols(hl.len(sample_qc_ht[mt.col_key].filters) > 0, keep=False)\n", - "mt = mt.filter_cols(hl.is_defined(relateds_to_drop_ht[mt.col_key]), keep=False)\n", - "mt = mt.filter_rows(hl.len(vqsr_ht[mt.row_key].filters) > 0, keep=False)" + "mt = mt.filter_cols(hl.len(sample_qc_ht[mt.col_key].filters) > 0, keep=False) # sample-level filtering\n", + "mt = mt.filter_cols(hl.is_defined(relateds_to_drop_ht[mt.col_key]), keep=False) # sample-level filtering\n", + "mt = mt.filter_rows(hl.len(vqsr_ht[mt.row_key].filters) > 0, keep=False) # row-level filtering" + ] + }, + { + "cell_type": "markdown", + "id": "18736b06", + "metadata": {}, + "source": [ + "#### Annotation\n", + "\n", + "Annotating our `MatrixTable` means adding `column fields` to the `MatrixTable`'s `columns` `Hail Table` that contain information about the samples and, vice versa, adding new `row fields` to the `rows` `Hail Table` that contains additional information about the calls. We will annotate our `MatrixTable` with the following information:\n", + "- Sample QC metrics from the `sample_qc_ht` table\n", + "- Sample population label from the `pop_ht` table\n", + "- Call information from the `vqsr` table\n", + "- Call frequency information from the `freq_ht` table" ] }, { @@ -248,6 +407,22 @@ "show(p)" ] }, + { + "cell_type": "markdown", + "id": "f7c23590", + "metadata": {}, + "source": [ + "## Conducting PCA on Public Dataset\n", + "\n", + "In this section, we will be conducting a Principal Component Analysis (PCA) using Hail on the publicly available Human Genome Diversity Project - 1000 Genomes Project (HGDP-1KG) dataset. As we've seen, PCA ican be used to identify and visualise genetic variation between individuals. It can help us understand the genetic structure of our dataset and identify potential population stratification, which is crucial for many genetic analyses.\n", + "\n", + "The HGDP-1KG dataset is a comprehensive collection of human genetic variation from globally diverse populations. It combines data from two significant projects: the Human Genome Diversity Project (HGDP) and the 1000 Genomes Project (1KG). This dataset is readily accessible through Hail's interface as it is hosted by `gnomAD`.\n", + "\n", + "Our goal in this section is to compare the ancestry of our dataset with the ancestry of the HGDP-1KG dataset. By projecting our samples onto the genetic variation patterns identified in the HGDP-1KG dataset, we can see where our samples fit in the global population structure. This can provide valuable insights into the genetic background of our samples and inform subsequent analyses.\n", + "\n", + "Let's get started with the PCA." + ] + }, { "cell_type": "code", "execution_count": 10, From 3aab5ca7e300f655aea6df034c56d4505c9094fc Mon Sep 17 00:00:00 2001 From: Michael Harper Date: Tue, 9 Jan 2024 18:48:53 +1100 Subject: [PATCH 6/8] cleared cell outputs --- .../Final_analyst_tutorial.ipynb | 237 ++---------------- 1 file changed, 21 insertions(+), 216 deletions(-) diff --git a/onboarding_documentation/technical_documentation/analyst_tutorial/Final_analyst_tutorial.ipynb b/onboarding_documentation/technical_documentation/analyst_tutorial/Final_analyst_tutorial.ipynb index 496a3bf..1f9a272 100644 --- a/onboarding_documentation/technical_documentation/analyst_tutorial/Final_analyst_tutorial.ipynb +++ b/onboarding_documentation/technical_documentation/analyst_tutorial/Final_analyst_tutorial.ipynb @@ -86,69 +86,12 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "0d6ccf14-c2b7-4d1c-8c1f-642d2ed2aba5", "metadata": { "tags": [] }, - "outputs": [ - { - "data": { - "text/html": [ - " \n", - "
\n", - " \n", - " Loading BokehJS ...\n", - "
\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": "(function(root) {\n function now() {\n return new Date();\n }\n\n const force = true;\n\n if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\nconst JS_MIME_TYPE = 'application/javascript';\n const HTML_MIME_TYPE = 'text/html';\n const EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n const CLASS_NAME = 'output_bokeh rendered_html';\n\n /**\n * Render data to the DOM node\n */\n function render(props, node) {\n const script = document.createElement(\"script\");\n node.appendChild(script);\n }\n\n /**\n * Handle when an output is cleared or removed\n */\n function handleClearOutput(event, handle) {\n function drop(id) {\n const view = Bokeh.index.get_by_id(id)\n if (view != null) {\n view.model.document.clear()\n Bokeh.index.delete(view)\n }\n }\n\n const cell = handle.cell;\n\n const id = cell.output_area._bokeh_element_id;\n const server_id = cell.output_area._bokeh_server_id;\n\n // Clean up Bokeh references\n if (id != null) {\n drop(id)\n }\n\n if (server_id !== undefined) {\n // Clean up Bokeh references\n const cmd_clean = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n cell.notebook.kernel.execute(cmd_clean, {\n iopub: {\n output: function(msg) {\n const id = msg.content.text.trim()\n drop(id)\n }\n }\n });\n // Destroy server and session\n const cmd_destroy = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n cell.notebook.kernel.execute(cmd_destroy);\n }\n }\n\n /**\n * Handle when a new output is added\n */\n function handleAddOutput(event, handle) {\n const output_area = handle.output_area;\n const output = handle.output;\n\n // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n if ((output.output_type != \"display_data\") || (!Object.prototype.hasOwnProperty.call(output.data, EXEC_MIME_TYPE))) {\n return\n }\n\n const toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n\n if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n // store reference to embed id on output_area\n output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n }\n if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n const bk_div = document.createElement(\"div\");\n bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n const script_attrs = bk_div.children[0].attributes;\n for (let i = 0; i < script_attrs.length; i++) {\n toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n toinsert[toinsert.length - 1].firstChild.textContent = bk_div.children[0].textContent\n }\n // store reference to server id on output_area\n output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n }\n }\n\n function register_renderer(events, OutputArea) {\n\n function append_mime(data, metadata, element) {\n // create a DOM node to render to\n const toinsert = this.create_output_subarea(\n metadata,\n CLASS_NAME,\n EXEC_MIME_TYPE\n );\n this.keyboard_manager.register_events(toinsert);\n // Render to node\n const props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n render(props, toinsert[toinsert.length - 1]);\n element.append(toinsert);\n return toinsert\n }\n\n /* Handle when an output is cleared or removed */\n events.on('clear_output.CodeCell', handleClearOutput);\n events.on('delete.Cell', handleClearOutput);\n\n /* Handle when a new output is added */\n events.on('output_added.OutputArea', handleAddOutput);\n\n /**\n * Register the mime type and append_mime function with output_area\n */\n OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n /* Is output safe? */\n safe: true,\n /* Index of renderer in `output_area.display_order` */\n index: 0\n });\n }\n\n // register the mime type if in Jupyter Notebook environment and previously unregistered\n if (root.Jupyter !== undefined) {\n const events = require('base/js/events');\n const OutputArea = require('notebook/js/outputarea').OutputArea;\n\n if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n register_renderer(events, OutputArea);\n }\n }\n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n const NB_LOAD_WARNING = {'data': {'text/html':\n \"
\\n\"+\n \"

\\n\"+\n \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n \"

\\n\"+\n \"
    \\n\"+\n \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n \"
  • use INLINE resources instead, as so:
    • \\n\"+\n \"
\\n\"+\n \"\\n\"+\n \"from bokeh.resources import INLINE\\n\"+\n \"output_notebook(resources=INLINE)\\n\"+\n \"\\n\"+\n \"
\"}};\n\n function display_loaded() {\n const el = document.getElementById(\"cc7e9275-9ef1-4dfd-8d37-4a188441122d\");\n if (el != null) {\n el.textContent = \"BokehJS is loading...\";\n }\n if (root.Bokeh !== undefined) {\n if (el != null) {\n el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(display_loaded, 100)\n }\n }\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls == null || js_urls.length === 0) {\n run_callbacks();\n return null;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n\n function on_error(url) {\n console.error(\"failed to load \" + url);\n }\n\n for (let i = 0; i < css_urls.length; i++) {\n const url = css_urls[i];\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error.bind(null, url);\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n }\n\n for (let i = 0; i < js_urls.length; i++) {\n const url = js_urls[i];\n const element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error.bind(null, url);\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n const js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.3.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.3.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.3.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.3.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-3.3.2.min.js\"];\n const css_urls = [];\n\n const inline_js = [ function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\nfunction(Bokeh) {\n }\n ];\n\n function run_inline_js() {\n if (root.Bokeh !== undefined || force === true) {\n for (let i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }\nif (force === true) {\n display_loaded();\n }} else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n } else if (force !== true) {\n const cell = $(document.getElementById(\"cc7e9275-9ef1-4dfd-8d37-4a188441122d\")).parents('.cell').data().cell;\n cell.output_area.append_execute_result(NB_LOAD_WARNING)\n }\n }\n\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(css_urls, js_urls, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));", - "application/vnd.bokehjs_load.v0+json": "" - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - " \n", - "
\n", - " \n", - " Loading BokehJS ...\n", - "
\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": "(function(root) {\n function now() {\n return new Date();\n }\n\n const force = true;\n\n if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\nconst JS_MIME_TYPE = 'application/javascript';\n const HTML_MIME_TYPE = 'text/html';\n const EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n const CLASS_NAME = 'output_bokeh rendered_html';\n\n /**\n * Render data to the DOM node\n */\n function render(props, node) {\n const script = document.createElement(\"script\");\n node.appendChild(script);\n }\n\n /**\n * Handle when an output is cleared or removed\n */\n function handleClearOutput(event, handle) {\n function drop(id) {\n const view = Bokeh.index.get_by_id(id)\n if (view != null) {\n view.model.document.clear()\n Bokeh.index.delete(view)\n }\n }\n\n const cell = handle.cell;\n\n const id = cell.output_area._bokeh_element_id;\n const server_id = cell.output_area._bokeh_server_id;\n\n // Clean up Bokeh references\n if (id != null) {\n drop(id)\n }\n\n if (server_id !== undefined) {\n // Clean up Bokeh references\n const cmd_clean = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n cell.notebook.kernel.execute(cmd_clean, {\n iopub: {\n output: function(msg) {\n const id = msg.content.text.trim()\n drop(id)\n }\n }\n });\n // Destroy server and session\n const cmd_destroy = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n cell.notebook.kernel.execute(cmd_destroy);\n }\n }\n\n /**\n * Handle when a new output is added\n */\n function handleAddOutput(event, handle) {\n const output_area = handle.output_area;\n const output = handle.output;\n\n // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n if ((output.output_type != \"display_data\") || (!Object.prototype.hasOwnProperty.call(output.data, EXEC_MIME_TYPE))) {\n return\n }\n\n const toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n\n if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n // store reference to embed id on output_area\n output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n }\n if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n const bk_div = document.createElement(\"div\");\n bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n const script_attrs = bk_div.children[0].attributes;\n for (let i = 0; i < script_attrs.length; i++) {\n toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n toinsert[toinsert.length - 1].firstChild.textContent = bk_div.children[0].textContent\n }\n // store reference to server id on output_area\n output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n }\n }\n\n function register_renderer(events, OutputArea) {\n\n function append_mime(data, metadata, element) {\n // create a DOM node to render to\n const toinsert = this.create_output_subarea(\n metadata,\n CLASS_NAME,\n EXEC_MIME_TYPE\n );\n this.keyboard_manager.register_events(toinsert);\n // Render to node\n const props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n render(props, toinsert[toinsert.length - 1]);\n element.append(toinsert);\n return toinsert\n }\n\n /* Handle when an output is cleared or removed */\n events.on('clear_output.CodeCell', handleClearOutput);\n events.on('delete.Cell', handleClearOutput);\n\n /* Handle when a new output is added */\n events.on('output_added.OutputArea', handleAddOutput);\n\n /**\n * Register the mime type and append_mime function with output_area\n */\n OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n /* Is output safe? */\n safe: true,\n /* Index of renderer in `output_area.display_order` */\n index: 0\n });\n }\n\n // register the mime type if in Jupyter Notebook environment and previously unregistered\n if (root.Jupyter !== undefined) {\n const events = require('base/js/events');\n const OutputArea = require('notebook/js/outputarea').OutputArea;\n\n if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n register_renderer(events, OutputArea);\n }\n }\n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n const NB_LOAD_WARNING = {'data': {'text/html':\n \"
\\n\"+\n \"

\\n\"+\n \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n \"

\\n\"+\n \"
    \\n\"+\n \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n \"
  • use INLINE resources instead, as so:
    • \\n\"+\n \"
\\n\"+\n \"\\n\"+\n \"from bokeh.resources import INLINE\\n\"+\n \"output_notebook(resources=INLINE)\\n\"+\n \"\\n\"+\n \"
\"}};\n\n function display_loaded() {\n const el = document.getElementById(\"f4f32713-7290-47db-90d0-9f7f5246a434\");\n if (el != null) {\n el.textContent = \"BokehJS is loading...\";\n }\n if (root.Bokeh !== undefined) {\n if (el != null) {\n el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(display_loaded, 100)\n }\n }\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls == null || js_urls.length === 0) {\n run_callbacks();\n return null;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n\n function on_error(url) {\n console.error(\"failed to load \" + url);\n }\n\n for (let i = 0; i < css_urls.length; i++) {\n const url = css_urls[i];\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error.bind(null, url);\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n }\n\n for (let i = 0; i < js_urls.length; i++) {\n const url = js_urls[i];\n const element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error.bind(null, url);\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n const js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.3.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.3.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.3.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.3.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-3.3.2.min.js\"];\n const css_urls = [];\n\n const inline_js = [ function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\nfunction(Bokeh) {\n }\n ];\n\n function run_inline_js() {\n if (root.Bokeh !== undefined || force === true) {\n for (let i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }\nif (force === true) {\n display_loaded();\n }} else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n } else if (force !== true) {\n const cell = $(document.getElementById(\"f4f32713-7290-47db-90d0-9f7f5246a434\")).parents('.cell').data().cell;\n cell.output_area.append_execute_result(NB_LOAD_WARNING)\n }\n }\n\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(css_urls, js_urls, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));", - "application/vnd.bokehjs_load.v0+json": "" - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "import hail as hl\n", "import pprint\n", @@ -189,36 +132,12 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "fe84e1cc-0b8d-473b-a239-24e7ae14187b", "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Initializing Hail with default parameters...\n", - "SLF4J: No SLF4J providers were found.\n", - "SLF4J: Defaulting to no-operation (NOP) logger implementation\n", - "SLF4J: See https://www.slf4j.org/codes.html#noProviders for further details.\n", - "SLF4J: Class path contains SLF4J bindings targeting slf4j-api versions 1.7.x or earlier.\n", - "SLF4J: Ignoring binding found at [jar:file:/opt/conda/envs/python310/lib/python3.10/site-packages/pyspark/jars/log4j-slf4j-impl-2.17.2.jar!/org/slf4j/impl/StaticLoggerBinder.class]\n", - "SLF4J: See https://www.slf4j.org/codes.html#ignoredBindings for an explanation.\n", - "Setting default log level to \"WARN\".\n", - "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", - "Running on Apache Spark version 3.3.4\n", - "SparkUI available at http://dd7b7b83e829:4040\n", - "Welcome to\n", - " __ __ <>__\n", - " / /_/ /__ __/ /\n", - " / __ / _ `/ / /\n", - " /_/ /_/\\_,_/_/_/ version 0.2.126-ee77707f4fab\n", - "LOGGING: writing to /home/jupyter/hail-20240104-2101-0.2.126-ee77707f4fab.log\n" - ] - } - ], + "outputs": [], "source": [ "vds = hl.vds.read_vds('gs://cpg-bioheart-test/vds/5-0.vds/')\n", "sample_qc_ht = hl.read_table('gs://cpg-bioheart-test/large_cohort/5-0/sample_qc.ht/')\n", @@ -249,7 +168,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "4289f74d-60a4-4dfb-b17b-e198df09acff", "metadata": { "tags": [] @@ -275,7 +194,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "1e74f5e6-1c91-445e-af95-fd5ff131e912", "metadata": { "tags": [] @@ -304,7 +223,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "510813e0-e22b-4be2-9d7b-c6f4e715be31", "metadata": { "tags": [] @@ -321,24 +240,12 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "b6ae5753-b90a-48d5-a435-9d76929cb758", "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2024-01-04 21:02:29.088 Hail: INFO: Reading table to impute column types\n", - "2024-01-04 21:02:30.624 Hail: INFO: Finished type imputation\n", - " Loading field 'Sample' as type str (imputed)\n", - " Loading field 'population' as type str (imputed)\n", - " Loading field 'superpopulation' as type str (imputed)\n" - ] - } - ], + "outputs": [], "source": [ "# import annotations table with ancestry information\n", "# even though we have an 'ancestry' output from the pipeline, we won't use it\n", @@ -347,7 +254,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "049c9540-1e6d-470e-877c-f17db3a215df", "metadata": { "tags": [] @@ -361,43 +268,12 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "2f286ecc-0784-478c-b84e-76254f36aac2", "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2024-01-04 21:02:55.654 Hail: WARN: aggregate_cols(): Aggregates over cols ordered by 'col_key'.\n", - " To preserve matrix table column order, first unkey columns with 'key_cols_by()'\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - "
\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": "(function(root) {\n function embed_document(root) {\n const docs_json = {\"73a1dfc6-515f-48ef-98b1-6f59889e5062\":{\"version\":\"3.3.2\",\"title\":\"Bokeh Application\",\"roots\":[{\"type\":\"object\",\"name\":\"Figure\",\"id\":\"p1001\",\"attributes\":{\"width\":800,\"height\":800,\"x_range\":{\"type\":\"object\",\"name\":\"DataRange1d\",\"id\":\"p1002\"},\"y_range\":{\"type\":\"object\",\"name\":\"DataRange1d\",\"id\":\"p1003\"},\"x_scale\":{\"type\":\"object\",\"name\":\"LinearScale\",\"id\":\"p1011\"},\"y_scale\":{\"type\":\"object\",\"name\":\"LinearScale\",\"id\":\"p1012\"},\"title\":{\"type\":\"object\",\"name\":\"Title\",\"id\":\"p1004\",\"attributes\":{\"text\":\"PCA\"}},\"renderers\":[{\"type\":\"object\",\"name\":\"GlyphRenderer\",\"id\":\"p1049\",\"attributes\":{\"data_source\":{\"type\":\"object\",\"name\":\"ColumnDataSource\",\"id\":\"p1035\",\"attributes\":{\"selected\":{\"type\":\"object\",\"name\":\"Selection\",\"id\":\"p1036\",\"attributes\":{\"indices\":[],\"line_indices\":[]}},\"selection_policy\":{\"type\":\"object\",\"name\":\"UnionRenderers\",\"id\":\"p1037\"},\"data\":{\"type\":\"map\",\"entries\":[[\"index\",{\"type\":\"ndarray\",\"array\":{\"type\":\"bytes\",\"data\":\"AAAAAAEAAAACAAAAAwAAAAQAAAA=\"},\"shape\":[5],\"dtype\":\"int32\",\"order\":\"little\"}],[\"x\",{\"type\":\"ndarray\",\"array\":{\"type\":\"bytes\",\"data\":\"/5bzoNJX1L/ocxozGmrQv1sAadQxh8y/7L6DYfwsx78zNYL3AU7vPw==\"},\"shape\":[5],\"dtype\":\"float64\",\"order\":\"little\"}],[\"y\",{\"type\":\"ndarray\",\"array\":{\"type\":\"bytes\",\"data\":\"DRAK+CIZ4b+lpfkOFV3oP2frKWnzkdy/owBruc3ezz94/qsWelWevw==\"},\"shape\":[5],\"dtype\":\"float64\",\"order\":\"little\"}],[\"label\",{\"type\":\"ndarray\",\"array\":[\"AMR\",\"EUR\",\"AMR\",\"EAS\",\"EUR\"],\"shape\":[5],\"dtype\":\"object\",\"order\":\"little\"}]]}}},\"view\":{\"type\":\"object\",\"name\":\"CDSView\",\"id\":\"p1041\",\"attributes\":{\"filter\":{\"type\":\"object\",\"name\":\"IntersectionFilter\",\"id\":\"p1040\",\"attributes\":{\"operands\":[{\"type\":\"object\",\"name\":\"GroupFilter\",\"id\":\"p1039\",\"attributes\":{\"column_name\":\"label\",\"group\":\"AMR\"}}]}}}},\"glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1046\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"type\":\"object\",\"name\":\"CategoricalColorMapper\",\"id\":\"p1038\",\"attributes\":{\"palette\":[\"#1f77b4\",\"#ff7f0e\",\"#2ca02c\",\"#d62728\",\"#9467bd\",\"#8c564b\",\"#e377c2\",\"#7f7f7f\",\"#bcbd22\",\"#17becf\"],\"factors\":[\"EUR\",\"AMR\",\"EAS\"]}}},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}}}},\"nonselection_glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1047\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"line_alpha\":{\"type\":\"value\",\"value\":0.1},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"fill_alpha\":{\"type\":\"value\",\"value\":0.1},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.1}}},\"muted_glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1048\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"line_alpha\":{\"type\":\"value\",\"value\":0.2},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"fill_alpha\":{\"type\":\"value\",\"value\":0.2},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.2}}}}},{\"type\":\"object\",\"name\":\"GlyphRenderer\",\"id\":\"p1062\",\"attributes\":{\"data_source\":{\"id\":\"p1035\"},\"view\":{\"type\":\"object\",\"name\":\"CDSView\",\"id\":\"p1054\",\"attributes\":{\"filter\":{\"type\":\"object\",\"name\":\"IntersectionFilter\",\"id\":\"p1053\",\"attributes\":{\"operands\":[{\"type\":\"object\",\"name\":\"GroupFilter\",\"id\":\"p1052\",\"attributes\":{\"column_name\":\"label\",\"group\":\"EAS\"}}]}}}},\"glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1059\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}}}},\"nonselection_glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1060\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"line_alpha\":{\"type\":\"value\",\"value\":0.1},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"fill_alpha\":{\"type\":\"value\",\"value\":0.1},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.1}}},\"muted_glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1061\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"line_alpha\":{\"type\":\"value\",\"value\":0.2},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"fill_alpha\":{\"type\":\"value\",\"value\":0.2},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.2}}}}},{\"type\":\"object\",\"name\":\"GlyphRenderer\",\"id\":\"p1075\",\"attributes\":{\"data_source\":{\"id\":\"p1035\"},\"view\":{\"type\":\"object\",\"name\":\"CDSView\",\"id\":\"p1067\",\"attributes\":{\"filter\":{\"type\":\"object\",\"name\":\"IntersectionFilter\",\"id\":\"p1066\",\"attributes\":{\"operands\":[{\"type\":\"object\",\"name\":\"GroupFilter\",\"id\":\"p1065\",\"attributes\":{\"column_name\":\"label\",\"group\":\"EUR\"}}]}}}},\"glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1072\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}}}},\"nonselection_glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1073\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"line_alpha\":{\"type\":\"value\",\"value\":0.1},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"fill_alpha\":{\"type\":\"value\",\"value\":0.1},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.1}}},\"muted_glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1074\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"line_alpha\":{\"type\":\"value\",\"value\":0.2},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"fill_alpha\":{\"type\":\"value\",\"value\":0.2},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1038\"}},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.2}}}}}],\"toolbar\":{\"type\":\"object\",\"name\":\"Toolbar\",\"id\":\"p1010\",\"attributes\":{\"tools\":[{\"type\":\"object\",\"name\":\"PanTool\",\"id\":\"p1023\"},{\"type\":\"object\",\"name\":\"WheelZoomTool\",\"id\":\"p1024\",\"attributes\":{\"renderers\":\"auto\"}},{\"type\":\"object\",\"name\":\"BoxZoomTool\",\"id\":\"p1025\",\"attributes\":{\"overlay\":{\"type\":\"object\",\"name\":\"BoxAnnotation\",\"id\":\"p1026\",\"attributes\":{\"syncable\":false,\"level\":\"overlay\",\"visible\":false,\"left\":{\"type\":\"number\",\"value\":\"nan\"},\"right\":{\"type\":\"number\",\"value\":\"nan\"},\"top\":{\"type\":\"number\",\"value\":\"nan\"},\"bottom\":{\"type\":\"number\",\"value\":\"nan\"},\"left_units\":\"canvas\",\"right_units\":\"canvas\",\"top_units\":\"canvas\",\"bottom_units\":\"canvas\",\"line_color\":\"black\",\"line_alpha\":1.0,\"line_width\":2,\"line_dash\":[4,4],\"fill_color\":\"lightgrey\",\"fill_alpha\":0.5}}}},{\"type\":\"object\",\"name\":\"SaveTool\",\"id\":\"p1031\"},{\"type\":\"object\",\"name\":\"ResetTool\",\"id\":\"p1032\"},{\"type\":\"object\",\"name\":\"HelpTool\",\"id\":\"p1033\"},{\"type\":\"object\",\"name\":\"HoverTool\",\"id\":\"p1034\",\"attributes\":{\"renderers\":\"auto\",\"tooltips\":[[\"x\",\"@x\"],[\"y\",\"@y\"]]}}]}},\"left\":[{\"type\":\"object\",\"name\":\"LinearAxis\",\"id\":\"p1018\",\"attributes\":{\"ticker\":{\"type\":\"object\",\"name\":\"BasicTicker\",\"id\":\"p1019\",\"attributes\":{\"mantissas\":[1,2,5]}},\"formatter\":{\"type\":\"object\",\"name\":\"BasicTickFormatter\",\"id\":\"p1020\"},\"axis_label\":\"PC2\",\"major_label_policy\":{\"type\":\"object\",\"name\":\"AllLabels\",\"id\":\"p1021\"}}},{\"type\":\"object\",\"name\":\"Legend\",\"id\":\"p1081\",\"attributes\":{\"click_policy\":\"hide\",\"items\":[{\"type\":\"object\",\"name\":\"LegendItem\",\"id\":\"p1078\",\"attributes\":{\"label\":{\"type\":\"value\",\"value\":\"AMR\"},\"renderers\":[{\"id\":\"p1049\"}]}},{\"type\":\"object\",\"name\":\"LegendItem\",\"id\":\"p1079\",\"attributes\":{\"label\":{\"type\":\"value\",\"value\":\"EAS\"},\"renderers\":[{\"id\":\"p1062\"}]}},{\"type\":\"object\",\"name\":\"LegendItem\",\"id\":\"p1080\",\"attributes\":{\"label\":{\"type\":\"value\",\"value\":\"EUR\"},\"renderers\":[{\"id\":\"p1075\"}]}}]}},{\"type\":\"object\",\"name\":\"ColorBar\",\"id\":\"p1082\",\"attributes\":{\"visible\":false,\"major_label_policy\":{\"type\":\"object\",\"name\":\"NoOverlap\",\"id\":\"p1083\"},\"color_mapper\":{\"id\":\"p1038\"}}}],\"below\":[{\"type\":\"object\",\"name\":\"LinearAxis\",\"id\":\"p1013\",\"attributes\":{\"ticker\":{\"type\":\"object\",\"name\":\"BasicTicker\",\"id\":\"p1014\",\"attributes\":{\"mantissas\":[1,2,5]}},\"formatter\":{\"type\":\"object\",\"name\":\"BasicTickFormatter\",\"id\":\"p1015\"},\"axis_label\":\"PC1\",\"major_label_policy\":{\"type\":\"object\",\"name\":\"AllLabels\",\"id\":\"p1016\"}}}],\"center\":[{\"type\":\"object\",\"name\":\"Grid\",\"id\":\"p1017\",\"attributes\":{\"axis\":{\"id\":\"p1013\"}}},{\"type\":\"object\",\"name\":\"Grid\",\"id\":\"p1022\",\"attributes\":{\"dimension\":1,\"axis\":{\"id\":\"p1018\"}}}]}}]}};\n const render_items = [{\"docid\":\"73a1dfc6-515f-48ef-98b1-6f59889e5062\",\"roots\":{\"p1001\":\"f725dce6-7696-42f2-bb79-81f286875e0b\"},\"root_ids\":[\"p1001\"]}];\n root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n }\n if (root.Bokeh !== undefined) {\n embed_document(root);\n } else {\n let attempts = 0;\n const timer = setInterval(function(root) {\n if (root.Bokeh !== undefined) {\n clearInterval(timer);\n embed_document(root);\n } else {\n attempts++;\n if (attempts > 100) {\n clearInterval(timer);\n console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n }\n }\n }, 10, root)\n }\n})(window);", - "application/vnd.bokehjs_exec.v0+json": "" - }, - "metadata": { - "application/vnd.bokehjs_exec.v0+json": { - "id": "p1001" - } - }, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "# plot ancestry\n", "p = hl.plot.scatter(mt.pca_scores[0],\n", @@ -425,37 +301,19 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "f86783b0-340c-4e23-a7c7-fd1b47127ad5", "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2024-01-04 21:14:29.256 Hail: INFO: downloading 1KG VCF ...\n", - " Source: https://storage.googleapis.com/hail-tutorial/1kg.vcf.bgz\n", - "2024-01-04 21:14:31.451 Hail: INFO: importing VCF and writing to matrix table...\n", - "2024-01-04 21:14:32.133 Hail: INFO: scanning VCF for sortedness...\n", - "2024-01-04 21:14:35.146 Hail: INFO: Coerced sorted VCF - no additional import work to do\n", - "2024-01-04 21:14:40.409 Hail: INFO: wrote matrix table with 10879 rows and 284 columns in 16 partitions to data/1kg.mt\n", - "2024-01-04 21:14:40.510 Hail: INFO: downloading 1KG annotations ...\n", - " Source: https://storage.googleapis.com/hail-tutorial/1kg_annotations.txt\n", - "2024-01-04 21:14:41.056 Hail: INFO: downloading Ensembl gene annotations ...\n", - " Source: https://storage.googleapis.com/hail-tutorial/ensembl_gene_annotations.txt\n", - "2024-01-04 21:14:42.298 Hail: INFO: Done!\n" - ] - } - ], + "outputs": [], "source": [ "hl.utils.get_1kg('data/')" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "d22599dd-007c-4fc4-b246-948ba059c680", "metadata": { "tags": [] @@ -467,27 +325,12 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "id": "ddd6d2ba-0609-4752-99dc-47c33b5b2d23", "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2024-01-04 21:25:13.742 Hail: INFO: Reading table to impute column types\n", - "2024-01-04 21:25:14.248 Hail: INFO: Finished type imputation\n", - " Loading field 'Sample' as type str (imputed)\n", - " Loading field 'Population' as type str (imputed)\n", - " Loading field 'SuperPopulation' as type str (imputed)\n", - " Loading field 'isFemale' as type bool (imputed)\n", - " Loading field 'PurpleHair' as type bool (imputed)\n", - " Loading field 'CaffeineConsumption' as type int32 (imputed)\n" - ] - } - ], + "outputs": [], "source": [ "table_1kg = (hl.import_table('data/1kg_annotations.txt', impute=True)\n", " .key_by('Sample'))\n", @@ -496,20 +339,12 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "72024335-3aa6-4039-8c00-0b3c212f8a27", "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Filtering 3.60% entries out of downstream analysis.\n" - ] - } - ], + "outputs": [], "source": [ "# Quality control\n", "mt_1kg = hl.sample_qc(mt_1kg)\n", @@ -547,42 +382,12 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "id": "61b2ad7e-517f-4a3b-afc5-5952c00d4952", "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Stage 221:==================================> (10 + 4) / 16]\r" - ] - }, - { - "data": { - "text/html": [ - "\n", - "
\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": "(function(root) {\n function embed_document(root) {\n const docs_json = {\"2c4bc5ec-810f-42fb-aa3c-79a3e0026f5f\":{\"version\":\"3.3.2\",\"title\":\"Bokeh Application\",\"roots\":[{\"type\":\"object\",\"name\":\"Figure\",\"id\":\"p1087\",\"attributes\":{\"width\":800,\"height\":800,\"x_range\":{\"type\":\"object\",\"name\":\"DataRange1d\",\"id\":\"p1088\"},\"y_range\":{\"type\":\"object\",\"name\":\"DataRange1d\",\"id\":\"p1089\"},\"x_scale\":{\"type\":\"object\",\"name\":\"LinearScale\",\"id\":\"p1097\"},\"y_scale\":{\"type\":\"object\",\"name\":\"LinearScale\",\"id\":\"p1098\"},\"title\":{\"type\":\"object\",\"name\":\"Title\",\"id\":\"p1090\",\"attributes\":{\"text\":\"PCA\"}},\"renderers\":[{\"type\":\"object\",\"name\":\"GlyphRenderer\",\"id\":\"p1135\",\"attributes\":{\"data_source\":{\"type\":\"object\",\"name\":\"ColumnDataSource\",\"id\":\"p1121\",\"attributes\":{\"selected\":{\"type\":\"object\",\"name\":\"Selection\",\"id\":\"p1122\",\"attributes\":{\"indices\":[],\"line_indices\":[]}},\"selection_policy\":{\"type\":\"object\",\"name\":\"UnionRenderers\",\"id\":\"p1123\"},\"data\":{\"type\":\"map\",\"entries\":[[\"index\",{\"type\":\"ndarray\",\"array\":{\"type\":\"bytes\",\"data\":\"AAAAAAEAAAACAAAAAwAAAAQAAAAFAAAABgAAAAcAAAAIAAAACQAAAAoAAAALAAAADAAAAA0AAAAOAAAADwAAABAAAAARAAAAEgAAABMAAAAUAAAAFQAAABYAAAAXAAAAGAAAABkAAAAaAAAAGwAAABwAAAAdAAAAHgAAAB8AAAAgAAAAIQAAACIAAAAjAAAAJAAAACUAAAAmAAAAJwAAACgAAAApAAAAKgAAACsAAAAsAAAALQAAAC4AAAAvAAAAMAAAADEAAAAyAAAAMwAAADQAAAA1AAAANgAAADcAAAA4AAAAOQAAADoAAAA7AAAAPAAAAD0AAAA+AAAAPwAAAEAAAABBAAAAQgAAAEMAAABEAAAARQAAAEYAAABHAAAASAAAAEkAAABKAAAASwAAAEwAAABNAAAATgAAAE8AAABQAAAAUQAAAFIAAABTAAAAVAAAAFUAAABWAAAAVwAAAFgAAABZAAAAWgAAAFsAAABcAAAAXQAAAF4AAABfAAAAYAAAAGEAAABiAAAAYwAAAGQAAABlAAAAZgAAAGcAAABoAAAAaQAAAGoAAABrAAAAbAAAAG0AAABuAAAAbwAAAHAAAABxAAAAcgAAAHMAAAB0AAAAdQAAAHYAAAB3AAAAeAAAAHkAAAB6AAAAewAAAHwAAAB9AAAAfgAAAH8AAACAAAAAgQAAAIIAAACDAAAAhAAAAIUAAACGAAAAhwAAAIgAAACJAAAAigAAAIsAAACMAAAAjQAAAI4AAACPAAAAkAAAAJEAAACSAAAAkwAAAJQAAACVAAAAlgAAAJcAAACYAAAAmQAAAJoAAACbAAAAnAAAAJ0AAACeAAAAnwAAAKAAAAChAAAAogAAAKMAAACkAAAApQAAAKYAAACnAAAAqAAAAKkAAACqAAAAqwAAAKwAAACtAAAArgAAAK8AAACwAAAAsQAAALIAAACzAAAAtAAAALUAAAC2AAAAtwAAALgAAAC5AAAAugAAALsAAAC8AAAAvQAAAL4AAAC/AAAAwAAAAMEAAADCAAAAwwAAAMQAAADFAAAAxgAAAMcAAADIAAAAyQAAAMoAAADLAAAAzAAAAM0AAADOAAAAzwAAANAAAADRAAAA0gAAANMAAADUAAAA1QAAANYAAADXAAAA2AAAANkAAADaAAAA2wAAANwAAADdAAAA3gAAAN8AAADgAAAA4QAAAOIAAADjAAAA5AAAAOUAAADmAAAA5wAAAOgAAADpAAAA6gAAAOsAAADsAAAA7QAAAO4AAADvAAAA8AAAAPEAAADyAAAA8wAAAPQAAAD1AAAA9gAAAPcAAAD4AAAA\"},\"shape\":[249],\"dtype\":\"int32\",\"order\":\"little\"}],[\"x\",{\"type\":\"ndarray\",\"array\":{\"type\":\"bytes\",\"data\":\"h4HgrWMY0L/KAy8D+fTPv7PtPLjOOs+/8GR6Muv2zr+riJ13YenOvw6wsN0y5c6/PANUDvTwzr84GfKVvfjOv8d+o6q/qs6/ObqiJjCnzr9JipmpNpPOv7uBQex5X86/7vmIMR9ozr/5WRR3d3bOv6PKCUmXbc6/CazYbJQ9zr/z2yn+TTfOv7vp3RGVOs6/JhbPIVn8zb9Z+OvPI/TNv820IsS9zM2/+/9/nBLDzb+H1AuX4bnNv7lfoTnJtM2/gakxveuNzb/2Yx4ZIarNv+Eh1q9Dq82/AFnBHZWEzb8FHAJL1YrNv3Yzyxa5Us2/QvfeZ1Efzb/anUh2u0PNv1qFvuRYD82/zZYPN0Edzb+d7/hQCtLMv5VtR7yg7cy/VKdn2sGfzL9pPOhjXKTMvzoqlhwojcy/W/llGsKXzL/FFPnHBjPMvxwliyR3U8y/Q0OcgSUQzL8uFIlxah7Mv0bt9uPK/su/ZmVRSAfry7/EMpW/AfTLv72eW6fLzcu/cySQom6hy78jKthYTZjLvzS24b6Hn8u/JClpVslty7/nNlaYJHTLv7g1pbWWbMu/Pw3MHQg9y790nZkZ2APLv2eSi6h//sq/Gn4VlGfwyr/WHdu+L93Jv78q7yMogcm/ekxp2rD1yL9f+289P+PIv9ycAs/PtMa/QbhuTTkSxr8SKwglXWDFv+IDQrjm6sS/22zvVk/HxL+JA5IP1FzEvxIGLr3fx8O/RJ6Z02hkw78l4rt1kyzDv6r5huhNS8O/x9qSJ+o0w7+q/AN+zujCvxpLxapF88K/txOtJ67Twr/NqenOEXvCv8lY4y5ZScK/VO0vX4dfwr+ua9Bg0BHCvzCwLK8K7sG/8EtNbEeBwb/Uq8C8N4jBv+BV25iMTsG/epbdLpBfwb90RDbsE0DBv7vWwzAdOcG/gpTxL80Jwb8X3qwuijHBv0vcXMcA+cC/J0mWcZHtwL+udOoYM+vAv8qayzUX48C/ekn8GvbBwL/OHB1QG73Av4xyTnZMqcC/r1Y7gNDUwL9EZ+NHrbrAv5IUJ3940MC/vFQkt2SzwL+qkR1rHITAv+K4Tzy7oMC/bmRiweSOwL+pQJpptZHAv+hcWFW9ccC/1CnP9mxJwL9m2+gGQWHAv+3KHoPlX8C/vsm4azYuwL8Asy24oznAv3FfpVpmK8C/MYcsgrAowL+o3KJFgzPAv0/Z0mkMFcC/N0On6K0RwL8sk51AgxHAv6MmJs/SAsC/0RM8613Yv7/GOHKVYLW/v0yy2l5rsL+/LC8id89ov7+aIzyFO2W/v12sP1CCeb+/AfpFLo1Yv7+WK/R6/Su/v5Hx57BqOb+/WzDD1ilfv7/FhfTFzqW+v75tGWvaq76/Json9nPAvr+jvPnHeFW+v1id4jk9Sr6/AsOdKOhOvr9vR2cp0tq9v6m17YbwI76/9XcByDcbvr+Z9oD1zjO+vxf0CpRlNb6/uOx6bcEnvr85ZpsV77K9vz1QkKwUmb2/aoNCuKjTvb+H0MyVj869vxjguEpAqL2/uMLqdwCivb+FPVRPki29v7ttTJ0zWL2/EdiVGVD+vL8P+ZJBnLq8v4hHKVscD72/tdj8tEGLvL+jYP7Gp2y8v+TF5iTc1Lu/MBB13TDPu788DoPgHVi7v+2iTrLKhru/sWMcKfZAu79Z77hosdS6v4+kPMQQ2bq/ojdei9cVu78ccGvGBw67v+Rt0zyaurq/Jq8349z4ub/aVFwH6X25v5ATeEAP/ri/Uv2oEWizuL/fIpvpbIK4v2DnsJpv0bi/6X8A3gVLuL8BhN3jqPy3vz+Eza0xfbe/LLBPMzqdt79l5MEnqZ+3vwGuQhiSSLe/r4IdbBO/tr9ehCn2QS+2v6V0PTdZzrW/dvW8m4Ijtr88UkKocTa0v/bViBH+D7O/9erqzPJjsb96GlN7/Ryov7ipLS3d66W/T0wZeXjQkr+uLsZ3ytvOP56hwBW4cNE/Btq3DBYQ0j+r6g+N5q/VPzaEjcDQzNY/xqHQ6HHV1z8t+XrAd2vYP03dBOztRdo/DKkxVCmW2j83YUltjcPaP7WqN0ruGts/c1MThkJm2z8hbcQBLmPbP9xzMGkIANw/VoifdFwv3D/VngLZ40vcP3aL0cgsPtw/x0U6FU1e3D+YxAk4aHncP2tI/PrGj9w/gq/QXsqU3D81F6EDlZvcP8CLNUhjvdw/P9bUDNvW3D9I+dugge7cP7pKyDRD/Nw/GyK+CL8L3T8ae8shVgDdP5bazZ7NC90/zcqHtoE93T8trIVEEFLdP4IsKMRmc90/Nih29EN13T8teBkCuHTdP4381JZ6hN0/yTNF69Wi3T/jFDbb6bvdP0gS/CeBsd0/SugQHPK43T9QevsBlK7dP8Fiy1M9w90/NSm7TSns3T//s1Os7PHdP9uL5PeM8N0/UHS3Janv3T9rJCE8rPrdPynXt8KoCN4/jQUFlqD/3T9ImqmabPzdPyp4slqv/N0/3Crk3VdL3j+dA9pO51TePyRMk1SxY94/6Nzpr5xa3j8PJEoYz5PeP6ZQFA2Mpt4/lagfr2XF3j/35+FT0cveP/aIOLaf094/MwrgSizu3j+lfXuC9wjfP8htghcRB98//15qSCM/3z8bLkllW1ffPzsMDG1M3t8/\"},\"shape\":[249],\"dtype\":\"float64\",\"order\":\"little\"}],[\"y\",{\"type\":\"ndarray\",\"array\":{\"type\":\"bytes\",\"data\":\"d7LZF8r40T9HJ5QTSLPSP/WDK6pT8tE/UPj8zADp0T8c70YGWFzSP2RNklOTv9I/+2/DjxsE0z9Iualfi6fTP73CXqbrrdI/gOlsih4d0j9X26AvOCfSP1O0dFEZRdE/JedfGTa10T8Y+63gi3fTP9rlXW/ujNQ/Cmrw6wtK0j8LCbfjztDSPzuq8jhzUtM/r2WJk1GV0T8mKSyehCXSP6q+iBAr99A/4KaqdFnJ0T/6XbqJ3vTRP2atlnpAJdI/rh/ZW15N0T8qC5qVeGfRP/bcbXCBgdE/WWSz7Jr40j81Ep3XmRPTP8T5f2yepdI/vxr+3kX00D+u1yy1533RP+P/j4F+EtM/fzFnY4N30z9gxsiqdCfSP6fbJyvR5dI/PnPLVL090T+zcrVI6/7RP+pE2b/ettI/Y2HhCg3y0j9SVqDCnw3SP5280Lz6B9M/f8TWjcE00j9+MhbXyVPSP4NNhnlJqtM/kT9oUgyEsT+nCGBYUZbNP6zC+kca/dE/XnqH665i0T/DB/q01KXSP1tRTo8wk9Q/CQQk2eYq0j9J/73FFvLSP1C5MGKzSdM/2LAIHQ/T0j/+D7r2c/XRP/IFenXxdNE/8tMPCofp0j/DBBngJV7TPyKaW2iRS9E/3+9XDtourT+u01KBWGjTPwm7xqzd35E/+OY1HZcjrb+0BmlQftarv08NYg0+y7O/1G8Xb2yRur81Of7T+fuyv0xBPFpQzau/nEqX1KjTvb+sxLj4yFe7v+U1+xVQAbi/59yoikFgsL9iUcNeAFq3v7xqxIbeQra/FBsn2CgDpL9N8gEkmHuiv9a4njSIfLu/9mJQAPP7q79NN4E3il3Av/25AbBAcsK/JAseQuQXx7/2ftsvmwG5v2zHi15xQ9C/jfA5FdO0v79LrBLBFUixvzokx2MgL9O/5UyzfX+K0b+oSONr6Q7Gv0nQ2rEX+cy/pR/Sv5S3wL+Ww5wc/Oi7v/pIB8jrVLu/TMtONmWs07+iK/Tl19bHv9chFXCyhMW/9wQiriXwvr/FXQOu0pOwv3RNmZTX1Ka/jd5x8CuIpb/Af79sYBLUv8y1tdQac8S/tuMS2hEKvr+en9qAfIy7v9LFI6MJKM+/PB3vIF3ewb9oq1jPrhnAv02WAklc97q/NbQ2yKYEz79vTRJ8CXjFvyt4sVVNTb2/RVJmDsrcvL+cSiS1MG+7v7b856Moube/qJYwfrDf0r9rG2fUumvSv2nq5Ib6lbq/s45g2VYok78zmwSv3cTSv/Wf/L3I49G/50e+7Mvmv78qfDZQML29v56UuiVqZLa/cyIazDb10b8uMXNRSezGvxRaMuxmsrq/ZPLE9lVYtL/6mO2nMELUv9SOnEWJWsC/2hA4A+yspb+aP3KQg17Rv1YT0vPFSLq/hfBNadQgq79BbEAR+vzTv7Hq64IeY9K/O9yu+yJJ0b8meaG41YC9v91wDBcW9ry/bPEV0ME3sr9xlwazGAHNvz/WVagiBMy/3lkhACNHxr9687Zy3DbEv9dpG6KZyKK/xaTJWrbJjb9NqkcUVoXSvxArM02EdNK/jhtyZ9Vx1L8BGH//HLjTv86T9NFdZbm/ZF74rz/jwb9sqPcRLGa8vz+AjorWptK/Ez8SJIfa0b+r+ojDwJ3TvzQHPta+IdO/np7VVw5Q0r8HF+qWBhvUvx9ZzE4RZ9K/33m/NDlE0r9GVu84Hsa+vxtk8T7596A/Gov2yW3U0r9AOj4uStzRv211dwkOWNG/spk+a8Ws0r9KVfR7lZDQv+/TigIV482/o/wYVKM1079Odke4+bfRvzpbgPvQotK/DmC9QUcSzr8eCWVWZN+/v8BzRasTh8e/b9xctQoMx78igE0+Q6vSv9GgZUnIkdK/Rj/TX4Id0b/tBhNO32rSv4AsoJPRAs6/a/fX3+y6yr/dUYk9/rHKvwjsW/4qhsC/2nMgDjGjw7+3rD2sN4mkv+SNYkAdh5G/sauOFI7sn7/mLwolJSR/P3lj5q/CQ0a/o1kYsHenjD+bw1gswaGQP34qYERkCK0/tDGMCuEylz9SxjgXAfSmP1XxjviwDbA/KPA11Y/umz/xD+2SpgimP7nazXh1Cqo/10O9GtpmqT9cjAwQ2HytP93vmkLxlq8/4dCkolhMqD+TZzCRPX6wP88eBgdqb7A/SXMz5RNfsT9JOZedenShP0brwN9FmbE/4HD5uZMfpD/hb+kkGqeeP5Dc8fuCSrA/BQW6fGcFqj/BzhWHjaOqP3SlT4FXIbE/H8X+UI88rD++A8WJnPiyP2ReHxAOwKc/r0z3fOBdqD8cQipemVi1P/M+hpgAPrA/B6byVa9ArD+NIihlIuGoP8ryss0OFK4/400j8VCtsD/iJPz3E0exPxTfbkSqsK8/Zt7JMTqKrT/W48uW6QWzPwoE0iEMW7M/iDiIAiD0uT8LmDSRUwGuP7dmt7J+q64/Z2/2N8mgsj+keN1p1LG0P2W51Z898bg/xknyU5yNrD8I1NEilQewP3n1eFLaSbE/hwHHAJ0ctT9OfGGsd2qvPxohF3PHk6s/naA1zS9PqD+aIw/YTXKqP9UO9u2ST7E/UhcyH2y2sj8BRYOKRXKpPzplKCR6BrA/LV8s+Wo/sT8aIxkUq22zP6XqGNGrnLI/\"},\"shape\":[249],\"dtype\":\"float64\",\"order\":\"little\"}],[\"label\",{\"type\":\"ndarray\",\"array\":[\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"AMR\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"EAS\",\"AMR\",\"EAS\",\"AMR\",\"AMR\",\"AMR\",\"SAS\",\"SAS\",\"SAS\",\"SAS\",\"SAS\",\"SAS\",\"SAS\",\"SAS\",\"SAS\",\"AMR\",\"SAS\",\"SAS\",\"SAS\",\"SAS\",\"SAS\",\"SAS\",\"SAS\",\"SAS\",\"EUR\",\"SAS\",\"SAS\",\"EUR\",\"EUR\",\"AMR\",\"AMR\",\"SAS\",\"SAS\",\"SAS\",\"EUR\",\"SAS\",\"SAS\",\"SAS\",\"AMR\",\"SAS\",\"SAS\",\"EUR\",\"SAS\",\"SAS\",\"SAS\",\"EUR\",\"SAS\",\"SAS\",\"SAS\",\"EUR\",\"SAS\",\"SAS\",\"SAS\",\"SAS\",\"AMR\",\"EUR\",\"EUR\",\"SAS\",\"AMR\",\"EUR\",\"EUR\",\"SAS\",\"SAS\",\"SAS\",\"EUR\",\"SAS\",\"SAS\",\"SAS\",\"EUR\",\"SAS\",\"SAS\",\"EUR\",\"AMR\",\"AMR\",\"EUR\",\"EUR\",\"EUR\",\"SAS\",\"SAS\",\"AMR\",\"AMR\",\"AMR\",\"SAS\",\"SAS\",\"AMR\",\"AMR\",\"EUR\",\"EUR\",\"EUR\",\"EUR\",\"SAS\",\"SAS\",\"SAS\",\"EUR\",\"EUR\",\"EUR\",\"EUR\",\"EUR\",\"EUR\",\"EUR\",\"EUR\",\"SAS\",\"AMR\",\"EUR\",\"EUR\",\"EUR\",\"EUR\",\"AMR\",\"AMR\",\"EUR\",\"EUR\",\"EUR\",\"AMR\",\"AMR\",\"AMR\",\"AMR\",\"EUR\",\"EUR\",\"AMR\",\"EUR\",\"AMR\",\"AMR\",\"AMR\",\"AMR\",\"AMR\",\"AFR\",\"AMR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\",\"AFR\"],\"shape\":[249],\"dtype\":\"object\",\"order\":\"little\"}]]}}},\"view\":{\"type\":\"object\",\"name\":\"CDSView\",\"id\":\"p1127\",\"attributes\":{\"filter\":{\"type\":\"object\",\"name\":\"IntersectionFilter\",\"id\":\"p1126\",\"attributes\":{\"operands\":[{\"type\":\"object\",\"name\":\"GroupFilter\",\"id\":\"p1125\",\"attributes\":{\"column_name\":\"label\",\"group\":\"AFR\"}}]}}}},\"glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1132\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"type\":\"object\",\"name\":\"CategoricalColorMapper\",\"id\":\"p1124\",\"attributes\":{\"palette\":[\"#1f77b4\",\"#ff7f0e\",\"#2ca02c\",\"#d62728\",\"#9467bd\",\"#8c564b\",\"#e377c2\",\"#7f7f7f\",\"#bcbd22\",\"#17becf\"],\"factors\":[\"EAS\",\"AMR\",\"EUR\",\"SAS\",\"AFR\"]}}},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}}}},\"nonselection_glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1133\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"line_alpha\":{\"type\":\"value\",\"value\":0.1},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"fill_alpha\":{\"type\":\"value\",\"value\":0.1},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.1}}},\"muted_glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1134\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"line_alpha\":{\"type\":\"value\",\"value\":0.2},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"fill_alpha\":{\"type\":\"value\",\"value\":0.2},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.2}}}}},{\"type\":\"object\",\"name\":\"GlyphRenderer\",\"id\":\"p1148\",\"attributes\":{\"data_source\":{\"id\":\"p1121\"},\"view\":{\"type\":\"object\",\"name\":\"CDSView\",\"id\":\"p1140\",\"attributes\":{\"filter\":{\"type\":\"object\",\"name\":\"IntersectionFilter\",\"id\":\"p1139\",\"attributes\":{\"operands\":[{\"type\":\"object\",\"name\":\"GroupFilter\",\"id\":\"p1138\",\"attributes\":{\"column_name\":\"label\",\"group\":\"AMR\"}}]}}}},\"glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1145\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}}}},\"nonselection_glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1146\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"line_alpha\":{\"type\":\"value\",\"value\":0.1},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"fill_alpha\":{\"type\":\"value\",\"value\":0.1},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.1}}},\"muted_glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1147\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"line_alpha\":{\"type\":\"value\",\"value\":0.2},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"fill_alpha\":{\"type\":\"value\",\"value\":0.2},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.2}}}}},{\"type\":\"object\",\"name\":\"GlyphRenderer\",\"id\":\"p1161\",\"attributes\":{\"data_source\":{\"id\":\"p1121\"},\"view\":{\"type\":\"object\",\"name\":\"CDSView\",\"id\":\"p1153\",\"attributes\":{\"filter\":{\"type\":\"object\",\"name\":\"IntersectionFilter\",\"id\":\"p1152\",\"attributes\":{\"operands\":[{\"type\":\"object\",\"name\":\"GroupFilter\",\"id\":\"p1151\",\"attributes\":{\"column_name\":\"label\",\"group\":\"EAS\"}}]}}}},\"glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1158\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}}}},\"nonselection_glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1159\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"line_alpha\":{\"type\":\"value\",\"value\":0.1},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"fill_alpha\":{\"type\":\"value\",\"value\":0.1},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.1}}},\"muted_glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1160\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"line_alpha\":{\"type\":\"value\",\"value\":0.2},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"fill_alpha\":{\"type\":\"value\",\"value\":0.2},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.2}}}}},{\"type\":\"object\",\"name\":\"GlyphRenderer\",\"id\":\"p1174\",\"attributes\":{\"data_source\":{\"id\":\"p1121\"},\"view\":{\"type\":\"object\",\"name\":\"CDSView\",\"id\":\"p1166\",\"attributes\":{\"filter\":{\"type\":\"object\",\"name\":\"IntersectionFilter\",\"id\":\"p1165\",\"attributes\":{\"operands\":[{\"type\":\"object\",\"name\":\"GroupFilter\",\"id\":\"p1164\",\"attributes\":{\"column_name\":\"label\",\"group\":\"EUR\"}}]}}}},\"glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1171\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}}}},\"nonselection_glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1172\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"line_alpha\":{\"type\":\"value\",\"value\":0.1},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"fill_alpha\":{\"type\":\"value\",\"value\":0.1},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.1}}},\"muted_glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1173\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"line_alpha\":{\"type\":\"value\",\"value\":0.2},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"fill_alpha\":{\"type\":\"value\",\"value\":0.2},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.2}}}}},{\"type\":\"object\",\"name\":\"GlyphRenderer\",\"id\":\"p1187\",\"attributes\":{\"data_source\":{\"id\":\"p1121\"},\"view\":{\"type\":\"object\",\"name\":\"CDSView\",\"id\":\"p1179\",\"attributes\":{\"filter\":{\"type\":\"object\",\"name\":\"IntersectionFilter\",\"id\":\"p1178\",\"attributes\":{\"operands\":[{\"type\":\"object\",\"name\":\"GroupFilter\",\"id\":\"p1177\",\"attributes\":{\"column_name\":\"label\",\"group\":\"SAS\"}}]}}}},\"glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1184\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}}}},\"nonselection_glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1185\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"line_alpha\":{\"type\":\"value\",\"value\":0.1},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"fill_alpha\":{\"type\":\"value\",\"value\":0.1},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.1}}},\"muted_glyph\":{\"type\":\"object\",\"name\":\"Circle\",\"id\":\"p1186\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"line_alpha\":{\"type\":\"value\",\"value\":0.2},\"fill_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"fill_alpha\":{\"type\":\"value\",\"value\":0.2},\"hatch_color\":{\"type\":\"field\",\"field\":\"label\",\"transform\":{\"id\":\"p1124\"}},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.2}}}}}],\"toolbar\":{\"type\":\"object\",\"name\":\"Toolbar\",\"id\":\"p1096\",\"attributes\":{\"tools\":[{\"type\":\"object\",\"name\":\"PanTool\",\"id\":\"p1109\"},{\"type\":\"object\",\"name\":\"WheelZoomTool\",\"id\":\"p1110\",\"attributes\":{\"renderers\":\"auto\"}},{\"type\":\"object\",\"name\":\"BoxZoomTool\",\"id\":\"p1111\",\"attributes\":{\"overlay\":{\"type\":\"object\",\"name\":\"BoxAnnotation\",\"id\":\"p1112\",\"attributes\":{\"syncable\":false,\"level\":\"overlay\",\"visible\":false,\"left\":{\"type\":\"number\",\"value\":\"nan\"},\"right\":{\"type\":\"number\",\"value\":\"nan\"},\"top\":{\"type\":\"number\",\"value\":\"nan\"},\"bottom\":{\"type\":\"number\",\"value\":\"nan\"},\"left_units\":\"canvas\",\"right_units\":\"canvas\",\"top_units\":\"canvas\",\"bottom_units\":\"canvas\",\"line_color\":\"black\",\"line_alpha\":1.0,\"line_width\":2,\"line_dash\":[4,4],\"fill_color\":\"lightgrey\",\"fill_alpha\":0.5}}}},{\"type\":\"object\",\"name\":\"SaveTool\",\"id\":\"p1117\"},{\"type\":\"object\",\"name\":\"ResetTool\",\"id\":\"p1118\"},{\"type\":\"object\",\"name\":\"HelpTool\",\"id\":\"p1119\"},{\"type\":\"object\",\"name\":\"HoverTool\",\"id\":\"p1120\",\"attributes\":{\"renderers\":\"auto\",\"tooltips\":[[\"x\",\"@x\"],[\"y\",\"@y\"]]}}]}},\"left\":[{\"type\":\"object\",\"name\":\"LinearAxis\",\"id\":\"p1104\",\"attributes\":{\"ticker\":{\"type\":\"object\",\"name\":\"BasicTicker\",\"id\":\"p1105\",\"attributes\":{\"mantissas\":[1,2,5]}},\"formatter\":{\"type\":\"object\",\"name\":\"BasicTickFormatter\",\"id\":\"p1106\"},\"axis_label\":\"PC2\",\"major_label_policy\":{\"type\":\"object\",\"name\":\"AllLabels\",\"id\":\"p1107\"}}},{\"type\":\"object\",\"name\":\"Legend\",\"id\":\"p1195\",\"attributes\":{\"click_policy\":\"hide\",\"items\":[{\"type\":\"object\",\"name\":\"LegendItem\",\"id\":\"p1190\",\"attributes\":{\"label\":{\"type\":\"value\",\"value\":\"AFR\"},\"renderers\":[{\"id\":\"p1135\"}]}},{\"type\":\"object\",\"name\":\"LegendItem\",\"id\":\"p1191\",\"attributes\":{\"label\":{\"type\":\"value\",\"value\":\"AMR\"},\"renderers\":[{\"id\":\"p1148\"}]}},{\"type\":\"object\",\"name\":\"LegendItem\",\"id\":\"p1192\",\"attributes\":{\"label\":{\"type\":\"value\",\"value\":\"EAS\"},\"renderers\":[{\"id\":\"p1161\"}]}},{\"type\":\"object\",\"name\":\"LegendItem\",\"id\":\"p1193\",\"attributes\":{\"label\":{\"type\":\"value\",\"value\":\"EUR\"},\"renderers\":[{\"id\":\"p1174\"}]}},{\"type\":\"object\",\"name\":\"LegendItem\",\"id\":\"p1194\",\"attributes\":{\"label\":{\"type\":\"value\",\"value\":\"SAS\"},\"renderers\":[{\"id\":\"p1187\"}]}}]}},{\"type\":\"object\",\"name\":\"ColorBar\",\"id\":\"p1196\",\"attributes\":{\"visible\":false,\"major_label_policy\":{\"type\":\"object\",\"name\":\"NoOverlap\",\"id\":\"p1197\"},\"color_mapper\":{\"id\":\"p1124\"}}}],\"below\":[{\"type\":\"object\",\"name\":\"LinearAxis\",\"id\":\"p1099\",\"attributes\":{\"ticker\":{\"type\":\"object\",\"name\":\"BasicTicker\",\"id\":\"p1100\",\"attributes\":{\"mantissas\":[1,2,5]}},\"formatter\":{\"type\":\"object\",\"name\":\"BasicTickFormatter\",\"id\":\"p1101\"},\"axis_label\":\"PC1\",\"major_label_policy\":{\"type\":\"object\",\"name\":\"AllLabels\",\"id\":\"p1102\"}}}],\"center\":[{\"type\":\"object\",\"name\":\"Grid\",\"id\":\"p1103\",\"attributes\":{\"axis\":{\"id\":\"p1099\"}}},{\"type\":\"object\",\"name\":\"Grid\",\"id\":\"p1108\",\"attributes\":{\"dimension\":1,\"axis\":{\"id\":\"p1104\"}}}]}}]}};\n const render_items = [{\"docid\":\"2c4bc5ec-810f-42fb-aa3c-79a3e0026f5f\",\"roots\":{\"p1087\":\"e436bca5-7b35-4012-bf85-0ce527df74ae\"},\"root_ids\":[\"p1087\"]}];\n root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n }\n if (root.Bokeh !== undefined) {\n embed_document(root);\n } else {\n let attempts = 0;\n const timer = setInterval(function(root) {\n if (root.Bokeh !== undefined) {\n clearInterval(timer);\n embed_document(root);\n } else {\n attempts++;\n if (attempts > 100) {\n clearInterval(timer);\n console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n }\n }\n }, 10, root)\n }\n})(window);", - "application/vnd.bokehjs_exec.v0+json": "" - }, - "metadata": { - "application/vnd.bokehjs_exec.v0+json": { - "id": "p1087" - } - }, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "p = hl.plot.scatter(mt_1kg.scores[0],\n", " mt_1kg.scores[1],\n", From 7282c962cad20d4aa626b6faca87d09e6eed9f62 Mon Sep 17 00:00:00 2001 From: Michael Harper Date: Wed, 10 Jan 2024 07:25:24 +1100 Subject: [PATCH 7/8] tidied up formatting --- .../analyst_tutorial/analyst_tutorial.md | 48 ++++++++++++------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/onboarding_documentation/technical_documentation/analyst_tutorial/analyst_tutorial.md b/onboarding_documentation/technical_documentation/analyst_tutorial/analyst_tutorial.md index 1bd1c52..f9e14c4 100644 --- a/onboarding_documentation/technical_documentation/analyst_tutorial/analyst_tutorial.md +++ b/onboarding_documentation/technical_documentation/analyst_tutorial/analyst_tutorial.md @@ -1,6 +1,6 @@ As an analyst you will be running analysis jobs via the `analysis-runner`, producing output, and then analysing the output. This tutorial will walk you through the process of running an analysis job, and then analysing the output. -In this tutorial, we will begin with raw fastq files and process them using the large-cohort pipeline. This pipeline performs several steps: +In this tutorial, we will begin with raw fastq files and process them using the `large cohort` pipeline. This pipeline performs several steps: 1. Aligns the reads to the reference genome. 2. Conducts quality control checks. @@ -8,9 +8,9 @@ In this tutorial, we will begin with raw fastq files and process them using the 4. Performs an ancestry analysis. 5. Plots the results of the ancestry analysis. -While the large-cohort pipeline includes additional stages, for the purpose of this tutorial, we will only execute up to and including the Ancestry stage, which carries out Principal Component Analysis (PCA). +While the `large cohort` pipeline includes additional stages, for the purpose of this tutorial, we will only execute up to and including the Ancestry stage, which carries out Principal Component Analysis (PCA). -For a description of what each stage of the `large-cohort` pipeline does, please see the `large-cohort` pipeline documentation [here](https://github.com/populationgenomics/production-pipelines#large-cohort-workflow) +For a description of what each stage of the `large cohort` pipeline does, please see the `large cohort` pipeline documentation [here](https://github.com/populationgenomics/production-pipelines#large-cohort-workflow) ## Table of Contents - [1. Getting test data](#getting-test-data) @@ -21,7 +21,9 @@ For a description of what each stage of the `large-cohort` pipeline does, please ## 1. Getting test data As an analyst, it's crucial to initially run our analysis jobs on test data before executing them on the actual data. The primary reason for this approach is cost efficiency. Large datasets can be incredibly expensive to process and debug. By using test data, we can estimate the time and cost of running the job on the full dataset, allowing us to optimise resources. Additionally, this process helps us ensure that the pipeline is functioning correctly and producing the expected output. While these checks can also be performed on the main dataset, using test data allows for quicker iterations and adjustments. It's also a good practice to run the analysis job on test data whenever the pipeline is updated, to verify that the updates haven't introduced any issues. -In this tutorial, we will be working with the bioheart project dataset. The BiohEART project aims to identify individuals at risk of atherosclerosis by detecting genetic markers in blood before the disease manifests. Our first step is to verify your access to the bioheart dataset. To do this, execute the following command: +In this tutorial, we will be working with the bioheart project dataset. The BiohEART project aims to identify individuals at risk of atherosclerosis by detecting genetic markers in blood before the disease manifests. + +Our first step is to verify your access to the bioheart dataset. To do this, execute the following command: ```bash gsutil ls gs://cpg-bioheart-main-upload/ @@ -41,7 +43,7 @@ For more information on the analysis-runner please check the repo [here](https:/ If you haven't done so already, clone the `metamist` repository to your local machine. Once you've done that, navigate to your `metamist` repository. Make sure you're on the `main` branch of `metamist`, then execute the following command from the CLI: -**TODO: We will need to set up more genomes in the actual bucket we're using** +**TODO: We will need to set up more genomes in the actual bucket we're using for downstream ancestry analysis to work** ```bash analysis-runner \ --dataset bioheart --description "populate bioheart test subset" --output-dir "bioheart-test" \ @@ -50,19 +52,25 @@ scripts/create_test_subset.py --project bioheart --samples XPG280371 XPG280389 X ``` **FOR REFERENCE: The above was taken from [this](https://centrepopgen.slack.com/archives/C03FA2M1MR9/p1700020527448029?thread_ts=1699935103.776929&cid=C03FA2M1MR9) Slack thread** +Note that `create_test_subset.py` script takes `samples` as an input and not `sequencingGroups`. `samples` start with `XPG` while `sequencingGroups` start with `CPG`. + ## 2. Writing a config file -Config files, written in TOML format, are used to specify the parameters of an analysis job for the large cohort pipeline. They contain all the necessary parameters to run the job, such as images, references, specific stages of the pipeline to run, and the input data. You can find a comprehensive description of config files and their usage in the `team-docs` repo [here](https://github.com/populationgenomics/team-docs/blob/13755bd51356b50ce11e6be78a76e53ed0a3ccb1/cpg_utils_config.md). +Now that we have test data available in the `bioheart-test` bucket, we can begin writing a config file. -Please note that while config files are essential for the large cohort pipeline, they are not required to start the `analysis-runner`, which is a wrapper that can be used independently. +Config files, written in TOML format, are used to specify the parameters of an analysis job for the `large cohort` pipeline. They contain all the necessary parameters to run the job, such as images, references, specific stages of the pipeline to run, and the input data. You can find a comprehensive description of config files and their usage in the `team-docs` repo [here](https://github.com/populationgenomics/team-docs/blob/13755bd51356b50ce11e6be78a76e53ed0a3ccb1/cpg_utils_config.md). -When running stages in the large cohort pipeline, there are set config files that are used, one of which is the defaults file you can see [here](https://github.com/populationgenomics/production-pipelines/blob/0bcf9775206f10ee91ac197c8c178f844ecad447/cpg_workflows/defaults.toml). It's important to understand that this defaults file does not cover all possible parameters. It provides a base set of parameters, but it may not include every parameter that could be relevant for your specific analysis. User-defined parameters in your specific config file can override these defaults as necessary. You can find the default config file for the large cohort pipeline [here](https://github.com/populationgenomics/production-pipelines/blob/main/configs/defaults/large_cohort.toml), where you can see a description of some of the parameters and their uses. +Please note that while config files are essential for the `large cohort` pipeline, they are not required to start the `analysis-runner`, which is a wrapper that can be used independently. -#### Guide: Write a config file +When running stages in the `large cohort` pipeline, there are set config files that are used, one of which is the defaults file you can see [here](https://github.com/populationgenomics/production-pipelines/blob/0bcf9775206f10ee91ac197c8c178f844ecad447/cpg_workflows/defaults.toml). It's important to understand that this defaults file does not cover all possible parameters. It provides a base set of parameters, but it may not include every parameter that could be relevant for your specific analysis. User-defined parameters in your specific config file can override these defaults as necessary. You can find the default config file for the `large cohort` pipeline [here](https://github.com/populationgenomics/production-pipelines/blob/main/configs/defaults/large_cohort.toml), where you can see a description of some of the parameters and their uses. + +#### Guide: Writing a config file If we want to run the `large cohort `pipeline on the `bioheart-test` dataset, we need to create a config file that is capable of doing this. Here's a step-by-step guide on how to do it: -Since the default config file serves as our base, we only need to define the parameters that deviate from these defaults. This includes parameters that are left undefined in the default file due to their dataset-specific nature. In this case, we need to specify the input dataset, the sequencing type, and the output version. We also need to specify the sequencing groups we want to run on. We can do this by creating a new config file specifying the following paramters in the `[workflow]` of the `toml` file: +Since the default config file serves as our base, we only need to define the parameters that deviate from these defaults. This includes parameters that are left undefined in the default file due to their dataset-specific nature. In this case, we need to specify the input dataset, the sequencing type, and the output version. We also have the flexibility to specify the sequencing groups we want to run on. + +We can do the above by creating a new config file specifying the following paramters in the `[workflow]` of the `toml` file: ```TOML [workflow] @@ -71,19 +79,19 @@ sequencing_type = 'genome' output_version = '1.0' # TODO: Do we want them to specify output_version? only_sgs = [] # to be used to demonstrate how to run on a subset of samples ``` -- `input_datasets`: This parameter specifies the datasets to be loaded as inputs for the analysis. If this parameter is not provided, the datasets will be determined automatically based on the input of `analysis-runner`. It's an array that can contain multiple dataset names e.g. `['bioheart', 'tob-wgs']`. +- `input_datasets`: This parameter specifies the datasets to be loaded as inputs for the analysis. If this parameter is not provided, the datasets will be determined automatically based on the input of `analysis-runner`. It's an array that can contain multiple dataset names e.g. `input_datasets = ['bioheart', 'tob-wgs']`. - `only_sgs`: This parameter is used to limit the analysis to specific `sequencingGroups`. If you want to run the analysis on a subset of sequencing groups, you can specify their IDs in this array. -- `sequencing_type`: This parameter is used to limit the data to a specific sequencing type. The default value is 'genome', but it can be changed to other sequencing types like 'exome' depending on the analysis. -- `output_version`: This parameter is used to suffix the location of the workflow outputs (get_workflow().prefix) with a specific string. By default, the hash of all input paths will be used. This can be useful for versioning your outputs, especially when you run the same analysis multiple times with different parameters or input data. +- `sequencing_type`: This parameter is used to limit the data to a specific sequencing type. The default value is 'genome', but it can be changed to other sequencing types like 'exome' depending on the analysis. The `bioheart` dataset is WGS, so we will use 'genome' as the value for this parameter. +- `output_version`: This parameter is used to suffix the location of the workflow outputs (`get_workflow().prefix`) with a specific string. By default, the hash of all input paths will be used, though we can adopt a 'version' approach that can be useful for versioning your outputs, especially when you run the same analysis multiple times with different parameters or input data. -- Please note that you do not need to change anything in the default `large cohort` config file. The changes you make should be in the new config file you're creating for the `bioheart-test` analysis. +- Please note that you do not need to change anything in the default `large cohort` config file. The changes you make should be in the new config file you're creating for the `bioheart-test` analysis that will eventually overwrite parameters in default files. Once the config file has been written, save it with an appropriate name (e.g. `bioheart-test.toml`). Instead of saving it on your local machine, push it to the `production-pipelines-configuration` repository, which is a private repository. In the following step, we will use the `analysis-runner` to submit the job and point it to the config file we have just pushed. ## 3. Submitting an analysis job Now that we have a config file, we can submit an analysis job. To do this we will use the `analysis-runner` tool. The `analysis-runner` tool is a command line tool that is used to submit analysis jobs to the cloud. -Remember, we are using the `large cohort` pipeline to conduct an analysis of several genomes in the `Bioheart` dataset. As the `large cohort` pipeline is a component of the `Production-pipelines` repository, we need to be within this repository to run it. This means that our current working directory should be the `Production-pipelines` repository when executing the pipeline. So our next step is to clone the `Production-pipelines` repository to our local machine. To do this, execute the following command in a directory of your choosing: +Remember, we are using the `large cohort` pipeline to conduct an analysis of several genomes in the `bioheart` dataset. As the `large cohort` pipeline is a component of the `Production-pipelines` repository, we need to be within this repository to run it. This means that our current working directory should be the `Production-pipelines` repository when executing the pipeline. So our next step is to clone the `Production-pipelines` repository to our local machine. To do this, execute the following command in a directory of your choosing: ```bash git clone --recurse-submodules git@github.com:populationgenomics/production-pipelines.git @@ -123,16 +131,20 @@ analysis-runner \ --config configs/defaults/large_cohort.toml \ --config path/to/you/config/config_filename.toml \ --image australia-southeast1-docker.pkg.dev/cpg-common/images/cpg_workflows:latest \ -main.py large_cohort +main.py large_cohort # python script to be execusted plus arguments (i.e. pipeline name) ``` - The output of the `analysis-runner` command is link to the Hail Batch driver job. Something like this: ``` Submitting analysis-runner@ for dataset "bioheart" Request submitted successfully: https://batch.hail.populationgenomics.org.au/batches/ ``` -This driver job is responsible for setting up all the subsequent jobs to run the analysis. When this driver job is complete, a link to another Hail Batch service will be provided. This is where all the individual jobs of the pipeline will be listed as well as their status. Once this secondary Batch is completed, the output will be available in `bioheart-test`. +This driver job is responsible for setting up all the subsequent jobs to run the analysis. When this driver job is complete, a link to another Hail Batch service will be provided at the bottom of the `Logs`. It will look something like this: +``` +Submitted batch , see https://batch.hail.populationgenomics.org.au/batches/ +``` + +This is where all the individual jobs of the pipeline will be listed as well as their status. Once this secondary Batch is completed, the output will be available in `bioheart-test`. ## 4. Analysing the output From 06eb98034f6b20d46c0b78276c595887d927ce0c Mon Sep 17 00:00:00 2001 From: Michael Harper Date: Wed, 10 Jan 2024 07:37:50 +1100 Subject: [PATCH 8/8] slight typo fixed --- .../analyst_tutorial/Final_analyst_tutorial.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onboarding_documentation/technical_documentation/analyst_tutorial/Final_analyst_tutorial.ipynb b/onboarding_documentation/technical_documentation/analyst_tutorial/Final_analyst_tutorial.ipynb index 1f9a272..0612ca4 100644 --- a/onboarding_documentation/technical_documentation/analyst_tutorial/Final_analyst_tutorial.ipynb +++ b/onboarding_documentation/technical_documentation/analyst_tutorial/Final_analyst_tutorial.ipynb @@ -290,7 +290,7 @@ "source": [ "## Conducting PCA on Public Dataset\n", "\n", - "In this section, we will be conducting a Principal Component Analysis (PCA) using Hail on the publicly available Human Genome Diversity Project - 1000 Genomes Project (HGDP-1KG) dataset. As we've seen, PCA ican be used to identify and visualise genetic variation between individuals. It can help us understand the genetic structure of our dataset and identify potential population stratification, which is crucial for many genetic analyses.\n", + "In this section, we will be conducting a Principal Component Analysis (PCA) using Hail on the publicly available Human Genome Diversity Project - 1000 Genomes Project (HGDP-1KG) dataset. As we've seen, PCA can be used to identify and visualise genetic variation between individuals. It can help us understand the genetic structure of our dataset and identify potential population stratification, which is crucial for many genetic analyses.\n", "\n", "The HGDP-1KG dataset is a comprehensive collection of human genetic variation from globally diverse populations. It combines data from two significant projects: the Human Genome Diversity Project (HGDP) and the 1000 Genomes Project (1KG). This dataset is readily accessible through Hail's interface as it is hosted by `gnomAD`.\n", "\n",