Vcf import

.env-sample

@@ -44,3 +44,33 @@ FRONTEND_PORT=3000
 #BACKEND_ROOT=http://localhost:8888 # use if running django on a different port
 
 
+####### VCF IMPORT SETTINGS (WIP) #######
+
+VCF_FILE=/Users/username/path/to/joint/file.vcf
+
+#   you can compare to a previous set of TSV output files and show hash differences per file
+#HASH_COMPARE=/Users/username/path/to/tsv_files
+
+
+## VARIOME joint vcf format example settings ##
+
+# include only these ranges per chromosome (eg: 22:27010000-27020000,X:2702000-2802000)
+RANGES=
+#   whether to keep 'chr' prefix in chromosome names
+OUT_CHR=True
+#   whether to use hyphen '-' in variant IDs instead of underscores '_'
+OUT_HYPHENS=True
+
+#   CADD phred score threshold (if greater than or equal to this number, counts as "damaging")
+CADD_DAMAGING_THRESHOLD=20
+#   fallback None value filler
+NA=.
+#   Default transcript source if unknown (E = Ensembl, R = Refseq)
+DEFAULT_TRANSCRIPT_SOURCE=E
+#   Path to dir containing severity table file
+INPUT_TSV_PATH=data/fixtures
+
+
+## IBVL joint vcf format example settings ##
+
+#CADD_DAMAGING_THRESHOLD=15 # verify ?

.gitignore

@@ -176,3 +176,5 @@ data/backups/*
 
 # because it depends on OS and is generated dynamically
 package-lock.json
+
+vcf_import/output

.vscode/launch.json

@@ -1,100 +1,143 @@
 {
-    // Use IntelliSense to learn about possible attributes.
-    // Hover to view descriptions of existing attributes.
-    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-    "version": "0.2.0",
-    "configurations": [
-        {
-            "name": "Debug mange.py import_bvl ",
-            "type": "debugpy",
-            "request": "launch",
-            "program": "${workspaceFolder}/manage.py",
-            "args": [
-                "import_bvl",
-                "--failfast"
-            ],
-            "django": true
-
-        },
-        {
-            "name": "Debug mange.py import_bvl (delete first)",
-            "type": "debugpy",
-            "request": "launch",
-            "program": "${workspaceFolder}/manage.py",
-            "args": [
-                "import_bvl",
-                "--failfast",
-                "--delete"
-            ],
-            "django": true
-
-        },
-        {
-            "name": "Debug mange.py import_bvl (update consequences only)",
-            "type": "debugpy",
-            "request": "launch",
-            "program": "${workspaceFolder}/manage.py",
-            "args": [
-                "import_bvl",
-                "--failfast",
-                "--no-severities",
-                "--no-genes",
-                "--no-variants",
-                "--no-transcripts",
-                "--no-snvs",
-                "--no-gvfs",
-                "--no-ggfs",
-                "--no-annotations",
-                "--no-vts",
-                "--no-progress"
-            ],
-            "django": true
-
-        },
-        {
-            "name": "Python Debugger: Django",
-            "type": "debugpy",
-            "request": "launch",
-            "args": [
-                "runserver"
-            ],
-            "django": true,
-            "autoStartBrowser": false,
-            "program": "${workspaceFolder}/manage.py"
-        }
-        ,{
-                    "name": "debug migrate",
-                    "type": "debugpy",
-                    "request": "launch",
-                    "python": "${workspaceFolder}/.venv/bin/python",
-                    "program": "${workspaceFolder}/manage.py",
-                    "args": [
-                        "migrate"
-                    ],
-                    "console": "integratedTerminal",
-                    "justMyCode": true,
-                    "env": {
-                        "PYTHONPATH": "${workspaceFolder}"
-                    },
-                    "cwd": "${workspaceFolder}"
-                },
-            {
-                    "name": "debug a migration",
-                    "type": "debugpy",
-                    "request": "launch",
-                    "python": "${workspaceFolder}/.venv/bin/python",
-                    "program": "${workspaceFolder}/manage.py",
-                    "args": [
-                        "migrate",
-                        "library_access",
-                        "0002_migrate_pghistory_to_auditlog"
-                    ],
-                    "console": "integratedTerminal",
-                    "justMyCode": true,
-                    "env": {
-                        "PYTHONPATH": "${workspaceFolder}"
-                    },
-                    "cwd": "${workspaceFolder}"
-                }
-        ]
-    }
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "import_bvl TSV ",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${workspaceFolder}/manage.py",
+      "args": [
+        "import_bvl",
+        "--failfast",
+        "--no-progress"
+      ],
+      "django": true
+    },
+    {
+      "name": "import_bvl TSV (delete first)",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${workspaceFolder}/manage.py",
+      "args": [
+        "import_bvl",
+        "--failfast",
+        "--no-progress",
+        "--delete"
+      ],
+      "django": true
+    },
+    {
+      "name": "import_bvl TSV (update consequences only)",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${workspaceFolder}/manage.py",
+      "args": [
+        "import_bvl",
+        "--failfast",
+        "--no-severities",
+        "--no-genes",
+        "--no-variants",
+        "--no-transcripts",
+        "--no-snvs",
+        "--no-gvfs",
+        "--no-ggfs",
+        "--no-annotations",
+        "--no-vts",
+        "--no-progress"
+      ],
+      "django": true
+    },
+    {
+      "name": "Import VCF (delete first)",
+      "type": "python",
+      "request": "launch",
+      "python": "${workspaceFolder}/.venv/bin/python",
+      "program": "${workspaceFolder}/manage.py",
+      "args": [
+        "import_bvl_vcf",
+        "--vcf",
+        "data/fixtures/vcf/variome.vcf",
+        "--delete",
+        "--no-progress"
+      ],
+      "console": "integratedTerminal",
+      "cwd": "${workspaceFolder}"
+    },
+    {
+      "name": "Import VCF",
+      "type": "python",
+      "request": "launch",
+      "python": "${workspaceFolder}/.venv/bin/python",
+      "program": "${workspaceFolder}/manage.py",
+      "args": [
+        "import_bvl_vcf",
+        "--vcf",
+        "data/fixtures/vcf/variome.vcf",
+        "--no-progress"
+      ],
+      "console": "integratedTerminal",
+      "cwd": "${workspaceFolder}"
+    },
+    {
+      "name": "Debug vcf import tests",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "pytest",
+      "args": [
+        "-s",
+        "--log-cli-level=DEBUG",
+        "vcf_import/VCF_test.py"
+      ],
+      "console": "integratedTerminal",
+      "justMyCode": false,
+    },
+    {
+      "name": "Python Debugger: Django",
+      "type": "debugpy",
+      "request": "launch",
+      "args": [
+        "runserver"
+      ],
+      "django": true,
+      "autoStartBrowser": false,
+      "program": "${workspaceFolder}/manage.py"
+    },
+    {
+      "name": "debug migrate",
+      "type": "debugpy",
+      "request": "launch",
+      "python": "${workspaceFolder}/.venv/bin/python",
+      "program": "${workspaceFolder}/manage.py",
+      "args": [
+        "migrate"
+      ],
+      "console": "integratedTerminal",
+      "justMyCode": true,
+      "env": {
+        "PYTHONPATH": "${workspaceFolder}"
+      },
+      "cwd": "${workspaceFolder}"
+    },
+    {
+      "name": "debug a migration",
+      "type": "debugpy",
+      "request": "launch",
+      "python": "${workspaceFolder}/.venv/bin/python",
+      "program": "${workspaceFolder}/manage.py",
+      "args": [
+        "migrate",
+        "library_access",
+        "0002_migrate_pghistory_to_auditlog"
+      ],
+      "console": "integratedTerminal",
+      "justMyCode": true,
+      "env": {
+        "PYTHONPATH": "${workspaceFolder}"
+      },
+      "cwd": "${workspaceFolder}"
+    }
+  ]
+}

README.md

@@ -146,6 +146,94 @@ Markdown format files in the /content folder are treated as full pages, availabl
 
 Markdown format files in /content/Home are individual, "hard-coded" page elements on the Home page, so you shouldn't add more or rename these.
 
+### config reference
+PUBLIC_BVL - opens the database for public access, as a demonstration app. For this to work, a shared user account must also be created ( username: `public_demo_user`, email: `public_demo_user@example.com` )
+
+
+# Importing from a VCF file
+
+Use the `import_bvl_vcf` management command to import data directly from a VCF file into the database.
+
+Mitochondrial tables are omitted because these are not part of the currently used VCF reference files.
+
+## Quickstart
+
+```
+python manage.py import_bvl_vcf --vcf /path/to/joint.vcf
+python manage.py create_indices
+```
+
+Run `python manage.py import_bvl_vcf --help` for a full list of options.
+
+## Key options
+
+| Option | Default | Description |
+|--------|---------|-------------|
+| `--vcf` | *(required)* | Path to the input VCF file (plain text or `.gz`) |
+| `--input-tsv-dir` | `data/fixtures` | Path to the dir containing severities.tsv |
+| `--na` | `.` | Value used to represent missing/null data |
+| `--out-chr` | enabled | Prefix chromosome names with `chr` |
+| `--out-hyphens` | enabled | Use hyphens in variant IDs (e.g. `1-100-A-G`) if false, uses underscores instead |
+| `--cadd-threshold` | `20` | cadd_intr will be classified as "Damaging" if cadd_score is greater than or equal to this number |
+| `--default-transcript-source` | `E` | Fallback transcript source when unknown (`E`=Ensembl, `R`=RefSeq) |
+| `--ranges` | *(all)* | Restrict processing to specific regions, e.g. `22:27010000-27020000,X:2702000-2802000` |
+| `--convert-to-tsv` | disabled | Write TSV files instead of importing into the database |
+| `--tsv-output-dir` | `data/vcf_output` | Directory for TSV output (requires `--convert-to-tsv`) |
+| `--hash-compare` | *(none)* | Directory of an existing TSV set to compare output hashes against (requires `--convert-to-tsv`) |
+| `--dry-run` / `-n` | disabled | Parse and validate without writing to the database |
+| `--delete` | disabled | Delete all existing data before importing |
+| `--no-genes`, `--no-variants`, etc. | *(all enabled)* | Skip specific tables |
+
+## Comparing output against a reference TSV set
+
+When `--convert-to-tsv` is active, pass `--hash-compare` to verify that the generated TSV files
+match a previously known-good set ( this is useful for development of this project, ensuring consistency with legacy tsv-producting tools):
+
+```
+python manage.py import_bvl_vcf \
+  --vcf /path/to/joint.vcf \
+  --convert-to-tsv \
+  --tsv-output-dir data/vcf_output \
+  --hash-compare data/reference_tsvs
+```
+
+Per-file hash differences are reported in the log output.
+
+## Running backend tests
+
+All backend tests (middleware and VCF import filters) live in `variome_backend/tests/`.
+
+First, install dev dependencies:
+
+```
+uv sync --dev
+```
+
+Run all backend tests once:
+
+```
+DB= uv run python manage.py test variome_backend.tests --verbosity=2
+```
+
+Tests run against an in-memory SQLite database (no Postgres connection needed). If the `DB` environment variable is set in your shell, prefix the command with `DB=` to clear it for the test run, or simply use the watch script below which handles this automatically.
+
+Run all backend tests and automatically re-run whenever a `.py` file changes (recommended during development):
+
+```
+bash run_backend_tests.sh
+```
+
+Or inline, without the script:
+
+```
+uv run watchmedo shell-command \
+    --patterns="*.py" \
+    --recursive \
+    --drop \
+    --command='uv run python manage.py test variome_backend.tests --verbosity=2' \
+    .
+```
+
 ### attributions
 
 example DNA image: https://commons.wikimedia.org/wiki/File:202104_Laboratory_instrument_dna.svg