Skip to content

Type-check notebooks in CI and fix typing/library usage issues #74

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
darthtrevino merged 3 commits into from
May 4, 2026
Merged

Type-check notebooks in CI and fix typing/library usage issues #74

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bc61a2d
add checks for notebooks
darthtrevino May 1, 2026
6ae5a63
restore ruff ignore
darthtrevino May 1, 2026
1a69bdf
add semversioner
darthtrevino May 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .semversioner/next-release/patch-20260501174720978257.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"type": "patch",
"description": "Add type-checking to Jupyter Notebooks, update Pyright"
}
3 changes: 3 additions & 0 deletions benchmark_qed/autoe/utils/stats.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -550,6 +550,9 @@ def _apply_correction_and_build_result(
p_values = [c[3] for c in comparisons_raw]

_, corrected_pvals, _, _ = multipletests(p_values, alpha=alpha, method=correction)
if corrected_pvals is None:
msg = f"Invalid correction method: {correction}"
raise ValueError(msg)

comparisons = [
PairwiseComparison(
Expand Down
59 changes: 38 additions & 21 deletions docs/notebooks/assertion_gen.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
"metadata": {},
"outputs": [],
"source": [
"import asyncio\n",
"import logging\n",
"import os\n",
"\n",
Expand Down Expand Up @@ -173,13 +174,16 @@
"metadata": {},
"outputs": [],
"source": [
"from graphrag_storage.file_storage import FileStorage\n",
"\n",
"from benchmark_qed.autoq.question_gen.data_questions.assertion_gen.local_claim_assertion_gen import (\n",
" LocalClaimAssertionGenerator,\n",
")\n",
"\n",
"# Load existing data-local questions from disk\n",
"existing_local_questions = load_questions(\n",
" f\"{OUTPUT_QUESTIONS_PATH}/data_local_questions/selected_questions.json\"\n",
"local_question_storage = FileStorage(f\"{OUTPUT_QUESTIONS_PATH}/data_local_questions/\")\n",
"existing_local_questions = asyncio.run(\n",
" load_questions(local_question_storage, \"selected_questions.json\")\n",
")\n",
"\n",
"print(f\"Loaded {len(existing_local_questions)} existing data-local questions\")\n",
Expand Down Expand Up @@ -209,25 +213,29 @@
")\n",
"\n",
"# Generate assertions for all questions with claims\n",
"await local_assertion_generator.agenerate_assertions_for_questions(\n",
" questions_with_claims\n",
"asyncio.run(\n",
" local_assertion_generator.agenerate_assertions_for_questions(questions_with_claims)\n",
")\n",
"\n",
"# Combine back with questions that had no claims\n",
"updated_local_questions = questions_with_claims + questions_without_claims\n",
"\n",
"# Save updated questions with assertions\n",
"save_questions(\n",
" updated_local_questions,\n",
" f\"{OUTPUT_QUESTIONS_PATH}/data_local_questions/\",\n",
" \"selected_questions_with_assertions\",\n",
"asyncio.run(\n",
" save_questions(\n",
" updated_local_questions,\n",
" local_question_storage,\n",
" \"selected_questions_with_assertions\",\n",
" )\n",
")\n",
"\n",
"# Show summary\n",
"print(\"\\n=== SUMMARY ===\")\n",
"print(f\"Processed {len(questions_with_claims)} questions with claims\")\n",
"total_assertions = sum(\n",
" len(q.attributes.get(\"assertions\", [])) for q in questions_with_claims\n",
" len(q.attributes.get(\"assertions\", []))\n",
" for q in questions_with_claims\n",
" if q.attributes is not None\n",
")\n",
"print(f\"Total assertions generated: {total_assertions}\")\n",
"print(\n",
Expand Down Expand Up @@ -366,8 +374,9 @@
")\n",
"\n",
"# Load existing data-global questions from disk\n",
"existing_global_questions = load_questions(\n",
" f\"{OUTPUT_QUESTIONS_PATH}/data_global_questions/selected_questions.json\"\n",
"global_question_storage = FileStorage(f\"{OUTPUT_QUESTIONS_PATH}/data_global_questions/\")\n",
"existing_global_questions = asyncio.run(\n",
" load_questions(global_question_storage, \"selected_questions.json\")\n",
")\n",
"\n",
"print(f\"Loaded {len(existing_global_questions)} existing data-global questions\")\n",
Expand Down Expand Up @@ -406,28 +415,34 @@
")\n",
"\n",
"# Generate assertions for ALL questions with claims\n",
"await global_assertion_generator.agenerate_assertions_for_questions(\n",
" questions_with_claims\n",
"asyncio.run(\n",
" global_assertion_generator.agenerate_assertions_for_questions(questions_with_claims)\n",
")\n",
"\n",
"# Combine back with questions that had no claims\n",
"updated_global_questions = questions_with_claims + questions_without_claims\n",
"\n",
"# Save updated questions with assertions\n",
"save_questions(\n",
" updated_global_questions,\n",
" f\"{OUTPUT_QUESTIONS_PATH}/data_global_questions/\",\n",
" \"selected_questions_with_assertions\",\n",
"asyncio.run(\n",
" save_questions(\n",
" updated_global_questions,\n",
" global_question_storage,\n",
" \"selected_questions_with_assertions\",\n",
" )\n",
")\n",
"\n",
"# Show summary\n",
"print(\"\\n=== SUMMARY ===\")\n",
"print(f\"Processed {len(questions_with_claims)} questions with claims\")\n",
"total_assertions = sum(\n",
" len(q.attributes.get(\"assertions\", [])) for q in questions_with_claims\n",
" len(q.attributes.get(\"assertions\", []))\n",
" for q in questions_with_claims\n",
" if q.attributes is not None\n",
")\n",
"total_map_assertions = sum(\n",
" len(q.attributes.get(\"map_assertions\", [])) for q in questions_with_claims\n",
" len(q.attributes.get(\"map_assertions\", []))\n",
" for q in questions_with_claims\n",
" if q.attributes is not None\n",
")\n",
"print(f\"Total map assertions generated: {total_map_assertions}\")\n",
"print(f\"Total final assertions generated: {total_assertions}\")\n",
Expand All @@ -438,8 +453,10 @@
"# Show per-question breakdown\n",
"print(\"\\n=== PER-QUESTION BREAKDOWN ===\")\n",
"for i, q in enumerate(questions_with_claims, 1):\n",
" assertions = q.attributes.get(\"assertions\", [])\n",
" map_assertions = q.attributes.get(\"map_assertions\", [])\n",
" assertions = q.attributes.get(\"assertions\", []) if q.attributes is not None else []\n",
" map_assertions = (\n",
" q.attributes.get(\"map_assertions\", []) if q.attributes is not None else []\n",
" )\n",
" print(\n",
" f\"{i}. [{len(map_assertions)} map → {len(assertions)} final] {q.text[:60]}...\"\n",
" )"
Expand Down
41 changes: 25 additions & 16 deletions docs/notebooks/autod.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,11 @@
"metadata": {},
"outputs": [],
"source": [
"import asyncio\n",
"import logging\n",
"import os\n",
"\n",
"from graphrag_storage.file_storage import FileStorage\n",
"from pydantic import SecretStr\n",
"\n",
"from benchmark_qed.autod.data_processor.embedding import TextEmbedder\n",
Expand Down Expand Up @@ -101,14 +103,17 @@
"metadata": {},
"outputs": [],
"source": [
"documents = create_documents(\n",
" input_path=INPUT_DATA_PATH,\n",
" input_type=\"json\",\n",
" text_tag=TEXT_COLUMN,\n",
" metadata_tags=METADATA_COLUMNS,\n",
" encoding=JSON_ENCODING,\n",
"documents = asyncio.run(\n",
" create_documents(\n",
" input_path=INPUT_DATA_PATH,\n",
" input_type=\"json\",\n",
" text_tag=TEXT_COLUMN,\n",
" metadata_tags=METADATA_COLUMNS,\n",
" encoding=JSON_ENCODING,\n",
" )\n",
")\n",
"document_df = save_documents(documents, OUTPUT_DATA_PATH)\n",
"output_storage = FileStorage(OUTPUT_DATA_PATH)\n",
"document_df = asyncio.run(save_documents(documents, output_storage))\n",
"print(f\"Document count: {len(document_df)}\")\n",
"document_df.head()"
]
Expand Down Expand Up @@ -144,14 +149,16 @@
" )\n",
")\n",
"\n",
"text_units = await create_text_units(\n",
" documents=documents,\n",
" metadata_tags=METADATA_COLUMNS,\n",
" text_splitter=text_splitter,\n",
" text_embedder=text_embedder,\n",
" embed_text=True,\n",
"text_units = asyncio.run(\n",
" create_text_units(\n",
" documents=documents,\n",
" metadata_tags=METADATA_COLUMNS,\n",
" text_splitter=text_splitter,\n",
" text_embedder=text_embedder,\n",
" embed_text=True,\n",
" )\n",
")\n",
"text_unit_df = save_text_units(text_units, OUTPUT_DATA_PATH)\n",
"text_unit_df = asyncio.run(save_text_units(text_units, output_storage))\n",
"print(f\"Text unit count: {len(text_unit_df)}\")\n",
"text_unit_df.head()"
]
Expand Down Expand Up @@ -273,8 +280,10 @@
" concurrent_coroutines=32,\n",
")\n",
"\n",
"summary_result = await summarizer.asummarize(\n",
" text_units=sampled_text_units,\n",
"summary_result = asyncio.run(\n",
" summarizer.asummarize(\n",
" text_units=sampled_text_units,\n",
" )\n",
")\n",
"print(f\"Summary: {summary_result.summary}\")"
]
Expand Down
Loading
Loading