diff --git a/pyproject.toml b/pyproject.toml index 69e9694e..2fde6dac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,8 +30,10 @@ jaxcpu = [ dev = [ "build", "flake8", + "google-genai", "isort", "mypy>=1.0.0", + "pydantic", "pytest", "regex", "toml", @@ -59,8 +61,10 @@ budoux = ["models/*.json", "skip_nodes.json", "py.typed"] dev = [ "build", "flake8", + "google-genai", "isort", "mypy>=1.0.0", + "pydantic", "pytest", "regex", "toml", diff --git a/scripts/synthesize_samples.py b/scripts/synthesize_samples.py new file mode 100644 index 00000000..83b4d1b3 --- /dev/null +++ b/scripts/synthesize_samples.py @@ -0,0 +1,381 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Agentic candidate sentence synthesis and pre-filtering utility. + +Implements a compact 3-agent structure dedicated to synthesizing natural +training sentences and repairing segmentation discrepancies: + 1. Intent & Repro Agent: Extracts target break boundaries and confirms live + bug reproduction against current BudouX models. + 2. Example Generator Agent: Synthesizes high-density realistic sentences + containing the targeted phrases or break expressions across any language. + 3. Base Parser Overlay & Linguistic Expert Agent: Maps surrounding sentence + breaks strictly to existing BudouX model behavior while accurately pruning + awkward linguistic expressions. +""" + +import argparse +import json +import os +import re +import sys +from typing import Any, List, Optional + +# Module hack to allow importing budoux from repository root when run from CLI +LIB_PATH = os.path.join(os.path.dirname(__file__), '..') +sys.path.insert(0, os.path.abspath(LIB_PATH)) + +import requests # noqa: E402 +from google import genai # noqa: E402 +from pydantic import BaseModel, Field # noqa: E402 + +import budoux # noqa: E402 + +SEP = budoux.utils.SEP + +# ============================================================================== +# Structured Pydantic Models (Contractual Type Boundaries) +# ============================================================================== + + +class IntentTarget(BaseModel): + """Target bug remediation boundaries identified by Intent Understanding.""" + target_phrase: str = Field(..., description="Raw target phrase string.") + expected_split: str = Field(..., + description="Expected string separated with ▁.") + is_positive: bool = Field( + ..., + description="True if forced break across slash, False if cohesive word.") + is_reproducible: bool = Field( + default=True, + description="Whether current runtime model exhibits the reported bug.") + + +class CandidateBatch(BaseModel): + """Container for oversampled candidate sentences generated by the Example Generator Agent.""" + candidates: List[str] + + +class ExpertEvaluation(BaseModel): + """Evaluation assessment and local boundary polish for a candidate sentence.""" + sentence: str = Field(..., + description="Candidate sentence string under audit.") + is_fluent: bool = Field( + ..., description="True if candidate can be polished/retained naturally.") + corrected_sentence: str = Field( + "", + description="The polished candidate string with proper native boundary splits applied." + ) + rejection_reason: str = Field("", description="Explanation if rejected.") + + +class EvaluationBatch(BaseModel): + """Container for linguistic expert audit decisions.""" + evaluations: List[ExpertEvaluation] + + +# ============================================================================== +# Agent Utilities & Transformations +# ============================================================================== + + +def parse_direct_input(input_str: str) -> IntentTarget: + """Parses a direct --input parameter into an IntentTarget representation.""" + normalized = input_str.strip().replace(" / ", SEP).replace("/", SEP) + is_positive = SEP in normalized + raw = normalized.replace(SEP, "") + return IntentTarget( + target_phrase=raw, + expected_split=normalized, + is_positive=is_positive, + is_reproducible=True, + ) + + +def fetch_github_issue_context(issue_id: str, + requests_mod: Any = requests) -> str: + """Safely retrieves public GitHub issue description content.""" + issue_num = re.sub(r"[^\d]", "", str(issue_id)) + if not issue_num: + raise ValueError(f"Invalid issue parameter value: {issue_id}") + url = f"https://api.github.com/repos/google/budoux/issues/{issue_num}" + response = requests_mod.get(url, timeout=10) + response.raise_for_status() + data = response.json() + raw_body = str(data.get("body") or data.get("title") or "") + return raw_body.strip() + + +def verify_bug_reproduction(target: IntentTarget, + parser: budoux.Parser) -> bool: + """Tests whether the live parser currently reproduces the reported breaking defect. + + Returns True if the current model segments the string improperly (bug confirmed). + Returns False if the current model already achieves expected_split. + """ + clean_text = target.target_phrase.replace(SEP, "") + current_splits = parser.parse(clean_text) + actual_output = SEP.join(current_splits) + return actual_output != target.expected_split + + +def align_to_base_parser_splits( + candidates: List[str], + target: IntentTarget, + parser: budoux.Parser, +) -> List[str]: + """Aligns candidate boundaries outside the target phrase to base parser splits. + + Args: + candidates: List of raw candidate sentence strings (`List[str]`). + target: IntentTarget parameter declaring `target_phrase` and `expected_split`. + parser: Loaded `budoux.Parser` instance (`ja.json`). + + Returns: + List of valid candidate strings aligned to base model segmentations outside the target interval. + """ + surviving: List[str] = [] + raw_target = target.target_phrase.replace(SEP, "") + if not raw_target: + return surviving + + # Build the regex pattern once outside the loop to locate the target phrase in the parsed string + char_patterns = [re.escape(char) for char in raw_target] + target_pattern = re.compile(f"[{SEP}]*".join(char_patterns)) + + for sentence in candidates: + if target.expected_split not in sentence: + continue # Filter: Discard if the targeted split sequence is missing + + # Align: Parse sentence with base model to get default production splits + clean_sentence = sentence.replace(SEP, "") + base_parsed = SEP.join(parser.parse(clean_sentence)) + + # Overlay: Replace the target phrase region in base splits with the corrected split + aligned_sentence = target_pattern.sub( + lambda m: target.expected_split, base_parsed, count=1) + surviving.append(aligned_sentence) + + return surviving + + +# ============================================================================== +# Agent Roles +# ============================================================================== + + +def extract_intent_target( + issue_text: str, + client: genai.Client, + model_name: str = "gemini-3.1-flash-lite", +) -> IntentTarget: + """Intent Understanding Agent: diagnoses target bug parameters from issue text.""" + prompt = f"""You are an expert BudouX segmentation analyst. +Examine this GitHub bug issue description: +--- +{issue_text} +--- +Determine the primary problematic phrase and its desired canonical split formatted with '▁'. +If a phrase was mistakenly split internally and should stay unbroken (e.g., 'もはや'), return target_phrase='もはや', expected_split='もはや', is_positive=False. +If a phrase needed a boundary across separate words that was missing or shifted (e.g., 'いよいよ/はじまる'), format expected_split inserting '▁' at the intended break location and set is_positive=True. +""" + config = genai.types.GenerateContentConfig( + response_mime_type="application/json", + response_schema=IntentTarget, + temperature=0.1, + ) + resp = client.models.generate_content( + model=model_name, contents=prompt, config=config) + return IntentTarget.model_validate_json(resp.text or "") + + +def generate_oversample_candidates( + target: IntentTarget, + client: genai.Client, + num_candidates: int = 30, + lang: str = "ja", + model_name: str = "gemini-3.1-flash-lite", +) -> List[str]: + """Example Generator Agent: synthesizes diverse natural sentences cleanly across target lang.""" + prompt = f"""You are a skilled corpus synthesizer for target language code '{lang}'. +Generate exactly {num_candidates} highly natural, realistic sentences across language '{lang}' where the phrase '{target.expected_split}' appears cleanly within distinct realistic contexts (e.g., news headlines, casual speech, essays, formal letters). + +CRITICAL CONSTRAINTS: +1. Every candidate MUST contain the exact substring '{target.expected_split}' strictly as formatted. +2. Maintain natural grammatical structures and standard Bunsetsu word boundaries across language '{lang}'. +""" + config = genai.types.GenerateContentConfig( + response_mime_type="application/json", + response_schema=CandidateBatch, + temperature=0.4, + ) + resp = client.models.generate_content( + model=model_name, contents=prompt, config=config) + batch = CandidateBatch.model_validate_json(resp.text or "") + return batch.candidates + + +def prune_linguistic_anomalies( + candidates: List[str], + client: genai.Client, + max_keep: int = 15, + lang: str = "ja", + model_name: str = "gemini-3.1-flash-lite", +) -> List[str]: + """Linguistic Expert Agent: polishes boundary alignment across candidates and audits fluency.""" + if not candidates: + return [] + + lines_summary = [f"{i}. {s}" for i, s in enumerate(candidates)] + prompt = f"""You are an expert linguistic polisher and boundary arbiter for language '{lang}'. +Review these candidate training sentences containing existing boundary splits marked by '▁': +{chr(10).join(lines_summary)} + +Your critical mission across three synthesized layers (existing base parser behavior, intent understanding, and linguistic fluency): +1. Evaluate whether the sentence is natural and well-structured across target language '{lang}' (is_fluent). Immediately reject (is_fluent=False) malformed syntax or broken words. +2. For fluent sentences (is_fluent=True), perform precise boundary polishing and return the perfected string in 'corrected_sentence': + - Keep quotation framing clean without forcing splits directly after opening brackets or before closing quotes (e.g., maintain '「こんにちは」と▁...' without cutting around quotation marks). + - If an independent vocabulary word or bunsetsu was improperly fused due to base model deficiencies (e.g., '彼の▁才能はもはや誰にも▁...' where words like 'もはや' lack surrounding breaks), restore clean natural bunsetsu gaps around the word ('彼の▁才能は▁もはや▁誰にも▁...'). + - Never alter or remove the canonical splits established inside the targeted phrase itself. +""" + config = genai.types.GenerateContentConfig( + response_mime_type="application/json", + response_schema=EvaluationBatch, + temperature=0.1, + ) + resp = client.models.generate_content( + model=model_name, contents=prompt, config=config) + eval_batch = EvaluationBatch.model_validate_json(resp.text or "") + + kept: List[str] = [] + for e in eval_batch.evaluations: + if e.is_fluent: + kept.append(e.corrected_sentence.strip() or e.sentence) + if not kept: + kept = candidates + return kept[:max_keep] + + +# ============================================================================== +# Executive Driver & CLI Setup +# ============================================================================== + + +def run_agentic_synthesis_pipeline( + input_str: Optional[str] = None, + issue_id: Optional[str] = None, + num_candidates: int = 30, + max_keep: int = 15, + lang: str = "ja", + outfile: str = "staging_raw.txt", + client: Optional[genai.Client] = None, + parser: Optional[budoux.Parser] = None, +) -> List[str]: + """Orchestrates Intent Understanding, Oversample Generation, and Expert Pruning.""" + if not parser: + model_path = os.path.join( + os.path.dirname(__file__), "..", "budoux", "models", f"{lang}.json") + with open(model_path, "r", encoding="utf-8") as f: + parser = budoux.Parser(json.load(f)) + + if not client and (issue_id or num_candidates > 0): + api_key = os.environ.get("GEMINI_API_KEY", "").strip() + if not api_key: + raise RuntimeError( + "GEMINI_API_KEY environment variable not set or empty.") + client = genai.Client(api_key=api_key) + + # Step 1: Intent & Bug Verification + if input_str: + target = parse_direct_input(input_str) + elif issue_id: + assert client is not None + context = fetch_github_issue_context(issue_id) + target = extract_intent_target(context, client) + + bug_exists = verify_bug_reproduction(target, parser) + target.is_reproducible = bug_exists + if not bug_exists: + print( + f"[Intent Agent] No bug found: model already parses '{target.expected_split}' correctly." + ) + + # Step 2: Generator Oversampling + assert client is not None + raw_candidates = generate_oversample_candidates( + target, client, num_candidates=num_candidates, lang=lang) + print( + f"[Generator Agent] Synthesized {len(raw_candidates)} candidate variations for lang '{lang}'." + ) + + # Step 3: Base Parser Overlay & Linguistic Expert Pruning + aligned_candidates = align_to_base_parser_splits(raw_candidates, target, + parser) + final_candidates = prune_linguistic_anomalies( + aligned_candidates, client, max_keep=max_keep, lang=lang) + print( + f"[Expert Agent] Pruned to {len(final_candidates)} high-confidence rows.") + + output_lines = final_candidates + if outfile: + with open(outfile, "w", encoding="utf-8") as f: + f.write("\n".join(output_lines) + "\n") + print(f"[Staging] Saved {len(output_lines)} lines directly to {outfile}.") + + return output_lines + + +def build_parser() -> argparse.ArgumentParser: + """Constructs command-line flags for agentic candidate sample synthesis.""" + p = argparse.ArgumentParser(description="Agentic candidate sample synthesis.") + group = p.add_mutually_exclusive_group(required=True) + group.add_argument( + "--input", "-i", type=str, help="Target string (e.g. 'いよいよ/はじまる')") + group.add_argument( + "--issue", type=str, help="GitHub bug report number or URL ID") + p.add_argument( + "--lang", + type=str, + default="ja", + help="Target model language (default: ja)") + p.add_argument( + "--num-samples", + "-n", + type=int, + default=30, + help="Initial generation count") + p.add_argument( + "--max-keep", type=int, default=15, help="Pruned final row target") + p.add_argument( + "--output", + "-o", + type=str, + default="staging_raw.txt", + help="Staging destination file") + return p + + +def main() -> None: + args = build_parser().parse_args() + run_agentic_synthesis_pipeline( + input_str=args.input, + issue_id=args.issue, + num_candidates=args.num_samples, + max_keep=args.max_keep, + lang=args.lang, + outfile=args.output, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/tests/test_synthesize_samples.py b/scripts/tests/test_synthesize_samples.py new file mode 100644 index 00000000..e814d556 --- /dev/null +++ b/scripts/tests/test_synthesize_samples.py @@ -0,0 +1,168 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test suite for agentic training data synthesis.""" + +import os +import sys +import tempfile +import unittest +from typing import Any +from unittest.mock import MagicMock, patch + +# Module hack to allow importing scripts and budoux from workspace root +LIB_PATH = os.path.join(os.path.dirname(__file__), '..', '..') +sys.path.insert(0, os.path.abspath(LIB_PATH)) + +import budoux # noqa: E402 +from scripts import synthesize_samples # noqa: E402 + +SEP = budoux.utils.SEP # Canonical character separator '▁' + + +class TestAlignToBaseParserSplits(unittest.TestCase): + + def test_align_to_base_parser_splits_negative_phrase(self) -> None: + target = synthesize_samples.IntentTarget( + target_phrase="もはや", + expected_split="もはや", + is_positive=False, + is_reproducible=True) + mock_parser = MagicMock(spec=budoux.Parser) + mock_parser.parse.return_value = ["彼の", "才能は", "もは", "や", "誰にも", "超えられない。"] + + cands = [f"彼の{SEP}才能は{SEP}もはや{SEP}誰にも超えられない。"] + result = synthesize_samples.align_to_base_parser_splits( + cands, target, mock_parser) + + self.assertEqual(len(result), 1) + expected_sentence = f"彼の{SEP}才能は{SEP}もはや{SEP}誰にも{SEP}超えられない。" + self.assertEqual(result[0], expected_sentence) + + def test_align_to_base_parser_splits_positive_phrase(self) -> None: + target = synthesize_samples.IntentTarget( + target_phrase="いよいよはじまる", + expected_split=f"いよいよ{SEP}はじまる", + is_positive=True, + is_reproducible=True, + ) + mock_parser = MagicMock(spec=budoux.Parser) + mock_parser.parse.return_value = ["いよいよは", "じまる", "決戦だ。"] + + cands = ["いよいよ▁はじまる決戦だ。"] + result = synthesize_samples.align_to_base_parser_splits( + cands, target, mock_parser) + + self.assertEqual(len(result), 1) + expected_sentence = f"いよいよ{SEP}はじまる{SEP}決戦だ。" + self.assertEqual(result[0], expected_sentence) + + def test_align_to_base_parser_splits_discards_missing_target(self) -> None: + target = synthesize_samples.IntentTarget( + target_phrase="もはや", + expected_split="もはや", + is_positive=False, + is_reproducible=True) + mock_parser = MagicMock(spec=budoux.Parser) + cands = ["関係のない完全に異なる文章です。"] + result = synthesize_samples.align_to_base_parser_splits( + cands, target, mock_parser) + self.assertEqual(len(result), 0) + + +class TestParseDirectInput(unittest.TestCase): + + def test_parse_direct_input_positive(self) -> None: + target = synthesize_samples.parse_direct_input("いよいよ/はじまる") + self.assertEqual(target.target_phrase, "いよいよはじまる") + self.assertEqual(target.expected_split, f"いよいよ{SEP}はじまる") + self.assertTrue(target.is_positive) + self.assertTrue(target.is_reproducible) + + def test_parse_direct_input_negative(self) -> None: + target = synthesize_samples.parse_direct_input("もはや") + self.assertEqual(target.target_phrase, "もはや") + self.assertEqual(target.expected_split, "もはや") + self.assertFalse(target.is_positive) + self.assertTrue(target.is_reproducible) + + +class TestVerifyBugReproduction(unittest.TestCase): + + def test_verify_bug_reproduction_detects_bug(self) -> None: + target = synthesize_samples.IntentTarget( + target_phrase="いよいよはじまる", + expected_split=f"いよいよ{SEP}はじまる", + is_positive=True, + ) + mock_parser = MagicMock(spec=budoux.Parser) + mock_parser.parse.return_value = ["いよいよは", "じまる"] + + self.assertTrue( + synthesize_samples.verify_bug_reproduction(target, mock_parser)) + mock_parser.parse.assert_called_once_with("いよいよはじまる") + + def test_verify_bug_reproduction_no_bug(self) -> None: + target = synthesize_samples.IntentTarget( + target_phrase="もはや", expected_split="もはや", is_positive=False) + mock_parser = MagicMock(spec=budoux.Parser) + mock_parser.parse.return_value = ["もはや"] + self.assertFalse( + synthesize_samples.verify_bug_reproduction(target, mock_parser)) + + +class TestRunAgenticSynthesisPipeline(unittest.TestCase): + + def test_run_pipeline_raises_without_env_or_client(self) -> None: + mock_parser = MagicMock(spec=budoux.Parser) + with patch.dict(os.environ, {"GEMINI_API_KEY": ""}, clear=True): + with self.assertRaises(RuntimeError): + synthesize_samples.run_agentic_synthesis_pipeline( + input_str="いよいよ/はじまる", client=None, parser=mock_parser) + + @patch("scripts.synthesize_samples.generate_oversample_candidates") + @patch("scripts.synthesize_samples.prune_linguistic_anomalies") + def test_run_agentic_synthesis_pipeline_end_to_end( + self, mock_prune: Any, mock_generate: Any) -> None: + mock_client = MagicMock() + mock_parser = MagicMock(spec=budoux.Parser) + mock_parser.parse.return_value = ["いよいよ", "はじまる", "決戦だ。"] + + raw_sentence = f"いよいよ{SEP}はじまる決戦だ。" + mock_generate.return_value = [raw_sentence] + expected_overlay = f"いよいよ{SEP}はじまる{SEP}決戦だ。" + mock_prune.return_value = [expected_overlay] + + outfile = os.path.join(self.test_dir, "staging_ja.txt") + lines = synthesize_samples.run_agentic_synthesis_pipeline( + input_str="いよいよ/はじまる", + num_candidates=5, + max_keep=3, + lang="ja", + outfile=outfile, + client=mock_client, + parser=mock_parser, + ) + self.assertEqual(lines, [expected_overlay]) + self.assertTrue(os.path.exists(outfile)) + + def setUp(self) -> None: + self.temp_dir_obj = tempfile.TemporaryDirectory() + self.test_dir = self.temp_dir_obj.name + + def tearDown(self) -> None: + self.temp_dir_obj.cleanup() + + +if __name__ == "__main__": + unittest.main()