diff --git a/pyproject.toml b/pyproject.toml index 69e9694e..b372e7e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,9 +29,11 @@ jaxcpu = [ ] dev = [ "build", + "google-genai", "flake8", "isort", "mypy>=1.0.0", + "pydantic", "pytest", "regex", "toml", @@ -58,9 +60,11 @@ budoux = ["models/*.json", "skip_nodes.json", "py.typed"] [dependency-groups] dev = [ "build", + "google-genai", "flake8", "isort", "mypy>=1.0.0", + "pydantic", "pytest", "regex", "toml", diff --git a/scripts/synthesize_samples.py b/scripts/synthesize_samples.py new file mode 100644 index 00000000..669b4d00 --- /dev/null +++ b/scripts/synthesize_samples.py @@ -0,0 +1,491 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Candidate sample synthesis utility using Gemini from GitHub issues.""" + +import argparse +import os +import sys +import typing + +import pydantic +import requests +from google import genai + +# module hack to allow importing budoux from parent directory +LIB_PATH = os.path.join(os.path.dirname(__file__), '..') +sys.path.insert(0, os.path.abspath(LIB_PATH)) + +import budoux # noqa: E402 + + +class Analysis(pydantic.BaseModel): + negative_phrases: typing.List[str] + positive_phrases: typing.List[str] + + +class NegativeSentenceGroup(pydantic.BaseModel): + phrase: str + sentences: typing.List[str] + + +class PositiveSentenceGroup(pydantic.BaseModel): + char1: str + char2: str + sentences: typing.List[str] + + +class SynthesisResponse(pydantic.BaseModel): + analysis: Analysis + negative_sentences: typing.List[NegativeSentenceGroup] + positive_sentences: typing.List[PositiveSentenceGroup] + + +def parse_issue(issue_id: str) -> str: + """Fetches and sanitizes a GitHub issue body. + + Args: + issue_id: The GitHub issue ID to fetch. + + Returns: + The sanitized issue body text. + """ + url = f'https://api.github.com/repos/google/budoux/issues/{issue_id}' + + try: + res = requests.get(url, timeout=10) + except requests.exceptions.RequestException as e: + sys.exit(f'Error: Failed to connect to GitHub API: {e}') + + if res.status_code != 200: + sys.exit( + f'Error: Failed to fetch GitHub issue #{issue_id} (status: {res.status_code})' + ) + body = res.json().get('body', '') + return typing.cast(str, body).strip() + + +def get_separator_indices(text_with_seps: str) -> typing.Set[int]: + """Computes the character boundary indices where separators are inserted. + + Args: + text_with_seps: String containing separator characters. + + Returns: + A set of character indices representing splits in the plain text. + """ + indices = set() + curr_idx = 0 + for char in text_with_seps: + if char == budoux.utils.SEP: + indices.add(curr_idx) + else: + curr_idx += 1 + return indices + + +def align_to_baseline_model( + clean_text: str, + baseline_breaks: typing.Set[int], + llm_breaks: typing.Set[int], + target_sequence: str, + split_offset: typing.Optional[int] = None, +) -> str: + """Aligns background phrase boundaries to baseline model predictions with target overrides. + + This function overrides baseline boundary splits within and around a target sequence: + 1. Forces target_sequence to be unbroken internally, except at split_offset if provided. + 2. Syncs the splits immediately before and after target_sequence with the LLM's choices + (llm_breaks), allowing context-dependent boundary splits (e.g. keeping particles attached). + 3. Preserves baseline breaks for all other background positions. + + Args: + clean_text: The sentence plain text without any separators. + baseline_breaks: Set of boundary indices predicted by the baseline model. + llm_breaks: Set of boundary indices chosen by the LLM in its raw output. + target_sequence: The exact substring in clean_text to apply overrides to. + split_offset: Offset within target_sequence where a split must be forced. + If None, the entire target_sequence is kept completely unsplit. + + Returns: + The reconstructed sentence containing aligned separator characters. + """ + predicted_breaks = set(baseline_breaks) + + # Apply overrides on the target sequence local neighborhood + idx = clean_text.find(target_sequence) + if idx != -1: + target_len = len(target_sequence) + # Clear any baseline splits inside the target sequence + for i in range(idx + 1, idx + target_len): + predicted_breaks.discard(i) + + # Force the split at the target offset if requested + if split_offset is not None: + predicted_breaks.add(idx + split_offset) + + # Sync splits immediately before and after the target sequence with the LLM's choices + if idx > 0: + if idx in llm_breaks: + predicted_breaks.add(idx) + else: + predicted_breaks.discard(idx) + + if idx + target_len < len(clean_text): + outer_end_idx = idx + target_len + if outer_end_idx in llm_breaks: + predicted_breaks.add(outer_end_idx) + else: + predicted_breaks.discard(outer_end_idx) + + # Reconstruct the sentence with separators inserted at the break indices + result = [] + for i, c in enumerate(clean_text): + if i in predicted_breaks and i > 0: + result.append(budoux.utils.SEP) + result.append(c) + return ''.join(result) + + +def generate_synthesis_prompt(issue_context: str, num_samples: int) -> str: + """Generates the LLM synthesis prompt based on the issue context and sample count. + + Args: + issue_context: Body of the GitHub issue. + num_samples: Number of candidates to generate. + + Returns: + The prompt string. + """ + return f"""You are an expert Japanese linguist training a BudouX segmenter model. +Analyze the following GitHub issue bug report: +{issue_context} + +Your task is to analyze the issue and identify: +1. "negative_phrases": List of phrases/compounds that were incorrectly split internally and should remain as a single unsplit unit (e.g. adverbs like "もはや" or compounds like "こんにちは"). +2. "positive_phrases": List of phrases containing a split separator "/" representing boundaries that were incorrectly kept unsplit and MUST be split (e.g. "いよいよ/はじまる" meaning a split is forced between "いよいよ" and "はじまる"). + +Then, generate natural Japanese training sentences for each: +- For each negative phrase (e.g. "もはや"): Generate {num_samples} natural Japanese sentences where the phrase is completely unsplit internally, but has splits before and after it. Mark the boundaries before and after with the separator "▁" (e.g. "もはや▁これまで" or "技術は▁もはや▁時代遅れ"). +- For each adjacent character transition within the negative phrases (for example, if the phrase is "もはや", the transitions are "も" and "は", and "は" and "や"): Generate {num_samples} natural Japanese sentences where the two characters appear adjacent but split across a semantic boundary (e.g. "私も▁はやく走る" or "いつも▁はれる"). Do NOT include the negative phrases themselves in these sentences. +- For each positive phrase containing a "/" boundary (e.g. "いよいよ/はじまる"): Generate {num_samples} natural Japanese sentences where the boundary is split. Mark the split with the separator "▁" (e.g. "いよいよ▁はじまる。"). +CRITICAL RULES FOR TRANSITION SENTENCES: +1. The two characters of the transition ('char1' and 'char2') MUST appear immediately adjacent as literal characters in the written text of the sentence. Do NOT confuse pronunciation or hiragana readings with written characters. For example, if the transition is 'も' and 'は', the character 'は' must literally appear in the Japanese text (e.g. 'もはっきり'); a kanji like '晴' pronounced 'は' is INVALID. +2. The split separator '▁' must be placed EXACTLY between 'char1' and 'char2' (e.g. '私も▁はやく'). +3. The split separator '▁' MUST represent a valid word or phrase boundary. You MUST NOT split a single word internally (e.g. do NOT split "ありえない" as "あ▁りえない", or "ありがたい" as "あ▁りがたい", or "ございます" as "ご▁ざいます"). The split must only occur between two separate words, particles, or clauses. +Format your output strictly as a JSON object matching this schema: +{{ + "analysis": {{ + "negative_phrases": ["もはや"], + "positive_phrases": ["いよいよ/はじまる"] + }}, + "negative_sentences": [ + {{ + "phrase": "もはや", + "sentences": [ + "もはや▁これまでだ。", + "技術は▁もはや▁時代遅れだ。" + ] + }} + ], + "positive_sentences": [ + {{ + "char1": "も", + "char2": "は", + "sentences": [ + "私も▁はやく走りたい。", + "いつも▁はれる。" + ] + }}, + {{ + "char1": "は", + "char2": "や", + "sentences": [ + "これは▁やはり面白い。", + "うちわは▁やはり便利だ。" + ] + }}, + {{ + "char1": "よ", + "char2": "は", + "sentences": [ + "試合がいよいよ▁はじまる。" + ] + }} + ] +}} + +Ensure all sentences are grammatically correct and natural Japanese. +Do not include any other markdown formatting outside the raw JSON object. +""" + + +def get_synthesis_config() -> genai.types.GenerateContentConfig: + """Configures the structured output using Pydantic model. + + Returns: + The GenerateContentConfig object. + """ + return genai.types.GenerateContentConfig( + response_mime_type='application/json', + response_schema=SynthesisResponse, + ) + + +def validate_negative_candidate(candidate_with_breaks: str, + target_phrase: str) -> bool: + """Validates that a negative candidate doesn't have internal splits within target_phrase. + + Args: + candidate_with_breaks: Sentence generated by LLM, possibly containing breaks. + target_phrase: The phrase that must remain unsplit. + + Returns: + True if the candidate contains no internal breaks inside the phrase. + """ + internal_char_pairs = [ + target_phrase[i:i + 2] for i in range(len(target_phrase) - 1) + ] + for pair in internal_char_pairs: + if f'{pair[0]}{budoux.utils.SEP}{pair[1]}' in candidate_with_breaks: + return False + return True + + +def validate_positive_candidate( + candidate_with_breaks: str, + left_char: str, + right_char: str, + negative_phrases: typing.List[str], +) -> bool: + """Validates that a positive candidate is split at the transition and has no parent contamination. + + Args: + candidate_with_breaks: Sentence generated by LLM, possibly containing breaks. + left_char: Character before the transition. + right_char: Character after the transition. + negative_phrases: List of negative target phrases to prevent contamination. + + Returns: + True if the candidate is clean and correctly split. + """ + for phrase in negative_phrases: + if phrase in candidate_with_breaks: + return False + return f'{left_char}{budoux.utils.SEP}{right_char}' in candidate_with_breaks + + +def filter_and_align_negatives( + negative_sentences_data: typing.List[NegativeSentenceGroup], + negative_phrases: typing.List[str], + parser: budoux.Parser, +) -> typing.List[str]: + """Filters and aligns raw LLM negative candidate sentences. + + Args: + negative_sentences_data: List of negative candidate groups. + negative_phrases: List of target negative phrases to align. + parser: Baseline parser for predicting background breaks. + + Returns: + List of aligned negative training sentences. + """ + results = [] + for item in negative_sentences_data: + phrase = item.phrase + if phrase not in negative_phrases: + continue + for llm_candidate in item.sentences: + clean_text = llm_candidate.replace(budoux.utils.SEP, '') + if phrase not in clean_text: + continue + + if not validate_negative_candidate(llm_candidate, phrase): + continue + + baseline_breaks = get_separator_indices( + budoux.utils.SEP.join(parser.parse(clean_text))) + llm_breaks = get_separator_indices(llm_candidate) + aligned = align_to_baseline_model( + clean_text=clean_text, + baseline_breaks=baseline_breaks, + llm_breaks=llm_breaks, + target_sequence=phrase, + split_offset=None, + ) + results.append(aligned) + return results + + +def filter_and_align_positives( + positive_sentences_data: typing.List[PositiveSentenceGroup], + negative_phrases: typing.List[str], + parser: budoux.Parser, +) -> typing.List[str]: + """Filters and aligns raw LLM positive candidate sentences. + + Args: + positive_sentences_data: List of split transition candidate objects. + negative_phrases: List of negative target phrases to prevent contamination. + parser: Baseline parser for predicting background breaks. + + Returns: + List of aligned positive training sentences. + """ + results = [] + for item in positive_sentences_data: + left_char = item.char1 + right_char = item.char2 + target_transition = left_char + right_char + for llm_candidate in item.sentences: + clean_text = llm_candidate.replace(budoux.utils.SEP, '') + if target_transition not in clean_text: + continue + + if not validate_positive_candidate(llm_candidate, left_char, right_char, + negative_phrases): + continue + + baseline_breaks = get_separator_indices( + budoux.utils.SEP.join(parser.parse(clean_text))) + llm_breaks = get_separator_indices(llm_candidate) + aligned = align_to_baseline_model( + clean_text=clean_text, + baseline_breaks=baseline_breaks, + llm_breaks=llm_breaks, + target_sequence=target_transition, + split_offset=len(left_char), + ) + results.append(aligned) + return results + + +def synthesize( + issue_context: str, + api_key: str, + parser: budoux.Parser, + num_samples: int = 15, + model: str = 'gemini-3.1-flash-lite', +) -> typing.Tuple[typing.List[str], typing.List[str], typing.List[str], + typing.List[str]]: + """Generates positive and negative Japanese training sentences using Gemini based on a GitHub issue. + + Args: + issue_context: The raw text/body of the GitHub issue context. + api_key: Gemini API developer credentials key. + parser: The baseline BudouX Parser to align background separators against. + num_samples: The number of candidate sentences to generate per phrase or transition. + model: The Gemini model ID to use for content generation. + + Returns: + A tuple containing four lists: + (positive_sentences, negative_sentences, negative_phrases, positive_phrases). + """ + client = genai.Client(api_key=api_key) + prompt = generate_synthesis_prompt(issue_context, num_samples) + config = get_synthesis_config() + + try: + response = client.models.generate_content( + model=model, contents=prompt, config=config) + except Exception as e: + sys.exit(f'Error: Gemini API call failed: {e}') + + result_data = response.parsed + if not isinstance(result_data, SynthesisResponse): + sys.exit('Error: Gemini API returned empty or unparseable response.') + + negative_phrases = result_data.analysis.negative_phrases + positive_phrases = result_data.analysis.positive_phrases + + pos_aligned = [] + neg_aligned = [] + + # 1. Filter and align negative sentences for target phrases + neg_aligned.extend( + filter_and_align_negatives( + negative_sentences_data=result_data.negative_sentences, + negative_phrases=negative_phrases, + parser=parser, + )) + + # 2. Filter and align positive sentences for forced splits + pos_aligned.extend( + filter_and_align_positives( + positive_sentences_data=result_data.positive_sentences, + negative_phrases=negative_phrases, + parser=parser, + )) + + return ( + list(dict.fromkeys(pos_aligned)), + list(dict.fromkeys(neg_aligned)), + negative_phrases, + positive_phrases, + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + '-i', + '--issue', + required=True, + help='GitHub issue ID to fetch context and target definitions from.') + parser.add_argument( + '-n', + '--num-samples', + type=int, + default=15, + help='Number of training samples to synthesize (default: 15).') + parser.add_argument( + '-o', + '--output', + default='staging_raw.txt', + help='Output staged file path (default: staging_raw.txt).') + parser.add_argument( + '-m', + '--model', + default='gemini-3.1-flash-lite', + help='Gemini model to use (default: gemini-3.1-flash-lite).') + args = parser.parse_args() + + api_key = os.environ.get('GEMINI_API_KEY') + if not api_key: + sys.exit('Error: GEMINI_API_KEY environment variable is not set.') + + # Load the current default parser for Japanese to align background separators + parser_instance = budoux.load_default_japanese_parser() + + issue_context = parse_issue(args.issue) + + pos, neg, _, _ = synthesize( + issue_context, + api_key, + parser_instance, + args.num_samples, + args.model, + ) + + # Write output candidates to staging file + with open(args.output, 'w', encoding='utf-8') as f: + for sent in pos + neg: + f.write(sent + '\n') + + print( + f'Candidate synthesis complete. Staged {len(pos)} positive and {len(neg)} negative rows in {args.output}.' + ) + + +if __name__ == '__main__': + main() diff --git a/scripts/tests/test_synthesize_samples.py b/scripts/tests/test_synthesize_samples.py new file mode 100644 index 00000000..43f11f04 --- /dev/null +++ b/scripts/tests/test_synthesize_samples.py @@ -0,0 +1,345 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests the candidate synthesis script and its granular units.""" + +import os +import sys +import tempfile +import unittest +from unittest.mock import MagicMock, patch + +import requests + +# module hack to allow importing scripts and budoux from workspace root +LIB_PATH = os.path.join(os.path.dirname(__file__), '..', '..') +sys.path.insert(0, os.path.abspath(LIB_PATH)) + +# Mock google modules before importing the production script +mock_genai = MagicMock() +sys.modules['google'] = mock_genai +sys.modules['google.genai'] = mock_genai + +import budoux # noqa: E402 +from scripts import synthesize_samples # noqa (module hack) + + +class TestGetSeparatorIndices(unittest.TestCase): + """Unit tests for get_separator_indices.""" + + def test_standard(self) -> None: + text_with_seps = f'事態は{budoux.utils.SEP}もはや{budoux.utils.SEP}深刻だ。' + self.assertEqual( + synthesize_samples.get_separator_indices(text_with_seps), {3, 6}) + + def test_no_separators(self) -> None: + self.assertEqual( + synthesize_samples.get_separator_indices('事態はもはや深刻だ。'), set()) + + def test_empty(self) -> None: + self.assertEqual(synthesize_samples.get_separator_indices(''), set()) + + +class TestAlignToBaselineModel(unittest.TestCase): + """Unit tests for align_to_baseline_model.""" + + def test_unsplit_with_buggy_splits(self) -> None: + # Under buggy model: '事態はもは/や/深刻だ。' (split at 3, 5, 6) + # Target 'もはや' (starts at 3, ends at 6). + # LLM broke at 3 and 6: '事態は▁もはや▁深刻だ。' + clean_text = '事態はもはや深刻だ。' + baseline_breaks = {0, 3, 5, 6, 10} + llm_breaks = {3, 6} + aligned = synthesize_samples.align_to_baseline_model( + clean_text=clean_text, + baseline_breaks=baseline_breaks, + llm_breaks=llm_breaks, + target_sequence='もはや', + split_offset=None) + # The buggy internal split at index 5 must be discarded, + # and outer boundaries 3 and 6 should be forced because they are in llm_breaks. + self.assertEqual(aligned, f'事態は{budoux.utils.SEP}もはや{budoux.utils.SEP}深刻だ。') + + def test_unsplit_already_unsplit(self) -> None: + # Baseline has correct outer splits and no internal splits. + clean_text = '事態はもはや深刻だ。' + baseline_breaks = {0, 3, 6, 10} + llm_breaks = {3, 6} + aligned = synthesize_samples.align_to_baseline_model( + clean_text=clean_text, + baseline_breaks=baseline_breaks, + llm_breaks=llm_breaks, + target_sequence='もはや', + split_offset=None) + self.assertEqual(aligned, f'事態は{budoux.utils.SEP}もはや{budoux.utils.SEP}深刻だ。') + + def test_unsplit_at_sentence_start(self) -> None: + # Target phrase is at the start of sentence (idx = 0). + clean_text = 'もはや深刻だ。' + baseline_breaks = {0, 7} + llm_breaks = {3} # Only split after 'もはや' + aligned = synthesize_samples.align_to_baseline_model( + clean_text=clean_text, + baseline_breaks=baseline_breaks, + llm_breaks=llm_breaks, + target_sequence='もはや', + split_offset=None) + # Outer split before 'もはや' is skipped because it's at index 0 + self.assertEqual(aligned, f'もはや{budoux.utils.SEP}深刻だ。') + + def test_unsplit_with_particle(self) -> None: + # Target 'ありがとう' is followed by particle 'と': 'ありがとうと言った。' (idx 0-5) + # LLM output: 'ありがとうと▁言った。' -> no split at index 5, split at index 6. + # Baseline parser predicted: 'ありが/とう/と/言った。' -> splits at {3, 5, 6, 10} + clean_text = 'ありがとうと言った。' + baseline_breaks = {0, 3, 5, 6, 10} + llm_breaks = {6} + aligned = synthesize_samples.align_to_baseline_model( + clean_text=clean_text, + baseline_breaks=baseline_breaks, + llm_breaks=llm_breaks, + target_sequence='ありがとう', + split_offset=None) + # Buggy internal splits (3) are discarded. + # Split at 5 (between 'ありがとう' and 'と') is discarded because it is not in llm_breaks. + # Background split at 6 (between 'と' and '言った') is preserved. + self.assertEqual(aligned, f'ありがとうと{budoux.utils.SEP}言った。') + + def test_unsplit_force_boundary_split(self) -> None: + # Target 'ありがとう' followed by word '感謝': 'ありがとう感謝。' (idx 0-5) + # LLM output: 'ありがとう▁感謝。' -> split at 5. + clean_text = 'ありがとう感謝。' + baseline_breaks = {0} + llm_breaks = {5} + aligned = synthesize_samples.align_to_baseline_model( + clean_text=clean_text, + baseline_breaks=baseline_breaks, + llm_breaks=llm_breaks, + target_sequence='ありがとう', + split_offset=None) + # Split at index 5 is forced because LLM chose to split there, even if baseline had no split. + self.assertEqual(aligned, f'ありがとう{budoux.utils.SEP}感謝。') + + def test_split_standard(self) -> None: + # Positive sample for transition 'もは' in '誰もはっきりと答えない。' + clean_text = '誰もはっきりと答えない。' + baseline_breaks = {0, 7, 12} + llm_breaks = {2} # LLM split after 'も' + aligned = synthesize_samples.align_to_baseline_model( + clean_text=clean_text, + baseline_breaks=baseline_breaks, + llm_breaks=llm_breaks, + target_sequence='もは', + split_offset=1) + # Should force break between 'も' and 'は' (index 2) and preserve baseline break at index 7. + self.assertEqual(aligned, + f'誰も{budoux.utils.SEP}はっきりと{budoux.utils.SEP}答えない。') + + +class TestValidateNegativeCandidate(unittest.TestCase): + """Unit tests for validate_negative_candidate.""" + + def test_valid_unsplit(self) -> None: + candidate = f'事態は{budoux.utils.SEP}もはや{budoux.utils.SEP}深刻だ。' + self.assertTrue( + synthesize_samples.validate_negative_candidate(candidate, 'もはや')) + + def test_invalid_split_inside(self) -> None: + candidate = f'事態は{budoux.utils.SEP}もは{budoux.utils.SEP}や{budoux.utils.SEP}深刻だ。' + self.assertFalse( + synthesize_samples.validate_negative_candidate(candidate, 'もはや')) + + +class TestValidatePositiveCandidate(unittest.TestCase): + """Unit tests for validate_positive_candidate.""" + + def test_valid_split(self) -> None: + candidate = f'誰も{budoux.utils.SEP}はっきりと答えない。' + self.assertTrue( + synthesize_samples.validate_positive_candidate(candidate, 'も', 'は', + ['もはや'])) + + def test_invalid_no_split(self) -> None: + candidate = '誰もはっきりと答えない。' + self.assertFalse( + synthesize_samples.validate_positive_candidate(candidate, 'も', 'は', + ['もはや'])) + + def test_invalid_contaminated(self) -> None: + candidate = f'私は{budoux.utils.SEP}もはや{budoux.utils.SEP}限界だ。' + self.assertFalse( + synthesize_samples.validate_positive_candidate(candidate, 'も', 'は', + ['もはや'])) + + +class TestParseIssue(unittest.TestCase): + """Unit tests for parse_issue.""" + + @patch('requests.get') + def test_parse_issue_happy_path(self, mock_get: MagicMock) -> None: + mock_res = MagicMock() + mock_res.status_code = 200 + mock_res.json.return_value = {'body': ' もはや / これまでべきが細切れに '} + mock_get.return_value = mock_res + + sanitized = synthesize_samples.parse_issue('841') + self.assertEqual(sanitized, 'もはや / これまでべきが細切れに') + + @patch('requests.get') + def test_parse_issue_connection_error(self, mock_get: MagicMock) -> None: + mock_get.side_effect = requests.exceptions.ConnectionError('Network down') + with self.assertRaises(SystemExit): + synthesize_samples.parse_issue('841') + + +class TestSynthesize(unittest.TestCase): + """Integration verification for LLM response integration and filtering orchestration.""" + + def setUp(self) -> None: + self.temp_dir = tempfile.TemporaryDirectory() + self.api_key = 'fake_key_123' + + # A tiny baseline model structured specifically to verify background chunk alignment and bug recovery: + # - TW1: splits after '事態は' and '今日は' + # - UW4: splits before '深', '答', 'あ' + # - BW2: buggy feature causing over-segmentation inside 'もはや' ('もは/や') + # - UW1: balances base_score to -1500 + self.tiny_model = { + 'TW1': { + '事態は': 3000, + '今日は': 3000 + }, + 'UW4': { + '深': 3000, + '答': 3000, + 'あ': 3000 + }, + 'BW2': { + 'はや': 3000 + }, + 'UW1': { + 'dummy': -15000 + } + } + self.parser = budoux.Parser(self.tiny_model) + + def tearDown(self) -> None: + self.temp_dir.cleanup() + + @patch('scripts.synthesize_samples.genai') + def test_synthesize_happy_path(self, mock_genai_module: MagicMock) -> None: + mock_client = MagicMock() + mock_response = MagicMock() + parsed_response = synthesize_samples.SynthesisResponse( + analysis=synthesize_samples.Analysis( + negative_phrases=['もはや'], + positive_phrases=['いよいよ/はじまる'], + ), + negative_sentences=[ + synthesize_samples.NegativeSentenceGroup( + phrase='もはや', + sentences=[ + '事態は▁もはや▁深刻だ。', + '彼の▁実力は▁もは▁や▁止められない。', # Invalid: contains split inside "もはや" + '本日は▁とても▁良い▁天気だ。' # Invalid: missing target phrase "もはや" + ]) + ], + positive_sentences=[ + synthesize_samples.PositiveSentenceGroup( + char1='も', + char2='は', + sentences=[ + '誰も▁はっきりと答えない。', + '私は▁もはや▁限界だ。' # Invalid: contaminated with unsplit phrase "もはや" + ]), + synthesize_samples.PositiveSentenceGroup( + char1='は', + char2='や', + sentences=[ + '今日は▁やるべきことがある。', + '部屋に▁入る。' # Invalid: does not contain target transition "は" + "や" + ]), + synthesize_samples.PositiveSentenceGroup( + char1='よ', char2='は', sentences=[ + 'いよいよ▁はじまる。', + ]) + ]) + mock_response.parsed = parsed_response + mock_client.models.generate_content.return_value = mock_response + mock_genai_module.Client.return_value = mock_client + + pos, neg, neg_phrases, pos_phrases = synthesize_samples.synthesize( + '「もはや」が細切れになってしまう。あと、「いよいよはじまる」も「いよいよ」の後で切れるべき。', + self.api_key, + self.parser, + num_samples=5, + ) + + self.assertEqual(neg_phrases, ['もはや']) + self.assertEqual(pos_phrases, ['いよいよ/はじまる']) + + # Verify positive samples preserve realistic background breaks while strictly splitting targeted transitions + self.assertEqual(pos, [ + f'誰も{budoux.utils.SEP}はっきりと{budoux.utils.SEP}答えない。', + f'今日は{budoux.utils.SEP}やるべきことが{budoux.utils.SEP}ある。', + f'いよいよ{budoux.utils.SEP}はじまる。' + ]) + + # Verify negative sample eliminates internal over-segmentation defect + self.assertEqual(neg, [f'事態は{budoux.utils.SEP}もはや{budoux.utils.SEP}深刻だ。']) + + # Verify default model parameter passed to API client + mock_client.models.generate_content.assert_called_once_with( + model='gemini-3.1-flash-lite', + contents=unittest.mock.ANY, + config=unittest.mock.ANY, + ) + + @patch('scripts.synthesize_samples.genai') + def test_synthesize_custom_model(self, mock_genai_module: MagicMock) -> None: + mock_client = MagicMock() + mock_response = MagicMock() + mock_response.parsed = synthesize_samples.SynthesisResponse( + analysis=synthesize_samples.Analysis( + negative_phrases=[], positive_phrases=[]), + negative_sentences=[], + positive_sentences=[], + ) + mock_client.models.generate_content.return_value = mock_response + mock_genai_module.Client.return_value = mock_client + + synthesize_samples.synthesize( + 'Fake issue context', + self.api_key, + self.parser, + model='gemini-2.0-pro-experimental', + ) + mock_client.models.generate_content.assert_called_once_with( + model='gemini-2.0-pro-experimental', + contents=unittest.mock.ANY, + config=unittest.mock.ANY, + ) + + @patch('scripts.synthesize_samples.genai') + def test_synthesize_api_error(self, mock_genai_module: MagicMock) -> None: + mock_client = MagicMock() + mock_client.models.generate_content.side_effect = Exception( + 'Quota exceeded') + mock_genai_module.Client.return_value = mock_client + with self.assertRaises(SystemExit): + synthesize_samples.synthesize('Fake Issue Body', self.api_key, + self.parser) + + +if __name__ == '__main__': + unittest.main()