From 9f4acb43888c75adaa3dae8564cad9bc2fea03a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= Date: Thu, 9 Jul 2026 10:00:47 +0200 Subject: [PATCH 1/4] add speedup_ttfn to OnnxDiscrepancyCheck --- olive/passes/onnx/discrepancy_check.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/olive/passes/onnx/discrepancy_check.py b/olive/passes/onnx/discrepancy_check.py index ab282d617..a6f3edd05 100644 --- a/olive/passes/onnx/discrepancy_check.py +++ b/olive/passes/onnx/discrepancy_check.py @@ -396,9 +396,19 @@ def _run_for_config( self._run_llama_cpp_comparison(model, config, ref_model, ref_path, report_dir, generation_metrics, results) + self._compute_final_metrics(results) + self._save_results(model, results, report_dir) return model + def _compute_final_metrics(self, results): + if "transformers_ttfn_s" in results and "genai_ttfn_s" in results: + results["speedup_ttfn_genai_torch"] = results["transformers_ttfn_s"] / results["genai_ttfn_s"] + if "transformers_ttfn_s" in results and "llama_cpp_ttfn_s" in results: + results["speedup_ttfn_llama_cpp_torch"] = results["transformers_ttfn_s"] / results["llama_cpp_ttfn_s"] + if "llama_cpp_ttfn_s" in results and "genai_ttfn_s" in results: + results["speedup_ttfn_genai_llama_cpp"] = results["llama_cpp_ttfn_s"] / results["genai_ttfn_s"] + def _prepare_dataloader(self, model: ONNXModelHandler): from olive.common.config_utils import validate_config from olive.data.template import dummy_data_config_template From ee23f85b0c7a82f61307a36a925adf139f5dec05 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 9 Jul 2026 13:26:08 +0000 Subject: [PATCH 2/4] Add unit tests for speedup_ttfn metrics and fix test assertion --- test/passes/onnx/test_discrepancy_check.py | 103 ++++++++++++++++++ .../passes/onnx/test_session_params_tuning.py | 5 +- 2 files changed, 107 insertions(+), 1 deletion(-) diff --git a/test/passes/onnx/test_discrepancy_check.py b/test/passes/onnx/test_discrepancy_check.py index 2f003c236..d9c270800 100644 --- a/test/passes/onnx/test_discrepancy_check.py +++ b/test/passes/onnx/test_discrepancy_check.py @@ -825,3 +825,106 @@ def test_compare_llama_cpp_returns_stderr_and_stdout_on_helper_error(self, tmp_p ) assert result == {"llama_cpp_out": "stderr text", "llama_cpp_err": "stdout text"} + + +class TestComputeFinalMetrics: + """Unit tests for OnnxDiscrepancyCheck._compute_final_metrics.""" + + def test_compute_final_metrics_all_speedups(self): + """Test that all speedup metrics are computed when all base metrics are present.""" + from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck + + pass_instance = OnnxDiscrepancyCheck.__new__(OnnxDiscrepancyCheck) + results = { + "transformers_ttfn_s": 0.8, + "genai_ttfn_s": 0.4, + "llama_cpp_ttfn_s": 0.2, + } + + pass_instance._compute_final_metrics(results) + + assert "speedup_ttfn_genai_torch" in results + assert results["speedup_ttfn_genai_torch"] == pytest.approx(2.0) + assert "speedup_ttfn_llama_cpp_torch" in results + assert results["speedup_ttfn_llama_cpp_torch"] == pytest.approx(4.0) + assert "speedup_ttfn_genai_llama_cpp" in results + assert results["speedup_ttfn_genai_llama_cpp"] == pytest.approx(0.5) + + def test_compute_final_metrics_genai_torch_only(self): + """Test speedup_ttfn_genai_torch is computed when only transformers and genai are present.""" + from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck + + pass_instance = OnnxDiscrepancyCheck.__new__(OnnxDiscrepancyCheck) + results = { + "transformers_ttfn_s": 1.0, + "genai_ttfn_s": 0.5, + } + + pass_instance._compute_final_metrics(results) + + assert "speedup_ttfn_genai_torch" in results + assert results["speedup_ttfn_genai_torch"] == pytest.approx(2.0) + assert "speedup_ttfn_llama_cpp_torch" not in results + assert "speedup_ttfn_genai_llama_cpp" not in results + + def test_compute_final_metrics_llama_cpp_torch_only(self): + """Test speedup_ttfn_llama_cpp_torch is computed when only transformers and llama_cpp are present.""" + from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck + + pass_instance = OnnxDiscrepancyCheck.__new__(OnnxDiscrepancyCheck) + results = { + "transformers_ttfn_s": 1.0, + "llama_cpp_ttfn_s": 0.25, + } + + pass_instance._compute_final_metrics(results) + + assert "speedup_ttfn_llama_cpp_torch" in results + assert results["speedup_ttfn_llama_cpp_torch"] == pytest.approx(4.0) + assert "speedup_ttfn_genai_torch" not in results + assert "speedup_ttfn_genai_llama_cpp" not in results + + def test_compute_final_metrics_genai_llama_cpp_only(self): + """Test speedup_ttfn_genai_llama_cpp is computed when only llama_cpp and genai are present.""" + from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck + + pass_instance = OnnxDiscrepancyCheck.__new__(OnnxDiscrepancyCheck) + results = { + "llama_cpp_ttfn_s": 0.3, + "genai_ttfn_s": 0.6, + } + + pass_instance._compute_final_metrics(results) + + assert "speedup_ttfn_genai_llama_cpp" in results + assert results["speedup_ttfn_genai_llama_cpp"] == pytest.approx(0.5) + assert "speedup_ttfn_genai_torch" not in results + assert "speedup_ttfn_llama_cpp_torch" not in results + + def test_compute_final_metrics_no_metrics(self): + """Test that no speedup metrics are computed when base metrics are missing.""" + from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck + + pass_instance = OnnxDiscrepancyCheck.__new__(OnnxDiscrepancyCheck) + results = {} + + pass_instance._compute_final_metrics(results) + + assert "speedup_ttfn_genai_torch" not in results + assert "speedup_ttfn_llama_cpp_torch" not in results + assert "speedup_ttfn_genai_llama_cpp" not in results + + def test_compute_final_metrics_only_transformers(self): + """Test that no speedup metrics are computed when only transformers metric is present.""" + from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck + + pass_instance = OnnxDiscrepancyCheck.__new__(OnnxDiscrepancyCheck) + results = { + "transformers_ttfn_s": 1.0, + } + + pass_instance._compute_final_metrics(results) + + assert "speedup_ttfn_genai_torch" not in results + assert "speedup_ttfn_llama_cpp_torch" not in results + assert "speedup_ttfn_genai_llama_cpp" not in results diff --git a/test/passes/onnx/test_session_params_tuning.py b/test/passes/onnx/test_session_params_tuning.py index 4eed0653a..1a46128b5 100644 --- a/test/passes/onnx/test_session_params_tuning.py +++ b/test/passes/onnx/test_session_params_tuning.py @@ -171,4 +171,7 @@ def test_ort_session_params_tuning_pass_with_dynamic_shapes(mock_get_io_config, with pytest.raises(TypeError) as e: # execute p.run(input_model, output_folder) - assert "ones() received an invalid combination of arguments" in str(e.value) + # Error message changed in newer PyTorch versions + error_msg = str(e.value) + assert "ones()" in error_msg + assert "invalid combination of arguments" in error_msg or "must be tuple of ints" in error_msg From 7531eb1f855adce14055d918156066d863658d30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= Date: Thu, 9 Jul 2026 18:54:08 +0200 Subject: [PATCH 3/4] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- olive/passes/onnx/discrepancy_check.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/olive/passes/onnx/discrepancy_check.py b/olive/passes/onnx/discrepancy_check.py index a6f3edd05..edd77b484 100644 --- a/olive/passes/onnx/discrepancy_check.py +++ b/olive/passes/onnx/discrepancy_check.py @@ -401,13 +401,17 @@ def _run_for_config( self._save_results(model, results, report_dir) return model - def _compute_final_metrics(self, results): - if "transformers_ttfn_s" in results and "genai_ttfn_s" in results: - results["speedup_ttfn_genai_torch"] = results["transformers_ttfn_s"] / results["genai_ttfn_s"] - if "transformers_ttfn_s" in results and "llama_cpp_ttfn_s" in results: - results["speedup_ttfn_llama_cpp_torch"] = results["transformers_ttfn_s"] / results["llama_cpp_ttfn_s"] - if "llama_cpp_ttfn_s" in results and "genai_ttfn_s" in results: - results["speedup_ttfn_genai_llama_cpp"] = results["llama_cpp_ttfn_s"] / results["genai_ttfn_s"] + def _compute_final_metrics(self, results: dict) -> None: + def _ratio(numer_key: str, denom_key: str, out_key: str) -> None: + numer = results.get(numer_key) + denom = results.get(denom_key) + if numer is None or denom is None or denom == 0: + return + results[out_key] = numer / denom + + _ratio("transformers_ttfn_s", "genai_ttfn_s", "speedup_ttfn_genai_torch") + _ratio("transformers_ttfn_s", "llama_cpp_ttfn_s", "speedup_ttfn_llama_cpp_torch") + _ratio("llama_cpp_ttfn_s", "genai_ttfn_s", "speedup_ttfn_genai_llama_cpp") def _prepare_dataloader(self, model: ONNXModelHandler): from olive.common.config_utils import validate_config From 232b327837aae5db3fa4800fba825fd3bd63aabe Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:58:29 +0000 Subject: [PATCH 4/4] Add None/zero denominator test cases for _compute_final_metrics --- test/passes/onnx/test_discrepancy_check.py | 53 ++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/test/passes/onnx/test_discrepancy_check.py b/test/passes/onnx/test_discrepancy_check.py index d9c270800..9d0c7adf6 100644 --- a/test/passes/onnx/test_discrepancy_check.py +++ b/test/passes/onnx/test_discrepancy_check.py @@ -928,3 +928,56 @@ def test_compute_final_metrics_only_transformers(self): assert "speedup_ttfn_genai_torch" not in results assert "speedup_ttfn_llama_cpp_torch" not in results assert "speedup_ttfn_genai_llama_cpp" not in results + + def test_compute_final_metrics_none_denominator(self): + """Test that no speedup metric is created when the denominator (genai or llama_cpp) is None.""" + from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck + + pass_instance = OnnxDiscrepancyCheck.__new__(OnnxDiscrepancyCheck) + results = { + "transformers_ttfn_s": 1.0, + "genai_ttfn_s": None, + "llama_cpp_ttfn_s": None, + } + + pass_instance._compute_final_metrics(results) + + assert "speedup_ttfn_genai_torch" not in results + assert "speedup_ttfn_llama_cpp_torch" not in results + assert "speedup_ttfn_genai_llama_cpp" not in results + + def test_compute_final_metrics_zero_denominator(self): + """Test that no speedup metric is created when the denominator (genai or llama_cpp) is 0.""" + from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck + + pass_instance = OnnxDiscrepancyCheck.__new__(OnnxDiscrepancyCheck) + results = { + "transformers_ttfn_s": 1.0, + "genai_ttfn_s": 0.0, + "llama_cpp_ttfn_s": 0.0, + } + + pass_instance._compute_final_metrics(results) + + assert "speedup_ttfn_genai_torch" not in results + assert "speedup_ttfn_llama_cpp_torch" not in results + assert "speedup_ttfn_genai_llama_cpp" not in results + + def test_compute_final_metrics_none_numerator(self): + """Test that no speedup metric is created when the numerator (transformers or llama_cpp) is None.""" + from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck + + pass_instance = OnnxDiscrepancyCheck.__new__(OnnxDiscrepancyCheck) + results = { + "transformers_ttfn_s": None, + "genai_ttfn_s": 0.4, + "llama_cpp_ttfn_s": 0.2, + } + + pass_instance._compute_final_metrics(results) + + assert "speedup_ttfn_genai_torch" not in results + assert "speedup_ttfn_llama_cpp_torch" not in results + # llama_cpp / genai ratio: numerator (llama_cpp) is valid, denominator (genai) is valid + assert "speedup_ttfn_genai_llama_cpp" in results + assert results["speedup_ttfn_genai_llama_cpp"] == pytest.approx(0.5)