diff --git a/olive/passes/onnx/discrepancy_check.py b/olive/passes/onnx/discrepancy_check.py index ab282d617..edd77b484 100644 --- a/olive/passes/onnx/discrepancy_check.py +++ b/olive/passes/onnx/discrepancy_check.py @@ -396,9 +396,23 @@ def _run_for_config( self._run_llama_cpp_comparison(model, config, ref_model, ref_path, report_dir, generation_metrics, results) + self._compute_final_metrics(results) + self._save_results(model, results, report_dir) return model + def _compute_final_metrics(self, results: dict) -> None: + def _ratio(numer_key: str, denom_key: str, out_key: str) -> None: + numer = results.get(numer_key) + denom = results.get(denom_key) + if numer is None or denom is None or denom == 0: + return + results[out_key] = numer / denom + + _ratio("transformers_ttfn_s", "genai_ttfn_s", "speedup_ttfn_genai_torch") + _ratio("transformers_ttfn_s", "llama_cpp_ttfn_s", "speedup_ttfn_llama_cpp_torch") + _ratio("llama_cpp_ttfn_s", "genai_ttfn_s", "speedup_ttfn_genai_llama_cpp") + def _prepare_dataloader(self, model: ONNXModelHandler): from olive.common.config_utils import validate_config from olive.data.template import dummy_data_config_template diff --git a/test/passes/onnx/test_discrepancy_check.py b/test/passes/onnx/test_discrepancy_check.py index 2f003c236..9d0c7adf6 100644 --- a/test/passes/onnx/test_discrepancy_check.py +++ b/test/passes/onnx/test_discrepancy_check.py @@ -825,3 +825,159 @@ def test_compare_llama_cpp_returns_stderr_and_stdout_on_helper_error(self, tmp_p ) assert result == {"llama_cpp_out": "stderr text", "llama_cpp_err": "stdout text"} + + +class TestComputeFinalMetrics: + """Unit tests for OnnxDiscrepancyCheck._compute_final_metrics.""" + + def test_compute_final_metrics_all_speedups(self): + """Test that all speedup metrics are computed when all base metrics are present.""" + from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck + + pass_instance = OnnxDiscrepancyCheck.__new__(OnnxDiscrepancyCheck) + results = { + "transformers_ttfn_s": 0.8, + "genai_ttfn_s": 0.4, + "llama_cpp_ttfn_s": 0.2, + } + + pass_instance._compute_final_metrics(results) + + assert "speedup_ttfn_genai_torch" in results + assert results["speedup_ttfn_genai_torch"] == pytest.approx(2.0) + assert "speedup_ttfn_llama_cpp_torch" in results + assert results["speedup_ttfn_llama_cpp_torch"] == pytest.approx(4.0) + assert "speedup_ttfn_genai_llama_cpp" in results + assert results["speedup_ttfn_genai_llama_cpp"] == pytest.approx(0.5) + + def test_compute_final_metrics_genai_torch_only(self): + """Test speedup_ttfn_genai_torch is computed when only transformers and genai are present.""" + from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck + + pass_instance = OnnxDiscrepancyCheck.__new__(OnnxDiscrepancyCheck) + results = { + "transformers_ttfn_s": 1.0, + "genai_ttfn_s": 0.5, + } + + pass_instance._compute_final_metrics(results) + + assert "speedup_ttfn_genai_torch" in results + assert results["speedup_ttfn_genai_torch"] == pytest.approx(2.0) + assert "speedup_ttfn_llama_cpp_torch" not in results + assert "speedup_ttfn_genai_llama_cpp" not in results + + def test_compute_final_metrics_llama_cpp_torch_only(self): + """Test speedup_ttfn_llama_cpp_torch is computed when only transformers and llama_cpp are present.""" + from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck + + pass_instance = OnnxDiscrepancyCheck.__new__(OnnxDiscrepancyCheck) + results = { + "transformers_ttfn_s": 1.0, + "llama_cpp_ttfn_s": 0.25, + } + + pass_instance._compute_final_metrics(results) + + assert "speedup_ttfn_llama_cpp_torch" in results + assert results["speedup_ttfn_llama_cpp_torch"] == pytest.approx(4.0) + assert "speedup_ttfn_genai_torch" not in results + assert "speedup_ttfn_genai_llama_cpp" not in results + + def test_compute_final_metrics_genai_llama_cpp_only(self): + """Test speedup_ttfn_genai_llama_cpp is computed when only llama_cpp and genai are present.""" + from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck + + pass_instance = OnnxDiscrepancyCheck.__new__(OnnxDiscrepancyCheck) + results = { + "llama_cpp_ttfn_s": 0.3, + "genai_ttfn_s": 0.6, + } + + pass_instance._compute_final_metrics(results) + + assert "speedup_ttfn_genai_llama_cpp" in results + assert results["speedup_ttfn_genai_llama_cpp"] == pytest.approx(0.5) + assert "speedup_ttfn_genai_torch" not in results + assert "speedup_ttfn_llama_cpp_torch" not in results + + def test_compute_final_metrics_no_metrics(self): + """Test that no speedup metrics are computed when base metrics are missing.""" + from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck + + pass_instance = OnnxDiscrepancyCheck.__new__(OnnxDiscrepancyCheck) + results = {} + + pass_instance._compute_final_metrics(results) + + assert "speedup_ttfn_genai_torch" not in results + assert "speedup_ttfn_llama_cpp_torch" not in results + assert "speedup_ttfn_genai_llama_cpp" not in results + + def test_compute_final_metrics_only_transformers(self): + """Test that no speedup metrics are computed when only transformers metric is present.""" + from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck + + pass_instance = OnnxDiscrepancyCheck.__new__(OnnxDiscrepancyCheck) + results = { + "transformers_ttfn_s": 1.0, + } + + pass_instance._compute_final_metrics(results) + + assert "speedup_ttfn_genai_torch" not in results + assert "speedup_ttfn_llama_cpp_torch" not in results + assert "speedup_ttfn_genai_llama_cpp" not in results + + def test_compute_final_metrics_none_denominator(self): + """Test that no speedup metric is created when the denominator (genai or llama_cpp) is None.""" + from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck + + pass_instance = OnnxDiscrepancyCheck.__new__(OnnxDiscrepancyCheck) + results = { + "transformers_ttfn_s": 1.0, + "genai_ttfn_s": None, + "llama_cpp_ttfn_s": None, + } + + pass_instance._compute_final_metrics(results) + + assert "speedup_ttfn_genai_torch" not in results + assert "speedup_ttfn_llama_cpp_torch" not in results + assert "speedup_ttfn_genai_llama_cpp" not in results + + def test_compute_final_metrics_zero_denominator(self): + """Test that no speedup metric is created when the denominator (genai or llama_cpp) is 0.""" + from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck + + pass_instance = OnnxDiscrepancyCheck.__new__(OnnxDiscrepancyCheck) + results = { + "transformers_ttfn_s": 1.0, + "genai_ttfn_s": 0.0, + "llama_cpp_ttfn_s": 0.0, + } + + pass_instance._compute_final_metrics(results) + + assert "speedup_ttfn_genai_torch" not in results + assert "speedup_ttfn_llama_cpp_torch" not in results + assert "speedup_ttfn_genai_llama_cpp" not in results + + def test_compute_final_metrics_none_numerator(self): + """Test that no speedup metric is created when the numerator (transformers or llama_cpp) is None.""" + from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck + + pass_instance = OnnxDiscrepancyCheck.__new__(OnnxDiscrepancyCheck) + results = { + "transformers_ttfn_s": None, + "genai_ttfn_s": 0.4, + "llama_cpp_ttfn_s": 0.2, + } + + pass_instance._compute_final_metrics(results) + + assert "speedup_ttfn_genai_torch" not in results + assert "speedup_ttfn_llama_cpp_torch" not in results + # llama_cpp / genai ratio: numerator (llama_cpp) is valid, denominator (genai) is valid + assert "speedup_ttfn_genai_llama_cpp" in results + assert results["speedup_ttfn_genai_llama_cpp"] == pytest.approx(0.5)