Normalize LLM judge scores in result JSON

README.md

@@ -43,6 +43,8 @@ MCP-Bench is a comprehensive evaluation framework designed to assess Large Langu
 
 *Overall Score represents the average performance across all evaluation dimensions including rule-based schema understanding, LLM-judged (o4-mini as judge model) task completion, tool usage, and planning effectiveness. Scores are averaged across single-server and multi-server settings.*
 
+LLM judge prompts use a 0-10 raw scoring rubric. MCP-Bench normalizes LLM judge subdimension and aggregate scores to the 0-1 range in result JSON, while rule-based rate metrics such as schema compliance are already reported on the 0-1 scale.
+
 ## Quick Start
 
 ### Installation

benchmark/evaluator.py

@@ -15,6 +15,10 @@
 import jsonschema
 from jsonschema import ValidationError
 import config.config_loader as config_loader
+from benchmark.score_normalization import (
+    LLM_JUDGE_SUBDIMENSION_SCORE_FIELDS,
+    normalize_raw_llm_judge_scores,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -384,7 +388,12 @@ def _calculate_average_scores(self, all_scores: List[Dict[str, Any]]) -> Dict[st
                 logger.warning(f"No valid scores found for {field}, using 0")
                 averaged_result[field] = 0
 
-        # Recalculate aggregate scores from averaged subdimension scores
+        averaged_result = normalize_raw_llm_judge_scores(
+            averaged_result,
+            LLM_JUDGE_SUBDIMENSION_SCORE_FIELDS,
+        )
+
+        # Recalculate aggregate scores from normalized subdimension scores
         task_completion_scores = [
             averaged_result['task_fulfillment'], 
             averaged_result['grounding']
@@ -880,15 +889,19 @@ async def _perform_evaluation(self, task: str, final_solution: str,
             logger.info(f"   Parsing time: {parse_time:.3f}s")
             logger.debug(f"Parsed result: {result}")
 
-            # Extract 6 subdimension scores
-            task_fulfillment = result.get('task_fulfillment')
-            grounding = result.get('grounding')
+            # Extract and normalize 6 raw subdimension scores from 0-10 to 0-1.
+            normalized_result = normalize_raw_llm_judge_scores(
+                result,
+                LLM_JUDGE_SUBDIMENSION_SCORE_FIELDS,
+            )
+            task_fulfillment = normalized_result.get('task_fulfillment')
+            grounding = normalized_result.get('grounding')
 
-            tool_appropriateness = result.get('tool_appropriateness')
-            parameter_accuracy = result.get('parameter_accuracy')
+            tool_appropriateness = normalized_result.get('tool_appropriateness')
+            parameter_accuracy = normalized_result.get('parameter_accuracy')
 
-            dependency_awareness = result.get('dependency_awareness')
-            parallelism_and_efficiency = result.get('parallelism_and_efficiency')
+            dependency_awareness = normalized_result.get('dependency_awareness')
+            parallelism_and_efficiency = normalized_result.get('parallelism_and_efficiency')
 
             # Calculate aggregate scores (2 scores per category)
             task_completion_scores = [task_fulfillment, grounding]

benchmark/score_normalization.py

@@ -0,0 +1,38 @@
+"""
+Utilities for normalizing MCP-Bench LLM judge scores.
+
+The judge prompt asks for 0-10 scores. Benchmark result JSON stores these
+scores on a 0-1 scale so they are comparable with the other rate metrics.
+"""
+
+from numbers import Real
+from typing import Any, Dict, Iterable
+
+
+LLM_JUDGE_SUBDIMENSION_SCORE_FIELDS = (
+    "task_fulfillment",
+    "grounding",
+    "tool_appropriateness",
+    "parameter_accuracy",
+    "dependency_awareness",
+    "parallelism_and_efficiency",
+)
+
+
+def normalize_raw_llm_judge_score(value: Any) -> Any:
+    """Convert a raw 0-10 LLM judge score to the benchmark's 0-1 scale."""
+    if isinstance(value, bool) or not isinstance(value, Real):
+        return value
+    return value / 10
+
+
+def normalize_raw_llm_judge_scores(
+    scores: Dict[str, Any],
+    fields: Iterable[str] = LLM_JUDGE_SUBDIMENSION_SCORE_FIELDS,
+) -> Dict[str, Any]:
+    """Return a copy with raw LLM judge score fields normalized to 0-1."""
+    normalized = dict(scores)
+    for field in fields:
+        if field in normalized:
+            normalized[field] = normalize_raw_llm_judge_score(normalized[field])
+    return normalized

tests/test_score_normalization.py

@@ -0,0 +1,115 @@
+import json
+import unittest
+
+from benchmark.evaluator import LLMJudge, TaskEvaluator
+from benchmark.score_normalization import normalize_raw_llm_judge_scores
+
+
+class StubLLM:
+    async def get_completion(self, system_prompt, user_prompt, max_tokens):
+        return json.dumps({
+            "task_fulfillment_reasoning": "ok",
+            "grounding_reasoning": "ok",
+            "tool_appropriateness_reasoning": "ok",
+            "parameter_accuracy_reasoning": "ok",
+            "dependency_awareness_reasoning": "ok",
+            "parallelism_efficiency_reasoning": "ok",
+            "task_fulfillment": 8,
+            "grounding": 6,
+            "tool_appropriateness": 10,
+            "parameter_accuracy": 4,
+            "dependency_awareness": 7,
+            "parallelism_and_efficiency": 3,
+        })
+
+    def clean_and_parse_json(self, raw_json):
+        return json.loads(raw_json)
+
+
+class ScoreNormalizationTest(unittest.TestCase):
+    def test_normalizes_raw_llm_judge_scores_to_zero_one_scale(self):
+        normalized = normalize_raw_llm_judge_scores({
+            "task_fulfillment": 1,
+            "grounding": 8.5,
+            "tool_appropriateness": 10,
+            "parameter_accuracy": 0,
+            "input_schema_compliance": 0.95,
+            "task_fulfillment_reasoning": "raw judge score",
+        })
+
+        self.assertEqual(normalized["task_fulfillment"], 0.1)
+        self.assertEqual(normalized["grounding"], 0.85)
+        self.assertEqual(normalized["tool_appropriateness"], 1.0)
+        self.assertEqual(normalized["parameter_accuracy"], 0)
+        self.assertEqual(normalized["input_schema_compliance"], 0.95)
+        self.assertEqual(normalized["task_fulfillment_reasoning"], "raw judge score")
+
+    def test_stability_average_scores_are_normalized_before_aggregation(self):
+        judge = LLMJudge(llm_provider=None)
+
+        averaged = judge._calculate_average_scores([
+            {
+                "task_fulfillment": 8,
+                "grounding": 6,
+                "tool_appropriateness": 10,
+                "parameter_accuracy": 4,
+                "dependency_awareness": 7,
+                "parallelism_and_efficiency": 3,
+                "task_completion_analysis": "first",
+            },
+            {
+                "task_fulfillment": 6,
+                "grounding": 4,
+                "tool_appropriateness": 8,
+                "parameter_accuracy": 6,
+                "dependency_awareness": 5,
+                "parallelism_and_efficiency": 5,
+                "task_completion_analysis": "second",
+            },
+        ])
+
+        self.assertAlmostEqual(averaged["task_fulfillment"], 0.7)
+        self.assertAlmostEqual(averaged["grounding"], 0.5)
+        self.assertAlmostEqual(averaged["tool_appropriateness"], 0.9)
+        self.assertAlmostEqual(averaged["parameter_accuracy"], 0.5)
+        self.assertAlmostEqual(averaged["dependency_awareness"], 0.6)
+        self.assertAlmostEqual(averaged["parallelism_and_efficiency"], 0.4)
+        self.assertAlmostEqual(averaged["task_completion_score"], 0.6)
+        self.assertAlmostEqual(averaged["tool_selection_score"], 0.7)
+        self.assertAlmostEqual(
+            averaged["planning_effectiveness_and_efficiency_score"],
+            0.5,
+        )
+        self.assertEqual(averaged["task_completion_analysis"], "first")
+
+
+class TaskEvaluatorScoreNormalizationTest(unittest.IsolatedAsyncioTestCase):
+    async def test_task_evaluator_returns_normalized_llm_judge_scores(self):
+        evaluator = TaskEvaluator(StubLLM())
+
+        evaluation = await evaluator.evaluate(
+            task="test task",
+            execution_results=[],
+            final_solution="done",
+            total_rounds=1,
+            available_tools={},
+            planning_json_compliance=1.0,
+        )
+
+        self.assertAlmostEqual(evaluation["task_fulfillment"], 0.8)
+        self.assertAlmostEqual(evaluation["grounding"], 0.6)
+        self.assertAlmostEqual(evaluation["tool_appropriateness"], 1.0)
+        self.assertAlmostEqual(evaluation["parameter_accuracy"], 0.4)
+        self.assertAlmostEqual(evaluation["dependency_awareness"], 0.7)
+        self.assertAlmostEqual(evaluation["parallelism_and_efficiency"], 0.3)
+        self.assertAlmostEqual(evaluation["task_completion_score"], 0.7)
+        self.assertAlmostEqual(evaluation["tool_selection_score"], 0.7)
+        self.assertAlmostEqual(
+            evaluation["planning_effectiveness_and_efficiency_score"],
+            0.5,
+        )
+        self.assertIsNone(evaluation["input_schema_compliance"])
+
+
+if __name__ == "__main__":
+    unittest.main()