Add benchmark results for x-ai/grok-code-fast-1 (concurrency 20) and in-progress checkpoints

locodiff-250425/results/aider_aider___init__.py/x-ai_grok-code-fast-1/20250909_153753/extracted_output.txt

@@ -0,0 +1,22 @@
+from packaging import version
+
+__version__ = "0.82.3.dev"
+safe_version = __version__
+
+try:
+    from aider._version import __version__
+except Exception:
+    __version__ = safe_version + "+import"
+
+if type(__version__) is not str:
+    __version__ = safe_version + "+type"
+else:
+    try:
+        print(__version__)
+        print(safe_version)
+        if version.parse(__version__) < version.parse(safe_version):
+            __version__ = safe_version + "+less"
+    except Exception:
+        __version__ = safe_version + "+parse"
+
+__all__ = [__version__]

locodiff-250425/results/aider_aider___init__.py/x-ai_grok-code-fast-1/20250909_153753/metadata.json

@@ -0,0 +1,25 @@
+{
+    "model": "x-ai/grok-code-fast-1",
+    "benchmark_case": "aider_aider___init__.py",
+    "prompts_dir": "locodiff-250425/prompts",
+    "prompt_file": "locodiff-250425/prompts/aider_aider___init__.py_prompt.txt",
+    "expected_file": "locodiff-250425/prompts/aider_aider___init__.py_expectedoutput.txt",
+    "timestamp_utc": "2025-09-09T15:37:53.111816+00:00",
+    "success": false,
+    "error": "Output mismatch",
+    "api_error": false,
+    "raw_response_length": 558,
+    "extracted_output_length": 550,
+    "expected_output_length": 496,
+    "results_dir": "locodiff-250425/results/aider_aider___init__.py/x-ai_grok-code-fast-1/20250909_153753",
+    "generation_id": "gen-1757432273-nJo7FnL7gk0OpBpnxgoN",
+    "cost_usd": 0.01752618,
+    "prompt_tokens": 59709,
+    "completion_tokens": 1710,
+    "total_tokens": 61419,
+    "native_prompt_tokens": 58824,
+    "native_completion_tokens": 3887,
+    "native_finish_reason": "stop",
+    "stats_error": null,
+    "native_tokens_reasoning": 3752
+}

locodiff-250425/results/aider_aider___init__.py/x-ai_grok-code-fast-1/20250909_153753/output.diff

@@ -0,0 +1,13 @@
+
+index 50f38daf5..ef401e2d0 100644
+--- a/aider_aider___init__.py_expectedoutput.txt (expected):tmp/tmpcq_pjert_expected.txt	
++++ b/aider_aider___init__.py_extracted.txt (actual):tmp/tmp0lwd29oe_actual.txt	
+@@ -12,6 +12,8 @@ if type(__version__) is not str:
+     __version__ = safe_version + "+type"
+ else:
+     try:
++        print(__version__)
++        print(safe_version)
+         if version.parse(__version__) < version.parse(safe_version):
+             __version__ = safe_version + "+less"
+     except Exception:

locodiff-250425/results/aider_aider___init__.py/x-ai_grok-code-fast-1/20250909_153753/raw_response.txt

@@ -0,0 +1,24 @@
+```
+from packaging import version
+
+__version__ = "0.82.3.dev"
+safe_version = __version__
+
+try:
+    from aider._version import __version__
+except Exception:
+    __version__ = safe_version + "+import"
+
+if type(__version__) is not str:
+    __version__ = safe_version + "+type"
+else:
+    try:
+        print(__version__)
+        print(safe_version)
+        if version.parse(__version__) < version.parse(safe_version):
+            __version__ = safe_version + "+less"
+    except Exception:
+        __version__ = safe_version + "+parse"
+
+__all__ = [__version__]
+```

locodiff-250425/results/aider_aider_analytics.py/x-ai_grok-code-fast-1/20250909_153841/extracted_output.txt

@@ -0,0 +1,72 @@
+import json
+import platform
+import sys
+import time
+import uuid
+from pathlib import Path
+
+from mixpanel import MixpanelException
+from posthog import Posthog
+
+from aider import __version__
+from aider.dump import dump  # noqa: F401
+from aider.models import model_info_manager
+
+PERCENT = 10
+
+def compute_hex_threshold(percent):
+    """Convert percentage to 6-digit hex threshold.
+
+    Args:
+        percent: Percentage threshold (0-100)
+
+    Returns:
+        str: 6-digit hex threshold
+    """
+    return format(int(0xFFFFFF * percent / 100), "06x")
+
+
+def is_uuid_in_percentage(uuid_str, percent):
+    """Check if a UUID string falls within the first X percent of the UUID space.
+
+    Args:
+        uuid_str: UUID string to test
+        percent: Percentage threshold (0-100)
+
+    Returns:
+        bool: True if UUID falls within the first X percent
+    """
+    if not (0 <= percent <= 100):
+        raise ValueError("Percentage must be between 0 and 100")
+
+    if not uuid_str:
+        return False
+
+    # Convert percentage to hex threshold (1% = "04...", 10% = "1a...", etc)
+    # Using first 6 hex digits
+    if percent == 0:
+        return False
+
+    threshold = compute_hex_threshold(percent)
+    return uuid_str[:6] <= threshold
+
+
+mixpanel_project_token = "6da9a43058a5d1b9f3353153921fb04d"
+posthog_project_api_key = "phc_99T7muzafUMMZX15H8XePbMSreEUzahHbtWjy3l5Qbv"
+posthog_host = "https://us.i.posthog.com"
+
+
+class Analytics:
+    # providers
+    mp = None
+    ph = None
+
+    # saved
+    user_id = None
+    permanently_disable = None
+    asked_opt_in = None
+
+    # ephemeral
+    logfile = None
+
+    def __init__(self, logfile=None,

locodiff-250425/results/aider_aider_analytics.py/x-ai_grok-code-fast-1/20250909_153841/metadata.json

@@ -0,0 +1,25 @@
+{
+    "model": "x-ai/grok-code-fast-1",
+    "benchmark_case": "aider_aider_analytics.py",
+    "prompts_dir": "locodiff-250425/prompts",
+    "prompt_file": "locodiff-250425/prompts/aider_aider_analytics.py_prompt.txt",
+    "expected_file": "locodiff-250425/prompts/aider_aider_analytics.py_expectedoutput.txt",
+    "timestamp_utc": "2025-09-09T15:38:41.149572+00:00",
+    "success": false,
+    "error": "Output mismatch",
+    "api_error": false,
+    "raw_response_length": 1644,
+    "extracted_output_length": 1634,
+    "expected_output_length": 7203,
+    "results_dir": "locodiff-250425/results/aider_aider_analytics.py/x-ai_grok-code-fast-1/20250909_153841",
+    "generation_id": "gen-1757432321-6X7xfuS2m5TdVVbPCbiT",
+    "cost_usd": 0.01980674,
+    "prompt_tokens": 24526,
+    "completion_tokens": 3378,
+    "total_tokens": 27904,
+    "native_prompt_tokens": 24214,
+    "native_completion_tokens": 9999,
+    "native_finish_reason": "length",
+    "stats_error": null,
+    "native_tokens_reasoning": 9563
+}

locodiff-250425/results/aider_aider_analytics.py/x-ai_grok-code-fast-1/20250909_153841/output.diff

@@ -0,0 +1,196 @@
+
+index bcb21745a..b670b870d 100644
+--- a/aider_aider_analytics.py_expectedoutput.txt (expected):tmp/tmpy8e02rew_expected.txt	
++++ b/aider_aider_analytics.py_extracted.txt (actual):tmp/tmpend_fxpn_actual.txt	
+@@ -14,7 +14,6 @@ from aider.models import model_info_manager
+
+ PERCENT = 10
+
+-
+ def compute_hex_threshold(percent):
+     """Convert percentage to 6-digit hex threshold.
+
+@@ -70,181 +69,4 @@ class Analytics:
+     # ephemeral
+     logfile = None
+
+-    def __init__(self, logfile=None, permanently_disable=False):
+-        self.logfile = logfile
+-        self.get_or_create_uuid()
+-
+-        if self.permanently_disable or permanently_disable or not self.asked_opt_in:
+-            self.disable(permanently_disable)
+-
+-    def enable(self):
+-        if not self.user_id:
+-            self.disable(False)
+-            return
+-
+-        if self.permanently_disable:
+-            self.disable(True)
+-            return
+-
+-        if not self.asked_opt_in:
+-            self.disable(False)
+-            return
+-
+-        # self.mp = Mixpanel(mixpanel_project_token)
+-        self.ph = Posthog(
+-            project_api_key=posthog_project_api_key,
+-            host=posthog_host,
+-            on_error=self.posthog_error,
+-            enable_exception_autocapture=True,
+-            super_properties=self.get_system_info(),  # Add system info to all events
+-        )
+-
+-    def disable(self, permanently):
+-        self.mp = None
+-        self.ph = None
+-
+-        if permanently:
+-            self.asked_opt_in = True
+-            self.permanently_disable = True
+-            self.save_data()
+-
+-    def need_to_ask(self, args_analytics):
+-        if args_analytics is False:
+-            return False
+-
+-        could_ask = not self.asked_opt_in and not self.permanently_disable
+-        if not could_ask:
+-            return False
+-
+-        if args_analytics is True:
+-            return True
+-
+-        assert args_analytics is None, args_analytics
+-
+-        if not self.user_id:
+-            return False
+-
+-        return is_uuid_in_percentage(self.user_id, PERCENT)
+-
+-    def get_data_file_path(self):
+-        try:
+-            data_file = Path.home() / ".aider" / "analytics.json"
+-            data_file.parent.mkdir(parents=True, exist_ok=True)
+-            return data_file
+-        except OSError:
+-            # If we can't create/access the directory, just disable analytics
+-            self.disable(permanently=False)
+-            return None
+-
+-    def get_or_create_uuid(self):
+-        self.load_data()
+-        if self.user_id:
+-            return
+-
+-        self.user_id = str(uuid.uuid4())
+-        self.save_data()
+-
+-    def load_data(self):
+-        data_file = self.get_data_file_path()
+-        if not data_file:
+-            return
+-
+-        if data_file.exists():
+-            try:
+-                data = json.loads(data_file.read_text())
+-                self.permanently_disable = data.get("permanently_disable")
+-                self.user_id = data.get("uuid")
+-                self.asked_opt_in = data.get("asked_opt_in", False)
+-            except (json.decoder.JSONDecodeError, OSError):
+-                self.disable(permanently=False)
+-
+-    def save_data(self):
+-        data_file = self.get_data_file_path()
+-        if not data_file:
+-            return
+-
+-        data = dict(
+-            uuid=self.user_id,
+-            permanently_disable=self.permanently_disable,
+-            asked_opt_in=self.asked_opt_in,
+-        )
+-
+-        try:
+-            data_file.write_text(json.dumps(data, indent=4))
+-        except OSError:
+-            # If we can't write the file, just disable analytics
+-            self.disable(permanently=False)
+-
+-    def get_system_info(self):
+-        return {
+-            "python_version": sys.version.split()[0],
+-            "os_platform": platform.system(),
+-            "os_release": platform.release(),
+-            "machine": platform.machine(),
+-            "aider_version": __version__,
+-        }
+-
+-    def _redact_model_name(self, model):
+-        if not model:
+-            return None
+-
+-        info = model_info_manager.get_model_from_cached_json_db(model.name)
+-        if info:
+-            return model.name
+-        elif "/" in model.name:
+-            return model.name.split("/")[0] + "/REDACTED"
+-        return None
+-
+-    def posthog_error(self):
+-        """disable posthog if we get an error"""
+-        print("X" * 100)
+-        # https://github.com/PostHog/posthog-python/blob/aider_aider_analytics.py_extracted.txt (actual)://github.com/Aider-AI/aider/issues/2532
+-        self.ph = None
+-
+-    def event(self, event_name, main_model=None, **kwargs):
+-        if not self.mp and not self.ph and not self.logfile:
+-            return
+-
+-        properties = {}
+-
+-        if main_model:
+-            properties["main_model"] = self._redact_model_name(main_model)
+-            properties["weak_model"] = self._redact_model_name(main_model.weak_model)
+-            properties["editor_model"] = self._redact_model_name(main_model.editor_model)
+-
+-        properties.update(kwargs)
+-
+-        # Handle numeric values
+-        for key, value in properties.items():
+-            if isinstance(value, (int, float)):
+-                properties[key] = value
+-            else:
+-                properties[key] = str(value)
+-
+-        if self.mp:
+-            try:
+-                self.mp.track(self.user_id, event_name, dict(properties))
+-            except MixpanelException:
+-                self.mp = None  # Disable mixpanel on connection errors
+-
+-        if self.ph:
+-            self.ph.capture(self.user_id, event_name, dict(properties))
+-
+-        if self.logfile:
+-            log_entry = {
+-                "event": event_name,
+-                "properties": properties,
+-                "user_id": self.user_id,
+-                "time": int(time.time()),
+-            }
+-            try:
+-                with open(self.logfile, "a") as f:
+-                    json.dump(log_entry, f)
+-                    f.write("\n")
+-            except OSError:
+-                pass  # Ignore OS errors when writing to logfile
+-
+-
+-if __name__ == "__main__":
+-    dump(compute_hex_threshold(PERCENT))
+\ No newline at end of file
++    def __init__(self, logfile=None,
+\ No newline at end of file