From b0f71ee4b4caf1142948624f6bc6113812478f3b Mon Sep 17 00:00:00 2001 From: Felix Jonas Kroner Date: Mon, 13 Jul 2026 13:18:24 +0200 Subject: [PATCH 01/10] [bench] add reproducible GPU benchmark suite (ExaBoost vs LightGBM/XGBoost/CatBoost) Adds benchmarks/: a self-contained, resumable harness comparing ExaBoost (built from the checkout) against upstream LightGBM (CUDA), XGBoost and CatBoost on GPU, across the classic gbm-bench datasets (fraud, covtype, year, higgs, epsilon, airline) plus optionally Numerai v5 all-data with official per-era corr/Sharpe metrics. Measures construction/train time, peak GPU/host memory, quality metrics and time-to-quality curves; writes a markdown report with charts. See benchmarks/README.md for reproduction. Co-Authored-By: Claude Fable 5 --- .gitignore | 1 + benchmarks/README.md | 70 ++++++ benchmarks/bench.py | 442 ++++++++++++++++++++++++++++++++++++ benchmarks/common.py | 79 +++++++ benchmarks/datasets.py | 304 +++++++++++++++++++++++++ benchmarks/orchestrate.py | 145 ++++++++++++ benchmarks/report.py | 199 ++++++++++++++++ benchmarks/requirements.txt | 7 + benchmarks/setup_envs.sh | 46 ++++ 9 files changed, 1293 insertions(+) create mode 100644 benchmarks/README.md create mode 100644 benchmarks/bench.py create mode 100644 benchmarks/common.py create mode 100644 benchmarks/datasets.py create mode 100644 benchmarks/orchestrate.py create mode 100644 benchmarks/report.py create mode 100644 benchmarks/requirements.txt create mode 100755 benchmarks/setup_envs.sh diff --git a/.gitignore b/.gitignore index 458ad4ebffe5..e18e11cab3df 100644 --- a/.gitignore +++ b/.gitignore @@ -484,3 +484,4 @@ dmypy.json # files created by release tasks /release-artifacts +benchmarks/workspace/ diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 000000000000..831c38b32026 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,70 @@ +# ExaBoost GPU benchmark suite + +Reproducible comparison of **ExaBoost** (this repo, `device_type=cuda`) against +**upstream LightGBM** (CUDA build), **XGBoost** (`device=cuda`, +`tree_method=hist`) and **CatBoost** (`task_type=GPU`) — training speed *and* +model quality on a single GPU. + +## Datasets + +The classic GBDT speed suite (NVIDIA gbm-bench lineage) plus Numerai: + +| dataset | shape | task | split | +|---|---|---|---| +| fraud (OpenML creditcard) | 285K × 29 | binary, imbalanced | stratified 20% | +| covtype (UCI) | 581K × 54 | 7-class | stratified 20% | +| year (UCI YearPredictionMSD) | 515K × 90 | regression | canonical last 51,630 | +| higgs (UCI) | 11M × 28 | binary | canonical last 500K | +| epsilon (LIBSVM) | 500K × 2000 | binary | official train/test | +| airline (Ikonomovska) | 115M × 13 | binary (ArrDelay>0) | random 20%, seed 42 | +| numerai (v5 "all data", optional) | ~6.7M × ~2.7K | regression + era metrics | last 200 eras, 10-era embargo | + +All datasets are cached as float32 arrays so every library trains from +identical bits. Numerai requires the v5 training parquet (get it with +`numerapi`) via the `NUMERAI_PARQUET` env var; its quality metrics are the +official Numerai correlation (rank → gaussianize → power 1.5 → Pearson), +per-era mean/std/Sharpe and max drawdown. + +## Run matrix + +Six library configs — `exaboost`, `exaboost-quant` (`use_quantized_grad`), +`lightgbm`, `lightgbm-quant`, `xgboost`, `catboost` — across two aligned +hyperparameter regimes (gbm-bench convention: 500 rounds, lr 0.1, 255 bins): + +- **shallow**: depth 6 / 63 leaves +- **deep**: depth 10 / 1023 leaves + +plus the official Numerai example-model config (2000 trees, lr 0.01, depth 5, +32 leaves, colsample 0.1). Structural caveat: LightGBM-family grows leaf-wise +(`num_leaves` + `max_depth` cap), XGBoost depth-wise (`max_depth`), CatBoost +symmetric (`depth`) — the regimes align the tree budget, not the tree shape. + +Each cell runs 1 discarded warmup + 3 timed repeats (median reported) + 1 +`curve` run that evaluates the held-out set periodically for time-to-quality +plots. Every run is an isolated subprocess; peak GPU memory and host RSS are +polled throughout. See the docstring in `bench.py` for exact timing semantics. + +## Reproducing + +```bash +# 1. build both environments (needs CUDA toolkit + driver, cmake, compiler) +./benchmarks/setup_envs.sh + +# 2. download + preprocess datasets (largest: airline ~6GB download, ~30GB RAM) +./benchmarks/workspace/env-competitors/bin/python benchmarks/datasets.py all +NUMERAI_PARQUET=/path/to/v5_all_data.parquet \ + ./benchmarks/workspace/env-competitors/bin/python benchmarks/datasets.py numerai # optional + +# 3. run the matrix (resumable — interrupt and relaunch freely) +python3 benchmarks/orchestrate.py + +# 4. aggregate results into workspace/report/REPORT.md + charts +./benchmarks/workspace/env-competitors/bin/python benchmarks/report.py +``` + +Everything lands under `benchmarks/workspace/` (override with +`EXABOOST_BENCH_ROOT`). `EXABOOST_CUDA_ARCHS` overrides +`CMAKE_CUDA_ARCHITECTURES` (default `native`; e.g. `120-real;120-virtual` for +Blackwell). Expect ~25GB of downloads and, with all datasets, a day-plus of +GPU time for the full matrix; `--only fraud,covtype,year` gives a quick +signal in under an hour. diff --git a/benchmarks/bench.py b/benchmarks/bench.py new file mode 100644 index 000000000000..6d0bab3826e6 --- /dev/null +++ b/benchmarks/bench.py @@ -0,0 +1,442 @@ +"""Run exactly one benchmark cell: (library, dataset, regime, kind). + +Appends one JSON line to ``/results/runs.jsonl`` and exits. Meant +to be launched by orchestrate.py in a fresh subprocess using the venv python +that owns the requested library, so GPU state and library versions are +isolated per run. + +Timing semantics (reported separately, because they differ per library): + +- ``construct_s``: building the library's training data structure — + ``lgb.Dataset`` (binning happens here), ``xgb.QuantileDMatrix`` (quantile + sketch), ``catboost.Pool`` (thin wrapper; CatBoost quantizes inside fit). +- ``train_s``: the boosting loop, including any remaining device transfer. +- ``curve`` runs additionally evaluate the held-out set every + ``eval_every`` iterations; their timings are NOT comparable to plain runs + (CatBoost's curve time axis is linearly approximated — no per-iteration + wall clock is exposed — and is flagged with ``curve_time_approx``). +""" + +import argparse +import json +import os +import subprocess +import sys +import threading +import time +import traceback + +import numpy as np + +from common import CACHE_DIR, DATASETS, REGIMES, RUNS_JSONL, SEED + + +class ResourceMonitor: + """Polls peak GPU memory (nvidia-smi) and host RSS in a thread.""" + + def __init__(self, interval=0.25): + self.interval = interval + self.gpu_peak_mb = 0 + self.rss_peak_mb = 0 + self._stop = threading.Event() + self._thread = threading.Thread(target=self._run, daemon=True) + + def _run(self): + import psutil + + proc = psutil.Process() + while not self._stop.is_set(): + try: + out = subprocess.run( + [ + "nvidia-smi", + "--query-gpu=memory.used", + "--format=csv,noheader,nounits", + ], + capture_output=True, + text=True, + timeout=5, + ).stdout.strip() + self.gpu_peak_mb = max(self.gpu_peak_mb, int(out.splitlines()[0])) + except Exception: + pass + try: + rss = proc.memory_info().rss + for c in proc.children(recursive=True): + try: + rss += c.memory_info().rss + except Exception: + pass + self.rss_peak_mb = max(self.rss_peak_mb, rss // (1024 * 1024)) + except Exception: + pass + self._stop.wait(self.interval) + + def __enter__(self): + self._thread.start() + return self + + def __exit__(self, *a): + self._stop.set() + self._thread.join(timeout=5) + + +def load_data(name): + d = os.path.join(CACHE_DIR, name) + if name == "numerai": + with open(os.path.join(d, "meta.json")) as fh: + meta = json.load(fh) + x = np.memmap( + os.path.join(d, "X.f32.mem"), + dtype=np.float32, + mode="r", + shape=(meta["n_rows"], meta["n_features"]), + ) + y = np.load(os.path.join(d, "y.npy")) + era = np.load(os.path.join(d, "era_codes.npy")) + tr_end, te_start = meta["train_end"], meta["test_start"] + return ( + x[:tr_end], + y[:tr_end], + x[te_start:], + y[te_start:], + {"era_test": era[te_start:]}, + ) + x_tr = np.load(os.path.join(d, "X_train.npy"), mmap_mode="r") + y_tr = np.load(os.path.join(d, "y_train.npy")) + x_te = np.load(os.path.join(d, "X_test.npy"), mmap_mode="r") + y_te = np.load(os.path.join(d, "y_test.npy")) + return x_tr, y_tr, x_te, y_te, {} + + +def numerai_corr_np(preds, targets): + """Official Numerai correlation (rank -> gaussianize -> pow 1.5 -> Pearson).""" + from scipy.stats import norm, rankdata + + n = len(preds) + ranked = (rankdata(preds, method="average") - 0.5) / n + gauss = norm.ppf(ranked) + preds_p15 = np.sign(gauss) * np.abs(gauss) ** 1.5 + centered = targets - targets.mean() + targets_p15 = np.sign(centered) * np.abs(centered) ** 1.5 + return float(np.corrcoef(preds_p15, targets_p15)[0, 1]) + + +def numerai_metrics(preds, y, era): + corrs = np.array( + [numerai_corr_np(preds[era == e], y[era == e]) for e in np.unique(era)] + ) + mean, std = corrs.mean(), corrs.std(ddof=0) + cumulative = np.cumprod(1 + corrs) + rolling_max = np.maximum.accumulate(cumulative) + drawdown = ((rolling_max - cumulative) / rolling_max).max() + return { + "corr_mean": float(mean), + "corr_std": float(std), + "corr_sharpe": float(mean / std) if std > 0 else None, + "max_drawdown": float(-drawdown), + "n_eras": len(corrs), + } + + +def quality_metrics(task, preds, y, extra): + from sklearn.metrics import accuracy_score, log_loss, roc_auc_score + + if task == "binary": + auc = roc_auc_score(y, preds) + return {"auc": float(auc), "sane": bool(auc > 0.55)} + if task == "multiclass": + acc = accuracy_score(y, preds.argmax(axis=1)) + ll = log_loss(y, preds, labels=list(range(preds.shape[1]))) + return {"accuracy": float(acc), "mlogloss": float(ll), "sane": bool(acc > 0.5)} + if task == "regression": + rmse = float(np.sqrt(np.mean((preds - y) ** 2))) + return {"rmse": rmse, "sane": bool(rmse < float(np.std(y)))} + if task == "numerai": + m = numerai_metrics(preds, y, extra["era_test"]) + m["rmse"] = float(np.sqrt(np.mean((preds - y) ** 2))) + m["sane"] = bool(m["corr_mean"] > 0) + return m + raise ValueError(task) + + +def run_lightgbm(task, x_tr, y_tr, x_te, y_te, reg, quantized, curve): + import lightgbm as lgb + + params = { + "objective": { + "binary": "binary", + "multiclass": "multiclass", + "regression": "regression", + "numerai": "regression", + }[task], + "learning_rate": reg["lr"], + "num_leaves": reg["leaves"], + "max_depth": reg["depth"], + "max_bin": 255, + "device_type": "cuda", + "num_threads": os.cpu_count(), + "seed": SEED, + "verbose": -1, + "metric": "None", + } + if task == "multiclass": + params["num_class"] = DATASETS["covtype"]["num_class"] + if "colsample" in reg: + params["feature_fraction"] = reg["colsample"] + if quantized: + params["use_quantized_grad"] = True + + t0 = time.perf_counter() + # the Dataset must be created with the final params (incl. device_type) + dtrain = lgb.Dataset(x_tr, label=y_tr, params=params) + dtrain.construct() + construct_s = time.perf_counter() - t0 + + curve_pts = [] + t0 = time.perf_counter() + if curve: + dvalid = lgb.Dataset(x_te, label=y_te, reference=dtrain) + bst = lgb.Booster(params=params, train_set=dtrain) + bst.add_valid(dvalid, "test") + for i in range(reg["rounds"]): + bst.update() + if (i + 1) % reg["eval_every"] == 0 or i + 1 == reg["rounds"]: + res = bst.eval_valid() + curve_pts.append( + [i + 1, time.perf_counter() - t0, res[0][2] if res else None] + ) + else: + bst = lgb.train(params, dtrain, num_boost_round=reg["rounds"]) + train_s = time.perf_counter() - t0 + + preds = bst.predict(x_te) + return { + "construct_s": construct_s, + "train_s": train_s, + "preds": preds, + "version": lgb.__version__, + "curve": curve_pts, + } + + +def run_xgboost(task, x_tr, y_tr, x_te, y_te, reg, curve): + import xgboost as xgb + + params = { + "objective": { + "binary": "binary:logistic", + "multiclass": "multi:softprob", + "regression": "reg:squarederror", + "numerai": "reg:squarederror", + }[task], + "eta": reg["lr"], + "max_depth": reg["depth"], + "max_bin": 255, + "device": "cuda", + "tree_method": "hist", + "nthread": os.cpu_count(), + "seed": SEED, + } + if task == "multiclass": + params["num_class"] = DATASETS["covtype"]["num_class"] + if "colsample" in reg: + params["colsample_bytree"] = reg["colsample"] + + t0 = time.perf_counter() + dtrain = xgb.QuantileDMatrix(np.asarray(x_tr), label=y_tr, max_bin=255) + construct_s = time.perf_counter() - t0 + + curve_pts = [] + t0 = time.perf_counter() + if curve: + dtest = xgb.DMatrix(np.asarray(x_te), label=y_te) + params["eval_metric"] = { + "binary": "auc", + "multiclass": "mlogloss", + "regression": "rmse", + "numerai": "rmse", + }[task] + bst = xgb.Booster(params, [dtrain]) + for i in range(reg["rounds"]): + bst.update(dtrain, i) + if (i + 1) % reg["eval_every"] == 0 or i + 1 == reg["rounds"]: + res = bst.eval_set([(dtest, "test")], i) + curve_pts.append( + [i + 1, time.perf_counter() - t0, float(res.split(":")[-1])] + ) + else: + bst = xgb.train(params, dtrain, num_boost_round=reg["rounds"]) + train_s = time.perf_counter() - t0 + + bst.set_param({"device": "cuda"}) + preds = bst.predict(xgb.DMatrix(np.asarray(x_te))) + return { + "construct_s": construct_s, + "train_s": train_s, + "preds": preds, + "version": xgb.__version__, + "curve": curve_pts, + } + + +def run_catboost(task, x_tr, y_tr, x_te, y_te, reg, curve): + import catboost as cb + + loss = { + "binary": "Logloss", + "multiclass": "MultiClass", + "regression": "RMSE", + "numerai": "RMSE", + }[task] + kw = { + "iterations": reg["rounds"], + "learning_rate": reg["lr"], + "depth": reg["depth"], + "border_count": 254, + "task_type": "GPU", + "devices": "0", + "random_seed": SEED, + "verbose": False, + "allow_writing_files": False, + "loss_function": loss, + } + rsm_dropped = False + if "colsample" in reg: + kw["rsm"] = reg["colsample"] + + t0 = time.perf_counter() + train_pool = cb.Pool(np.asarray(x_tr), label=y_tr) + construct_s = time.perf_counter() - t0 + + cls = ( + cb.CatBoostClassifier + if task in ("binary", "multiclass") + else cb.CatBoostRegressor + ) + + def fit(kw): + model = cls(**kw) + t0 = time.perf_counter() + if curve: + model.fit( + train_pool, + eval_set=cb.Pool(np.asarray(x_te), label=y_te), + metric_period=reg["eval_every"], + ) + else: + model.fit(train_pool) + return model, time.perf_counter() - t0 + + try: + model, train_s = fit(kw) + except Exception as e: + # rsm is not supported for every GPU loss; retry without and flag it + if "rsm" in kw and "rsm" in str(e).lower(): + kw.pop("rsm") + rsm_dropped = True + model, train_s = fit(kw) + else: + raise + + curve_pts = [] + if curve: + vals = model.get_evals_result().get("validation", {}) + if vals: + series = vals[next(iter(vals))] + n = len(series) + # no per-iteration wall clock exposed; approximate linearly + curve_pts = [ + [i + 1, train_s * (i + 1) / n, series[i]] + for i in range(0, n, reg["eval_every"]) + ] + + if task == "binary": + preds = model.predict_proba(np.asarray(x_te))[:, 1] + elif task == "multiclass": + preds = model.predict_proba(np.asarray(x_te)) + else: + preds = model.predict(np.asarray(x_te)) + return { + "construct_s": construct_s, + "train_s": train_s, + "preds": preds, + "version": cb.__version__, + "curve": curve_pts, + "rsm_dropped": rsm_dropped, + "curve_time_approx": bool(curve), + } + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument( + "--library", + required=True, + choices=[ + "exaboost", + "exaboost-quant", + "lightgbm", + "lightgbm-quant", + "xgboost", + "catboost", + ], + ) + ap.add_argument("--dataset", required=True, choices=list(DATASETS)) + ap.add_argument("--regime", required=True, choices=list(REGIMES)) + ap.add_argument("--kind", required=True) # warmup | timed1..3 | curve + ap.add_argument("--out", default=RUNS_JSONL) + args = ap.parse_args() + + task = DATASETS[args.dataset]["task"] + reg = REGIMES[args.regime] + curve = args.kind == "curve" + + rec = { + "library": args.library, + "dataset": args.dataset, + "regime": args.regime, + "kind": args.kind, + "status": "ok", + } + try: + x_tr, y_tr, x_te, y_te, extra = load_data(args.dataset) + rec["n_train"], rec["n_features"] = int(x_tr.shape[0]), int(x_tr.shape[1]) + with ResourceMonitor() as mon: + t_total = time.perf_counter() + if args.library.startswith(("exaboost", "lightgbm")): + r = run_lightgbm( + task, + x_tr, + y_tr, + x_te, + y_te, + reg, + quantized=args.library.endswith("-quant"), + curve=curve, + ) + elif args.library == "xgboost": + r = run_xgboost(task, x_tr, y_tr, x_te, y_te, reg, curve) + else: + r = run_catboost(task, x_tr, y_tr, x_te, y_te, reg, curve) + total_s = time.perf_counter() - t_total + preds = r.pop("preds") + rec.update(r) + rec["total_s"] = total_s + rec["trees_per_s"] = reg["rounds"] / r["train_s"] + rec["gpu_mem_peak_mb"] = mon.gpu_peak_mb + rec["rss_peak_mb"] = mon.rss_peak_mb + rec["metrics"] = quality_metrics(task, np.asarray(preds), y_te, extra) + except Exception: + rec["status"] = "failed" + rec["error"] = traceback.format_exc()[-3000:] + + os.makedirs(os.path.dirname(args.out), exist_ok=True) + with open(args.out, "a") as f: + f.write(json.dumps(rec) + "\n") + print(json.dumps({k: v for k, v in rec.items() if k != "curve"})[:2000]) + sys.exit(0 if rec["status"] == "ok" else 1) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/common.py b/benchmarks/common.py new file mode 100644 index 000000000000..0ab110cf57f8 --- /dev/null +++ b/benchmarks/common.py @@ -0,0 +1,79 @@ +"""Shared configuration for the ExaBoost GPU benchmark suite. + +All artifacts (raw downloads, preprocessed caches, results, report) live under +a single workspace directory, resolved from the ``EXABOOST_BENCH_ROOT`` +environment variable and defaulting to ``benchmarks/workspace`` inside the +repository checkout. +""" + +import os + +ROOT = os.environ.get( + "EXABOOST_BENCH_ROOT", + os.path.join(os.path.dirname(os.path.abspath(__file__)), "workspace"), +) +DATA_DIR = os.path.join(ROOT, "data") +CACHE_DIR = os.path.join(ROOT, "data", "cache") +RESULTS_DIR = os.path.join(ROOT, "results") +REPORT_DIR = os.path.join(ROOT, "report") +RUNS_JSONL = os.path.join(RESULTS_DIR, "runs.jsonl") + +SEED = 42 + +#: benchmark datasets; ``task`` drives objective/metric selection in bench.py +DATASETS = { + "higgs": {"task": "binary"}, + "epsilon": {"task": "binary"}, + "airline": {"task": "binary"}, + "covtype": {"task": "multiclass", "num_class": 7}, + "year": {"task": "regression"}, + "fraud": {"task": "binary"}, + "numerai": {"task": "numerai"}, +} + +#: hyperparameter regimes, aligned across libraries (gbm-bench convention); +#: LightGBM-family maps depth/leaves to (max_depth, num_leaves), XGBoost uses +#: max_depth (native depth-wise growth), CatBoost uses symmetric depth. +REGIMES = { + "shallow": {"rounds": 500, "lr": 0.1, "depth": 6, "leaves": 63, "eval_every": 25}, + "deep": {"rounds": 500, "lr": 0.1, "depth": 10, "leaves": 1023, "eval_every": 25}, + # official Numerai example-model parameters + "numerai": { + "rounds": 2000, + "lr": 0.01, + "depth": 5, + "leaves": 32, + "colsample": 0.1, + "eval_every": 250, + }, + # tiny config for smoke-testing the library/GPU integration + "smoke": {"rounds": 10, "lr": 0.1, "depth": 6, "leaves": 63, "eval_every": 5}, +} + +#: exaboost/lightgbm both install as package "lightgbm", hence two venvs +LIBRARIES = [ + "exaboost", + "exaboost-quant", + "lightgbm", + "lightgbm-quant", + "xgboost", + "catboost", +] + + +def venv_python(library: str) -> str: + """Path of the venv python that owns ``library`` (see setup_envs.sh).""" + env = "env-exaboost" if library.startswith("exaboost") else "env-competitors" + override = os.environ.get(f"EXABOOST_BENCH_PY_{env.replace('-', '_').upper()}") + return override or os.path.join(ROOT, env, "bin", "python") + + +def regimes_for(dataset: str): + """Regimes to benchmark for a dataset.""" + return ["numerai"] if dataset == "numerai" else ["shallow", "deep"] + + +def dataset_ready(dataset: str) -> bool: + """Whether the preprocessed cache for ``dataset`` exists.""" + marker = "meta.json" if dataset == "numerai" else "y_test.npy" + return os.path.exists(os.path.join(CACHE_DIR, dataset, marker)) diff --git a/benchmarks/datasets.py b/benchmarks/datasets.py new file mode 100644 index 000000000000..c4bfd0190e84 --- /dev/null +++ b/benchmarks/datasets.py @@ -0,0 +1,304 @@ +"""Download and preprocess benchmark datasets into ``/data/cache/``. + +Each dataset is cached as float32 ``X_train/y_train/X_test/y_test`` .npy files +(Numerai as one era-ordered float32 memmap plus metadata) so every library +trains from identical bits. Raw files are downloaded on demand with resume +support. + +Run inside the competitors venv (needs sklearn, pandas, pyarrow):: + + python benchmarks/datasets.py all # everything except numerai + python benchmarks/datasets.py higgs airline + NUMERAI_PARQUET=/path/to/v5_all_data.parquet python benchmarks/datasets.py numerai + +The Numerai parquet must contain ``feature*`` columns (int8), a ``target`` +column, and a string ``era`` column, sorted by era — the "all data" training +file distributed by Numerai (v5) has exactly this layout. +""" + +import json +import os +import subprocess +import sys +import zipfile + +import numpy as np + +from common import CACHE_DIR, DATA_DIR, SEED, dataset_ready + +URLS = { + "higgs.zip": "https://archive.ics.uci.edu/static/public/280/higgs.zip", + "epsilon_train.bz2": "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/epsilon_normalized.bz2", + "epsilon_test.bz2": "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/epsilon_normalized.t.bz2", + "airline.data.bz2": "http://kt.ijs.si/elena_ikonomovska/datasets/airline_14col.data.bz2", + "year.zip": "https://archive.ics.uci.edu/static/public/203/yearpredictionmsd.zip", + "covtype.data.gz": "https://archive.ics.uci.edu/ml/machine-learning-databases/covtype/covtype.data.gz", +} + +NUMERAI_TEST_ERAS = 200 +NUMERAI_EMBARGO_ERAS = 10 + + +def fetch(filename: str) -> str: + """Download ``filename`` into the data dir if missing; returns its path.""" + os.makedirs(DATA_DIR, exist_ok=True) + path = os.path.join(DATA_DIR, filename) + if os.path.exists(path): + return path + part = path + ".part" + cmd = [ + "curl", + "-SL", + "--retry", + "10", + "--retry-all-errors", + "-C", + "-", + "--fail", + "-o", + part, + URLS[filename], + ] + print(f"downloading {URLS[filename]}", flush=True) + subprocess.run(cmd, check=True) + os.rename(part, path) + return path + + +def save(name, x_tr, y_tr, x_te, y_te): + d = os.path.join(CACHE_DIR, name) + os.makedirs(d, exist_ok=True) + np.save( + os.path.join(d, "X_train.npy"), np.ascontiguousarray(x_tr, dtype=np.float32) + ) + np.save(os.path.join(d, "y_train.npy"), np.asarray(y_tr, dtype=np.float32)) + np.save(os.path.join(d, "X_test.npy"), np.ascontiguousarray(x_te, dtype=np.float32)) + np.save(os.path.join(d, "y_test.npy"), np.asarray(y_te, dtype=np.float32)) + print(f"{name}: train {x_tr.shape} test {x_te.shape} -> {d}", flush=True) + + +def random_split(x, y, frac=0.2, stratify=False): + from sklearn.model_selection import train_test_split + + return train_test_split( + x, y, test_size=frac, random_state=SEED, stratify=y if stratify else None + ) + + +def prep_higgs(): + import pandas as pd + + with zipfile.ZipFile(fetch("higgs.zip")) as z: + inner = z.namelist()[0] # HIGGS.csv.gz + with z.open(inner) as f: + df = pd.read_csv( + f, + header=None, + dtype=np.float32, + compression="gzip" if inner.endswith(".gz") else None, + ) + y = df[0].to_numpy() + x = df.drop(columns=[0]).to_numpy() + # canonical split: the last 500K rows are the test set + save("higgs", x[:-500_000], y[:-500_000], x[-500_000:], y[-500_000:]) + + +def prep_epsilon(): + from sklearn.datasets import load_svmlight_file + + x_tr, y_tr = load_svmlight_file(fetch("epsilon_train.bz2"), n_features=2000) + x_te, y_te = load_svmlight_file(fetch("epsilon_test.bz2"), n_features=2000) + save( + "epsilon", + x_tr.toarray(), + (y_tr > 0).astype(np.float32), + x_te.toarray(), + (y_te > 0).astype(np.float32), + ) + + +def prep_airline(): + import pandas as pd + + cols = [ + "Year", + "Month", + "DayofMonth", + "DayOfWeek", + "CRSDepTime", + "CRSArrTime", + "UniqueCarrier", + "FlightNum", + "ActualElapsedTime", + "Origin", + "Dest", + "Distance", + "Diverted", + "ArrDelay", + ] + cat_cols = ["UniqueCarrier", "Origin", "Dest"] + cat_maps = {c: {} for c in cat_cols} + chunks = [] + reader = pd.read_csv( + fetch("airline.data.bz2"), header=None, names=cols, chunksize=5_000_000 + ) + for i, ch in enumerate(reader): + for c in cat_cols: # ordinal-encode string categoricals + m = cat_maps[c] + vals = ch[c].astype(str) + for v in vals.unique(): + if v not in m: + m[v] = len(m) + ch[c] = vals.map(m) + y = (ch["ArrDelay"] > 0).astype(np.float32).to_numpy() + x = ch.drop(columns=["ArrDelay"]).to_numpy(dtype=np.float32) + chunks.append((x, y)) + print(f"airline chunk {i} ({len(ch)} rows)", flush=True) + x = np.concatenate([c[0] for c in chunks]) + y = np.concatenate([c[1] for c in chunks]) + del chunks + rng = np.random.default_rng(SEED) + perm = rng.permutation(len(x)) + n_test = int(0.2 * len(x)) + save( + "airline", + x[perm[n_test:]], + y[perm[n_test:]], + x[perm[:n_test]], + y[perm[:n_test]], + ) + + +def prep_covtype(): + import pandas as pd + + df = pd.read_csv(fetch("covtype.data.gz"), header=None) + y = df[54].to_numpy(dtype=np.float32) - 1 # labels 1..7 -> 0..6 + x = df.drop(columns=[54]).to_numpy(dtype=np.float32) + x_tr, x_te, y_tr, y_te = random_split(x, y, stratify=True) + save("covtype", x_tr, y_tr, x_te, y_te) + + +def prep_year(): + import pandas as pd + + with zipfile.ZipFile(fetch("year.zip")) as z: + with z.open(z.namelist()[0]) as f: + df = pd.read_csv(f, header=None, dtype=np.float32) + y = df[0].to_numpy() + x = df.drop(columns=[0]).to_numpy() + # canonical split: first 463,715 train / last 51,630 test + save("year", x[:463_715], y[:463_715], x[463_715:], y[463_715:]) + + +def prep_fraud(): + from sklearn.datasets import fetch_openml + + ds = fetch_openml("creditcard", version=1, as_frame=False, parser="auto") + x = ds.data.astype(np.float32) + y = ds.target.astype(np.float32) + x_tr, x_te, y_tr, y_te = random_split(x, y, stratify=True) + save("fraud", x_tr, y_tr, x_te, y_te) + + +def prep_numerai(): + """Era-ordered float32 memmap; last N eras held out with an embargo gap. + + Rows without a target are dropped. Train rows are ``X[:train_end]`` and + test rows ``X[test_start:]`` so both are zero-copy views of the memmap. + """ + import pyarrow.parquet as pq + + src = os.environ.get("NUMERAI_PARQUET") + if not src: + sys.exit("numerai: set NUMERAI_PARQUET to the v5 'all data' training parquet") + d = os.path.join(CACHE_DIR, "numerai") + os.makedirs(d, exist_ok=True) + f = pq.ParquetFile(src) + feat_cols = [c for c in f.schema_arrow.names if c.startswith("feature")] + + # pass 1: era + target only, to build the row filter and split boundaries + et = f.read(columns=["era", "target"]).to_pandas() + era_int = et["era"].astype(int).to_numpy() + keep = et["target"].notna().to_numpy() + if not (np.diff(era_int) >= 0).all(): + sys.exit("numerai: parquet must be sorted by era") + + kept_eras = era_int[keep] + uniq = np.unique(kept_eras) + test_eras = uniq[-NUMERAI_TEST_ERAS:] + embargo_eras = uniq[ + -(NUMERAI_TEST_ERAS + NUMERAI_EMBARGO_ERAS) : -NUMERAI_TEST_ERAS + ] + role = np.full( + len(kept_eras), 1, dtype=np.int8 + ) # 1 train, 0 embargo (drop), 2 test + role[np.isin(kept_eras, embargo_eras)] = 0 + role[np.isin(kept_eras, test_eras)] = 2 + + keep_within = role != 0 + n_rows = int(keep_within.sum()) + train_end = int((role == 1).sum()) + p = len(feat_cols) + print(f"numerai: {n_rows} rows x {p} features, train_end={train_end}", flush=True) + + x = np.memmap( + os.path.join(d, "X.f32.mem"), dtype=np.float32, mode="w+", shape=(n_rows, p) + ) + y = np.empty(n_rows, dtype=np.float32) + era_out = np.empty(n_rows, dtype=np.int32) + + # pass 2: stream feature batches into the memmap + keep_all = keep.copy() + keep_all[keep] = keep_within # absolute row filter + tgt_all = et["target"].to_numpy(dtype=np.float32) + row_abs = row_out = 0 + for batch in f.iter_batches(batch_size=200_000, columns=feat_cols): + nb = batch.num_rows + mask = keep_all[row_abs : row_abs + nb] + if mask.any(): + arr = batch.to_pandas().to_numpy(dtype=np.float32, na_value=np.nan) + sel = arr[mask] + x[row_out : row_out + len(sel)] = sel + y[row_out : row_out + len(sel)] = tgt_all[row_abs : row_abs + nb][mask] + era_out[row_out : row_out + len(sel)] = era_int[row_abs : row_abs + nb][ + mask + ] + row_out += len(sel) + row_abs += nb + assert row_out == n_rows, (row_out, n_rows) + x.flush() + np.save(os.path.join(d, "y.npy"), y) + np.save(os.path.join(d, "era_codes.npy"), era_out) + meta = { + "n_rows": n_rows, + "n_features": p, + "train_end": train_end, + "test_start": train_end, + "n_test_eras": NUMERAI_TEST_ERAS, + "embargo_eras": NUMERAI_EMBARGO_ERAS, + "source": src, + } + with open(os.path.join(d, "meta.json"), "w") as fh: + json.dump(meta, fh) + print(f"numerai done: {n_rows} x {p}", flush=True) + + +PREPS = { + "higgs": prep_higgs, + "epsilon": prep_epsilon, + "airline": prep_airline, + "covtype": prep_covtype, + "year": prep_year, + "fraud": prep_fraud, + "numerai": prep_numerai, +} + +if __name__ == "__main__": + names = sys.argv[1:] + targets = [n for n in PREPS if n != "numerai"] if names == ["all"] else names + for t in targets: + if dataset_ready(t): + print(f"{t}: cached, skipping", flush=True) + else: + PREPS[t]() diff --git a/benchmarks/orchestrate.py b/benchmarks/orchestrate.py new file mode 100644 index 000000000000..265afe0aba8f --- /dev/null +++ b/benchmarks/orchestrate.py @@ -0,0 +1,145 @@ +"""Resumable benchmark matrix runner. + +Executes bench.py cells sequentially (each run gets the GPU exclusively), +skipping cells already recorded in ``/results/runs.jsonl``, so it +is safe to interrupt and relaunch at any time. Datasets whose preprocessed +cache is missing are skipped with a note (run datasets.py first). + +Usage:: + + python benchmarks/orchestrate.py # everything available + python benchmarks/orchestrate.py --only higgs,epsilon + python benchmarks/orchestrate.py --dry-run +""" + +import argparse +import json +import os +import subprocess +import sys +import time + +from common import LIBRARIES, RUNS_JSONL, dataset_ready, regimes_for, venv_python + +BENCH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "bench.py") + +#: small/fast first for early signal; the huge ones last +DATASET_ORDER = ["fraud", "covtype", "year", "higgs", "epsilon", "numerai", "airline"] +KINDS = ["warmup", "timed1", "timed2", "timed3", "curve"] +TIMEOUT_S = { + "fraud": 1800, + "covtype": 1800, + "year": 1800, + "higgs": 7200, + "epsilon": 7200, + "numerai": 14400, + "airline": 14400, +} + + +def load_done(): + done = {} + if os.path.exists(RUNS_JSONL): + with open(RUNS_JSONL) as f: + for line in f: + try: + r = json.loads(line) + except json.JSONDecodeError: + continue + done[(r["library"], r["dataset"], r["regime"], r["kind"])] = r["status"] + return done + + +def record(status, lib, ds, reg, kind): + with open(RUNS_JSONL, "a") as f: + f.write( + json.dumps( + { + "library": lib, + "dataset": ds, + "regime": reg, + "kind": kind, + "status": status, + } + ) + + "\n" + ) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--only", default=None, help="comma-separated dataset subset") + ap.add_argument("--dry-run", action="store_true") + args = ap.parse_args() + datasets = args.only.split(",") if args.only else DATASET_ORDER + + cells = [] + for ds in datasets: + if not dataset_ready(ds): + print( + f"NOTE: dataset '{ds}' not prepared, skipping (run datasets.py)", + flush=True, + ) + continue + for reg in regimes_for(ds): + for lib in LIBRARIES: + for kind in KINDS: + cells.append((lib, ds, reg, kind)) + + done = load_done() + todo = [c for c in cells if c not in done] + print( + f"matrix: {len(cells)} cells, {len(cells) - len(todo)} done, {len(todo)} to run", + flush=True, + ) + if args.dry_run: + for c in todo: + print(c) + return + + os.makedirs(os.path.dirname(RUNS_JSONL), exist_ok=True) + for i, (lib, ds, reg, kind) in enumerate(todo): + done = load_done() + if (lib, ds, reg, kind) in done: + continue + # if the warmup for this combo failed, don't waste time on the rest + if kind != "warmup" and done.get((lib, ds, reg, "warmup")) == "failed": + print(f"SKIP {lib}/{ds}/{reg}/{kind} (warmup failed)", flush=True) + record("skipped_warmup_failed", lib, ds, reg, kind) + continue + cmd = [ + venv_python(lib), + BENCH, + "--library", + lib, + "--dataset", + ds, + "--regime", + reg, + "--kind", + kind, + ] + t0 = time.time() + print(f"[{i + 1}/{len(todo)}] RUN {lib}/{ds}/{reg}/{kind}", flush=True) + try: + p = subprocess.run( + cmd, + timeout=TIMEOUT_S.get(ds, 7200), + env={ + **os.environ, + "CUDA_VISIBLE_DEVICES": os.environ.get("CUDA_VISIBLE_DEVICES", "0"), + }, + capture_output=True, + text=True, + ) + status = "ok" if p.returncode == 0 else "failed" + if p.returncode != 0: + sys.stderr.write(p.stdout[-2000:] + p.stderr[-2000:] + "\n") + except subprocess.TimeoutExpired: + status = "timeout" + record("timeout", lib, ds, reg, kind) + print(f" -> {status} ({time.time() - t0:.0f}s)", flush=True) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/report.py b/benchmarks/report.py new file mode 100644 index 000000000000..a4aa36e42cda --- /dev/null +++ b/benchmarks/report.py @@ -0,0 +1,199 @@ +"""Aggregate ``/results/runs.jsonl`` into REPORT.md + charts.""" + +import json +import os +import subprocess + +import matplotlib +import numpy as np +import pandas as pd + +matplotlib.use("Agg") +import matplotlib.pyplot as plt # noqa: E402 + +from common import LIBRARIES, REPORT_DIR, RESULTS_DIR, RUNS_JSONL # noqa: E402 + +COLORS = { + "exaboost": "#d62728", + "exaboost-quant": "#ff9896", + "lightgbm": "#1f77b4", + "lightgbm-quant": "#aec7e8", + "xgboost": "#2ca02c", + "catboost": "#9467bd", +} +METRIC_KEY = { + "higgs": "auc", + "epsilon": "auc", + "airline": "auc", + "fraud": "auc", + "covtype": "accuracy", + "year": "rmse", + "numerai": "corr_mean", +} +NOTES = """ +## Measurement notes + +- `construct` covers each library's training-data structure: LightGBM `Dataset` + (binning), XGBoost `QuantileDMatrix` (quantile sketch), CatBoost `Pool` (thin + wrapper — CatBoost quantizes inside `fit`, i.e. inside `train`). +- CatBoost pre-reserves most of the GPU memory by default, so its "GPU peak" + reflects the reservation, not the working set. +- Time-to-quality curves for CatBoost use a linearly approximated time axis + (dashed) because it exposes no per-iteration wall clock. +- `xx-quant` = `use_quantized_grad=true`. Runs flagged `sane=false` produced + degenerate models and their timings should be ignored. +""" + + +def fmt(x, nd=1): + return "—" if x is None or (isinstance(x, float) and np.isnan(x)) else f"{x:.{nd}f}" + + +def main(): + os.makedirs(REPORT_DIR, exist_ok=True) + df = pd.DataFrame([json.loads(line) for line in open(RUNS_JSONL)]) + timed = df[df["kind"].str.startswith("timed") & (df["status"] == "ok")].copy() + for c in ("construct_s", "train_s", "total_s", "gpu_mem_peak_mb", "rss_peak_mb"): + timed[c] = pd.to_numeric(timed[c], errors="coerce") + + lines = ["# GPU GBDT benchmark: ExaBoost vs LightGBM vs XGBoost vs CatBoost\n"] + + sha_file = os.path.join(RESULTS_DIR, "exaboost_sha.txt") + sha = open(sha_file).read().strip()[:12] if os.path.exists(sha_file) else "unknown" + gpu = subprocess.run( + [ + "nvidia-smi", + "--query-gpu=name,memory.total,driver_version", + "--format=csv,noheader", + ], + capture_output=True, + text=True, + ).stdout.strip() + vers = ( + df[df["status"] == "ok"] + .groupby("library")["version"] + .agg(lambda s: s.dropna().iloc[0] if s.notna().any() else "?") + ) + lines += [ + "## Environment\n", + f"- GPU: {gpu}", + f"- ExaBoost: `{sha}`", + *[f"- {lib}: {v}" for lib, v in vers.items()], + NOTES, + ] + + # ---- per dataset x regime tables ------------------------------------ + for (ds, reg), g in timed.groupby(["dataset", "regime"], sort=False): + mkey = METRIC_KEY.get(ds, "auc") + lines.append(f"## {ds} — regime `{reg}`\n") + lines.append( + f"| library | construct (s) | train (s) | total (s) | {mkey} | GPU peak (MB) | RSS peak (MB) |" + ) + lines.append("|---|---|---|---|---|---|---|") + base = g[g["library"] == "exaboost"]["train_s"].median() + for lib in LIBRARIES: + gl = g[g["library"] == lib] + if gl.empty: + fails = df[ + (df.dataset == ds) + & (df.regime == reg) + & (df.library == lib) + & (df.status != "ok") + ] + note = fails["status"].iloc[0] if not fails.empty else "missing" + lines.append(f"| {lib} | {note} | | | | | |") + continue + met = gl["metrics"].iloc[0] or {} + spread = gl["train_s"].max() - gl["train_s"].min() + rel = ( + f" ({base / gl['train_s'].median():.2f}×)" + if lib != "exaboost" and base and gl["train_s"].median() + else "" + ) + flag = "" if met.get("sane", True) else " ⚠️insane" + lines.append( + f"| {lib} | {fmt(gl['construct_s'].median())} " + f"| {fmt(gl['train_s'].median())} ±{fmt(spread / 2)}{rel} " + f"| {fmt(gl['total_s'].median())} " + f"| {fmt(met.get(mkey), 4)}{flag} " + f"| {fmt(gl['gpu_mem_peak_mb'].median(), 0)} " + f"| {fmt(gl['rss_peak_mb'].median(), 0)} |" + ) + lines.append("") + + # ---- train-time bar charts ------------------------------------------ + for reg in ("shallow", "deep"): + sub = timed[timed["regime"] == reg] + if sub.empty: + continue + datasets = [ + d + for d in ["fraud", "covtype", "year", "higgs", "epsilon", "airline"] + if d in set(sub["dataset"]) + ] + x = np.arange(len(datasets)) + w = 0.13 + fig, ax = plt.subplots(figsize=(11, 5)) + for i, lib in enumerate(LIBRARIES): + vals = [ + sub[(sub.dataset == d) & (sub.library == lib)]["train_s"].median() + for d in datasets + ] + ax.bar(x + (i - 2.5) * w, vals, w, label=lib, color=COLORS[lib]) + ax.set_yscale("log") + ax.set_xticks(x, datasets) + ax.set_ylabel("train time (s, log)") + ax.set_title(f"GPU training time — regime {reg} (500 trees)") + ax.legend(ncol=3, fontsize=8) + fig.tight_layout() + fig.savefig(os.path.join(REPORT_DIR, f"train_time_{reg}.png"), dpi=120) + lines.append(f"![train time {reg}](train_time_{reg}.png)\n") + + # ---- time-to-quality curves ----------------------------------------- + curves = df[(df["kind"] == "curve") & (df["status"] == "ok")] + for (ds, reg), g in curves.groupby(["dataset", "regime"], sort=False): + fig, ax = plt.subplots(figsize=(7, 4.5)) + plotted = False + for _, r in g.iterrows(): + pts = [p for p in (r.get("curve") or []) if p[2] is not None] + if not pts: + continue + style = "--" if r.get("curve_time_approx") else "-" + ax.plot( + [p[1] for p in pts], + [p[2] for p in pts], + style, + label=r["library"], + color=COLORS[r["library"]], + ) + plotted = True + if not plotted: + plt.close(fig) + continue + ax.set_xlabel("wall time (s)") + ax.set_ylabel("held-out metric") + ax.set_title(f"time-to-quality — {ds} / {reg}") + ax.legend(fontsize=8) + fig.tight_layout() + name = f"curve_{ds}_{reg}.png" + fig.savefig(os.path.join(REPORT_DIR, name), dpi=120) + lines.append(f"![curve {ds} {reg}]({name})\n") + + fails = df[~df["status"].isin(["ok"])] + if not fails.empty: + lines.append("## Failed / skipped runs\n") + for _, r in fails.iterrows(): + err = (r.get("error") or "").strip().splitlines() + lines.append( + f"- {r['library']}/{r['dataset']}/{r['regime']}/{r['kind']}: {r['status']}" + + (f" — `{err[-1][:160]}`" if err else "") + ) + lines.append("") + + with open(os.path.join(REPORT_DIR, "REPORT.md"), "w") as f: + f.write("\n".join(lines)) + print(f"wrote {REPORT_DIR}/REPORT.md") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/requirements.txt b/benchmarks/requirements.txt new file mode 100644 index 000000000000..541826c08efc --- /dev/null +++ b/benchmarks/requirements.txt @@ -0,0 +1,7 @@ +numpy +pandas +pyarrow +scikit-learn +scipy +psutil +matplotlib diff --git a/benchmarks/setup_envs.sh b/benchmarks/setup_envs.sh new file mode 100755 index 000000000000..ceb7dcb648bd --- /dev/null +++ b/benchmarks/setup_envs.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Set up the two python environments for the GPU benchmark suite: +# env-exaboost ExaBoost built from THIS checkout (CUDA) +# env-competitors upstream LightGBM (CUDA, built from PyPI sdist), +# XGBoost and CatBoost (PyPI wheels, ship CUDA support) +# +# Environment variables: +# EXABOOST_BENCH_ROOT workspace dir (default: benchmarks/workspace) +# EXABOOST_CUDA_ARCHS CMAKE_CUDA_ARCHITECTURES (default: native) +# UPSTREAM_LGBM_VERSION upstream LightGBM version (default: latest on PyPI) +set -euo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO="$(dirname "$HERE")" +ROOT="${EXABOOST_BENCH_ROOT:-$HERE/workspace}" +ARCHS="${EXABOOST_CUDA_ARCHS:-native}" + +mkdir -p "$ROOT"/{data,results,report} + +echo "=== env-exaboost: building ExaBoost from $REPO (archs: $ARCHS)" +python3 -m venv "$ROOT/env-exaboost" +# shellcheck disable=SC1091 +source "$ROOT/env-exaboost/bin/activate" +pip install -q --upgrade pip +pip install -q -r "$HERE/requirements.txt" +# BUILD_WITH_SHARED_NCCL avoids nvlink failures against the static NCCL on +# arches it was not device-linked for (e.g. Blackwell) +(cd "$REPO" && CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=$ARCHS -DBUILD_WITH_SHARED_NCCL=ON" sh build-python.sh install --cuda) +git -C "$REPO" rev-parse HEAD > "$ROOT/results/exaboost_sha.txt" +python -c "import lightgbm as l; print('exaboost OK', l.__version__)" +deactivate + +echo "=== env-competitors: upstream LightGBM (CUDA) + XGBoost + CatBoost" +python3 -m venv "$ROOT/env-competitors" +# shellcheck disable=SC1091 +source "$ROOT/env-competitors/bin/activate" +pip install -q --upgrade pip +pip install -q -r "$HERE/requirements.txt" +pip install -q xgboost catboost +CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=$ARCHS" \ + pip install --no-binary lightgbm --config-settings=cmake.define.USE_CUDA=ON \ + "lightgbm${UPSTREAM_LGBM_VERSION:+==$UPSTREAM_LGBM_VERSION}" +python -c "import lightgbm as l, xgboost as x, catboost as c; print('upstream lgbm', l.__version__, '| xgb', x.__version__, '| catboost', c.__version__)" +deactivate + +echo "setup complete; workspace: $ROOT" From e8493afe0a54547f940883ee5e440d953b821405 Mon Sep 17 00:00:00 2001 From: Felix Jonas Kroner Date: Mon, 13 Jul 2026 13:51:03 +0200 Subject: [PATCH 02/10] [bench] record failures for runs that die before writing their record A bench.py subprocess that segfaults (observed: upstream LightGBM with use_quantized_grad on CUDA at HIGGS scale) leaves no line in runs.jsonl, so a resumed orchestrator would retry it forever and the warmup-failed skip never triggers. Record the failure from the orchestrator side when the child exits non-zero without having written its own record. Co-Authored-By: Claude Fable 5 --- benchmarks/orchestrate.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/benchmarks/orchestrate.py b/benchmarks/orchestrate.py index 265afe0aba8f..4060bcd9eec0 100644 --- a/benchmarks/orchestrate.py +++ b/benchmarks/orchestrate.py @@ -50,7 +50,7 @@ def load_done(): return done -def record(status, lib, ds, reg, kind): +def record(status, lib, ds, reg, kind, **extra): with open(RUNS_JSONL, "a") as f: f.write( json.dumps( @@ -60,6 +60,7 @@ def record(status, lib, ds, reg, kind): "regime": reg, "kind": kind, "status": status, + **extra, } ) + "\n" @@ -133,6 +134,17 @@ def main(): text=True, ) status = "ok" if p.returncode == 0 else "failed" + if p.returncode != 0 and (lib, ds, reg, kind) not in load_done(): + # the child died before writing its record (e.g. segfault); + # record the failure so resume doesn't retry it forever + record( + "failed", + lib, + ds, + reg, + kind, + error=f"exit code {p.returncode}: {p.stderr[-500:]}", + ) if p.returncode != 0: sys.stderr.write(p.stdout[-2000:] + p.stderr[-2000:] + "\n") except subprocess.TimeoutExpired: From 8ebd41b5c7347cc5c46b0807650d29a95e4471cf Mon Sep 17 00:00:00 2001 From: Felix Jonas Kroner Date: Mon, 13 Jul 2026 14:20:40 +0200 Subject: [PATCH 03/10] [bench] align L2 leaf regularization (1.0) across engines in speed regimes Engine defaults differ (xgboost lambda=1, lightgbm lambda_l2=0, catboost l2_leaf_reg=3). At lr 0.1 the unregularized LightGBM-family models degenerate on imbalanced data (creditcard fraud: 0.56 AUC vs 0.97 with lambda_l2=1, xgboost 0.98 with its default lambda=1), so unaligned runs measure default regularization choices rather than the engines. The numerai regime intentionally keeps each engine's own defaults, mirroring how the official example config is used in practice. Co-Authored-By: Claude Fable 5 --- benchmarks/README.md | 8 +++++++- benchmarks/bench.py | 6 ++++++ benchmarks/common.py | 21 +++++++++++++++++++-- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index 831c38b32026..a864dd5089ed 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -29,11 +29,17 @@ per-era mean/std/Sharpe and max drawdown. Six library configs — `exaboost`, `exaboost-quant` (`use_quantized_grad`), `lightgbm`, `lightgbm-quant`, `xgboost`, `catboost` — across two aligned -hyperparameter regimes (gbm-bench convention: 500 rounds, lr 0.1, 255 bins): +hyperparameter regimes (gbm-bench convention: 500 rounds, lr 0.1, 255 bins, +L2 leaf regularization 1.0): - **shallow**: depth 6 / 63 leaves - **deep**: depth 10 / 1023 leaves +L2 is aligned explicitly because engine defaults differ (XGBoost `lambda=1`, +LightGBM `lambda_l2=0`, CatBoost `l2_leaf_reg=3`) and at lr 0.1 an +unregularized leaf-wise model degenerates on imbalanced data (fraud drops to +~0.5 AUC), which would measure default choices rather than the engines. + plus the official Numerai example-model config (2000 trees, lr 0.01, depth 5, 32 leaves, colsample 0.1). Structural caveat: LightGBM-family grows leaf-wise (`num_leaves` + `max_depth` cap), XGBoost depth-wise (`max_depth`), CatBoost diff --git a/benchmarks/bench.py b/benchmarks/bench.py index 6d0bab3826e6..9c183f04dcf8 100644 --- a/benchmarks/bench.py +++ b/benchmarks/bench.py @@ -184,6 +184,8 @@ def run_lightgbm(task, x_tr, y_tr, x_te, y_te, reg, quantized, curve): params["num_class"] = DATASETS["covtype"]["num_class"] if "colsample" in reg: params["feature_fraction"] = reg["colsample"] + if "l2" in reg: + params["lambda_l2"] = reg["l2"] if quantized: params["use_quantized_grad"] = True @@ -242,6 +244,8 @@ def run_xgboost(task, x_tr, y_tr, x_te, y_te, reg, curve): params["num_class"] = DATASETS["covtype"]["num_class"] if "colsample" in reg: params["colsample_bytree"] = reg["colsample"] + if "l2" in reg: + params["lambda"] = reg["l2"] # xgboost default is already 1 t0 = time.perf_counter() dtrain = xgb.QuantileDMatrix(np.asarray(x_tr), label=y_tr, max_bin=255) @@ -304,6 +308,8 @@ def run_catboost(task, x_tr, y_tr, x_te, y_te, reg, curve): rsm_dropped = False if "colsample" in reg: kw["rsm"] = reg["colsample"] + if "l2" in reg: + kw["l2_leaf_reg"] = reg["l2"] # catboost default is 3 t0 = time.perf_counter() train_pool = cb.Pool(np.asarray(x_tr), label=y_tr) diff --git a/benchmarks/common.py b/benchmarks/common.py index 0ab110cf57f8..e91418c2d8bf 100644 --- a/benchmarks/common.py +++ b/benchmarks/common.py @@ -35,8 +35,25 @@ #: LightGBM-family maps depth/leaves to (max_depth, num_leaves), XGBoost uses #: max_depth (native depth-wise growth), CatBoost uses symmetric depth. REGIMES = { - "shallow": {"rounds": 500, "lr": 0.1, "depth": 6, "leaves": 63, "eval_every": 25}, - "deep": {"rounds": 500, "lr": 0.1, "depth": 10, "leaves": 1023, "eval_every": 25}, + # l2=1 aligns L2 leaf regularization across engines (defaults differ: + # xgboost lambda=1, lightgbm lambda_l2=0, catboost l2_leaf_reg=3); at + # lr 0.1 an unregularized leaf-wise model degenerates on imbalanced data + "shallow": { + "rounds": 500, + "lr": 0.1, + "depth": 6, + "leaves": 63, + "l2": 1.0, + "eval_every": 25, + }, + "deep": { + "rounds": 500, + "lr": 0.1, + "depth": 10, + "leaves": 1023, + "l2": 1.0, + "eval_every": 25, + }, # official Numerai example-model parameters "numerai": { "rounds": 2000, From 3e7dbca57bb8b3ec0bbac5b05f4dbb87392fc8e3 Mon Sep 17 00:00:00 2001 From: Felix Jonas Kroner Date: Mon, 13 Jul 2026 14:40:03 +0200 Subject: [PATCH 04/10] [bench] chunk xgboost test-set prediction to avoid device OOM Building a full GPU DMatrix of a wide test set (numerai: 1.3M x 2748) requested 26GB with the trained booster still resident on a 32GB card. Predict through inplace_predict in 200K-row chunks instead. Co-Authored-By: Claude Fable 5 --- benchmarks/bench.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/benchmarks/bench.py b/benchmarks/bench.py index 9c183f04dcf8..6c9119700f2d 100644 --- a/benchmarks/bench.py +++ b/benchmarks/bench.py @@ -273,8 +273,15 @@ def run_xgboost(task, x_tr, y_tr, x_te, y_te, reg, curve): bst = xgb.train(params, dtrain, num_boost_round=reg["rounds"]) train_s = time.perf_counter() - t0 - bst.set_param({"device": "cuda"}) - preds = bst.predict(xgb.DMatrix(np.asarray(x_te))) + # chunked inplace_predict: a full GPU DMatrix of a wide test set can OOM + # the device while the trained booster is still resident + step = 200_000 + preds = np.concatenate( + [ + bst.inplace_predict(np.ascontiguousarray(x_te[i : i + step])) + for i in range(0, x_te.shape[0], step) + ] + ) return { "construct_s": construct_s, "train_s": train_s, From 333a3fa619e388b061cc1e3224aee41cf1aa8892 Mon Sep 17 00:00:00 2001 From: Felix Jonas Kroner Date: Tue, 14 Jul 2026 07:00:55 +0200 Subject: [PATCH 05/10] [bench] report: tolerate non-string error fields from pandas NaN fill Co-Authored-By: Claude Fable 5 --- benchmarks/report.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/benchmarks/report.py b/benchmarks/report.py index a4aa36e42cda..b644759248ca 100644 --- a/benchmarks/report.py +++ b/benchmarks/report.py @@ -183,7 +183,8 @@ def main(): if not fails.empty: lines.append("## Failed / skipped runs\n") for _, r in fails.iterrows(): - err = (r.get("error") or "").strip().splitlines() + err_raw = r.get("error") + err = err_raw.strip().splitlines() if isinstance(err_raw, str) else [] lines.append( f"- {r['library']}/{r['dataset']}/{r['regime']}/{r['kind']}: {r['status']}" + (f" — `{err[-1][:160]}`" if err else "") From 5ed108c0564bf0bc8c58a6c9e1be5c069092bcd6 Mon Sep 17 00:00:00 2001 From: Felix Jonas Kroner Date: Tue, 14 Jul 2026 08:02:37 +0200 Subject: [PATCH 06/10] [bench] chart honesty: mark crashed configs and hatch degenerate-model timings An absent bar (e.g. upstream lightgbm-quant segfaulting on higgs) read as 'not measured'; a fast bar for a sane=false run (upstream quant garbage models on fraud/covtype/year) read as a legitimate win. Crashes now get an explicit marker and insane runs are hatched/dimmed. Co-Authored-By: Claude Fable 5 --- benchmarks/report.py | 47 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/benchmarks/report.py b/benchmarks/report.py index b644759248ca..8cbab9739384 100644 --- a/benchmarks/report.py +++ b/benchmarks/report.py @@ -135,15 +135,50 @@ def main(): w = 0.13 fig, ax = plt.subplots(figsize=(11, 5)) for i, lib in enumerate(LIBRARIES): - vals = [ - sub[(sub.dataset == d) & (sub.library == lib)]["train_s"].median() - for d in datasets - ] - ax.bar(x + (i - 2.5) * w, vals, w, label=lib, color=COLORS[lib]) + offs = x + (i - 2.5) * w + vals, hatches = [], [] + for d in datasets: + gl = sub[(sub.dataset == d) & (sub.library == lib)] + if gl.empty: + vals.append(np.nan) + hatches.append(None) + # crashed/failed configs get an explicit marker instead of + # silently-absent bars (e.g. upstream quantized on higgs) + crashed = df[ + (df.dataset == d) + & (df.regime == reg) + & (df.library == lib) + & (df.status != "ok") + ] + if not crashed.empty: + xi = offs[datasets.index(d)] + ax.annotate( + "✗", + (xi, ax.get_ylim()[0]), + xytext=(xi, 0.05), + ha="center", + fontsize=11, + color=COLORS[lib], + fontweight="bold", + annotation_clip=False, + ) + continue + vals.append(gl["train_s"].median()) + # degenerate models (sane=false): fast timings are meaningless + sane = all((m or {}).get("sane", True) for m in gl["metrics"]) + hatches.append(None if sane else "///") + bars = ax.bar(offs, vals, w, label=lib, color=COLORS[lib]) + for b, h in zip(bars, hatches): + if h: + b.set_hatch(h) + b.set_alpha(0.35) ax.set_yscale("log") ax.set_xticks(x, datasets) ax.set_ylabel("train time (s, log)") - ax.set_title(f"GPU training time — regime {reg} (500 trees)") + ax.set_title( + f"GPU training time — regime {reg} (500 trees)\n" + "✗ = crashed; hatched = degenerate model (timing not meaningful)" + ) ax.legend(ncol=3, fontsize=8) fig.tight_layout() fig.savefig(os.path.join(REPORT_DIR, f"train_time_{reg}.png"), dpi=120) From 18669c853a83fefb3dd4dce88a7f4c4b9f484253 Mon Sep 17 00:00:00 2001 From: Felix Jonas Kroner Date: Tue, 14 Jul 2026 08:10:37 +0200 Subject: [PATCH 07/10] [bench] chart: fix crash-marker placement, catch borderline-degenerate regressions The crash marker was annotated in data coordinates before the log scale and limits existed, so it floated outside the axes; use a blended transform (data x, axes y) drawn after the bars. Also hatch regression runs whose RMSE exceeds the xgboost reference by >15% even when they squeak past the std-based sanity flag (upstream quant on year: RMSE 10.85 vs std 10.9). Co-Authored-By: Claude Fable 5 --- benchmarks/report.py | 46 +++++++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/benchmarks/report.py b/benchmarks/report.py index 8cbab9739384..c0b0d4cb4a44 100644 --- a/benchmarks/report.py +++ b/benchmarks/report.py @@ -134,10 +134,11 @@ def main(): x = np.arange(len(datasets)) w = 0.13 fig, ax = plt.subplots(figsize=(11, 5)) + crash_marks = [] for i, lib in enumerate(LIBRARIES): offs = x + (i - 2.5) * w vals, hatches = [], [] - for d in datasets: + for j, d in enumerate(datasets): gl = sub[(sub.dataset == d) & (sub.library == lib)] if gl.empty: vals.append(np.nan) @@ -151,27 +152,42 @@ def main(): & (df.status != "ok") ] if not crashed.empty: - xi = offs[datasets.index(d)] - ax.annotate( - "✗", - (xi, ax.get_ylim()[0]), - xytext=(xi, 0.05), - ha="center", - fontsize=11, - color=COLORS[lib], - fontweight="bold", - annotation_clip=False, - ) + crash_marks.append((offs[j], COLORS[lib])) continue vals.append(gl["train_s"].median()) - # degenerate models (sane=false): fast timings are meaningless - sane = all((m or {}).get("sane", True) for m in gl["metrics"]) - hatches.append(None if sane else "///") + met = gl["metrics"].iloc[0] or {} + # degenerate models: sane=false, or a regression fit no better + # than 99% of the target's std (the year quant case squeaks past + # the sane threshold while still being useless) + degenerate = not all((m or {}).get("sane", True) for m in gl["metrics"]) + if not degenerate and "rmse" in met and d != "numerai": + ok_rmses = [ + (m or {}).get("rmse") + for m in sub[(sub.dataset == d) & (sub.library == "xgboost")][ + "metrics" + ] + ] + if ok_rmses and ok_rmses[0] and met["rmse"] > 1.15 * ok_rmses[0]: + degenerate = True + hatches.append("///" if degenerate else None) bars = ax.bar(offs, vals, w, label=lib, color=COLORS[lib]) for b, h in zip(bars, hatches): if h: b.set_hatch(h) b.set_alpha(0.35) + # crash markers: x in data coords, y just above the axis baseline + for xi, color in crash_marks: + ax.text( + xi, + 0.02, + "✗", + transform=ax.get_xaxis_transform(), + ha="center", + va="bottom", + fontsize=12, + color=color, + fontweight="bold", + ) ax.set_yscale("log") ax.set_xticks(x, datasets) ax.set_ylabel("train time (s, log)") From 4d82b9379233be4982b48aaa795ca4f7d06ca764 Mon Sep 17 00:00:00 2001 From: Felix Jonas Kroner Date: Tue, 14 Jul 2026 08:15:30 +0200 Subject: [PATCH 08/10] [bench] add speed-vs-quality scatter and numerai bar chart One scatter panel per dataset/regime (time log-x, quality oriented so up is better) makes degenerate-but-fast runs visually obvious; crash notes live in panel titles to avoid point collisions. The numerai regime now gets its own bar chart (it is neither shallow nor deep and appeared in no chart before). Co-Authored-By: Claude Fable 5 --- benchmarks/report.py | 113 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 110 insertions(+), 3 deletions(-) diff --git a/benchmarks/report.py b/benchmarks/report.py index c0b0d4cb4a44..6b49c90e61a1 100644 --- a/benchmarks/report.py +++ b/benchmarks/report.py @@ -122,13 +122,26 @@ def main(): lines.append("") # ---- train-time bar charts ------------------------------------------ - for reg in ("shallow", "deep"): + REG_TITLES = { + "shallow": "regime shallow (500 trees)", + "deep": "regime deep (500 trees)", + "numerai": "numerai example config (2000 trees)", + } + for reg in ("shallow", "deep", "numerai"): sub = timed[timed["regime"] == reg] if sub.empty: continue datasets = [ d - for d in ["fraud", "covtype", "year", "higgs", "epsilon", "airline"] + for d in [ + "fraud", + "covtype", + "year", + "higgs", + "epsilon", + "airline", + "numerai", + ] if d in set(sub["dataset"]) ] x = np.arange(len(datasets)) @@ -192,7 +205,7 @@ def main(): ax.set_xticks(x, datasets) ax.set_ylabel("train time (s, log)") ax.set_title( - f"GPU training time — regime {reg} (500 trees)\n" + f"GPU training time — {REG_TITLES[reg]}\n" "✗ = crashed; hatched = degenerate model (timing not meaningful)" ) ax.legend(ncol=3, fontsize=8) @@ -200,6 +213,100 @@ def main(): fig.savefig(os.path.join(REPORT_DIR, f"train_time_{reg}.png"), dpi=120) lines.append(f"![train time {reg}](train_time_{reg}.png)\n") + # ---- speed vs quality scatter ---------------------------------------- + panels = [] + for ds in ["fraud", "covtype", "year", "higgs", "epsilon", "airline", "numerai"]: + for reg in ["numerai"] if ds == "numerai" else ["shallow", "deep"]: + if not timed[(timed.dataset == ds) & (timed.regime == reg)].empty: + panels.append((ds, reg)) + if panels: + ncols = 3 + nrows = (len(panels) + ncols - 1) // ncols + fig, axes = plt.subplots(nrows, ncols, figsize=(14, 3.4 * nrows)) + axes = np.atleast_1d(axes).ravel() + for ax_i, (ds, reg) in zip(axes, panels): + mkey = METRIC_KEY.get(ds, "auc") + lower_better = mkey == "rmse" + for lib in LIBRARIES: + gl = timed[ + (timed.dataset == ds) + & (timed.regime == reg) + & (timed.library == lib) + ] + if gl.empty: + continue + met = gl["metrics"].iloc[0] or {} + q = met.get(mkey) + if q is None: + continue + degenerate = not all((m or {}).get("sane", True) for m in gl["metrics"]) + ax_i.scatter( + [gl["train_s"].median()], + [q], + s=110 if degenerate else 80, + color=COLORS[lib], + marker="X" if degenerate else "o", + edgecolor="black", + linewidth=0.6, + zorder=3, + ) + present = set( + timed[(timed.dataset == ds) & (timed.regime == reg)]["library"] + ) + crashed = sorted( + set( + df[(df.dataset == ds) & (df.regime == reg) & (df.status != "ok")][ + "library" + ] + ) + - present + ) + title = f"{ds} / {reg}" + if crashed: + title += "\n(crashed: " + ", ".join(crashed) + ")" + ax_i.set_xscale("log") + ax_i.xaxis.set_minor_formatter(matplotlib.ticker.NullFormatter()) + if lower_better: + ax_i.invert_yaxis() + ax_i.set_title(title, fontsize=10) + ax_i.set_xlabel("train time (s, log)", fontsize=8) + ax_i.set_ylabel( + mkey + (" (lower = better)" if lower_better else ""), fontsize=8 + ) + ax_i.tick_params(labelsize=7) + ax_i.grid(True, alpha=0.25) + for ax_i in axes[len(panels) :]: + ax_i.axis("off") + handles = [ + plt.Line2D( + [], + [], + marker="o", + linestyle="", + color=COLORS[lib], + label=lib, + markeredgecolor="black", + markeredgewidth=0.6, + ) + for lib in LIBRARIES + ] + handles.append( + plt.Line2D( + [], + [], + marker="X", + linestyle="", + color="gray", + label="degenerate model", + markeredgecolor="black", + ) + ) + fig.legend(handles=handles, ncol=7, loc="lower center", fontsize=9) + fig.suptitle("Speed vs quality — up and to the left is better", fontsize=12) + fig.tight_layout(rect=[0, 0.05, 1, 0.96]) + fig.savefig(os.path.join(REPORT_DIR, "speed_vs_quality.png"), dpi=120) + lines.append("![speed vs quality](speed_vs_quality.png)\n") + # ---- time-to-quality curves ----------------------------------------- curves = df[(df["kind"] == "curve") & (df["status"] == "ok")] for (ds, reg), g in curves.groupby(["dataset", "regime"], sort=False): From 3a615c074da0de1109d306bdfbc0ea136f71fce4 Mon Sep 17 00:00:00 2001 From: Felix Jonas Kroner Date: Tue, 14 Jul 2026 08:20:09 +0200 Subject: [PATCH 09/10] [bench] numerai chart: corr/sharpe labels on bars Co-Authored-By: Claude Fable 5 --- benchmarks/report.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/benchmarks/report.py b/benchmarks/report.py index 6b49c90e61a1..a06f6411402f 100644 --- a/benchmarks/report.py +++ b/benchmarks/report.py @@ -188,6 +188,19 @@ def main(): if h: b.set_hatch(h) b.set_alpha(0.35) + # single-dataset charts (numerai) have room for quality labels + if reg == "numerai" and len(datasets) == 1 and not np.isnan(vals[0]): + gl = sub[(sub.dataset == datasets[0]) & (sub.library == lib)] + met = gl["metrics"].iloc[0] or {} + if met.get("corr_mean") is not None: + ax.text( + offs[0], + vals[0] * 1.03, + f"corr {met['corr_mean']:.4f}\nsharpe {met['corr_sharpe']:.2f}", + ha="center", + va="bottom", + fontsize=8, + ) # crash markers: x in data coords, y just above the axis baseline for xi, color in crash_marks: ax.text( @@ -202,6 +215,9 @@ def main(): fontweight="bold", ) ax.set_yscale("log") + if reg == "numerai": + ymin, ymax = ax.get_ylim() + ax.set_ylim(ymin, ymax * 1.4) # headroom for the quality labels ax.set_xticks(x, datasets) ax.set_ylabel("train time (s, log)") ax.set_title( From 7b05f540cb21d20fb30028eda42229071d90003c Mon Sep 17 00:00:00 2001 From: Felix Jonas Kroner Date: Tue, 14 Jul 2026 18:22:20 +0200 Subject: [PATCH 10/10] [bench] optional int8 twin cache + native-ingestion benchmark (separate from the matrix) The cross-library matrix stays float32-fed for fairness; ingest_bench.py measures ExaBoost's native int8 path on numerai in its own results file. Reference on RTX 5090 (medians of 3): construct 38.9s -> 15.8s, peak host RSS 86.4GB -> 43.9GB, models md5-identical. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EYUXNtCAXDd3zYT71m8YSJ --- benchmarks/README.md | 16 +++++ benchmarks/datasets.py | 95 ++++++++++++++++++++----- benchmarks/ingest_bench.py | 138 +++++++++++++++++++++++++++++++++++++ 3 files changed, 233 insertions(+), 16 deletions(-) create mode 100644 benchmarks/ingest_bench.py diff --git a/benchmarks/README.md b/benchmarks/README.md index a864dd5089ed..846ffcadfd84 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -74,3 +74,19 @@ Everything lands under `benchmarks/workspace/` (override with Blackwell). Expect ~25GB of downloads and, with all datasets, a day-plus of GPU time for the full matrix; `--only fraud,covtype,year` gives a quick signal in under an hour. + +## Native int8 ingestion (optional, ExaBoost-only) + +The matrix feeds every library identical float32 bits, so ExaBoost's native +small-int path (int8/int16 matrices pass zero-copy through the C API and are +binned via per-column LUTs — bins byte-identical to the float path) is +measured separately: + +```bash +./benchmarks/workspace/env-competitors/bin/python benchmarks/datasets.py numerai-int8 +./benchmarks/workspace/env-exaboost/bin/python benchmarks/ingest_bench.py +``` + +Reference (RTX 5090, commit 9c0f5ffa, medians of 3): construct 38.9s (f32-fed) +vs **15.8s** (int8-fed), peak host RSS 86.4GB vs **43.9GB**; Booster create and +per-tree times identical, models md5-identical. diff --git a/benchmarks/datasets.py b/benchmarks/datasets.py index c4bfd0190e84..be5136474678 100644 --- a/benchmarks/datasets.py +++ b/benchmarks/datasets.py @@ -10,6 +10,7 @@ python benchmarks/datasets.py all # everything except numerai python benchmarks/datasets.py higgs airline NUMERAI_PARQUET=/path/to/v5_all_data.parquet python benchmarks/datasets.py numerai + python benchmarks/datasets.py numerai-int8 # optional int8 twin (ingest_bench.py) The Numerai parquet must contain ``feature*`` columns (int8), a ``target`` column, and a string ``era`` column, sorted by era — the "all data" training @@ -201,20 +202,14 @@ def prep_fraud(): save("fraud", x_tr, y_tr, x_te, y_te) -def prep_numerai(): - """Era-ordered float32 memmap; last N eras held out with an embargo gap. +def _numerai_roles(f): + """Row filter shared by the f32 and int8 numerai caches. - Rows without a target are dropped. Train rows are ``X[:train_end]`` and - test rows ``X[test_start:]`` so both are zero-copy views of the memmap. + Drops rows without a target, embargoes the eras before the test block, + and splits era-ordered. Returns ``(feat_cols, keep_all, n_rows, + train_end, era_int, targets)`` where ``keep_all`` is the absolute row + mask over the parquet. """ - import pyarrow.parquet as pq - - src = os.environ.get("NUMERAI_PARQUET") - if not src: - sys.exit("numerai: set NUMERAI_PARQUET to the v5 'all data' training parquet") - d = os.path.join(CACHE_DIR, "numerai") - os.makedirs(d, exist_ok=True) - f = pq.ParquetFile(src) feat_cols = [c for c in f.schema_arrow.names if c.startswith("feature")] # pass 1: era + target only, to build the row filter and split boundaries @@ -237,8 +232,29 @@ def prep_numerai(): role[np.isin(kept_eras, test_eras)] = 2 keep_within = role != 0 + keep_all = keep.copy() + keep_all[keep] = keep_within # absolute row filter n_rows = int(keep_within.sum()) train_end = int((role == 1).sum()) + targets = et["target"].to_numpy(dtype=np.float32) + return feat_cols, keep_all, n_rows, train_end, era_int, targets + + +def prep_numerai(): + """Era-ordered float32 memmap; last N eras held out with an embargo gap. + + Rows without a target are dropped. Train rows are ``X[:train_end]`` and + test rows ``X[test_start:]`` so both are zero-copy views of the memmap. + """ + import pyarrow.parquet as pq + + src = os.environ.get("NUMERAI_PARQUET") + if not src: + sys.exit("numerai: set NUMERAI_PARQUET to the v5 'all data' training parquet") + d = os.path.join(CACHE_DIR, "numerai") + os.makedirs(d, exist_ok=True) + f = pq.ParquetFile(src) + feat_cols, keep_all, n_rows, train_end, era_int, tgt_all = _numerai_roles(f) p = len(feat_cols) print(f"numerai: {n_rows} rows x {p} features, train_end={train_end}", flush=True) @@ -249,9 +265,6 @@ def prep_numerai(): era_out = np.empty(n_rows, dtype=np.int32) # pass 2: stream feature batches into the memmap - keep_all = keep.copy() - keep_all[keep] = keep_within # absolute row filter - tgt_all = et["target"].to_numpy(dtype=np.float32) row_abs = row_out = 0 for batch in f.iter_batches(batch_size=200_000, columns=feat_cols): nb = batch.num_rows @@ -284,6 +297,53 @@ def prep_numerai(): print(f"numerai done: {n_rows} x {p}", flush=True) +def prep_numerai_int8(): + """Optional int8 twin of the numerai cache (``X.i8.mem``), same rows/order. + + Feeds ExaBoost's native int8 ingestion path (see ingest_bench.py). The + main cross-library matrix stays float32-fed for fairness. Requires the + f32 cache to exist; sampled rows are verified against it so the two + caches cannot drift. + """ + import pyarrow.parquet as pq + + d = os.path.join(CACHE_DIR, "numerai") + meta_path = os.path.join(d, "meta.json") + if not os.path.exists(meta_path): + sys.exit("numerai-int8: build the numerai cache first") + meta = json.load(open(meta_path)) + src = os.environ.get("NUMERAI_PARQUET", meta["source"]) + f = pq.ParquetFile(src) + feat_cols, keep_all, n_rows, train_end, _, _ = _numerai_roles(f) + if n_rows != meta["n_rows"] or train_end != meta["train_end"]: + sys.exit("numerai-int8: row filter disagrees with the existing f32 cache") + p = len(feat_cols) + + x = np.memmap( + os.path.join(d, "X.i8.mem"), dtype=np.int8, mode="w+", shape=(n_rows, p) + ) + row_abs = row_out = 0 + for batch in f.iter_batches(batch_size=200_000, columns=feat_cols): + nb = batch.num_rows + mask = keep_all[row_abs : row_abs + nb] + if mask.any(): + sel = batch.to_pandas().to_numpy(dtype=np.int8)[mask] + x[row_out : row_out + len(sel)] = sel + row_out += len(sel) + row_abs += nb + assert row_out == n_rows, (row_out, n_rows) + x.flush() + + xf = np.memmap( + os.path.join(d, "X.f32.mem"), dtype=np.float32, mode="r", shape=(n_rows, p) + ) + rng = np.random.default_rng(SEED) + for r in rng.integers(0, n_rows, 50): + if not (x[r].astype(np.float32) == xf[r]).all(): + sys.exit(f"numerai-int8: row {r} mismatches the f32 cache") + print(f"numerai-int8 done: {n_rows} x {p}, sampled rows verified", flush=True) + + PREPS = { "higgs": prep_higgs, "epsilon": prep_epsilon, @@ -292,11 +352,14 @@ def prep_numerai(): "year": prep_year, "fraud": prep_fraud, "numerai": prep_numerai, + "numerai-int8": prep_numerai_int8, } if __name__ == "__main__": names = sys.argv[1:] - targets = [n for n in PREPS if n != "numerai"] if names == ["all"] else names + targets = ( + [n for n in PREPS if not n.startswith("numerai")] if names == ["all"] else names + ) for t in targets: if dataset_ready(t): print(f"{t}: cached, skipping", flush=True) diff --git a/benchmarks/ingest_bench.py b/benchmarks/ingest_bench.py new file mode 100644 index 000000000000..1f74dc282835 --- /dev/null +++ b/benchmarks/ingest_bench.py @@ -0,0 +1,138 @@ +"""Native int8 ingestion benchmark (ExaBoost-only, separate from the matrix). + +The cross-library matrix feeds every library the same float32 bits for +fairness, so ExaBoost's native small-int path (int8/int16 zero-copy through +the C API + LUT binning) is measured here instead: Dataset construct time, +Booster create time, first-trees time, and peak host RSS on numerai, fed from +the float32 memmap vs its int8 twin (``datasets.py numerai-int8``). The two +flows produce identical models (bins are byte-identical by construction). + +Run inside the ExaBoost venv:: + + python benchmarks/ingest_bench.py # 3 repeats each, medians + python benchmarks/ingest_bench.py --repeats 1 + +Each measurement runs in a fresh process (clean GPU + page-cache-warm parity +with the matrix); records append to ``/results/ingest.jsonl``. +""" + +import argparse +import json +import os +import statistics +import subprocess +import sys + +from common import CACHE_DIR, REGIMES, RESULTS_DIR, SEED + +INGEST_JSONL = os.path.join(RESULTS_DIR, "ingest.jsonl") +WARMUP_TREES = 5 + + +def measure_one(kind): + """Run in a fresh process: construct/create/first-trees timings.""" + import resource + import time + + import numpy as np + import lightgbm as lgb + + d = os.path.join(CACHE_DIR, "numerai") + meta = json.load(open(os.path.join(d, "meta.json"))) + shape = (meta["n_rows"], meta["n_features"]) + n = meta["train_end"] + if kind == "f32": + x = np.memmap(os.path.join(d, "X.f32.mem"), np.float32, "r", shape=shape) + else: + path = os.path.join(d, "X.i8.mem") + if not os.path.exists(path): + sys.exit("ingest_bench: run `datasets.py numerai-int8` first") + x = np.memmap(path, np.int8, "r", shape=shape) + y = np.load(os.path.join(d, "y.npy"))[:n] + + reg = REGIMES["numerai"] + params = { + "objective": "regression", + "learning_rate": reg["lr"], + "num_leaves": reg["leaves"], + "max_depth": reg["depth"], + "feature_fraction": reg["colsample"], + "max_bin": 255, + "device_type": "cuda", + "num_threads": os.cpu_count(), + "seed": SEED, + "verbose": -1, + "metric": "None", + } + + t0 = time.perf_counter() + dtrain = lgb.Dataset(x[:n], label=y, params=params) + dtrain.construct() + construct_s = time.perf_counter() - t0 + + t0 = time.perf_counter() + bst = lgb.Booster(params=params, train_set=dtrain) + create_s = time.perf_counter() - t0 + + t0 = time.perf_counter() + for _ in range(WARMUP_TREES): + bst.update() + trees_s = time.perf_counter() - t0 + + print( + json.dumps( + { + "kind": kind, + "construct_s": round(construct_s, 2), + "create_s": round(create_s, 2), + f"trees{WARMUP_TREES}_s": round(trees_s, 2), + "peak_rss_gb": round( + resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024**2, 1 + ), + "version": lgb.__version__, + } + ) + ) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--repeats", type=int, default=3) + ap.add_argument("--one", choices=["f32", "i8"], help=argparse.SUPPRESS) + args = ap.parse_args() + if args.one: + measure_one(args.one) + return + + os.makedirs(RESULTS_DIR, exist_ok=True) + results = {"f32": [], "i8": []} + for kind in ("f32", "i8"): + for rep in range(args.repeats): + out = ( + subprocess.run( + [sys.executable, os.path.abspath(__file__), "--one", kind], + capture_output=True, + text=True, + check=True, + ) + .stdout.strip() + .splitlines()[-1] + ) + rec = json.loads(out) + rec["repeat"] = rep + results[kind].append(rec) + with open(INGEST_JSONL, "a") as fh: + fh.write(json.dumps(rec) + "\n") + print(f"{kind} repeat {rep}: {out}", flush=True) + + print(f"\nmedians over {args.repeats} repeats (numerai, ExaBoost cuda):") + keys = ["construct_s", "create_s", f"trees{WARMUP_TREES}_s", "peak_rss_gb"] + header = "flow " + "".join(f"{k:>14}" for k in keys) + print(header) + for kind in ("f32", "i8"): + med = [statistics.median(r[k] for r in results[kind]) for k in keys] + print(f"{kind:<10}" + "".join(f"{v:>14.2f}" for v in med)) + + +if __name__ == "__main__": + main()