diff --git a/.gitignore b/.gitignore index 458ad4ebffe5..e18e11cab3df 100644 --- a/.gitignore +++ b/.gitignore @@ -484,3 +484,4 @@ dmypy.json # files created by release tasks /release-artifacts +benchmarks/workspace/ diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 000000000000..846ffcadfd84 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,92 @@ +# ExaBoost GPU benchmark suite + +Reproducible comparison of **ExaBoost** (this repo, `device_type=cuda`) against +**upstream LightGBM** (CUDA build), **XGBoost** (`device=cuda`, +`tree_method=hist`) and **CatBoost** (`task_type=GPU`) — training speed *and* +model quality on a single GPU. + +## Datasets + +The classic GBDT speed suite (NVIDIA gbm-bench lineage) plus Numerai: + +| dataset | shape | task | split | +|---|---|---|---| +| fraud (OpenML creditcard) | 285K × 29 | binary, imbalanced | stratified 20% | +| covtype (UCI) | 581K × 54 | 7-class | stratified 20% | +| year (UCI YearPredictionMSD) | 515K × 90 | regression | canonical last 51,630 | +| higgs (UCI) | 11M × 28 | binary | canonical last 500K | +| epsilon (LIBSVM) | 500K × 2000 | binary | official train/test | +| airline (Ikonomovska) | 115M × 13 | binary (ArrDelay>0) | random 20%, seed 42 | +| numerai (v5 "all data", optional) | ~6.7M × ~2.7K | regression + era metrics | last 200 eras, 10-era embargo | + +All datasets are cached as float32 arrays so every library trains from +identical bits. Numerai requires the v5 training parquet (get it with +`numerapi`) via the `NUMERAI_PARQUET` env var; its quality metrics are the +official Numerai correlation (rank → gaussianize → power 1.5 → Pearson), +per-era mean/std/Sharpe and max drawdown. + +## Run matrix + +Six library configs — `exaboost`, `exaboost-quant` (`use_quantized_grad`), +`lightgbm`, `lightgbm-quant`, `xgboost`, `catboost` — across two aligned +hyperparameter regimes (gbm-bench convention: 500 rounds, lr 0.1, 255 bins, +L2 leaf regularization 1.0): + +- **shallow**: depth 6 / 63 leaves +- **deep**: depth 10 / 1023 leaves + +L2 is aligned explicitly because engine defaults differ (XGBoost `lambda=1`, +LightGBM `lambda_l2=0`, CatBoost `l2_leaf_reg=3`) and at lr 0.1 an +unregularized leaf-wise model degenerates on imbalanced data (fraud drops to +~0.5 AUC), which would measure default choices rather than the engines. + +plus the official Numerai example-model config (2000 trees, lr 0.01, depth 5, +32 leaves, colsample 0.1). Structural caveat: LightGBM-family grows leaf-wise +(`num_leaves` + `max_depth` cap), XGBoost depth-wise (`max_depth`), CatBoost +symmetric (`depth`) — the regimes align the tree budget, not the tree shape. + +Each cell runs 1 discarded warmup + 3 timed repeats (median reported) + 1 +`curve` run that evaluates the held-out set periodically for time-to-quality +plots. Every run is an isolated subprocess; peak GPU memory and host RSS are +polled throughout. See the docstring in `bench.py` for exact timing semantics. + +## Reproducing + +```bash +# 1. build both environments (needs CUDA toolkit + driver, cmake, compiler) +./benchmarks/setup_envs.sh + +# 2. download + preprocess datasets (largest: airline ~6GB download, ~30GB RAM) +./benchmarks/workspace/env-competitors/bin/python benchmarks/datasets.py all +NUMERAI_PARQUET=/path/to/v5_all_data.parquet \ + ./benchmarks/workspace/env-competitors/bin/python benchmarks/datasets.py numerai # optional + +# 3. run the matrix (resumable — interrupt and relaunch freely) +python3 benchmarks/orchestrate.py + +# 4. aggregate results into workspace/report/REPORT.md + charts +./benchmarks/workspace/env-competitors/bin/python benchmarks/report.py +``` + +Everything lands under `benchmarks/workspace/` (override with +`EXABOOST_BENCH_ROOT`). `EXABOOST_CUDA_ARCHS` overrides +`CMAKE_CUDA_ARCHITECTURES` (default `native`; e.g. `120-real;120-virtual` for +Blackwell). Expect ~25GB of downloads and, with all datasets, a day-plus of +GPU time for the full matrix; `--only fraud,covtype,year` gives a quick +signal in under an hour. + +## Native int8 ingestion (optional, ExaBoost-only) + +The matrix feeds every library identical float32 bits, so ExaBoost's native +small-int path (int8/int16 matrices pass zero-copy through the C API and are +binned via per-column LUTs — bins byte-identical to the float path) is +measured separately: + +```bash +./benchmarks/workspace/env-competitors/bin/python benchmarks/datasets.py numerai-int8 +./benchmarks/workspace/env-exaboost/bin/python benchmarks/ingest_bench.py +``` + +Reference (RTX 5090, commit 9c0f5ffa, medians of 3): construct 38.9s (f32-fed) +vs **15.8s** (int8-fed), peak host RSS 86.4GB vs **43.9GB**; Booster create and +per-tree times identical, models md5-identical. diff --git a/benchmarks/bench.py b/benchmarks/bench.py new file mode 100644 index 000000000000..6c9119700f2d --- /dev/null +++ b/benchmarks/bench.py @@ -0,0 +1,455 @@ +"""Run exactly one benchmark cell: (library, dataset, regime, kind). + +Appends one JSON line to ``/results/runs.jsonl`` and exits. Meant +to be launched by orchestrate.py in a fresh subprocess using the venv python +that owns the requested library, so GPU state and library versions are +isolated per run. + +Timing semantics (reported separately, because they differ per library): + +- ``construct_s``: building the library's training data structure — + ``lgb.Dataset`` (binning happens here), ``xgb.QuantileDMatrix`` (quantile + sketch), ``catboost.Pool`` (thin wrapper; CatBoost quantizes inside fit). +- ``train_s``: the boosting loop, including any remaining device transfer. +- ``curve`` runs additionally evaluate the held-out set every + ``eval_every`` iterations; their timings are NOT comparable to plain runs + (CatBoost's curve time axis is linearly approximated — no per-iteration + wall clock is exposed — and is flagged with ``curve_time_approx``). +""" + +import argparse +import json +import os +import subprocess +import sys +import threading +import time +import traceback + +import numpy as np + +from common import CACHE_DIR, DATASETS, REGIMES, RUNS_JSONL, SEED + + +class ResourceMonitor: + """Polls peak GPU memory (nvidia-smi) and host RSS in a thread.""" + + def __init__(self, interval=0.25): + self.interval = interval + self.gpu_peak_mb = 0 + self.rss_peak_mb = 0 + self._stop = threading.Event() + self._thread = threading.Thread(target=self._run, daemon=True) + + def _run(self): + import psutil + + proc = psutil.Process() + while not self._stop.is_set(): + try: + out = subprocess.run( + [ + "nvidia-smi", + "--query-gpu=memory.used", + "--format=csv,noheader,nounits", + ], + capture_output=True, + text=True, + timeout=5, + ).stdout.strip() + self.gpu_peak_mb = max(self.gpu_peak_mb, int(out.splitlines()[0])) + except Exception: + pass + try: + rss = proc.memory_info().rss + for c in proc.children(recursive=True): + try: + rss += c.memory_info().rss + except Exception: + pass + self.rss_peak_mb = max(self.rss_peak_mb, rss // (1024 * 1024)) + except Exception: + pass + self._stop.wait(self.interval) + + def __enter__(self): + self._thread.start() + return self + + def __exit__(self, *a): + self._stop.set() + self._thread.join(timeout=5) + + +def load_data(name): + d = os.path.join(CACHE_DIR, name) + if name == "numerai": + with open(os.path.join(d, "meta.json")) as fh: + meta = json.load(fh) + x = np.memmap( + os.path.join(d, "X.f32.mem"), + dtype=np.float32, + mode="r", + shape=(meta["n_rows"], meta["n_features"]), + ) + y = np.load(os.path.join(d, "y.npy")) + era = np.load(os.path.join(d, "era_codes.npy")) + tr_end, te_start = meta["train_end"], meta["test_start"] + return ( + x[:tr_end], + y[:tr_end], + x[te_start:], + y[te_start:], + {"era_test": era[te_start:]}, + ) + x_tr = np.load(os.path.join(d, "X_train.npy"), mmap_mode="r") + y_tr = np.load(os.path.join(d, "y_train.npy")) + x_te = np.load(os.path.join(d, "X_test.npy"), mmap_mode="r") + y_te = np.load(os.path.join(d, "y_test.npy")) + return x_tr, y_tr, x_te, y_te, {} + + +def numerai_corr_np(preds, targets): + """Official Numerai correlation (rank -> gaussianize -> pow 1.5 -> Pearson).""" + from scipy.stats import norm, rankdata + + n = len(preds) + ranked = (rankdata(preds, method="average") - 0.5) / n + gauss = norm.ppf(ranked) + preds_p15 = np.sign(gauss) * np.abs(gauss) ** 1.5 + centered = targets - targets.mean() + targets_p15 = np.sign(centered) * np.abs(centered) ** 1.5 + return float(np.corrcoef(preds_p15, targets_p15)[0, 1]) + + +def numerai_metrics(preds, y, era): + corrs = np.array( + [numerai_corr_np(preds[era == e], y[era == e]) for e in np.unique(era)] + ) + mean, std = corrs.mean(), corrs.std(ddof=0) + cumulative = np.cumprod(1 + corrs) + rolling_max = np.maximum.accumulate(cumulative) + drawdown = ((rolling_max - cumulative) / rolling_max).max() + return { + "corr_mean": float(mean), + "corr_std": float(std), + "corr_sharpe": float(mean / std) if std > 0 else None, + "max_drawdown": float(-drawdown), + "n_eras": len(corrs), + } + + +def quality_metrics(task, preds, y, extra): + from sklearn.metrics import accuracy_score, log_loss, roc_auc_score + + if task == "binary": + auc = roc_auc_score(y, preds) + return {"auc": float(auc), "sane": bool(auc > 0.55)} + if task == "multiclass": + acc = accuracy_score(y, preds.argmax(axis=1)) + ll = log_loss(y, preds, labels=list(range(preds.shape[1]))) + return {"accuracy": float(acc), "mlogloss": float(ll), "sane": bool(acc > 0.5)} + if task == "regression": + rmse = float(np.sqrt(np.mean((preds - y) ** 2))) + return {"rmse": rmse, "sane": bool(rmse < float(np.std(y)))} + if task == "numerai": + m = numerai_metrics(preds, y, extra["era_test"]) + m["rmse"] = float(np.sqrt(np.mean((preds - y) ** 2))) + m["sane"] = bool(m["corr_mean"] > 0) + return m + raise ValueError(task) + + +def run_lightgbm(task, x_tr, y_tr, x_te, y_te, reg, quantized, curve): + import lightgbm as lgb + + params = { + "objective": { + "binary": "binary", + "multiclass": "multiclass", + "regression": "regression", + "numerai": "regression", + }[task], + "learning_rate": reg["lr"], + "num_leaves": reg["leaves"], + "max_depth": reg["depth"], + "max_bin": 255, + "device_type": "cuda", + "num_threads": os.cpu_count(), + "seed": SEED, + "verbose": -1, + "metric": "None", + } + if task == "multiclass": + params["num_class"] = DATASETS["covtype"]["num_class"] + if "colsample" in reg: + params["feature_fraction"] = reg["colsample"] + if "l2" in reg: + params["lambda_l2"] = reg["l2"] + if quantized: + params["use_quantized_grad"] = True + + t0 = time.perf_counter() + # the Dataset must be created with the final params (incl. device_type) + dtrain = lgb.Dataset(x_tr, label=y_tr, params=params) + dtrain.construct() + construct_s = time.perf_counter() - t0 + + curve_pts = [] + t0 = time.perf_counter() + if curve: + dvalid = lgb.Dataset(x_te, label=y_te, reference=dtrain) + bst = lgb.Booster(params=params, train_set=dtrain) + bst.add_valid(dvalid, "test") + for i in range(reg["rounds"]): + bst.update() + if (i + 1) % reg["eval_every"] == 0 or i + 1 == reg["rounds"]: + res = bst.eval_valid() + curve_pts.append( + [i + 1, time.perf_counter() - t0, res[0][2] if res else None] + ) + else: + bst = lgb.train(params, dtrain, num_boost_round=reg["rounds"]) + train_s = time.perf_counter() - t0 + + preds = bst.predict(x_te) + return { + "construct_s": construct_s, + "train_s": train_s, + "preds": preds, + "version": lgb.__version__, + "curve": curve_pts, + } + + +def run_xgboost(task, x_tr, y_tr, x_te, y_te, reg, curve): + import xgboost as xgb + + params = { + "objective": { + "binary": "binary:logistic", + "multiclass": "multi:softprob", + "regression": "reg:squarederror", + "numerai": "reg:squarederror", + }[task], + "eta": reg["lr"], + "max_depth": reg["depth"], + "max_bin": 255, + "device": "cuda", + "tree_method": "hist", + "nthread": os.cpu_count(), + "seed": SEED, + } + if task == "multiclass": + params["num_class"] = DATASETS["covtype"]["num_class"] + if "colsample" in reg: + params["colsample_bytree"] = reg["colsample"] + if "l2" in reg: + params["lambda"] = reg["l2"] # xgboost default is already 1 + + t0 = time.perf_counter() + dtrain = xgb.QuantileDMatrix(np.asarray(x_tr), label=y_tr, max_bin=255) + construct_s = time.perf_counter() - t0 + + curve_pts = [] + t0 = time.perf_counter() + if curve: + dtest = xgb.DMatrix(np.asarray(x_te), label=y_te) + params["eval_metric"] = { + "binary": "auc", + "multiclass": "mlogloss", + "regression": "rmse", + "numerai": "rmse", + }[task] + bst = xgb.Booster(params, [dtrain]) + for i in range(reg["rounds"]): + bst.update(dtrain, i) + if (i + 1) % reg["eval_every"] == 0 or i + 1 == reg["rounds"]: + res = bst.eval_set([(dtest, "test")], i) + curve_pts.append( + [i + 1, time.perf_counter() - t0, float(res.split(":")[-1])] + ) + else: + bst = xgb.train(params, dtrain, num_boost_round=reg["rounds"]) + train_s = time.perf_counter() - t0 + + # chunked inplace_predict: a full GPU DMatrix of a wide test set can OOM + # the device while the trained booster is still resident + step = 200_000 + preds = np.concatenate( + [ + bst.inplace_predict(np.ascontiguousarray(x_te[i : i + step])) + for i in range(0, x_te.shape[0], step) + ] + ) + return { + "construct_s": construct_s, + "train_s": train_s, + "preds": preds, + "version": xgb.__version__, + "curve": curve_pts, + } + + +def run_catboost(task, x_tr, y_tr, x_te, y_te, reg, curve): + import catboost as cb + + loss = { + "binary": "Logloss", + "multiclass": "MultiClass", + "regression": "RMSE", + "numerai": "RMSE", + }[task] + kw = { + "iterations": reg["rounds"], + "learning_rate": reg["lr"], + "depth": reg["depth"], + "border_count": 254, + "task_type": "GPU", + "devices": "0", + "random_seed": SEED, + "verbose": False, + "allow_writing_files": False, + "loss_function": loss, + } + rsm_dropped = False + if "colsample" in reg: + kw["rsm"] = reg["colsample"] + if "l2" in reg: + kw["l2_leaf_reg"] = reg["l2"] # catboost default is 3 + + t0 = time.perf_counter() + train_pool = cb.Pool(np.asarray(x_tr), label=y_tr) + construct_s = time.perf_counter() - t0 + + cls = ( + cb.CatBoostClassifier + if task in ("binary", "multiclass") + else cb.CatBoostRegressor + ) + + def fit(kw): + model = cls(**kw) + t0 = time.perf_counter() + if curve: + model.fit( + train_pool, + eval_set=cb.Pool(np.asarray(x_te), label=y_te), + metric_period=reg["eval_every"], + ) + else: + model.fit(train_pool) + return model, time.perf_counter() - t0 + + try: + model, train_s = fit(kw) + except Exception as e: + # rsm is not supported for every GPU loss; retry without and flag it + if "rsm" in kw and "rsm" in str(e).lower(): + kw.pop("rsm") + rsm_dropped = True + model, train_s = fit(kw) + else: + raise + + curve_pts = [] + if curve: + vals = model.get_evals_result().get("validation", {}) + if vals: + series = vals[next(iter(vals))] + n = len(series) + # no per-iteration wall clock exposed; approximate linearly + curve_pts = [ + [i + 1, train_s * (i + 1) / n, series[i]] + for i in range(0, n, reg["eval_every"]) + ] + + if task == "binary": + preds = model.predict_proba(np.asarray(x_te))[:, 1] + elif task == "multiclass": + preds = model.predict_proba(np.asarray(x_te)) + else: + preds = model.predict(np.asarray(x_te)) + return { + "construct_s": construct_s, + "train_s": train_s, + "preds": preds, + "version": cb.__version__, + "curve": curve_pts, + "rsm_dropped": rsm_dropped, + "curve_time_approx": bool(curve), + } + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument( + "--library", + required=True, + choices=[ + "exaboost", + "exaboost-quant", + "lightgbm", + "lightgbm-quant", + "xgboost", + "catboost", + ], + ) + ap.add_argument("--dataset", required=True, choices=list(DATASETS)) + ap.add_argument("--regime", required=True, choices=list(REGIMES)) + ap.add_argument("--kind", required=True) # warmup | timed1..3 | curve + ap.add_argument("--out", default=RUNS_JSONL) + args = ap.parse_args() + + task = DATASETS[args.dataset]["task"] + reg = REGIMES[args.regime] + curve = args.kind == "curve" + + rec = { + "library": args.library, + "dataset": args.dataset, + "regime": args.regime, + "kind": args.kind, + "status": "ok", + } + try: + x_tr, y_tr, x_te, y_te, extra = load_data(args.dataset) + rec["n_train"], rec["n_features"] = int(x_tr.shape[0]), int(x_tr.shape[1]) + with ResourceMonitor() as mon: + t_total = time.perf_counter() + if args.library.startswith(("exaboost", "lightgbm")): + r = run_lightgbm( + task, + x_tr, + y_tr, + x_te, + y_te, + reg, + quantized=args.library.endswith("-quant"), + curve=curve, + ) + elif args.library == "xgboost": + r = run_xgboost(task, x_tr, y_tr, x_te, y_te, reg, curve) + else: + r = run_catboost(task, x_tr, y_tr, x_te, y_te, reg, curve) + total_s = time.perf_counter() - t_total + preds = r.pop("preds") + rec.update(r) + rec["total_s"] = total_s + rec["trees_per_s"] = reg["rounds"] / r["train_s"] + rec["gpu_mem_peak_mb"] = mon.gpu_peak_mb + rec["rss_peak_mb"] = mon.rss_peak_mb + rec["metrics"] = quality_metrics(task, np.asarray(preds), y_te, extra) + except Exception: + rec["status"] = "failed" + rec["error"] = traceback.format_exc()[-3000:] + + os.makedirs(os.path.dirname(args.out), exist_ok=True) + with open(args.out, "a") as f: + f.write(json.dumps(rec) + "\n") + print(json.dumps({k: v for k, v in rec.items() if k != "curve"})[:2000]) + sys.exit(0 if rec["status"] == "ok" else 1) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/common.py b/benchmarks/common.py new file mode 100644 index 000000000000..e91418c2d8bf --- /dev/null +++ b/benchmarks/common.py @@ -0,0 +1,96 @@ +"""Shared configuration for the ExaBoost GPU benchmark suite. + +All artifacts (raw downloads, preprocessed caches, results, report) live under +a single workspace directory, resolved from the ``EXABOOST_BENCH_ROOT`` +environment variable and defaulting to ``benchmarks/workspace`` inside the +repository checkout. +""" + +import os + +ROOT = os.environ.get( + "EXABOOST_BENCH_ROOT", + os.path.join(os.path.dirname(os.path.abspath(__file__)), "workspace"), +) +DATA_DIR = os.path.join(ROOT, "data") +CACHE_DIR = os.path.join(ROOT, "data", "cache") +RESULTS_DIR = os.path.join(ROOT, "results") +REPORT_DIR = os.path.join(ROOT, "report") +RUNS_JSONL = os.path.join(RESULTS_DIR, "runs.jsonl") + +SEED = 42 + +#: benchmark datasets; ``task`` drives objective/metric selection in bench.py +DATASETS = { + "higgs": {"task": "binary"}, + "epsilon": {"task": "binary"}, + "airline": {"task": "binary"}, + "covtype": {"task": "multiclass", "num_class": 7}, + "year": {"task": "regression"}, + "fraud": {"task": "binary"}, + "numerai": {"task": "numerai"}, +} + +#: hyperparameter regimes, aligned across libraries (gbm-bench convention); +#: LightGBM-family maps depth/leaves to (max_depth, num_leaves), XGBoost uses +#: max_depth (native depth-wise growth), CatBoost uses symmetric depth. +REGIMES = { + # l2=1 aligns L2 leaf regularization across engines (defaults differ: + # xgboost lambda=1, lightgbm lambda_l2=0, catboost l2_leaf_reg=3); at + # lr 0.1 an unregularized leaf-wise model degenerates on imbalanced data + "shallow": { + "rounds": 500, + "lr": 0.1, + "depth": 6, + "leaves": 63, + "l2": 1.0, + "eval_every": 25, + }, + "deep": { + "rounds": 500, + "lr": 0.1, + "depth": 10, + "leaves": 1023, + "l2": 1.0, + "eval_every": 25, + }, + # official Numerai example-model parameters + "numerai": { + "rounds": 2000, + "lr": 0.01, + "depth": 5, + "leaves": 32, + "colsample": 0.1, + "eval_every": 250, + }, + # tiny config for smoke-testing the library/GPU integration + "smoke": {"rounds": 10, "lr": 0.1, "depth": 6, "leaves": 63, "eval_every": 5}, +} + +#: exaboost/lightgbm both install as package "lightgbm", hence two venvs +LIBRARIES = [ + "exaboost", + "exaboost-quant", + "lightgbm", + "lightgbm-quant", + "xgboost", + "catboost", +] + + +def venv_python(library: str) -> str: + """Path of the venv python that owns ``library`` (see setup_envs.sh).""" + env = "env-exaboost" if library.startswith("exaboost") else "env-competitors" + override = os.environ.get(f"EXABOOST_BENCH_PY_{env.replace('-', '_').upper()}") + return override or os.path.join(ROOT, env, "bin", "python") + + +def regimes_for(dataset: str): + """Regimes to benchmark for a dataset.""" + return ["numerai"] if dataset == "numerai" else ["shallow", "deep"] + + +def dataset_ready(dataset: str) -> bool: + """Whether the preprocessed cache for ``dataset`` exists.""" + marker = "meta.json" if dataset == "numerai" else "y_test.npy" + return os.path.exists(os.path.join(CACHE_DIR, dataset, marker)) diff --git a/benchmarks/datasets.py b/benchmarks/datasets.py new file mode 100644 index 000000000000..be5136474678 --- /dev/null +++ b/benchmarks/datasets.py @@ -0,0 +1,367 @@ +"""Download and preprocess benchmark datasets into ``/data/cache/``. + +Each dataset is cached as float32 ``X_train/y_train/X_test/y_test`` .npy files +(Numerai as one era-ordered float32 memmap plus metadata) so every library +trains from identical bits. Raw files are downloaded on demand with resume +support. + +Run inside the competitors venv (needs sklearn, pandas, pyarrow):: + + python benchmarks/datasets.py all # everything except numerai + python benchmarks/datasets.py higgs airline + NUMERAI_PARQUET=/path/to/v5_all_data.parquet python benchmarks/datasets.py numerai + python benchmarks/datasets.py numerai-int8 # optional int8 twin (ingest_bench.py) + +The Numerai parquet must contain ``feature*`` columns (int8), a ``target`` +column, and a string ``era`` column, sorted by era — the "all data" training +file distributed by Numerai (v5) has exactly this layout. +""" + +import json +import os +import subprocess +import sys +import zipfile + +import numpy as np + +from common import CACHE_DIR, DATA_DIR, SEED, dataset_ready + +URLS = { + "higgs.zip": "https://archive.ics.uci.edu/static/public/280/higgs.zip", + "epsilon_train.bz2": "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/epsilon_normalized.bz2", + "epsilon_test.bz2": "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/epsilon_normalized.t.bz2", + "airline.data.bz2": "http://kt.ijs.si/elena_ikonomovska/datasets/airline_14col.data.bz2", + "year.zip": "https://archive.ics.uci.edu/static/public/203/yearpredictionmsd.zip", + "covtype.data.gz": "https://archive.ics.uci.edu/ml/machine-learning-databases/covtype/covtype.data.gz", +} + +NUMERAI_TEST_ERAS = 200 +NUMERAI_EMBARGO_ERAS = 10 + + +def fetch(filename: str) -> str: + """Download ``filename`` into the data dir if missing; returns its path.""" + os.makedirs(DATA_DIR, exist_ok=True) + path = os.path.join(DATA_DIR, filename) + if os.path.exists(path): + return path + part = path + ".part" + cmd = [ + "curl", + "-SL", + "--retry", + "10", + "--retry-all-errors", + "-C", + "-", + "--fail", + "-o", + part, + URLS[filename], + ] + print(f"downloading {URLS[filename]}", flush=True) + subprocess.run(cmd, check=True) + os.rename(part, path) + return path + + +def save(name, x_tr, y_tr, x_te, y_te): + d = os.path.join(CACHE_DIR, name) + os.makedirs(d, exist_ok=True) + np.save( + os.path.join(d, "X_train.npy"), np.ascontiguousarray(x_tr, dtype=np.float32) + ) + np.save(os.path.join(d, "y_train.npy"), np.asarray(y_tr, dtype=np.float32)) + np.save(os.path.join(d, "X_test.npy"), np.ascontiguousarray(x_te, dtype=np.float32)) + np.save(os.path.join(d, "y_test.npy"), np.asarray(y_te, dtype=np.float32)) + print(f"{name}: train {x_tr.shape} test {x_te.shape} -> {d}", flush=True) + + +def random_split(x, y, frac=0.2, stratify=False): + from sklearn.model_selection import train_test_split + + return train_test_split( + x, y, test_size=frac, random_state=SEED, stratify=y if stratify else None + ) + + +def prep_higgs(): + import pandas as pd + + with zipfile.ZipFile(fetch("higgs.zip")) as z: + inner = z.namelist()[0] # HIGGS.csv.gz + with z.open(inner) as f: + df = pd.read_csv( + f, + header=None, + dtype=np.float32, + compression="gzip" if inner.endswith(".gz") else None, + ) + y = df[0].to_numpy() + x = df.drop(columns=[0]).to_numpy() + # canonical split: the last 500K rows are the test set + save("higgs", x[:-500_000], y[:-500_000], x[-500_000:], y[-500_000:]) + + +def prep_epsilon(): + from sklearn.datasets import load_svmlight_file + + x_tr, y_tr = load_svmlight_file(fetch("epsilon_train.bz2"), n_features=2000) + x_te, y_te = load_svmlight_file(fetch("epsilon_test.bz2"), n_features=2000) + save( + "epsilon", + x_tr.toarray(), + (y_tr > 0).astype(np.float32), + x_te.toarray(), + (y_te > 0).astype(np.float32), + ) + + +def prep_airline(): + import pandas as pd + + cols = [ + "Year", + "Month", + "DayofMonth", + "DayOfWeek", + "CRSDepTime", + "CRSArrTime", + "UniqueCarrier", + "FlightNum", + "ActualElapsedTime", + "Origin", + "Dest", + "Distance", + "Diverted", + "ArrDelay", + ] + cat_cols = ["UniqueCarrier", "Origin", "Dest"] + cat_maps = {c: {} for c in cat_cols} + chunks = [] + reader = pd.read_csv( + fetch("airline.data.bz2"), header=None, names=cols, chunksize=5_000_000 + ) + for i, ch in enumerate(reader): + for c in cat_cols: # ordinal-encode string categoricals + m = cat_maps[c] + vals = ch[c].astype(str) + for v in vals.unique(): + if v not in m: + m[v] = len(m) + ch[c] = vals.map(m) + y = (ch["ArrDelay"] > 0).astype(np.float32).to_numpy() + x = ch.drop(columns=["ArrDelay"]).to_numpy(dtype=np.float32) + chunks.append((x, y)) + print(f"airline chunk {i} ({len(ch)} rows)", flush=True) + x = np.concatenate([c[0] for c in chunks]) + y = np.concatenate([c[1] for c in chunks]) + del chunks + rng = np.random.default_rng(SEED) + perm = rng.permutation(len(x)) + n_test = int(0.2 * len(x)) + save( + "airline", + x[perm[n_test:]], + y[perm[n_test:]], + x[perm[:n_test]], + y[perm[:n_test]], + ) + + +def prep_covtype(): + import pandas as pd + + df = pd.read_csv(fetch("covtype.data.gz"), header=None) + y = df[54].to_numpy(dtype=np.float32) - 1 # labels 1..7 -> 0..6 + x = df.drop(columns=[54]).to_numpy(dtype=np.float32) + x_tr, x_te, y_tr, y_te = random_split(x, y, stratify=True) + save("covtype", x_tr, y_tr, x_te, y_te) + + +def prep_year(): + import pandas as pd + + with zipfile.ZipFile(fetch("year.zip")) as z: + with z.open(z.namelist()[0]) as f: + df = pd.read_csv(f, header=None, dtype=np.float32) + y = df[0].to_numpy() + x = df.drop(columns=[0]).to_numpy() + # canonical split: first 463,715 train / last 51,630 test + save("year", x[:463_715], y[:463_715], x[463_715:], y[463_715:]) + + +def prep_fraud(): + from sklearn.datasets import fetch_openml + + ds = fetch_openml("creditcard", version=1, as_frame=False, parser="auto") + x = ds.data.astype(np.float32) + y = ds.target.astype(np.float32) + x_tr, x_te, y_tr, y_te = random_split(x, y, stratify=True) + save("fraud", x_tr, y_tr, x_te, y_te) + + +def _numerai_roles(f): + """Row filter shared by the f32 and int8 numerai caches. + + Drops rows without a target, embargoes the eras before the test block, + and splits era-ordered. Returns ``(feat_cols, keep_all, n_rows, + train_end, era_int, targets)`` where ``keep_all`` is the absolute row + mask over the parquet. + """ + feat_cols = [c for c in f.schema_arrow.names if c.startswith("feature")] + + # pass 1: era + target only, to build the row filter and split boundaries + et = f.read(columns=["era", "target"]).to_pandas() + era_int = et["era"].astype(int).to_numpy() + keep = et["target"].notna().to_numpy() + if not (np.diff(era_int) >= 0).all(): + sys.exit("numerai: parquet must be sorted by era") + + kept_eras = era_int[keep] + uniq = np.unique(kept_eras) + test_eras = uniq[-NUMERAI_TEST_ERAS:] + embargo_eras = uniq[ + -(NUMERAI_TEST_ERAS + NUMERAI_EMBARGO_ERAS) : -NUMERAI_TEST_ERAS + ] + role = np.full( + len(kept_eras), 1, dtype=np.int8 + ) # 1 train, 0 embargo (drop), 2 test + role[np.isin(kept_eras, embargo_eras)] = 0 + role[np.isin(kept_eras, test_eras)] = 2 + + keep_within = role != 0 + keep_all = keep.copy() + keep_all[keep] = keep_within # absolute row filter + n_rows = int(keep_within.sum()) + train_end = int((role == 1).sum()) + targets = et["target"].to_numpy(dtype=np.float32) + return feat_cols, keep_all, n_rows, train_end, era_int, targets + + +def prep_numerai(): + """Era-ordered float32 memmap; last N eras held out with an embargo gap. + + Rows without a target are dropped. Train rows are ``X[:train_end]`` and + test rows ``X[test_start:]`` so both are zero-copy views of the memmap. + """ + import pyarrow.parquet as pq + + src = os.environ.get("NUMERAI_PARQUET") + if not src: + sys.exit("numerai: set NUMERAI_PARQUET to the v5 'all data' training parquet") + d = os.path.join(CACHE_DIR, "numerai") + os.makedirs(d, exist_ok=True) + f = pq.ParquetFile(src) + feat_cols, keep_all, n_rows, train_end, era_int, tgt_all = _numerai_roles(f) + p = len(feat_cols) + print(f"numerai: {n_rows} rows x {p} features, train_end={train_end}", flush=True) + + x = np.memmap( + os.path.join(d, "X.f32.mem"), dtype=np.float32, mode="w+", shape=(n_rows, p) + ) + y = np.empty(n_rows, dtype=np.float32) + era_out = np.empty(n_rows, dtype=np.int32) + + # pass 2: stream feature batches into the memmap + row_abs = row_out = 0 + for batch in f.iter_batches(batch_size=200_000, columns=feat_cols): + nb = batch.num_rows + mask = keep_all[row_abs : row_abs + nb] + if mask.any(): + arr = batch.to_pandas().to_numpy(dtype=np.float32, na_value=np.nan) + sel = arr[mask] + x[row_out : row_out + len(sel)] = sel + y[row_out : row_out + len(sel)] = tgt_all[row_abs : row_abs + nb][mask] + era_out[row_out : row_out + len(sel)] = era_int[row_abs : row_abs + nb][ + mask + ] + row_out += len(sel) + row_abs += nb + assert row_out == n_rows, (row_out, n_rows) + x.flush() + np.save(os.path.join(d, "y.npy"), y) + np.save(os.path.join(d, "era_codes.npy"), era_out) + meta = { + "n_rows": n_rows, + "n_features": p, + "train_end": train_end, + "test_start": train_end, + "n_test_eras": NUMERAI_TEST_ERAS, + "embargo_eras": NUMERAI_EMBARGO_ERAS, + "source": src, + } + with open(os.path.join(d, "meta.json"), "w") as fh: + json.dump(meta, fh) + print(f"numerai done: {n_rows} x {p}", flush=True) + + +def prep_numerai_int8(): + """Optional int8 twin of the numerai cache (``X.i8.mem``), same rows/order. + + Feeds ExaBoost's native int8 ingestion path (see ingest_bench.py). The + main cross-library matrix stays float32-fed for fairness. Requires the + f32 cache to exist; sampled rows are verified against it so the two + caches cannot drift. + """ + import pyarrow.parquet as pq + + d = os.path.join(CACHE_DIR, "numerai") + meta_path = os.path.join(d, "meta.json") + if not os.path.exists(meta_path): + sys.exit("numerai-int8: build the numerai cache first") + meta = json.load(open(meta_path)) + src = os.environ.get("NUMERAI_PARQUET", meta["source"]) + f = pq.ParquetFile(src) + feat_cols, keep_all, n_rows, train_end, _, _ = _numerai_roles(f) + if n_rows != meta["n_rows"] or train_end != meta["train_end"]: + sys.exit("numerai-int8: row filter disagrees with the existing f32 cache") + p = len(feat_cols) + + x = np.memmap( + os.path.join(d, "X.i8.mem"), dtype=np.int8, mode="w+", shape=(n_rows, p) + ) + row_abs = row_out = 0 + for batch in f.iter_batches(batch_size=200_000, columns=feat_cols): + nb = batch.num_rows + mask = keep_all[row_abs : row_abs + nb] + if mask.any(): + sel = batch.to_pandas().to_numpy(dtype=np.int8)[mask] + x[row_out : row_out + len(sel)] = sel + row_out += len(sel) + row_abs += nb + assert row_out == n_rows, (row_out, n_rows) + x.flush() + + xf = np.memmap( + os.path.join(d, "X.f32.mem"), dtype=np.float32, mode="r", shape=(n_rows, p) + ) + rng = np.random.default_rng(SEED) + for r in rng.integers(0, n_rows, 50): + if not (x[r].astype(np.float32) == xf[r]).all(): + sys.exit(f"numerai-int8: row {r} mismatches the f32 cache") + print(f"numerai-int8 done: {n_rows} x {p}, sampled rows verified", flush=True) + + +PREPS = { + "higgs": prep_higgs, + "epsilon": prep_epsilon, + "airline": prep_airline, + "covtype": prep_covtype, + "year": prep_year, + "fraud": prep_fraud, + "numerai": prep_numerai, + "numerai-int8": prep_numerai_int8, +} + +if __name__ == "__main__": + names = sys.argv[1:] + targets = ( + [n for n in PREPS if not n.startswith("numerai")] if names == ["all"] else names + ) + for t in targets: + if dataset_ready(t): + print(f"{t}: cached, skipping", flush=True) + else: + PREPS[t]() diff --git a/benchmarks/ingest_bench.py b/benchmarks/ingest_bench.py new file mode 100644 index 000000000000..1f74dc282835 --- /dev/null +++ b/benchmarks/ingest_bench.py @@ -0,0 +1,138 @@ +"""Native int8 ingestion benchmark (ExaBoost-only, separate from the matrix). + +The cross-library matrix feeds every library the same float32 bits for +fairness, so ExaBoost's native small-int path (int8/int16 zero-copy through +the C API + LUT binning) is measured here instead: Dataset construct time, +Booster create time, first-trees time, and peak host RSS on numerai, fed from +the float32 memmap vs its int8 twin (``datasets.py numerai-int8``). The two +flows produce identical models (bins are byte-identical by construction). + +Run inside the ExaBoost venv:: + + python benchmarks/ingest_bench.py # 3 repeats each, medians + python benchmarks/ingest_bench.py --repeats 1 + +Each measurement runs in a fresh process (clean GPU + page-cache-warm parity +with the matrix); records append to ``/results/ingest.jsonl``. +""" + +import argparse +import json +import os +import statistics +import subprocess +import sys + +from common import CACHE_DIR, REGIMES, RESULTS_DIR, SEED + +INGEST_JSONL = os.path.join(RESULTS_DIR, "ingest.jsonl") +WARMUP_TREES = 5 + + +def measure_one(kind): + """Run in a fresh process: construct/create/first-trees timings.""" + import resource + import time + + import numpy as np + import lightgbm as lgb + + d = os.path.join(CACHE_DIR, "numerai") + meta = json.load(open(os.path.join(d, "meta.json"))) + shape = (meta["n_rows"], meta["n_features"]) + n = meta["train_end"] + if kind == "f32": + x = np.memmap(os.path.join(d, "X.f32.mem"), np.float32, "r", shape=shape) + else: + path = os.path.join(d, "X.i8.mem") + if not os.path.exists(path): + sys.exit("ingest_bench: run `datasets.py numerai-int8` first") + x = np.memmap(path, np.int8, "r", shape=shape) + y = np.load(os.path.join(d, "y.npy"))[:n] + + reg = REGIMES["numerai"] + params = { + "objective": "regression", + "learning_rate": reg["lr"], + "num_leaves": reg["leaves"], + "max_depth": reg["depth"], + "feature_fraction": reg["colsample"], + "max_bin": 255, + "device_type": "cuda", + "num_threads": os.cpu_count(), + "seed": SEED, + "verbose": -1, + "metric": "None", + } + + t0 = time.perf_counter() + dtrain = lgb.Dataset(x[:n], label=y, params=params) + dtrain.construct() + construct_s = time.perf_counter() - t0 + + t0 = time.perf_counter() + bst = lgb.Booster(params=params, train_set=dtrain) + create_s = time.perf_counter() - t0 + + t0 = time.perf_counter() + for _ in range(WARMUP_TREES): + bst.update() + trees_s = time.perf_counter() - t0 + + print( + json.dumps( + { + "kind": kind, + "construct_s": round(construct_s, 2), + "create_s": round(create_s, 2), + f"trees{WARMUP_TREES}_s": round(trees_s, 2), + "peak_rss_gb": round( + resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024**2, 1 + ), + "version": lgb.__version__, + } + ) + ) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--repeats", type=int, default=3) + ap.add_argument("--one", choices=["f32", "i8"], help=argparse.SUPPRESS) + args = ap.parse_args() + if args.one: + measure_one(args.one) + return + + os.makedirs(RESULTS_DIR, exist_ok=True) + results = {"f32": [], "i8": []} + for kind in ("f32", "i8"): + for rep in range(args.repeats): + out = ( + subprocess.run( + [sys.executable, os.path.abspath(__file__), "--one", kind], + capture_output=True, + text=True, + check=True, + ) + .stdout.strip() + .splitlines()[-1] + ) + rec = json.loads(out) + rec["repeat"] = rep + results[kind].append(rec) + with open(INGEST_JSONL, "a") as fh: + fh.write(json.dumps(rec) + "\n") + print(f"{kind} repeat {rep}: {out}", flush=True) + + print(f"\nmedians over {args.repeats} repeats (numerai, ExaBoost cuda):") + keys = ["construct_s", "create_s", f"trees{WARMUP_TREES}_s", "peak_rss_gb"] + header = "flow " + "".join(f"{k:>14}" for k in keys) + print(header) + for kind in ("f32", "i8"): + med = [statistics.median(r[k] for r in results[kind]) for k in keys] + print(f"{kind:<10}" + "".join(f"{v:>14.2f}" for v in med)) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/orchestrate.py b/benchmarks/orchestrate.py new file mode 100644 index 000000000000..4060bcd9eec0 --- /dev/null +++ b/benchmarks/orchestrate.py @@ -0,0 +1,157 @@ +"""Resumable benchmark matrix runner. + +Executes bench.py cells sequentially (each run gets the GPU exclusively), +skipping cells already recorded in ``/results/runs.jsonl``, so it +is safe to interrupt and relaunch at any time. Datasets whose preprocessed +cache is missing are skipped with a note (run datasets.py first). + +Usage:: + + python benchmarks/orchestrate.py # everything available + python benchmarks/orchestrate.py --only higgs,epsilon + python benchmarks/orchestrate.py --dry-run +""" + +import argparse +import json +import os +import subprocess +import sys +import time + +from common import LIBRARIES, RUNS_JSONL, dataset_ready, regimes_for, venv_python + +BENCH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "bench.py") + +#: small/fast first for early signal; the huge ones last +DATASET_ORDER = ["fraud", "covtype", "year", "higgs", "epsilon", "numerai", "airline"] +KINDS = ["warmup", "timed1", "timed2", "timed3", "curve"] +TIMEOUT_S = { + "fraud": 1800, + "covtype": 1800, + "year": 1800, + "higgs": 7200, + "epsilon": 7200, + "numerai": 14400, + "airline": 14400, +} + + +def load_done(): + done = {} + if os.path.exists(RUNS_JSONL): + with open(RUNS_JSONL) as f: + for line in f: + try: + r = json.loads(line) + except json.JSONDecodeError: + continue + done[(r["library"], r["dataset"], r["regime"], r["kind"])] = r["status"] + return done + + +def record(status, lib, ds, reg, kind, **extra): + with open(RUNS_JSONL, "a") as f: + f.write( + json.dumps( + { + "library": lib, + "dataset": ds, + "regime": reg, + "kind": kind, + "status": status, + **extra, + } + ) + + "\n" + ) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--only", default=None, help="comma-separated dataset subset") + ap.add_argument("--dry-run", action="store_true") + args = ap.parse_args() + datasets = args.only.split(",") if args.only else DATASET_ORDER + + cells = [] + for ds in datasets: + if not dataset_ready(ds): + print( + f"NOTE: dataset '{ds}' not prepared, skipping (run datasets.py)", + flush=True, + ) + continue + for reg in regimes_for(ds): + for lib in LIBRARIES: + for kind in KINDS: + cells.append((lib, ds, reg, kind)) + + done = load_done() + todo = [c for c in cells if c not in done] + print( + f"matrix: {len(cells)} cells, {len(cells) - len(todo)} done, {len(todo)} to run", + flush=True, + ) + if args.dry_run: + for c in todo: + print(c) + return + + os.makedirs(os.path.dirname(RUNS_JSONL), exist_ok=True) + for i, (lib, ds, reg, kind) in enumerate(todo): + done = load_done() + if (lib, ds, reg, kind) in done: + continue + # if the warmup for this combo failed, don't waste time on the rest + if kind != "warmup" and done.get((lib, ds, reg, "warmup")) == "failed": + print(f"SKIP {lib}/{ds}/{reg}/{kind} (warmup failed)", flush=True) + record("skipped_warmup_failed", lib, ds, reg, kind) + continue + cmd = [ + venv_python(lib), + BENCH, + "--library", + lib, + "--dataset", + ds, + "--regime", + reg, + "--kind", + kind, + ] + t0 = time.time() + print(f"[{i + 1}/{len(todo)}] RUN {lib}/{ds}/{reg}/{kind}", flush=True) + try: + p = subprocess.run( + cmd, + timeout=TIMEOUT_S.get(ds, 7200), + env={ + **os.environ, + "CUDA_VISIBLE_DEVICES": os.environ.get("CUDA_VISIBLE_DEVICES", "0"), + }, + capture_output=True, + text=True, + ) + status = "ok" if p.returncode == 0 else "failed" + if p.returncode != 0 and (lib, ds, reg, kind) not in load_done(): + # the child died before writing its record (e.g. segfault); + # record the failure so resume doesn't retry it forever + record( + "failed", + lib, + ds, + reg, + kind, + error=f"exit code {p.returncode}: {p.stderr[-500:]}", + ) + if p.returncode != 0: + sys.stderr.write(p.stdout[-2000:] + p.stderr[-2000:] + "\n") + except subprocess.TimeoutExpired: + status = "timeout" + record("timeout", lib, ds, reg, kind) + print(f" -> {status} ({time.time() - t0:.0f}s)", flush=True) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/report.py b/benchmarks/report.py new file mode 100644 index 000000000000..a06f6411402f --- /dev/null +++ b/benchmarks/report.py @@ -0,0 +1,374 @@ +"""Aggregate ``/results/runs.jsonl`` into REPORT.md + charts.""" + +import json +import os +import subprocess + +import matplotlib +import numpy as np +import pandas as pd + +matplotlib.use("Agg") +import matplotlib.pyplot as plt # noqa: E402 + +from common import LIBRARIES, REPORT_DIR, RESULTS_DIR, RUNS_JSONL # noqa: E402 + +COLORS = { + "exaboost": "#d62728", + "exaboost-quant": "#ff9896", + "lightgbm": "#1f77b4", + "lightgbm-quant": "#aec7e8", + "xgboost": "#2ca02c", + "catboost": "#9467bd", +} +METRIC_KEY = { + "higgs": "auc", + "epsilon": "auc", + "airline": "auc", + "fraud": "auc", + "covtype": "accuracy", + "year": "rmse", + "numerai": "corr_mean", +} +NOTES = """ +## Measurement notes + +- `construct` covers each library's training-data structure: LightGBM `Dataset` + (binning), XGBoost `QuantileDMatrix` (quantile sketch), CatBoost `Pool` (thin + wrapper — CatBoost quantizes inside `fit`, i.e. inside `train`). +- CatBoost pre-reserves most of the GPU memory by default, so its "GPU peak" + reflects the reservation, not the working set. +- Time-to-quality curves for CatBoost use a linearly approximated time axis + (dashed) because it exposes no per-iteration wall clock. +- `xx-quant` = `use_quantized_grad=true`. Runs flagged `sane=false` produced + degenerate models and their timings should be ignored. +""" + + +def fmt(x, nd=1): + return "—" if x is None or (isinstance(x, float) and np.isnan(x)) else f"{x:.{nd}f}" + + +def main(): + os.makedirs(REPORT_DIR, exist_ok=True) + df = pd.DataFrame([json.loads(line) for line in open(RUNS_JSONL)]) + timed = df[df["kind"].str.startswith("timed") & (df["status"] == "ok")].copy() + for c in ("construct_s", "train_s", "total_s", "gpu_mem_peak_mb", "rss_peak_mb"): + timed[c] = pd.to_numeric(timed[c], errors="coerce") + + lines = ["# GPU GBDT benchmark: ExaBoost vs LightGBM vs XGBoost vs CatBoost\n"] + + sha_file = os.path.join(RESULTS_DIR, "exaboost_sha.txt") + sha = open(sha_file).read().strip()[:12] if os.path.exists(sha_file) else "unknown" + gpu = subprocess.run( + [ + "nvidia-smi", + "--query-gpu=name,memory.total,driver_version", + "--format=csv,noheader", + ], + capture_output=True, + text=True, + ).stdout.strip() + vers = ( + df[df["status"] == "ok"] + .groupby("library")["version"] + .agg(lambda s: s.dropna().iloc[0] if s.notna().any() else "?") + ) + lines += [ + "## Environment\n", + f"- GPU: {gpu}", + f"- ExaBoost: `{sha}`", + *[f"- {lib}: {v}" for lib, v in vers.items()], + NOTES, + ] + + # ---- per dataset x regime tables ------------------------------------ + for (ds, reg), g in timed.groupby(["dataset", "regime"], sort=False): + mkey = METRIC_KEY.get(ds, "auc") + lines.append(f"## {ds} — regime `{reg}`\n") + lines.append( + f"| library | construct (s) | train (s) | total (s) | {mkey} | GPU peak (MB) | RSS peak (MB) |" + ) + lines.append("|---|---|---|---|---|---|---|") + base = g[g["library"] == "exaboost"]["train_s"].median() + for lib in LIBRARIES: + gl = g[g["library"] == lib] + if gl.empty: + fails = df[ + (df.dataset == ds) + & (df.regime == reg) + & (df.library == lib) + & (df.status != "ok") + ] + note = fails["status"].iloc[0] if not fails.empty else "missing" + lines.append(f"| {lib} | {note} | | | | | |") + continue + met = gl["metrics"].iloc[0] or {} + spread = gl["train_s"].max() - gl["train_s"].min() + rel = ( + f" ({base / gl['train_s'].median():.2f}×)" + if lib != "exaboost" and base and gl["train_s"].median() + else "" + ) + flag = "" if met.get("sane", True) else " ⚠️insane" + lines.append( + f"| {lib} | {fmt(gl['construct_s'].median())} " + f"| {fmt(gl['train_s'].median())} ±{fmt(spread / 2)}{rel} " + f"| {fmt(gl['total_s'].median())} " + f"| {fmt(met.get(mkey), 4)}{flag} " + f"| {fmt(gl['gpu_mem_peak_mb'].median(), 0)} " + f"| {fmt(gl['rss_peak_mb'].median(), 0)} |" + ) + lines.append("") + + # ---- train-time bar charts ------------------------------------------ + REG_TITLES = { + "shallow": "regime shallow (500 trees)", + "deep": "regime deep (500 trees)", + "numerai": "numerai example config (2000 trees)", + } + for reg in ("shallow", "deep", "numerai"): + sub = timed[timed["regime"] == reg] + if sub.empty: + continue + datasets = [ + d + for d in [ + "fraud", + "covtype", + "year", + "higgs", + "epsilon", + "airline", + "numerai", + ] + if d in set(sub["dataset"]) + ] + x = np.arange(len(datasets)) + w = 0.13 + fig, ax = plt.subplots(figsize=(11, 5)) + crash_marks = [] + for i, lib in enumerate(LIBRARIES): + offs = x + (i - 2.5) * w + vals, hatches = [], [] + for j, d in enumerate(datasets): + gl = sub[(sub.dataset == d) & (sub.library == lib)] + if gl.empty: + vals.append(np.nan) + hatches.append(None) + # crashed/failed configs get an explicit marker instead of + # silently-absent bars (e.g. upstream quantized on higgs) + crashed = df[ + (df.dataset == d) + & (df.regime == reg) + & (df.library == lib) + & (df.status != "ok") + ] + if not crashed.empty: + crash_marks.append((offs[j], COLORS[lib])) + continue + vals.append(gl["train_s"].median()) + met = gl["metrics"].iloc[0] or {} + # degenerate models: sane=false, or a regression fit no better + # than 99% of the target's std (the year quant case squeaks past + # the sane threshold while still being useless) + degenerate = not all((m or {}).get("sane", True) for m in gl["metrics"]) + if not degenerate and "rmse" in met and d != "numerai": + ok_rmses = [ + (m or {}).get("rmse") + for m in sub[(sub.dataset == d) & (sub.library == "xgboost")][ + "metrics" + ] + ] + if ok_rmses and ok_rmses[0] and met["rmse"] > 1.15 * ok_rmses[0]: + degenerate = True + hatches.append("///" if degenerate else None) + bars = ax.bar(offs, vals, w, label=lib, color=COLORS[lib]) + for b, h in zip(bars, hatches): + if h: + b.set_hatch(h) + b.set_alpha(0.35) + # single-dataset charts (numerai) have room for quality labels + if reg == "numerai" and len(datasets) == 1 and not np.isnan(vals[0]): + gl = sub[(sub.dataset == datasets[0]) & (sub.library == lib)] + met = gl["metrics"].iloc[0] or {} + if met.get("corr_mean") is not None: + ax.text( + offs[0], + vals[0] * 1.03, + f"corr {met['corr_mean']:.4f}\nsharpe {met['corr_sharpe']:.2f}", + ha="center", + va="bottom", + fontsize=8, + ) + # crash markers: x in data coords, y just above the axis baseline + for xi, color in crash_marks: + ax.text( + xi, + 0.02, + "✗", + transform=ax.get_xaxis_transform(), + ha="center", + va="bottom", + fontsize=12, + color=color, + fontweight="bold", + ) + ax.set_yscale("log") + if reg == "numerai": + ymin, ymax = ax.get_ylim() + ax.set_ylim(ymin, ymax * 1.4) # headroom for the quality labels + ax.set_xticks(x, datasets) + ax.set_ylabel("train time (s, log)") + ax.set_title( + f"GPU training time — {REG_TITLES[reg]}\n" + "✗ = crashed; hatched = degenerate model (timing not meaningful)" + ) + ax.legend(ncol=3, fontsize=8) + fig.tight_layout() + fig.savefig(os.path.join(REPORT_DIR, f"train_time_{reg}.png"), dpi=120) + lines.append(f"![train time {reg}](train_time_{reg}.png)\n") + + # ---- speed vs quality scatter ---------------------------------------- + panels = [] + for ds in ["fraud", "covtype", "year", "higgs", "epsilon", "airline", "numerai"]: + for reg in ["numerai"] if ds == "numerai" else ["shallow", "deep"]: + if not timed[(timed.dataset == ds) & (timed.regime == reg)].empty: + panels.append((ds, reg)) + if panels: + ncols = 3 + nrows = (len(panels) + ncols - 1) // ncols + fig, axes = plt.subplots(nrows, ncols, figsize=(14, 3.4 * nrows)) + axes = np.atleast_1d(axes).ravel() + for ax_i, (ds, reg) in zip(axes, panels): + mkey = METRIC_KEY.get(ds, "auc") + lower_better = mkey == "rmse" + for lib in LIBRARIES: + gl = timed[ + (timed.dataset == ds) + & (timed.regime == reg) + & (timed.library == lib) + ] + if gl.empty: + continue + met = gl["metrics"].iloc[0] or {} + q = met.get(mkey) + if q is None: + continue + degenerate = not all((m or {}).get("sane", True) for m in gl["metrics"]) + ax_i.scatter( + [gl["train_s"].median()], + [q], + s=110 if degenerate else 80, + color=COLORS[lib], + marker="X" if degenerate else "o", + edgecolor="black", + linewidth=0.6, + zorder=3, + ) + present = set( + timed[(timed.dataset == ds) & (timed.regime == reg)]["library"] + ) + crashed = sorted( + set( + df[(df.dataset == ds) & (df.regime == reg) & (df.status != "ok")][ + "library" + ] + ) + - present + ) + title = f"{ds} / {reg}" + if crashed: + title += "\n(crashed: " + ", ".join(crashed) + ")" + ax_i.set_xscale("log") + ax_i.xaxis.set_minor_formatter(matplotlib.ticker.NullFormatter()) + if lower_better: + ax_i.invert_yaxis() + ax_i.set_title(title, fontsize=10) + ax_i.set_xlabel("train time (s, log)", fontsize=8) + ax_i.set_ylabel( + mkey + (" (lower = better)" if lower_better else ""), fontsize=8 + ) + ax_i.tick_params(labelsize=7) + ax_i.grid(True, alpha=0.25) + for ax_i in axes[len(panels) :]: + ax_i.axis("off") + handles = [ + plt.Line2D( + [], + [], + marker="o", + linestyle="", + color=COLORS[lib], + label=lib, + markeredgecolor="black", + markeredgewidth=0.6, + ) + for lib in LIBRARIES + ] + handles.append( + plt.Line2D( + [], + [], + marker="X", + linestyle="", + color="gray", + label="degenerate model", + markeredgecolor="black", + ) + ) + fig.legend(handles=handles, ncol=7, loc="lower center", fontsize=9) + fig.suptitle("Speed vs quality — up and to the left is better", fontsize=12) + fig.tight_layout(rect=[0, 0.05, 1, 0.96]) + fig.savefig(os.path.join(REPORT_DIR, "speed_vs_quality.png"), dpi=120) + lines.append("![speed vs quality](speed_vs_quality.png)\n") + + # ---- time-to-quality curves ----------------------------------------- + curves = df[(df["kind"] == "curve") & (df["status"] == "ok")] + for (ds, reg), g in curves.groupby(["dataset", "regime"], sort=False): + fig, ax = plt.subplots(figsize=(7, 4.5)) + plotted = False + for _, r in g.iterrows(): + pts = [p for p in (r.get("curve") or []) if p[2] is not None] + if not pts: + continue + style = "--" if r.get("curve_time_approx") else "-" + ax.plot( + [p[1] for p in pts], + [p[2] for p in pts], + style, + label=r["library"], + color=COLORS[r["library"]], + ) + plotted = True + if not plotted: + plt.close(fig) + continue + ax.set_xlabel("wall time (s)") + ax.set_ylabel("held-out metric") + ax.set_title(f"time-to-quality — {ds} / {reg}") + ax.legend(fontsize=8) + fig.tight_layout() + name = f"curve_{ds}_{reg}.png" + fig.savefig(os.path.join(REPORT_DIR, name), dpi=120) + lines.append(f"![curve {ds} {reg}]({name})\n") + + fails = df[~df["status"].isin(["ok"])] + if not fails.empty: + lines.append("## Failed / skipped runs\n") + for _, r in fails.iterrows(): + err_raw = r.get("error") + err = err_raw.strip().splitlines() if isinstance(err_raw, str) else [] + lines.append( + f"- {r['library']}/{r['dataset']}/{r['regime']}/{r['kind']}: {r['status']}" + + (f" — `{err[-1][:160]}`" if err else "") + ) + lines.append("") + + with open(os.path.join(REPORT_DIR, "REPORT.md"), "w") as f: + f.write("\n".join(lines)) + print(f"wrote {REPORT_DIR}/REPORT.md") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/requirements.txt b/benchmarks/requirements.txt new file mode 100644 index 000000000000..541826c08efc --- /dev/null +++ b/benchmarks/requirements.txt @@ -0,0 +1,7 @@ +numpy +pandas +pyarrow +scikit-learn +scipy +psutil +matplotlib diff --git a/benchmarks/setup_envs.sh b/benchmarks/setup_envs.sh new file mode 100755 index 000000000000..ceb7dcb648bd --- /dev/null +++ b/benchmarks/setup_envs.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Set up the two python environments for the GPU benchmark suite: +# env-exaboost ExaBoost built from THIS checkout (CUDA) +# env-competitors upstream LightGBM (CUDA, built from PyPI sdist), +# XGBoost and CatBoost (PyPI wheels, ship CUDA support) +# +# Environment variables: +# EXABOOST_BENCH_ROOT workspace dir (default: benchmarks/workspace) +# EXABOOST_CUDA_ARCHS CMAKE_CUDA_ARCHITECTURES (default: native) +# UPSTREAM_LGBM_VERSION upstream LightGBM version (default: latest on PyPI) +set -euo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO="$(dirname "$HERE")" +ROOT="${EXABOOST_BENCH_ROOT:-$HERE/workspace}" +ARCHS="${EXABOOST_CUDA_ARCHS:-native}" + +mkdir -p "$ROOT"/{data,results,report} + +echo "=== env-exaboost: building ExaBoost from $REPO (archs: $ARCHS)" +python3 -m venv "$ROOT/env-exaboost" +# shellcheck disable=SC1091 +source "$ROOT/env-exaboost/bin/activate" +pip install -q --upgrade pip +pip install -q -r "$HERE/requirements.txt" +# BUILD_WITH_SHARED_NCCL avoids nvlink failures against the static NCCL on +# arches it was not device-linked for (e.g. Blackwell) +(cd "$REPO" && CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=$ARCHS -DBUILD_WITH_SHARED_NCCL=ON" sh build-python.sh install --cuda) +git -C "$REPO" rev-parse HEAD > "$ROOT/results/exaboost_sha.txt" +python -c "import lightgbm as l; print('exaboost OK', l.__version__)" +deactivate + +echo "=== env-competitors: upstream LightGBM (CUDA) + XGBoost + CatBoost" +python3 -m venv "$ROOT/env-competitors" +# shellcheck disable=SC1091 +source "$ROOT/env-competitors/bin/activate" +pip install -q --upgrade pip +pip install -q -r "$HERE/requirements.txt" +pip install -q xgboost catboost +CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=$ARCHS" \ + pip install --no-binary lightgbm --config-settings=cmake.define.USE_CUDA=ON \ + "lightgbm${UPSTREAM_LGBM_VERSION:+==$UPSTREAM_LGBM_VERSION}" +python -c "import lightgbm as l, xgboost as x, catboost as c; print('upstream lgbm', l.__version__, '| xgb', x.__version__, '| catboost', c.__version__)" +deactivate + +echo "setup complete; workspace: $ROOT"