From 6619c6494a44b5738b3b9057d65d0e976e02da64 Mon Sep 17 00:00:00 2001 From: tcconnally Date: Wed, 15 Jul 2026 00:50:31 -0500 Subject: [PATCH] =?UTF-8?q?feat(playback):=20burst-aware=20budgets=20?= =?UTF-8?q?=E2=80=94=20fix=208-cell=20cache-starvation=20freezes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first 8-cell production run buffered ~9 min/2 hr (78 paused-for-cache freezes, 543 freeze-seconds), spread across ~15 files — systemic delivery starvation, not one bad asset and not decode (PERF stalls stayed <200ms). Root cause: 96.7% of plays go DIRECT (242 vs 8 transcode) since v10.3.0 dropped the resolution gate, so 8 cells pull raw high-bitrate files off greg at once, each with a 60s/1GB readahead. The fill-bursts starve each other's steady reads — 80% of freezes began within 8s of a stream-open. Two cell-count-aware budgets, both no-ops at <=4 cells (measured clean): - scale_readahead_s: demuxer_readahead 60s->30s at 8 cells (+cache_secs follows) — shrinks the per-open fill-burst. - scale_bitrate_budget_mbps: direct-play cap 60->~33 Mbps at 8 cells, so the rare 40-60+ Mbps outliers transcode (Emby throttles to ~realtime = smooth) while the library bulk stays direct. Explicit HYPERWALL_MAX_DIRECT_BITRATE_MBPS still wins verbatim. wall.py wires the resolved per-cell budget into the transcode heuristic and logs readahead + bitrate cap alongside the demuxer budget. Tests: 5 new (scale_readahead, graduated bitrate, env-override precedence, apply_cache_budget readahead scaling). Full suite green (v10.13.0). Note: scale_bitrate_budget_mbps assumes link_mbps=800 and caps per-file, not aggregate (8x33 ~= 264 Mbps sustained) — validate against greg's real disk/link ceiling before treating this as the complete fix. Co-Authored-By: Claude Opus 4.8 --- hyperwall/__init__.py | 2 +- hyperwall/constants.py | 26 ++++++++++++++++++++++- hyperwall/reliability.py | 33 +++++++++++++++++++++++++++++ hyperwall/wall.py | 23 +++++++++++++++++--- tests/run_repo_guards.py | 4 ++-- tests/test_reliability.py | 44 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 125 insertions(+), 7 deletions(-) diff --git a/hyperwall/__init__.py b/hyperwall/__init__.py index ef63ed9..44133a0 100644 --- a/hyperwall/__init__.py +++ b/hyperwall/__init__.py @@ -7,7 +7,7 @@ from __future__ import annotations -__version__ = "10.12.1" +__version__ = "10.13.0" # Short "major.minor" form, derived — used for User-Agent / Emby auth Version / # window titles so a version bump touches exactly ONE line (this file). VERSION_SHORT = ".".join(__version__.split(".")[:2]) diff --git a/hyperwall/constants.py b/hyperwall/constants.py index 923bf38..e001751 100644 --- a/hyperwall/constants.py +++ b/hyperwall/constants.py @@ -103,6 +103,23 @@ def _int_env(name: str, default: int, lo: int, hi: int) -> int: MAX_DIRECT_FPS = _int_env("HYPERWALL_MAX_DIRECT_FPS", 66, 0, 1_000) MAX_DIRECT_BITRATE_MBPS = _int_env("HYPERWALL_MAX_DIRECT_BITRATE_MBPS", 60, 0, 10_000) + +def effective_bitrate_budget_mbps(n_cells: int) -> int: + """Cell-count-aware direct-play bitrate cap. + + An explicit HYPERWALL_MAX_DIRECT_BITRATE_MBPS wins verbatim. Otherwise + the default 60 is scaled down as cells go up (8 cells → ~33 Mbps): the + graduated middle ground between transcode-everything and + direct-everything. High-bitrate outliers transcode server-side, where + Emby throttles delivery to ~realtime — smooth by construction — while + the bulk of the library stays direct. ≤4 cells resolve to the base + (measured clean at 4 cells). + """ + if os.environ.get("HYPERWALL_MAX_DIRECT_BITRATE_MBPS"): + return MAX_DIRECT_BITRATE_MBPS + from .reliability import scale_bitrate_budget_mbps + return scale_bitrate_budget_mbps(n_cells, MAX_DIRECT_BITRATE_MBPS) + # Memory-aware demuxer cache budget. Each cell wants PER_CELL demuxer bytes, but # the grid total is capped at CACHE_BUDGET_MB so large grids don't blow up RAM. # Sized for this box (32 GB): 1 GB/cell, 8 GB grid cap — deep enough that the @@ -223,7 +240,7 @@ def apply_cache_budget(opts: dict, n_cells: int) -> dict: Keeps the aggregate grid demuxer buffer within CACHE_BUDGET_MB so large grids (e.g. 6x6) don't exhaust RAM. Pure w.r.t. the reliability helper. """ - from .reliability import scale_demuxer_mb + from .reliability import scale_demuxer_mb, scale_readahead_s out = dict(opts) mb = scale_demuxer_mb( @@ -232,4 +249,11 @@ def apply_cache_budget(opts: dict, n_cells: int) -> dict: total_budget_mb=CACHE_BUDGET_MB, ) out["demuxer_max_bytes"] = f"{mb}MiB" + # Readahead depth = burst size on every track open; scale it down with + # cell count so 8 cells don't starve each other's steady reads + # (2026-07-14: 80% of freezes began within 8s of a stream-open). + base = int(out.get("demuxer_readahead_secs", 60) or 60) + ra = scale_readahead_s(n_cells, base_s=base) + out["demuxer_readahead_secs"] = ra + out["cache_secs"] = min(int(out.get("cache_secs", 60) or 60), ra) return out diff --git a/hyperwall/reliability.py b/hyperwall/reliability.py index 21acec7..a41d5c0 100644 --- a/hyperwall/reliability.py +++ b/hyperwall/reliability.py @@ -76,6 +76,39 @@ def scale_demuxer_mb( return int(max(floor_mb, per)) +def scale_readahead_s(n_cells: int, base_s: int = 60, floor_s: int = 10) -> int: + """Per-cell demuxer readahead (seconds), scaled down as cells go up. + + Readahead depth is burst SIZE: on every track open, mpv fills it at + line rate. Eight cells each slurping 60s of a 20 Mbps stream produce + fill-bursts that starve the other cells' steady reads — measured + 2026-07-14: 80% of freeze episodes began within 8s of a stream-open. + <=4 cells keep the full depth (measured clean); beyond that the depth + shrinks so aggregate burst demand stays roughly constant. + """ + n = max(1, int(n_cells)) + if n <= 4: + return base_s + return max(floor_s, int(base_s * 4 / n)) + + +def scale_bitrate_budget_mbps( + n_cells: int, base_mbps: int, link_mbps: int = 800, +) -> int: + """Cell-count-aware direct-play bitrate cap (the graduated middle + ground between transcode-everything and direct-everything). + + Items above the cap transcode server-side, where Emby throttles + delivery to ~realtime — smooth by construction. The cap divides a + conservative share of the link across cells with 3x burst headroom; + at <=4 cells it resolves to the configured base (measured clean), at + 8 cells ~33 Mbps so the rare 40-60+ Mbps outliers transcode while the + bulk of the library stays direct. + """ + n = max(1, int(n_cells)) + return min(base_mbps, max(8, link_mbps // (3 * n))) + + def apply_jitter(delay_s: float, rand: float) -> float: """Spread a retry delay over [0.75x, 1.25x] to desynchronize cells. diff --git a/hyperwall/wall.py b/hyperwall/wall.py index fc041e4..1391abb 100644 --- a/hyperwall/wall.py +++ b/hyperwall/wall.py @@ -37,6 +37,8 @@ from .cell import VideoCell from .perftrace import traced from .constants import ( + MAX_DIRECT_FPS, + effective_bitrate_budget_mbps, OUTAGE_MIN_CELLS, OUTAGE_WINDOW_S, STREAM_START_STAGGER_MS, @@ -48,7 +50,8 @@ MPV_OPTS, SCRIPT_DIR, ) -from .emby import EmbyClient, ContentLoader, needs_transcode +from .emby import EmbyClient, ContentLoader +from .urls import needs_transcode as _needs_transcode_pure from .reliability import is_systemic_outage from .urls import build_stream_url from .playlist import PlaylistManager, DEFAULT_GROUP @@ -184,9 +187,17 @@ def __init__( budgeted = apply_cache_budget(apply_env_overrides(MPV_OPTS), n_cells) for cell in self.cells: cell._mpv_opts = budgeted + # Burst-aware budgets (2026-07-14): 80% of freeze episodes began + # within 8s of a stream-open — the wall was starving itself with + # its own fill-bursts at 8 cells. Readahead depth is scaled inside + # apply_cache_budget; the direct-play bitrate cap scales here. + self._bitrate_budget_mbps = effective_bitrate_budget_mbps(n_cells) logger.info( - "MPV cache budget: %d cells → demuxer_max_bytes=%s each", + "MPV cache budget: %d cells → demuxer_max_bytes=%s, " + "readahead=%ss, direct-play bitrate cap=%s Mbps", n_cells, budgeted.get("demuxer_max_bytes"), + budgeted.get("demuxer_readahead_secs"), + self._bitrate_budget_mbps, ) for win in self.windows: @@ -282,7 +293,13 @@ def _build_url( base = self.client.server_url sid = uuid.uuid4().hex - auto_transcode = needs_transcode(item) + auto_transcode = _needs_transcode_pure( + item, + auto_transcode=os.environ.get( + "HYPERWALL_AUTO_TRANSCODE", "1") == "1", + max_fps=MAX_DIRECT_FPS, + max_bitrate_mbps=self._bitrate_budget_mbps, + ) transcode = bool(force_transcode or auto_transcode) url = build_stream_url( base=base, item_id=iid, api_key=key, diff --git a/tests/run_repo_guards.py b/tests/run_repo_guards.py index 9f9d90b..79c32da 100644 --- a/tests/run_repo_guards.py +++ b/tests/run_repo_guards.py @@ -46,10 +46,10 @@ def test_01_entry_point_imports(): def test_02_package_identity(): """Package has version and banner.""" from hyperwall import __version__, runtime_banner - assert __version__ == "10.12.1" + assert __version__ == "10.13.0" banner = runtime_banner() assert "Hyperwall" in banner - assert "10.12.1" in banner + assert "10.13.0" in banner def test_03_config_loads(): diff --git a/tests/test_reliability.py b/tests/test_reliability.py index 9e2f543..c9b523e 100644 --- a/tests/test_reliability.py +++ b/tests/test_reliability.py @@ -300,6 +300,39 @@ def test_constants_defaults_load(): assert c.CACHE_BUDGET_MB == 8_192 +def test_scale_readahead_shrinks_beyond_four_cells(): + from hyperwall.reliability import scale_readahead_s + assert scale_readahead_s(1) == 60 + assert scale_readahead_s(4) == 60 # measured clean — unchanged + assert scale_readahead_s(8) == 30 # burst size halves at 8 cells + assert scale_readahead_s(16) == 15 + assert scale_readahead_s(100) == 10 # floor + + +def test_scale_bitrate_budget_graduated(): + from hyperwall.reliability import scale_bitrate_budget_mbps + assert scale_bitrate_budget_mbps(4, 60) == 60 # <=4 cells: base wins + assert scale_bitrate_budget_mbps(8, 60) == 33 # 800/(3*8) + assert scale_bitrate_budget_mbps(16, 60) == 16 + assert scale_bitrate_budget_mbps(100, 60) == 8 # floor + + +def test_effective_budget_env_override_wins(monkey=None): + import os + from hyperwall import constants as c + old = os.environ.pop("HYPERWALL_MAX_DIRECT_BITRATE_MBPS", None) + try: + assert c.effective_bitrate_budget_mbps(8) == 33 + os.environ["HYPERWALL_MAX_DIRECT_BITRATE_MBPS"] = "60" + # explicit env wins verbatim even at 8 cells (module constant was + # parsed at import; presence of the var is the override signal) + assert c.effective_bitrate_budget_mbps(8) == c.MAX_DIRECT_BITRATE_MBPS + finally: + os.environ.pop("HYPERWALL_MAX_DIRECT_BITRATE_MBPS", None) + if old is not None: + os.environ["HYPERWALL_MAX_DIRECT_BITRATE_MBPS"] = old + + def test_apply_cache_budget_shape(): from hyperwall.constants import apply_cache_budget out = apply_cache_budget({"demuxer_max_bytes": "512MiB", "vo": "gpu-next"}, 36) @@ -308,6 +341,17 @@ def test_apply_cache_budget_shape(): assert int(out["demuxer_max_bytes"][:-3]) < 512 # scaled down for 36 cells +def test_apply_cache_budget_scales_readahead(): + from hyperwall.constants import apply_cache_budget + base = {"demuxer_max_bytes": "1024MiB", "demuxer_readahead_secs": 60, + "cache_secs": 60} + four = apply_cache_budget(dict(base), 4) + eight = apply_cache_budget(dict(base), 8) + assert four["demuxer_readahead_secs"] == 60 # unchanged at 4 cells + assert eight["demuxer_readahead_secs"] == 30 # burst-aware at 8 + assert eight["cache_secs"] == 30 # follows readahead + + def run_all() -> int: tests = [v for k, v in sorted(globals().items()) if k.startswith("test_")] passed = failed = 0