Archive delete: truthful results, unlock & confirm-through; UI/serving performance

tests/test_make_room_for.py

@@ -135,7 +135,7 @@ def test_make_room_evicts_until_under_then_true(env, monkeypatch):
     _patch_quota(monkeypatch, used_bytes=2 * (1 << 30))
     monkeypatch.setattr(
         "web.services.retention._delete_clip_files",
-        lambda row, recordings: 1 << 30,
+        lambda row, recordings: (1 << 30, True),
     )
     ok = ret.make_room_for(
         db, str(rec), size=0, before_ts=300,
@@ -190,7 +190,7 @@ def test_make_room_for_skips_locked_clip_protect_ro_false(env, monkeypatch):
     _patch_quota(monkeypatch, used_bytes=2 * (1 << 30))
     monkeypatch.setattr(
         "web.services.retention._delete_clip_files",
-        lambda row, recordings: 1 << 30,
+        lambda row, recordings: (1 << 30, True),
     )
 
     ok = ret.make_room_for(

tests/test_queue_delete.py

@@ -1,6 +1,7 @@
 """queue.delete_clips: local file + index delete, mark download_queue skipped."""
 from __future__ import annotations
 
+import os
 import time
 from pathlib import Path
 
@@ -49,7 +50,8 @@ def test_delete_removes_files_index_and_marks_skipped(tmp_path):
     rec, db = _env(tmp_path)
     cid, path = _make_clip(rec, db, basename="A.MP4")
     res = delete_clips(db, ["A.MP4"], str(rec))
-    assert res == {"deleted": 1, "skipped": 1, "protected": 0}
+    assert res == {"deleted": 1, "skipped": 1, "protected": 0,
+                   "protected_names": [], "failed": 0}
     assert not path.exists()
     assert not (path.parent / "A.MP4.gpx").exists()
     with db.conn() as c:
@@ -76,7 +78,8 @@ def test_delete_nonexistent_file_reports_zero(tmp_path):
     from web.services.queue import delete_clips
     rec, db = _env(tmp_path)
     res = delete_clips(db, ["GHOST.MP4"], str(rec))
-    assert res == {"deleted": 0, "skipped": 0, "protected": 0}
+    assert res == {"deleted": 0, "skipped": 0, "protected": 0,
+                   "protected_names": [], "failed": 0}
 
 
 def test_delete_skips_locked_clip(tmp_path):
@@ -97,7 +100,8 @@ def test_delete_undownloaded_marks_skipped_only(tmp_path):
     with db.write() as c:
         _insert_queue(c, "PEND.MP4", "pending")
     res = delete_clips(db, ["PEND.MP4"], str(rec))
-    assert res == {"deleted": 0, "skipped": 1, "protected": 0}
+    assert res == {"deleted": 0, "skipped": 1, "protected": 0,
+                   "protected_names": [], "failed": 0}
     with db.conn() as c:
         assert c.execute(
             "SELECT state FROM download_queue WHERE filename='PEND.MP4'"
@@ -116,7 +120,9 @@ def test_delete_missing_file_does_not_raise(tmp_path):
 def test_delete_empty_is_noop(tmp_path):
     from web.services.queue import delete_clips
     rec, db = _env(tmp_path)
-    assert delete_clips(db, [], str(rec)) == {"deleted": 0, "skipped": 0, "protected": 0}
+    assert delete_clips(db, [], str(rec)) == {
+        "deleted": 0, "skipped": 0, "protected": 0,
+        "protected_names": [], "failed": 0}
 
 
 def test_delete_logs_info_audit_line(tmp_path, caplog):
@@ -141,6 +147,60 @@ def test_delete_noop_does_not_log(tmp_path, caplog):
     assert not [r for r in caplog.records if r.name == "viofosync.queue"]
 
 
+def test_delete_failed_unlink_keeps_rows_and_reports_failed(tmp_path, monkeypatch):
+    # A file the container can't unlink (classic Unraid PUID/permissions
+    # mismatch) must NOT count as deleted: the index row stays, the queue row
+    # stays untouched, and the response says 'failed'. Dropping the row while
+    # the .mp4 survives would free nothing and resurrect the clip on rescan.
+    from web.services.queue import delete_clips
+    rec, db = _env(tmp_path)
+    _, path = _make_clip(rec, db, basename="STUCK.MP4")
+    real_remove = os.remove
+
+    def deny_mp4(p, *a, **kw):
+        if str(p) == str(path):
+            raise PermissionError(1, "Operation not permitted", str(p))
+        return real_remove(p, *a, **kw)
+
+    monkeypatch.setattr("web.services.retention.os.remove", deny_mp4)
+    res = delete_clips(db, ["STUCK.MP4"], str(rec))
+    assert res["deleted"] == 0 and res["failed"] == 1
+    assert path.exists()
+    with db.conn() as c:
+        assert c.execute("SELECT COUNT(*) AS n FROM clip_index").fetchone()["n"] == 1
+        row = c.execute(
+            "SELECT state FROM download_queue WHERE filename='STUCK.MP4'"
+        ).fetchone()
+    assert row["state"] == "done"  # not marked skipped — nothing went away
+
+
+def test_delete_force_removes_protected(tmp_path):
+    # force=True is the confirm-through path: the UI has already shown a
+    # second "delete anyway?" dialog, so RO and user-locked clips go too.
+    from web.services.queue import delete_clips
+    rec, db = _env(tmp_path)
+    _, ro_path = _make_clip(rec, db, basename="RO.MP4", event_type="ro")
+    _, lk_path = _make_clip(rec, db, basename="LOCKED.MP4")
+    with db.write() as c:
+        c.execute("UPDATE clip_index SET locked=1 WHERE basename='LOCKED.MP4'")
+    res = delete_clips(db, ["RO.MP4", "LOCKED.MP4"], str(rec), force=True)
+    assert res["deleted"] == 2 and res["protected"] == 0 and res["failed"] == 0
+    assert not ro_path.exists() and not lk_path.exists()
+    with db.conn() as c:
+        assert c.execute("SELECT COUNT(*) AS n FROM clip_index").fetchone()["n"] == 0
+
+
+def test_delete_reports_protected_names(tmp_path):
+    # The UI force-deletes exactly these names after its second confirm.
+    from web.services.queue import delete_clips
+    rec, db = _env(tmp_path)
+    _make_clip(rec, db, basename="RO.MP4", event_type="ro")
+    _, ok_path = _make_clip(rec, db, basename="OK.MP4")
+    res = delete_clips(db, ["RO.MP4", "OK.MP4"], str(rec))
+    assert res["deleted"] == 1 and not ok_path.exists()
+    assert res["protected"] == 1 and res["protected_names"] == ["RO.MP4"]
+
+
 @pytest.fixture
 def authed_client(tmp_config_dir: Path, tmp_recordings_dir: Path, monkeypatch):
     from web import app as app_mod
@@ -192,3 +252,31 @@ def test_delete_endpoint_empty_body(authed_client):
     r = authed_client.post("/api/queue/delete", json={})
     assert r.status_code == 200
     assert r.json()["deleted"] == 0 and r.json()["skipped"] == 0
+
+
+def test_delete_endpoint_force_confirm_through(authed_client, tmp_recordings_dir: Path):
+    # First call refuses the RO clip and names it; the forced follow-up
+    # (the UI's "delete anyway?" path) removes it.
+    db = authed_client.app.state.db
+    folder = tmp_recordings_dir / "2026-06-26"
+    folder.mkdir(exist_ok=True)
+    path = folder / "RO.MP4"
+    path.write_bytes(b"x" * 128)
+    with db.write() as c:
+        c.execute(
+            "INSERT INTO clip_index "
+            "(path, basename, group_name, timestamp, camera, sequence, "
+            " event_type, size_bytes, has_gpx, scanned_at) "
+            "VALUES (?, 'RO.MP4', '2026-06-26', 1, 'F', 1, 'ro', 128, 0, 1)",
+            (str(path),),
+        )
+    r = authed_client.post("/api/queue/delete", json={"filenames": ["RO.MP4"]})
+    body = r.json()
+    assert body["deleted"] == 0 and body["protected"] == 1
+    assert body["protected_names"] == ["RO.MP4"]
+    assert path.exists()
+    r = authed_client.post(
+        "/api/queue/delete", json={"filenames": ["RO.MP4"], "force": True})
+    body = r.json()
+    assert body["deleted"] == 1 and body["protected"] == 0
+    assert not path.exists()

tests/test_queue_lock.py

@@ -125,3 +125,26 @@ def test_lock_endpoint(authed_client):
         assert c.execute(
             "SELECT locked FROM download_queue WHERE filename='B.MP4'"
         ).fetchone()["locked"] == 1
+
+
+def test_unlock_endpoint(authed_client):
+    # /queue/unlock is the reverse of /queue/lock — before it existed a
+    # mark-read-only was permanent short of sqlite3 surgery on the DB.
+    db = authed_client.app.state.db
+    with db.write() as c:
+        _insert_queue(c, "U.MP4", "done")
+        _insert_index(c, "U.MP4")
+
+    authed_client.post("/api/queue/lock", json={"filenames": ["U.MP4"]})
+    r = authed_client.post("/api/queue/unlock", json={"filenames": ["U.MP4"]})
+    assert r.status_code == 200
+    body = r.json()
+    assert body["ok"] is True and body["updated"] == 1
+
+    with db.conn() as c:
+        assert c.execute(
+            "SELECT locked FROM clip_index WHERE basename='U.MP4'"
+        ).fetchone()["locked"] == 0
+        assert c.execute(
+            "SELECT locked FROM download_queue WHERE filename='U.MP4'"
+        ).fetchone()["locked"] == 0

tests/test_retention.py

@@ -238,7 +238,7 @@ def _patch_quota_scanner(monkeypatch, half_gib: int) -> None:
     orig_del = ret._delete_clip_files
     def del_returning(*a, **kw):
         orig_del(*a, **kw)
-        return half_gib
+        return (half_gib, True)
     monkeypatch.setattr(ret, "_delete_clip_files", del_returning)
 
 

tests/test_retention_export_guard.py

@@ -158,7 +158,7 @@ def test_make_room_for_skips_protected_clips(env, monkeypatch):
 
     def _del(*a, **kw):
         orig_del(*a, **kw)
-        return int(0.6 * gib)
+        return (int(0.6 * gib), True)
 
     monkeypatch.setattr(ret, "_delete_clip_files", _del)
     ok = ret.make_room_for(

tests/test_retention_failed_unlink.py

@@ -0,0 +1,127 @@
+"""Failed unlinks must not drop index rows, credit freed bytes, or wedge
+the retention loops.
+
+Regression tests for the Unraid permissions failure mode: the container
+user can't unlink in the recordings share, so os.remove raises
+PermissionError. Before the fix, delete_clip removed the clip_index row
+anyway (the clip vanished from the UI, no space freed, and it returned on
+the next rescan), and the disk-pressure pass credited the file's size as
+freed while looping over the same un-unlinkable candidates.
+"""
+from __future__ import annotations
+
+import collections
+import os
+from pathlib import Path
+
+import web.services.retention as ret
+
+
+def _env(tmp_path: Path):
+    from web.db import Database
+    rec = tmp_path / "rec"
+    rec.mkdir()
+    db = Database(str(rec / "v.db"))
+    return rec, db
+
+
+def _make_clip(rec: Path, db, *, basename: str, ts: int):
+    folder = rec / "2026-06-26"
+    folder.mkdir(exist_ok=True)
+    path = folder / basename
+    path.write_bytes(b"x" * 1024)
+    with db.write() as c:
+        cur = c.execute(
+            "INSERT INTO clip_index "
+            "(path, basename, group_name, timestamp, camera, sequence, "
+            " event_type, size_bytes, has_gpx, scanned_at) "
+            "VALUES (?, ?, '2026-06-26', ?, 'F', 1, 'normal', 1024, 0, 1)",
+            (str(path), basename, ts),
+        )
+        return cur.lastrowid, path
+
+
+def _deny_mp4(monkeypatch):
+    """Patch retention's os.remove so every .MP4 unlink raises the classic
+    share-permissions error; sidecars/caches still remove normally."""
+    real_remove = os.remove
+
+    def deny(p, *a, **kw):
+        if str(p).endswith(".MP4"):
+            raise PermissionError(1, "Operation not permitted", str(p))
+        return real_remove(p, *a, **kw)
+
+    monkeypatch.setattr("web.services.retention.os.remove", deny)
+
+
+def _index_count(db) -> int:
+    with db.conn() as c:
+        return c.execute("SELECT COUNT(*) AS n FROM clip_index").fetchone()["n"]
+
+
+def test_delete_clip_keeps_index_row_on_failed_unlink(tmp_path, monkeypatch):
+    rec, db = _env(tmp_path)
+    cid, path = _make_clip(rec, db, basename="A.MP4", ts=100)
+    _deny_mp4(monkeypatch)
+    freed, ok = ret.delete_clip(db, {"id": cid, "path": str(path)}, str(rec))
+    assert ok is False
+    assert freed == 0            # nothing was actually reclaimed
+    assert path.exists()
+    assert _index_count(db) == 1  # row survives → UI stays truthful
+
+
+def test_sweep_time_rule_failed_unlink_keeps_row(tmp_path, monkeypatch):
+    rec, db = _env(tmp_path)
+    _, path = _make_clip(rec, db, basename="OLD.MP4", ts=0)
+    _deny_mp4(monkeypatch)
+    summary = ret.sweep(
+        db, str(rec), max_days=1, disk_pct=0, protect_ro=True,
+        _now=86400 * 30,
+    )
+    assert summary["deleted_time"] == 0
+    assert summary["failed"] == 1
+    assert summary["bytes_freed"] == 0
+    assert path.exists()
+    assert _index_count(db) == 1
+
+
+def test_disk_pressure_pass_skips_failed_and_terminates(tmp_path, monkeypatch):
+    # Disk usage is pinned over threshold, and every unlink fails — the
+    # pass must still terminate (failed ids leave the candidate pool)
+    # with nothing deleted and nothing credited as freed.
+    rec, db = _env(tmp_path)
+    _make_clip(rec, db, basename="A.MP4", ts=100)
+    _make_clip(rec, db, basename="B.MP4", ts=200)
+
+    DU = collections.namedtuple("DU", "total used free")
+    monkeypatch.setattr(
+        "web.services.retention.shutil.disk_usage",
+        lambda p: DU(total=100, used=95, free=5),
+    )
+    _deny_mp4(monkeypatch)
+    deleted, bytes_freed, protected, failed = ret._disk_pressure_pass(
+        db, str(rec), disk_pct=80, quota_gb=0, protect_ro=True, sink=None,
+    )
+    assert deleted == 0
+    assert bytes_freed == 0
+    assert failed == 2
+    assert _index_count(db) == 2
+
+
+def test_make_room_for_failed_unlink_terminates_false(tmp_path, monkeypatch):
+    # Same wedge-guard for the import path: with every candidate
+    # un-unlinkable and the quota permanently breached, make_room_for
+    # must exhaust the pool and report False rather than spin.
+    rec, db = _env(tmp_path)
+    _make_clip(rec, db, basename="A.MP4", ts=100)
+    monkeypatch.setattr(
+        ret, "_scan_dir_bytes",
+        lambda p, exclude=frozenset(): 2 * (1 << 30),
+    )
+    _deny_mp4(monkeypatch)
+    ok = ret.make_room_for(
+        db, str(rec), size=0, before_ts=300,
+        disk_pct=0, quota_gb=1, protect_ro=True,
+    )
+    assert ok is False
+    assert _index_count(db) == 1

web/app.py

@@ -18,6 +18,7 @@
 from contextlib import asynccontextmanager, suppress
 
 from fastapi import FastAPI
+from fastapi.middleware.gzip import GZipMiddleware
 from fastapi.responses import FileResponse, Response
 from fastapi.staticfiles import StaticFiles
 
@@ -406,6 +407,10 @@ def create_app() -> FastAPI:
     app.state.log_handler = log_handler
 
     app.add_middleware(SetupModeMiddleware)
+    # Compress the SPA (app.js alone is ~200 KB) and the JSON day payloads
+    # (~80 KB for a 240-clip day, re-fetched on every re-render). ~5-8x
+    # smaller on the wire; minimum_size skips tiny responses and streams.
+    app.add_middleware(GZipMiddleware, minimum_size=1024)
 
     app.include_router(auth_router.router)
     app.include_router(archive_router.router)

web/routers/archive.py

@@ -1027,6 +1027,11 @@ async def geocode(
 
 
 def _fetch_clip(request: Request, clip_id: int) -> dict:
+    # Blocking: sqlite connect + an isfile() that can stall for hundreds
+    # of ms on a spun-down array. Async handlers must call this via
+    # asyncio.to_thread or they stall the event loop (and with it every
+    # other request and the progress WebSocket) — a day view issues ~240
+    # of these back-to-back.
     with _db(request).conn() as c:
         row = c.execute(
             "SELECT id, path, basename, size_bytes, duration_s "
@@ -1042,7 +1047,7 @@ def _fetch_clip(request: Request, clip_id: int) -> dict:
 
 @router.get("/clip/{clip_id}/thumb")
 async def clip_thumb(request: Request, clip_id: int):
-    clip = _fetch_clip(request, clip_id)
+    clip = await asyncio.to_thread(_fetch_clip, request, clip_id)
     s = _settings(request)
     path = await thumbs.ensure_thumb(
         s.recordings, clip_id, clip["path"]
@@ -1064,7 +1069,7 @@ async def clip_filmstrip(request: Request, clip_id: int):
     """Slicing metadata for the clip's filmstrip sprite (generates it
     on demand). 204 when ffmpeg is unavailable so the UI shows
     placeholder tiles."""
-    clip = _fetch_clip(request, clip_id)
+    clip = await asyncio.to_thread(_fetch_clip, request, clip_id)
     s = _settings(request)
     meta = await filmstrip.ensure_filmstrip(
         s.recordings, clip_id, clip["path"], clip.get("duration_s")
@@ -1083,7 +1088,7 @@ async def clip_filmstrip(request: Request, clip_id: int):
 
 @router.get("/clip/{clip_id}/filmstrip.jpg")
 async def clip_filmstrip_jpg(request: Request, clip_id: int):
-    clip = _fetch_clip(request, clip_id)
+    clip = await asyncio.to_thread(_fetch_clip, request, clip_id)
     s = _settings(request)
     meta = await filmstrip.ensure_filmstrip(
         s.recordings, clip_id, clip["path"], clip.get("duration_s")