Fix slow, fragile OpenClaw upgrade rollback in installer

deployer/assets/installer_template.html

@@ -187,6 +187,15 @@
         border-radius: 99px;
         transition: width 0.18s linear;
       }
+      .install-progress-status {
+        margin: 0 0 12px;
+        max-width: 260px;
+        font-size: 12px;
+        line-height: 1.4;
+        color: var(--text-secondary, #888);
+        text-align: center;
+        min-height: 16px;
+      }
 
       /* ---------- setup ---------- */
       .setup-wrap {
@@ -391,6 +400,7 @@ <h1 id="title" class="welcome-title"></h1>
           <div id="progressWrap" class="install-progress-wrap hidden">
             <div id="progressBar" class="install-progress-bar"></div>
           </div>
+          <p id="progressStatus" class="install-progress-status hidden"></p>
           <button id="quick" class="btn-primary"></button>
           <button id="custom" class="btn-ghost"></button>
         </div>
@@ -459,6 +469,7 @@ <h2 id="setupTitle" class="setup-title"></h2>
         desc: $("desc"),
         progressWrap: $("progressWrap"),
         progressBar: $("progressBar"),
+        progressStatus: $("progressStatus"),
         quick: $("quick"),
         custom: $("custom"),
         close: $("close"),
@@ -499,7 +510,11 @@ <h2 id="setupTitle" class="setup-title"></h2>
         els.custom.classList.toggle("hidden", on);
         els.desc.classList.toggle("hidden", !on);
         els.progressWrap.classList.toggle("hidden", !on);
-        if (!on) els.progressBar.style.width = "0%";
+        if (!on) {
+          els.progressBar.style.width = "0%";
+          els.progressStatus.classList.add("hidden");
+          els.progressStatus.textContent = "";
+        }
       }
 
       function showError(msg) {
@@ -638,11 +653,17 @@ <h2 id="setupTitle" class="setup-title"></h2>
               startTextCycle();
             }
             els.progressBar.style.width = snap.progress + "%";
+            const step = snap.progress_text || "";
+            const detail = snap.progress_detail || "";
+            const statusText = detail ? step + " — " + detail : step;
+            els.progressStatus.textContent = statusText;
+            els.progressStatus.classList.toggle("hidden", !statusText);
           }
           if (snap.status === "success") {
             stopImageCycle();
             stopTextCycle();
             els.progressBar.style.width = "100%";
+            els.progressStatus.classList.add("hidden");
             if (!state.closeTimer) {
               state.closeTimer = setTimeout(() => window.pywebview.api.close_window(), 2000);
             }

deployer/openclaw_upgrade.py

@@ -9,7 +9,8 @@
 import shutil
 import stat
 import uuid
-from collections.abc import Iterable
+from collections.abc import Callable, Iterable
+from concurrent.futures import ThreadPoolExecutor
 from dataclasses import asdict, dataclass, field
 from datetime import UTC, datetime
 from enum import StrEnum
@@ -110,8 +111,23 @@ def _flush_and_fsync(file: Any) -> None:
 
 
 def _fsync_file(path: Path) -> None:
-    with path.open("rb+") as file:
-        os.fsync(file.fileno())
+    try:
+        with path.open("rb+") as file:
+            os.fsync(file.fileno())
+    except PermissionError:
+        # On Windows a file carrying the read-only attribute cannot be opened
+        # for writing, which os.fsync (FlushFileBuffers) requires. npm packages
+        # frequently ship read-only files, so temporarily clear the attribute so
+        # the durability flush still happens, then restore the original mode.
+        if os.name != "nt":
+            raise
+        original_mode = path.stat().st_mode
+        os.chmod(path, stat.S_IWRITE)
+        try:
+            with path.open("rb+") as file:
+                os.fsync(file.fileno())
+        finally:
+            os.chmod(path, original_mode)
 
 
 def _directory_fsync_is_unsupported(error: OSError) -> bool:
@@ -276,15 +292,61 @@ def _durable_remove(path: Path) -> None:
     _fsync_directory(path.parent)
 
 
-def _fsync_payload_tree(root: Path) -> None:
+def _make_writable_and_retry(func: Any, target: Any, _exc: Any) -> None:
+    """rmtree onexc handler: clear the read-only attribute, then retry once.
+
+    npm packages ship many read-only files that otherwise make ``rmtree`` fail
+    on Windows with ``WinError 5`` (access denied).
+    """
+    try:
+        os.chmod(target, stat.S_IWRITE)
+        func(target)
+    except OSError:
+        pass
+
+
+def _force_remove_tree(path: Path) -> None:
+    """Best-effort recursive removal that tolerates read-only files."""
+    if not path.exists() and not path.is_symlink():
+        return
+    if path.is_dir() and not path.is_symlink():
+        shutil.rmtree(path, onexc=_make_writable_and_retry)
+        return
+    try:
+        os.chmod(path, stat.S_IWRITE)
+    except OSError:
+        pass
+    path.unlink(missing_ok=True)
+
+
+_FSYNC_MAX_WORKERS = 16
+
+
+def _fsync_payload_tree(
+    root: Path,
+    on_progress: Callable[[int, int], None] | None = None,
+) -> None:
     if not root.exists():
         return
+    files: list[Path] = []
     directories = [root]
     for path in root.rglob("*"):
         if path.is_file():
-            _fsync_file(path)
+            files.append(path)
         elif path.is_dir():
             directories.append(path)
+    total = len(files)
+    if total:
+        # fsync is a blocking C call that releases the GIL, so a thread pool
+        # turns the per-file durability flush from a serial ~90 files/sec crawl
+        # (22+ min for a 36k-file npm package) into a concurrent I/O batch.
+        done = 0
+        report_every = max(1, total // 100)
+        with ThreadPoolExecutor(max_workers=_FSYNC_MAX_WORKERS) as pool:
+            for _ in pool.map(_fsync_file, files):
+                done += 1
+                if on_progress is not None and (done % report_every == 0 or done == total):
+                    on_progress(done, total)
     for directory in sorted(directories, key=lambda path: len(path.parts), reverse=True):
         _fsync_directory(directory)
 
@@ -437,6 +499,10 @@ def __init__(
         self.microclaw_root = microclaw_root.resolve(strict=False)
         self.manifest = manifest
         self._held_lock = held_lock
+        # Optional UI hook: called with a human-readable status string during
+        # long per-file operations (backup / restore fsync) so the installer
+        # can show progress instead of appearing frozen.
+        self.progress_callback: Callable[[str], None] | None = None
         roots = trusted_openclaw_prefixes() if trusted_prefixes is None else trusted_prefixes
         self._trusted_prefixes = {_require_absolute(Path(path), "trusted prefix") for path in roots}
         self._validate_manifest()
@@ -666,6 +732,17 @@ def _shim_backup_path(self, shim: Path, backup_dir: Path | None = None) -> Path:
         )
         return (self.backup_dir if backup_dir is None else backup_dir) / "shims" / relative
 
+    def _payload_progress(self, action: str) -> Callable[[int, int], None] | None:
+        callback = self.progress_callback
+        if callback is None:
+            return None
+
+        def report(done: int, total: int) -> None:
+            if total:
+                callback(f"{action} ({done:,}/{total:,} files)")
+
+        return report
+
     def backup(self) -> None:
         self.set_phase(UpgradePhase.BACKING_UP)
         package_dir = Path(self.manifest.package_dir)
@@ -708,7 +785,7 @@ def backup(self) -> None:
             staging / "shims",
             staging / "state",
         ):
-            _fsync_payload_tree(payload_root)
+            _fsync_payload_tree(payload_root, self._payload_progress("Backing up OpenClaw files"))
 
         metadata_files = {
             staging / "transaction.json",
@@ -774,7 +851,7 @@ def _restore_tree(self, backup: Path, live: Path, failed: Path, existed: bool) -
             _durable_mkdir(live.parent)
             staging = live.with_name(f".{live.name}.{uuid.uuid4().hex}.restore")
             shutil.copytree(backup, staging)
-            _fsync_payload_tree(staging)
+            _fsync_payload_tree(staging, self._payload_progress("Restoring OpenClaw files"))
         self._move_to_failed(live, failed)
         if staging is not None:
             _durable_rename(staging, live)
@@ -859,6 +936,44 @@ def mark_rollback_failed(self) -> None:
             if held_lock is not None and not held_lock.released:
                 _RETAINED_FAILED_LOCKS[held_lock.owner_token] = held_lock
 
+    def discard(self) -> None:
+        """Abandon a transaction whose rollback could not complete.
+
+        Removes the on-disk manifest, lock file, and backup tree, and releases
+        the retained lock so future installs are not permanently blocked by a
+        ``rollback-failed`` transaction (which stays in ``RECOVERABLE_PHASES``
+        and re-acquires the lock on every ``load``). Because restores stage a
+        copy and only swap the live tree with a final atomic rename, a failure
+        during staging/fsync leaves the live installation intact; callers should
+        reinstall OpenClaw to repair any partial state. This never raises.
+        """
+        held_lock = self._held_lock
+        if held_lock is not None:
+            if _RETAINED_FAILED_LOCKS.get(held_lock.owner_token) is held_lock:
+                del _RETAINED_FAILED_LOCKS[held_lock.owner_token]
+            # Release (close) the lock handle before deleting the lock file so
+            # Windows does not refuse the unlink for an open file.
+            held_lock.release(held_lock.owner_token)
+            self._held_lock = None
+
+        backup_root = self.backup_root
+        for target in (self.manifest_path, self.lock_path, self.backup_dir):
+            try:
+                _force_remove_tree(target)
+            except OSError:
+                pass
+        # Sweep any orphaned staging/quarantine dirs left in the backup root.
+        try:
+            if backup_root.exists():
+                for child in backup_root.iterdir():
+                    if child.name.startswith("."):
+                        try:
+                            _force_remove_tree(child)
+                        except OSError:
+                            pass
+        except OSError:
+            pass
+
 
 def process_is_alive(pid: int) -> bool:
     if not isinstance(pid, int) or isinstance(pid, bool) or pid <= 0:

deployer/webview_bridge.py

@@ -207,6 +207,7 @@ def __init__(self, logger=None):
             "allow_read": self._default_allow_read,
             "progress": 0,
             "progress_text": "",
+            "progress_detail": "",
             "status": "idle",
             "running": False,
             "error": "",
@@ -329,11 +330,23 @@ def _set_progress(self, pct, text):
         with self._state_lock:
             self._state["progress"] = pct
             self._state["progress_text"] = text
+            # A new step supersedes any sub-status detail from the previous one.
+            self._state["progress_detail"] = ""
             self._state["status"] = "running"
             self._state["running"] = True
 
+    def _set_progress_detail(self, detail):
+        """Sub-status line fed by long-running file operations (backup/restore).
+
+        Keeps the installer from looking frozen during the multi-minute
+        per-file fsync of a large OpenClaw package.
+        """
+        with self._state_lock:
+            self._state["progress_detail"] = detail or ""
+
     def _install_thread(self):
         ws = WindowsSetup(self._config, self._logger)
+        ws.progress_callback = self._set_progress_detail
 
         steps = [
             (

deployer/windows_setup.py

@@ -133,6 +133,9 @@ def __init__(self, config, logger: DeployerLogger):
         self._git_bin: str | None = None  # path to git bin directory
         self._rollback_actions: list[tuple[str, Callable]] = []
         self._openclaw_transaction: OpenClawUpgradeTransaction | None = None
+        # Optional UI hook forwarded to upgrade transactions so long backup /
+        # restore file operations can report progress instead of looking frozen.
+        self.progress_callback: Callable[[str], None] | None = None
         self.appcontainer_enabled = True  # AppContainer sandbox (built-in)
         self.weixin_plugin_enabled = True  # Install by default
 
@@ -1133,7 +1136,28 @@ def _gateway_is_stopped_for_upgrade(self) -> bool:
             return False
         return True
 
+    def _discard_failed_transaction(
+        self, transaction: OpenClawUpgradeTransaction, reason: str
+    ) -> None:
+        """Abandon a transaction whose rollback failed so future installs work.
+
+        Leaving it in ``rollback-failed`` keeps the manifest recoverable and
+        retains the upgrade lock, which permanently blocks every subsequent
+        install with ``UpgradeInProgressError``. Discarding clears that state;
+        the live installation is left as-is and later steps reinstall OpenClaw.
+        """
+        self.log.warn(
+            f"OpenClaw upgrade rollback could not complete ({reason}); discarding "
+            "transaction state so future installs are not blocked. The existing "
+            "OpenClaw installation was left in place."
+        )
+        try:
+            transaction.discard()
+        except Exception as discard_error:
+            self.log.error(f"Failed to discard OpenClaw upgrade transaction: {discard_error}")
+
     def _rollback_openclaw_transaction(self, transaction: OpenClawUpgradeTransaction) -> bool:
+        transaction.progress_callback = self.progress_callback
         process: subprocess.Popen | None = None
         try:
             original_phase = transaction.manifest.phase
@@ -1150,19 +1174,17 @@ def _rollback_openclaw_transaction(self, transaction: OpenClawUpgradeTransaction
                     self.log.error(
                         "Previous OpenClaw Gateway did not become healthy after rollback"
                     )
-                    transaction.mark_rollback_failed()
+                    self._discard_failed_transaction(
+                        transaction, "restored gateway did not become healthy"
+                    )
                     return False
             transaction.complete_rollback()
             return True
         except Exception as error:
-            if transaction.manifest.phase == UpgradePhase.ROLLING_BACK:
-                try:
-                    transaction.mark_rollback_failed()
-                except Exception as persist_error:
-                    error.add_note(f"also failed to persist rollback-failed: {persist_error}")
             self.log.error(
                 f"Failed to restore OpenClaw backup at {transaction.backup_dir}: {error}"
             )
+            self._discard_failed_transaction(transaction, str(error) or error.__class__.__name__)
             return False
         finally:
             self._stop_validation_gateway(process)
@@ -1213,6 +1235,7 @@ def prepare_openclaw_upgrade(self) -> bool:
                 installation=source,
             )
             self._openclaw_transaction = transaction
+            transaction.progress_callback = self.progress_callback
             if not self._gateway_is_stopped_for_upgrade():
                 transaction.close()
                 self._openclaw_transaction = None