"""R2: every updater git runner shares ONE custody policy (hermes_cli.update_custody). Real git, real processes, no mocks: * every runner passes ``-c gc.autoDetach=false -c maintenance.auto=false`` (git reports the effective config it was started with), so no detached gc/maintenance child is ever forked; * POSIX: the checkout lock fd reaches only local mutators — a ``git fetch``'s upload-pack (where a credential-cache daemon would hang) never holds it, a ``git stash push``'s clean filter does. """ from __future__ import annotations import os import subprocess import sys import textwrap import time from pathlib import Path import pytest from hermes_cli import update_lock as ul REPO_ROOT = Path(__file__).resolve().parents[2] def _git(cwd: Path, *args: str) -> str: return subprocess.run(["git", "-C", str(cwd), *args], check=True, capture_output=True, text=True, encoding="utf-8", env=_env(cwd)).stdout.strip() def _env(base: Path) -> dict: return dict(os.environ, GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_GLOBAL=os.devnull, GIT_AUTHOR_NAME="t", GIT_AUTHOR_EMAIL="t@e", GIT_COMMITTER_NAME="t", GIT_COMMITTER_EMAIL="t@e") @pytest.fixture def repo(tmp_path, monkeypatch): for key, value in _env(tmp_path).items(): monkeypatch.setenv(key, value) root = tmp_path / "repo" root.mkdir() _git(root, "init", "-q", "-b", "main") (root / "f.txt").write_text("one\n", encoding="utf-8") _git(root, "add", "f.txt") _git(root, "commit", "-qm", "one") return root def _runners(repo: Path): """Every updater git runner, as ``args -> stdout``.""" from hermes_cli import gitlock, update_cmd, update_cmd_check, update_cmd_git, update_cmd_stash yield "update_cmd._git_run", lambda a: update_cmd._git_run(["git"], a, cwd=repo).stdout yield "update_cmd_git._git_run", lambda a: update_cmd_git._git_run(["git"], a, cwd=repo).stdout yield "update_cmd_stash._git_quiet", lambda a: update_cmd_stash._git_quiet(["git"], a, repo, text=True).stdout yield "update_cmd_check._git", lambda a: update_cmd_check._git(["git"], repo, a).stdout yield "gitlock._git_stdout_lines", lambda a: "\n".join(gitlock._git_stdout_lines(repo, a)) def test_every_updater_git_runner_forbids_detached_children(repo): missing = {} for name, run in _runners(repo): seen = {key: run(["config", "--get", key]).strip() for key in ("gc.autoDetach", "maintenance.auto")} if seen != {"gc.autoDetach": "false", "maintenance.auto": "false"}: missing[name] = seen assert not missing, f"updater git runners that may fork a detached gc/maintenance child: {missing}" # Records whether the checkout lock file is among this process's open fds, then hands over. _RECORDER = textwrap.dedent("""\ #!/bin/sh lock="$HERMES_TEST_LOCK"; out="$HERMES_TEST_OUT" held=no for fd in /proc/$$/fd/*; do [ "$(readlink "$fd")" = "$lock" ] && held=yes done echo "$held" >> "$out" exec "$@" """) @pytest.mark.skipif(not Path("/proc/self/fd").is_dir(), reason="needs /proc fd listing") def test_lock_fd_reaches_local_mutators_only(repo, tmp_path, monkeypatch): from hermes_cli import update_cmd recorder = tmp_path / "recorder.sh" recorder.write_text(_RECORDER, encoding="utf-8") recorder.chmod(0o755) remote = tmp_path / "remote.git" subprocess.run(["git", "clone", "-q", "--bare", str(repo), str(remote)], check=True, env=_env(tmp_path)) _git(repo, "remote", "add", "origin", str(remote)) _git(repo, "config", "filter.rec.clean", f"{recorder} cat") (repo / ".git" / "info" / "attributes").write_text("f.txt filter=rec\n", encoding="utf-8") out_fetch, out_stash = tmp_path / "fetch.out", tmp_path / "stash.out" lock = ul.UpdateLock(path=tmp_path / "marker", install_root=repo) assert lock.acquire() try: monkeypatch.setenv("HERMES_TEST_LOCK", os.path.realpath(ul.checkout_lock_path(repo))) monkeypatch.setenv("HERMES_TEST_OUT", str(out_fetch)) fetched = update_cmd._git_run(["git"], ["fetch", f"--upload-pack={recorder} git-upload-pack", "origin"], cwd=repo, network=True) assert fetched.returncode == 0, fetched.stderr monkeypatch.setenv("HERMES_TEST_OUT", str(out_stash)) (repo / "f.txt").write_text("two\n", encoding="utf-8") stashed = update_cmd._git_run(["git"], ["stash", "push", "-m", "custody"], cwd=repo) assert stashed.returncode == 0, stashed.stderr finally: lock.release() assert out_fetch.read_text(encoding="utf-8-sig").split() == ["no"], "a fetch child (credential/daemon territory) held the lock" assert "yes" in out_stash.read_text(encoding="utf-8-sig").split(), "a local mutator's child lost the checkout lock" @pytest.mark.platforms("posix") @pytest.mark.live_system_guard_bypass # kills the orphaned hook child it planted def test_a_background_hook_never_keeps_a_completed_update_locked(repo, tmp_path): """F1: a local mutator holds the lock fd, and a repository hook it ran inherited that fd; a hook that backgrounds a daemon kept the checkout locked after the merge returned and the owner released, so the next update was refused. Once the owner releases, a contender acquires.""" import signal base = _git(repo, "rev-parse", "HEAD") (repo / "f.txt").write_text("two\n", encoding="utf-8") _git(repo, "commit", "-qam", "two") target = _git(repo, "rev-parse", "HEAD") _git(repo, "reset", "-q", "--hard", base) pid_file = tmp_path / "hook-child.pid" hook = repo / ".git" / "hooks" / "post-merge" hook.write_text(f"#!/bin/sh\nsleep 60 /dev/null 2>&1 &\necho $! > {pid_file}\n", encoding="utf-8") hook.chmod(0o755) from hermes_cli.update_custody import run_git lock = ul.UpdateLock(path=tmp_path / "marker", install_root=repo) assert lock.acquire() try: merged = run_git(["git"], ["merge", "--ff-only", target], cwd=repo, capture_output=True, text=True, timeout=30) assert merged.returncode == 0, merged.stderr finally: lock.release() contender = ul.UpdateLock(path=tmp_path / "next-marker", install_root=repo) try: assert (repo / "f.txt").read_text(encoding="utf-8") == "two\n" assert contender.acquire(), f"a background git hook kept the completed update's checkout locked: {contender.holder}" finally: contender.release() if pid_file.exists(): os.kill(int(pid_file.read_text(encoding="utf-8")), signal.SIGKILL) # windows-footgun: ok - POSIX-only test @pytest.mark.skipif(not sys.platform.startswith("linux"), reason="parent-death signal is Linux-only") def test_network_git_dies_with_its_killed_owner(repo, tmp_path): """The fd-less fetch must not keep rewriting refs after the owner died (the next owner's lease).""" _assert_fetch_dies_with_killed_owner(repo, tmp_path, "") @pytest.mark.skipif(not sys.platform.startswith("linux"), reason="reads the fetch's pid from /proc") def test_network_git_dies_with_its_killed_owner_without_a_parent_death_signal(repo, tmp_path): """R3: where there is no parent-death signal (macOS), the fetch must still die with its owner.""" _assert_fetch_dies_with_killed_owner( repo, tmp_path, "from hermes_cli import update_custody\nupdate_custody._death_signal_preexec = lambda: None\n") def _assert_fetch_dies_with_killed_owner(repo, tmp_path, prelude: str) -> None: script = textwrap.dedent(f""" import sys sys.path.insert(0, {str(REPO_ROOT)!r}) from pathlib import Path from hermes_cli import update_lock as ul, update_cmd """) + prelude + textwrap.dedent(""" lock = ul.UpdateLock(path=Path(sys.argv[2]), install_root=sys.argv[1]) assert lock.acquire() update_cmd._git_run(["git"], ["fetch", "--upload-pack=" + sys.argv[3], "origin"], cwd=sys.argv[1], network=True) """) remote = tmp_path / "remote.git" subprocess.run(["git", "clone", "-q", "--bare", str(repo), str(remote)], check=True, env=_env(tmp_path)) _git(repo, "remote", "add", "origin", str(remote)) started = tmp_path / "upload-pack.pid" blocker = tmp_path / "block.sh" blocker.write_text(f"#!/bin/sh\necho $$ > {started}.self\necho $PPID > {started}\nsleep 60\nexec git-upload-pack \"$@\"\n", encoding="utf-8") blocker.chmod(0o755) owner = subprocess.Popen([sys.executable, "-c", script, str(repo), str(tmp_path / "marker"), str(blocker)], env=_env(tmp_path)) try: for _ in range(300): if started.exists() and started.read_text(encoding="utf-8-sig").strip(): break time.sleep(0.05) # git runs the upload-pack command through `sh -c`: the fetch is the shell's parent. shell = int(started.read_text(encoding="utf-8-sig")) fetch_pid = next(int(line.split()[1]) for line in Path(f"/proc/{shell}/status").read_text(encoding="utf-8-sig").splitlines() if line.startswith("PPid:")) assert "git" in Path(f"/proc/{fetch_pid}/cmdline").read_text(encoding="utf-8-sig", errors="replace") owner.kill() owner.wait(timeout=10) for _ in range(100): if not Path(f"/proc/{fetch_pid}").exists() or "Z" in _state(fetch_pid): break time.sleep(0.05) assert not Path(f"/proc/{fetch_pid}").exists() or "Z" in _state(fetch_pid), \ "git fetch outlived its killed update owner" finally: if owner.poll() is None: owner.kill() blocker_pid = Path(f"{started}.self") if blocker_pid.exists(): subprocess.run(["kill", "-9", blocker_pid.read_text(encoding="utf-8-sig").strip()], capture_output=True, check=False) def _state(pid: int) -> str: try: stat = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8-sig") except OSError: return "Z" return stat[stat.rfind(")") + 2:][:1] def test_a_refused_build_join_is_logged_and_noted_in_the_receipt(tmp_path, monkeypatch, caplog): """m1: the job-joining launcher's notice used to reach only the build's captured stderr (run_contained merges it into the output it drops on success). The report it leaves is now a warning (errors.log) and a receipt step; the build's outcome is untouched.""" import json import logging import hermes_cli.update_receipt as ur from hermes_cli import update_custody home = tmp_path / ".hermes" home.mkdir() monkeypatch.setenv("HERMES_HOME", str(home)) report = tmp_path / "report.txt" note = f"{update_custody._CUSTODY_UNAVAILABLE} (could not join the update job: 5); the command was not run" report.write_text(note, encoding="utf-8") with ur.update_receipt_scope(): ur.begin_update_receipt() with caplog.at_level(logging.WARNING, logger="hermes_cli.update_custody"): assert update_custody._report_refused_join(str(report), ["node", "scripts/build/tui.mjs"]) == note payload = json.loads(ur.finalize_update_receipt("success").read_text(encoding="utf-8-sig")) assert any(note in rec.getMessage() and "tui.mjs" in rec.getMessage() for rec in caplog.records) steps = [s for s in payload["steps"] if s["name"] == "update_custody"] assert steps and steps[0]["ok"] is False and note in steps[0]["detail"], payload["steps"] assert payload["outcome"] == "success" assert not report.exists() empty = tmp_path / "joined.txt" empty.write_text("", encoding="utf-8") assert update_custody._report_refused_join(str(empty), ["node"]) is None # joined: silent def test_a_swallowed_refusal_is_what_the_update_reports(tmp_path, monkeypatch, capsys): """R8 m2: readers swallow an OSError (``_git_stdout``, ``_capture_head_sha``), so a refused child used to end ``hermes update`` as whatever error followed downstream. The refusal is receipted and kept for the run, and the command's failure path prints it, actionably.""" import contextlib from types import SimpleNamespace import hermes_cli.update_receipt as ur from hermes_cli import main, update_cmd, update_custody, update_owning_install monkeypatch.setattr(update_custody, "_RUN", {"ran": False, "refused": None}) monkeypatch.setattr(update_owning_install, "retarget_to_owning_install", lambda root: None) monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes")) (tmp_path / "checkout").mkdir() monkeypatch.setattr(main, "PROJECT_ROOT", tmp_path / "checkout") monkeypatch.setattr(main, "_update_preflight_handled", lambda args: False) monkeypatch.setattr(main, "_install_hangup_protection", lambda **kw: None) monkeypatch.setattr(main, "_finalize_update_output", lambda state: None) def impl(args, gateway_mode): ur.begin_update_receipt() with contextlib.suppress(OSError): # a reader that swallows it, as _git_stdout does raise update_custody._refuse(["git", "rev-parse"], OSError(6, "The handle is invalid")) print("✗ Could not resolve HEAD") # the downstream error it turned into raise SystemExit(1) monkeypatch.setattr(update_cmd, "_cmd_update_impl", impl) with ur.update_receipt_scope(), pytest.raises(SystemExit) as stop: main.cmd_update(SimpleNamespace(gateway=False)) assert stop.value.code == 1 out = capsys.readouterr().out assert "`hermes update` stopped: Windows would not put `git` in this update's process job" in out, out assert "Nothing was changed" in out and "Run `hermes update` again from a regular terminal" in out, out payload = ur.read_latest_receipt() steps = [s for s in payload["steps"] if s["name"] == "update_custody"] assert steps and steps[0]["ok"] is False and "rev-parse" in steps[0]["detail"], payload["steps"] @pytest.mark.skipif(not Path("/proc/self/fd").is_dir(), reason="needs /proc fd listing") def test_a_partial_clone_move_never_fetches_under_the_lock_fd(repo, tmp_path, monkeypatch): """m3: installer checkouts are partial clones, so `reset --hard ` used to fetch the target's missing objects itself — as a descendant holding the lock fd, with the user's credential helper (a `git credential-cache--daemon` then held the checkout for 900 s). The objects are fetched first without the fd; the mutator runs with no helper and fetches nothing.""" recorder = tmp_path / "recorder.sh" recorder.write_text(_RECORDER, encoding="utf-8") recorder.chmod(0o755) origin = tmp_path / "origin.git" subprocess.run(["git", "clone", "-q", "--bare", str(repo), str(origin)], check=True, env=_env(tmp_path)) _git(origin, "config", "uploadpack.allowFilter", "true") _git(origin, "config", "uploadpack.allowAnySHA1InWant", "true") clone = tmp_path / "clone" subprocess.run(["git", "clone", "-q", "--filter=blob:none", f"file://{origin}", str(clone)], check=True, env=_env(tmp_path)) (repo / "new.txt").write_text("only on the remote\n", encoding="utf-8") _git(repo, "add", "new.txt") _git(repo, "commit", "-qm", "two") _git(repo, "push", "-q", str(origin), "main") _git(clone, "fetch", "-q", "origin") _git(clone, "config", "remote.origin.uploadpack", f"{recorder} git-upload-pack") missing = [o for o in _git(clone, "rev-list", "--objects", "--missing=print", "origin/main").split() if o[0] == "?"] assert missing, "fixture: the new blob must be missing locally" out = tmp_path / "upload-pack.out" monkeypatch.setenv("HERMES_TEST_LOCK", os.path.realpath(ul.checkout_lock_path(clone))) monkeypatch.setenv("HERMES_TEST_OUT", str(out)) from hermes_cli.update_custody import run_git lock = ul.UpdateLock(path=tmp_path / "marker", install_root=clone) assert lock.acquire() try: moved = run_git(["git"], ["reset", "--hard", "origin/main"], cwd=clone, capture_output=True, text=True) assert moved.returncode == 0, moved.stderr finally: lock.release() assert (clone / "new.txt").read_text(encoding="utf-8-sig") == "only on the remote\n" served = out.read_text(encoding="utf-8-sig").split() assert served and "yes" not in served, f"an upload-pack (a lazy fetch) ran holding the checkout lock: {served}" assert "credential.helper=" in moved.args, moved.args @pytest.mark.skipif(not Path("/proc/self/fd").is_dir(), reason="needs /proc fd listing") @pytest.mark.parametrize(("args", "holds"), [(["gc", "--auto"], True), (["rev-parse", "HEAD"], False)]) def test_run_git_hands_the_lock_fd_to_local_mutators_only(repo, tmp_path, monkeypatch, args, holds): """Through the production runner: `gc` (packs refs, repacks) is a local mutator and runs holding the checkout lock; a reader does not. `pull` is never one (its network half).""" from hermes_cli.update_custody import LOCAL_MUTATORS, run_git assert "pull" not in LOCAL_MUTATORS git = tmp_path / "git-recorder.sh" git.write_text(_RECORDER.replace('exec "$@"', 'exec git "$@"'), encoding="utf-8") git.chmod(0o755) out = tmp_path / "git.out" monkeypatch.setenv("HERMES_TEST_LOCK", os.path.realpath(ul.checkout_lock_path(repo))) monkeypatch.setenv("HERMES_TEST_OUT", str(out)) lock = ul.UpdateLock(path=tmp_path / "marker", install_root=repo) assert lock.acquire() try: ran = run_git([str(git)], args, cwd=repo, capture_output=True, text=True) finally: lock.release() assert ran.returncode == 0, ran.stderr assert out.read_text(encoding="utf-8-sig").split() == (["yes"] if holds else ["no"]), args class _Win32Sys: """``sys`` as one module sees it on Windows; everything but ``platform`` is the real module.""" platform = "win32" def __getattr__(self, name): return getattr(sys, name) @pytest.mark.platforms("posix") def test_a_timed_out_child_never_waits_on_a_grandchild_holding_its_pipes(monkeypatch): """N14: the in-update (Windows) run() killed only the direct child on a timeout, then waited unbounded for the pipes a grandchild still held (git-remote-https under a stalled fetch). Driven here through that branch with a POSIX grandchild the tree kill (taskkill) cannot reach.""" from hermes_cli import update_custody # Only update_custody sees win32: patching sys.platform itself would send every lazy import # in this window (the suite's Popen guard imports gateway.status) down its Windows branch. monkeypatch.setattr(update_custody, "sys", _Win32Sys()) monkeypatch.setattr(update_custody, "_held", lambda: {"fd": None}) monkeypatch.setattr(update_custody, "popen", lambda argv, **kw: subprocess.Popen(list(argv), **kw)) started = time.monotonic() with pytest.raises(subprocess.TimeoutExpired): update_custody.run(["sh", "-c", "sleep 20 & sleep 20"], capture_output=True, timeout=0.5) assert time.monotonic() - started < 15, "run() waited on a grandchild after the timeout" def _held_lock_recorder(tmp_path: Path, name: str, body: str) -> Path: """An executable on a fresh PATH dir that records whether it holds the checkout lock fd.""" bin_dir = tmp_path / "bin" bin_dir.mkdir(exist_ok=True) exe = bin_dir / name exe.write_text(_RECORDER.replace('exec "$@"', body), encoding="utf-8") exe.chmod(0o755) return exe @pytest.mark.skipif(not Path("/proc/self/fd").is_dir(), reason="needs /proc fd listing") def test_no_build_descendant_outlives_the_build_call(repo, tmp_path, monkeypatch): """N13: node marks the donated lock fd close-on-exec, so npm/esbuild under it never hold the checkout; custody rested on node alone. A node that dies leaving a writer behind must not leave it running once the build call returns: the next lock owner would share the checkout.""" from hermes_cli.source_build import run_source_script late = tmp_path / "late" writer = f"import os, time; os.closerange(3, 4096); time.sleep(2); __import__('pathlib').Path({str(late)!r}).touch()" node = _held_lock_recorder(tmp_path, "node", f"{sys.executable} -c \"{writer}\" >/dev/null 2>&1 &\nexit 1") monkeypatch.setenv("HERMES_TEST_LOCK", os.path.realpath(ul.checkout_lock_path(repo))) monkeypatch.setenv("HERMES_TEST_OUT", str(tmp_path / "node.out")) lock = ul.UpdateLock(path=tmp_path / "marker", install_root=repo) assert lock.acquire() try: with pytest.raises(subprocess.CalledProcessError): run_source_script(repo, "build.mjs", env={**os.environ, "PATH": f"{node.parent}{os.pathsep}{os.environ['PATH']}"}, label="probe") finally: lock.release() time.sleep(3) assert not late.exists(), "a build descendant kept writing after the build call returned" def _late_writer(late: Path, *, detach: bool = False, delay: float = 2.0) -> str: """Python source for a build descendant that writes ``late`` after ``delay`` seconds, holding no inherited fd and no pipe (so nothing but a kill stops it); ``detach`` starts its own session.""" detach_code = "os.setsid(); " if detach else "" # windows-footgun: ok - the callers are POSIX-only return (f"import os, pathlib, time; {detach_code}os.closerange(3, 4096); " f"time.sleep({delay}); pathlib.Path({str(late)!r}).touch()") def _heartbeat_writer(beat: Path) -> str: """Python source for a detached build writer (own session, no inherited fd) that rewrites ``beat`` with ``" "`` every 0.1 s. Each beat lands via a temp file + ``os.replace``: custody SIGKILLs this writer at an arbitrary instant, and an in-place ``write_text`` killed between its truncating open and its write leaves ``beat`` empty, so the test could no longer read the pid back out of it.""" return (f"import os, pathlib, time; os.setsid(); os.closerange(3, 4096); p = pathlib.Path({str(beat)!r})\n" # windows-footgun: ok - Linux-only callers "t = p.with_name(p.name + '.tmp')\n" "for i in range(300):\n t.write_text(f'{os.getpid()} {i}'); os.replace(t, p); time.sleep(0.1)") def _node_starting(writer: str, *, then: str) -> list[str]: """A stand-in for node: starts ``writer`` (stdio detached), then runs ``then``.""" return [sys.executable, "-c", "import subprocess, sys, time; d = subprocess.DEVNULL; " f"subprocess.Popen([sys.executable, '-c', {writer!r}], stdin=d, stdout=d, stderr=d); {then}"] def _launch_in_custody(repo: Path, tmp_path: Path, command: list[str], *, platform: str | None = None, **popen) -> subprocess.Popen: """Start ``command`` the way the build runner does (contained_command under a held checkout lock); ``platform`` makes the launcher see another ``sys.platform`` (its no-subreaper path).""" from hermes_cli.update_custody import contained_command lock = ul.UpdateLock(path=tmp_path / "marker", install_root=repo) assert lock.acquire() try: with contained_command(command, root=repo) as (argv, custody): assert argv[3] == "-c", argv if platform is not None: argv[4] = f"import sys; sys.platform = {platform!r}\n" + argv[4] return subprocess.Popen(argv, stdin=subprocess.DEVNULL, **custody, **popen) finally: lock.release() @pytest.mark.skipif(not sys.platform.startswith("linux"), reason="POSIX process groups + /proc") def test_a_group_kill_of_the_caller_stops_the_whole_build(repo, tmp_path): """L1: the completion child's Ctrl-C path ``killpg``s its own group and Desktop kills the backend's group; the build (node and what it starts) must be inside that group. A build moved into a session of its own outlived the kill and kept writing after the lock was free.""" import signal late_node, late_child = tmp_path / "late-node", tmp_path / "late-child" node = _node_starting(_late_writer(late_child), then=f"time.sleep(2); __import__('pathlib').Path({str(late_node)!r}).touch()") # The launcher leads a fresh group here, as the completion child does for its build. launcher = _launch_in_custody(repo, tmp_path, node, start_new_session=True) time.sleep(1) os.killpg(launcher.pid, signal.SIGKILL) # windows-footgun: ok - Linux-only test launcher.wait(timeout=10) time.sleep(3) assert not late_node.exists(), "node survived a group kill of its caller" assert not late_child.exists(), "a build descendant survived a group kill of its caller" @pytest.mark.skipif(not sys.platform.startswith("linux"), reason="POSIX sessions + /proc") @pytest.mark.live_system_guard_bypass # the caller's group kill is the scenario under test def test_a_group_kill_of_the_caller_keeps_custody_until_a_detached_writer_is_gone(repo, tmp_path): """E: the completion child ``killpg``s its own group on interruption; the launcher holding the lock fd for node's tree was in that group and died with it, so a descendant in a session of its own kept writing while a contender owned the checkout. A contender must only acquire once that writer has stopped.""" import contextlib import signal beat = tmp_path / "beat" writer = _heartbeat_writer(beat) caller = textwrap.dedent(f""" import subprocess, sys sys.path.insert(0, {str(REPO_ROOT)!r}) from pathlib import Path from hermes_cli import update_lock as ul from hermes_cli.update_custody import contained_command repo = Path({str(repo)!r}) lock = ul.UpdateLock(path=Path({str(tmp_path / "marker")!r}), install_root=repo) assert lock.acquire() with contained_command({_node_starting(writer, then="time.sleep(60)")!r}, root=repo) as (argv, custody): subprocess.Popen(argv, stdin=subprocess.DEVNULL, **custody).wait() """) owner = subprocess.Popen([sys.executable, "-c", caller], start_new_session=True) deadline = time.monotonic() + 20 while not beat.exists() and time.monotonic() < deadline: time.sleep(0.1) assert beat.exists(), "the detached writer never started" os.killpg(owner.pid, signal.SIGKILL) # windows-footgun: ok - Linux-only test owner.wait(timeout=10) contender = ul.UpdateLock(path=tmp_path / "next-marker", install_root=repo) deadline = time.monotonic() + 20 while not contender.acquire(): assert time.monotonic() < deadline, "the checkout stayed locked after the group kill" time.sleep(0.1) try: seen = beat.read_text(encoding="utf-8") time.sleep(1) assert beat.read_text(encoding="utf-8") == seen, "a detached build writer kept writing under the next owner" finally: contender.release() with contextlib.suppress(ProcessLookupError): os.kill(int(beat.read_text(encoding="utf-8").split()[0]), signal.SIGKILL) # windows-footgun: ok - Linux-only test @pytest.mark.skipif(not sys.platform.startswith("linux"), reason="POSIX sessions + /proc") @pytest.mark.live_system_guard_bypass # Ctrl-C of the caller's own group is the scenario under test @pytest.mark.parametrize(("verbose", "cleanup"), [("1", 2), ("0", 0)], ids=["verbose-slow-cleanup", "captured-instant-exit"]) def test_a_ctrl_c_keeps_custody_until_a_detached_writer_is_gone(repo, tmp_path, verbose, cleanup): """R4: on Ctrl-C the build runner's ``subprocess.run`` SIGKILLs its own child, which was the launcher holding custody; a command still cleaning up from the interrupt then exited and dropped the last lock fd while a writer in a session of its own went on writing under the next owner. A contender must only acquire once that writer has stopped.""" import contextlib import signal beat, ready = tmp_path / "beat", tmp_path / "ready" writer = _heartbeat_writer(beat) # A build that cleans up for `cleanup` s after SIGINT (2 s: longer than subprocess.run's interrupt wait). command = [sys.executable, "-c", textwrap.dedent(f""" import pathlib, signal, subprocess, sys, time signal.signal(signal.SIGINT, lambda *_: (time.sleep({cleanup}), sys.exit(130))) subprocess.Popen([sys.executable, '-c', {writer!r}], stdin=subprocess.DEVNULL) pathlib.Path({str(ready)!r}).touch() while True: time.sleep(0.1) """)] caller = textwrap.dedent(f""" import os, sys sys.path.insert(0, {str(REPO_ROOT)!r}) os.environ["HERMES_VERBOSE"] = {verbose!r} from pathlib import Path from hermes_cli import update_lock as ul from hermes_cli.source_build import run_in_custody repo = Path({str(repo)!r}) assert ul.UpdateLock(path=Path({str(tmp_path / "marker")!r}), install_root=repo).acquire() run_in_custody(repo, {command!r}, "probe build", stdin=__import__("subprocess").DEVNULL) """) owner = subprocess.Popen([sys.executable, "-c", caller], start_new_session=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) deadline = time.monotonic() + 20 while not (beat.exists() and ready.exists()) and time.monotonic() < deadline: time.sleep(0.1) assert beat.exists() and ready.exists(), "the build never started" os.killpg(owner.pid, signal.SIGINT) # windows-footgun: ok - Linux-only test owner.wait(timeout=10) contender = ul.UpdateLock(path=tmp_path / "next-marker", install_root=repo) deadline = time.monotonic() + 30 while not contender.acquire(): assert time.monotonic() < deadline, "the checkout stayed locked after the interrupted build settled" time.sleep(0.1) try: seen = beat.read_text(encoding="utf-8") time.sleep(1) assert beat.read_text(encoding="utf-8") == seen, "a detached build writer kept writing under the next owner" finally: contender.release() with contextlib.suppress(ProcessLookupError): os.kill(int(beat.read_text(encoding="utf-8").split()[0]), signal.SIGKILL) # windows-footgun: ok - Linux-only test @pytest.mark.skipif(not sys.platform.startswith("linux"), reason="POSIX sessions + /proc") @pytest.mark.parametrize("platform", [None, "darwin"], ids=["subreaper", "ps-walk"]) def test_a_detached_grandchild_outliving_node_dies_before_the_build_returns(repo, tmp_path, platform): """N13/L1: a descendant that left node's process group (its own session) and outlives node is still killed before the build call returns: Linux through the subreaper, elsewhere through the recorded descendant tree.""" late = tmp_path / "late" node = _node_starting(_late_writer(late, detach=True), then="time.sleep(1.2)") launcher = _launch_in_custody(repo, tmp_path, node, platform=platform) assert launcher.wait(timeout=30) == 0 time.sleep(3) assert not late.exists(), "a detached build descendant kept writing after the build call returned" @pytest.mark.skipif(not sys.platform.startswith("linux"), reason="POSIX signals + /proc") def test_a_terminated_launcher_takes_node_and_its_descendants_with_it(repo, tmp_path): """L1: SIGTERM to the launcher (a service manager, the caller's polite stop) is forwarded to node, and what node started is killed too, before the launcher exits.""" import signal late = tmp_path / "late" node = _node_starting(_late_writer(late), then="time.sleep(30)") launcher = _launch_in_custody(repo, tmp_path, node, start_new_session=True) time.sleep(1) launcher.send_signal(signal.SIGTERM) assert launcher.wait(timeout=20) == 128 + signal.SIGTERM time.sleep(3) assert not late.exists(), "a build descendant kept writing after its launcher was terminated" @pytest.mark.skipif(not Path("/proc/self/fd").is_dir(), reason="needs /proc fd listing") def test_the_desktop_build_runs_in_checkout_custody(repo, tmp_path, monkeypatch): """F04: the desktop app build (npm run build / builder) writes the checkout like the other product builds, so it holds the checkout lock while it runs.""" from hermes_cli import main_desktop npm = _held_lock_recorder(tmp_path, "npm", "exit 0") out = tmp_path / "npm.out" monkeypatch.setenv("HERMES_TEST_LOCK", os.path.realpath(ul.checkout_lock_path(repo))) monkeypatch.setenv("HERMES_TEST_OUT", str(out)) desktop = repo / "apps" / "desktop" desktop.mkdir(parents=True) lock = ul.UpdateLock(path=tmp_path / "marker", install_root=repo) assert lock.acquire() try: main_desktop.build_prepared_desktop(desktop, source_mode=True, npm=str(npm), env=dict(os.environ)) finally: lock.release() assert out.read_text(encoding="utf-8-sig").split() == ["yes"], "the desktop build ran without the checkout lock" # --- C3: the Windows build launcher returns only once the command's whole tree is gone --------- class _FakeWinCall: def __init__(self, name, events, answer): self.name, self.events, self.answer = name, events, answer self.argtypes = self.restype = None def __call__(self, *args): import ctypes self.events.append((self.name, *(a.value if isinstance(a, ctypes.c_void_p) else a for a in args))) return self.answer(*args) if callable(self.answer) else self.answer def _run_join_launcher(monkeypatch, *, stray_polls: int, escaped: bool = False, rebind: int = 1): """Execute the real ``_JOIN_JOB`` launcher source against a recording kernel32/ntdll and a fake leader process. The command's job reports ``stray_polls`` live processes (a build grandchild still writing) before its tree is empty. ``escaped``: the suspended command starts outside the update job (Store Python's breakaway), and assigning it there answers ``rebind``. Returns the ordered events.""" import ctypes import types from hermes_cli.update_custody import _JOIN_JOB events = [] live = {"left": stray_polls} def query(job, info_class, usage, size, _ret): usage._obj.active = 1 if live["left"] > 0 else 0 live["left"] -= 1 return 1 placed = {"update job": not escaped} def assign(job, proc): if tuple(getattr(a, "value", a) for a in (job, proc)) != (11, 33): return 1 placed["update job"] = bool(rebind) return rebind def in_job(proc, job, inside): inside._obj.value = int(placed["update job"]) return 1 kernel32 = {"AssignProcessToJobObject": assign, "GetCurrentProcess": 7, "CloseHandle": 1, "CreateJobObjectW": 22, "SetInformationJobObject": 1, "TerminateJobObject": 1, "QueryInformationJobObject": query, "IsProcessInJob": in_job} dlls = {"kernel32": types.SimpleNamespace(**{n: _FakeWinCall(n, events, a) for n, a in kernel32.items()}), "ntdll": types.SimpleNamespace(NtResumeProcess=_FakeWinCall("NtResumeProcess", events, 0))} # The real ctypes (its Structure, byref, c_* types), with Windows' DLL loader recorded. monkeypatch.setattr(ctypes, "WinDLL", lambda name, **_kw: dlls[name], raising=False) monkeypatch.setattr(ctypes, "get_last_error", lambda: 0, raising=False) class _Leader: _handle = 33 def __init__(self, argv, **kwargs): events.append(("Popen", tuple(argv), kwargs.get("creationflags"))) def wait(self): events.append(("leader exited",)) return 3 def kill(self): events.append(("kill",)) monkeypatch.setattr(subprocess, "Popen", _Leader) monkeypatch.setattr(sys, "argv", ["launcher", "11", "report.txt", "npm", "run", "build"]) with pytest.raises(SystemExit) as exited: exec(compile(_JOIN_JOB, "", "exec"), {"__name__": "__main__"}) events.append(("exit", exited.value.code)) return events def _index(events, name, *args): return next(i for i, event in enumerate(events) if event[0] == name and event[1:1 + len(args)] == args) def test_the_windows_build_launcher_reaps_the_command_tree_before_it_returns(monkeypatch): """C3: npm (the leader) exits while a build/builder descendant it started still writes. A normal return of the launcher lets the updater release the checkout, so the launcher must terminate the command's own job (never the update job, which holds the update's other children) and return only once no process of it is left, with the leader's exit status.""" events = _run_join_launcher(monkeypatch, stray_polls=2) tree = 22 assert _index(events, "AssignProcessToJobObject", tree, 33) < _index(events, "NtResumeProcess"), \ "the command ran before it was held in its own job" assert _index(events, "leader exited") < _index(events, "TerminateJobObject", tree), events assert not any(e[0] == "TerminateJobObject" and e[1] != tree for e in events), "terminated the update job" polls = [i for i, e in enumerate(events) if e[0] == "QueryInformationJobObject"] assert len(polls) == 3 and polls[0] > _index(events, "TerminateJobObject", tree), \ "the launcher returned while a build descendant was still alive" assert events[-1] == ("exit", 3), events def test_a_build_command_that_starts_outside_the_update_job_is_put_in_it_before_it_runs(monkeypatch, tmp_path): """F80: under Store Python the suspended node starts outside the update job (desktop-app breakaway through a job that permits breakaway); refusing it made every Node build of the update fail. It is assigned to the update job while suspended and runs once it is in.""" monkeypatch.chdir(tmp_path) # a refusal writes its report file (argv[2]) relative to here events = _run_join_launcher(monkeypatch, stray_polls=0, escaped=True) assert _index(events, "AssignProcessToJobObject", 11, 33) < _index(events, "NtResumeProcess"), \ "the command ran before it was in the update job" assert events[-1] == ("exit", 3), events def test_a_build_command_the_update_job_will_not_take_never_runs(monkeypatch, tmp_path): """F54 kept: a suspended command outside the job that Windows also refuses to assign to it is killed before its first instruction and the launcher refuses.""" from hermes_cli.update_custody import _REFUSED_EXIT monkeypatch.chdir(tmp_path) events = _run_join_launcher(monkeypatch, stray_polls=0, escaped=True, rebind=0) assert not any(e[0] == "NtResumeProcess" for e in events) and ("kill",) in events, events assert events[-1] == ("exit", _REFUSED_EXIT), events def test_the_windows_build_launcher_returns_at_once_when_nothing_was_left(monkeypatch): """C3 healthy control: a build whose descendants all exited with the leader costs one poll.""" events = _run_join_launcher(monkeypatch, stray_polls=0) assert [e[0] for e in events if e[0] == "QueryInformationJobObject"] == ["QueryInformationJobObject"] assert events[-1] == ("exit", 3), events