idle-draft: process-group child termination, unique tmp paths, stale sweep

Lock-contention test added; children die with the dispatcher (SIGTERM/
SIGINT handlers + atexit); tmp files are per-invocation (.tmp.<pid>) so
a SIGKILL-orphaned child can never collide with a new instance.

Claude-Session: https://claude.ai/code/session_01YQDoWNM7XPPii28khFWoMc
This commit is contained in:
Paul O'Reilly
2026-08-02 21:52:17 +12:00
parent 75add58246
commit 1f008eebcb
3 changed files with 363 additions and 13 deletions

View File

@@ -7,15 +7,19 @@ Design contract: ~/dev/claude/writing/oreillyconsulting/IDLE-DRAFT-PLAN.md
from __future__ import annotations from __future__ import annotations
import atexit
import concurrent.futures import concurrent.futures
import fcntl import fcntl
import hashlib import hashlib
import json import json
import os import os
import re import re
import signal
import subprocess import subprocess
import sys import sys
import tempfile import tempfile
import threading
import time
from datetime import datetime, timezone from datetime import datetime, timezone
from pathlib import Path from pathlib import Path
from string import Template from string import Template
@@ -828,7 +832,7 @@ def dispatch_preview(repo: Path, config: dict, candidate: dict, max_turns: int)
allowed_tools = resolve_allowed_tools(config, work_type) allowed_tools = resolve_allowed_tools(config, work_type)
argv = build_claude_argv(model_id, max_turns, add_dirs, allowed_tools) argv = build_claude_argv(model_id, max_turns, add_dirs, allowed_tools)
canonical = canonical_output_path(repo, candidate) canonical = canonical_output_path(repo, candidate)
tmp_path = canonical.with_name(canonical.name + ".tmp") tmp_path = tmp_output_path(canonical)
parsed_env = parse_provider_env(profile_dir / "provider.env") parsed_env = parse_provider_env(profile_dir / "provider.env")
return { return {
"dossier": dossier, "dossier": dossier,
@@ -846,6 +850,133 @@ def dispatch_preview(repo: Path, config: dict, candidate: dict, max_turns: int)
} }
# === Child process tracking, unique tmp paths, stale-tmp sweep ===
#
# Concurrency-protection hardening (see specs/idle-draft.spec.md "Concurrency
# protection"): the flock (§1) already keeps two dispatcher instances from
# running concurrently in the common case. The gap it does *not* close is a
# dispatcher that dies without releasing its children (SIGTERM/SIGKILL,
# machine reboot, OOM kill): the flock is released when the fd closes, but
# `claude` grandchildren spawned via subprocess survive as orphans and can
# still be mid-write to a tmp file when a fresh cron-fired instance starts.
# Two independent mechanisms close that gap:
# (a) children run in their own process group (`start_new_session=True`);
# a caught signal (SIGTERM/SIGINT) or normal interpreter exit kills
# every live child's group via os.killpg. This cannot catch SIGKILL of
# the dispatcher itself -- hence (b).
# (b) every task's tmp output path is unique per dispatcher invocation
# (`<canonical>.tmp.<dispatcher-pid>`), so even an orphan that outlives
# its dispatcher can never collide with the tmp path a *new* dispatcher
# instance uses for the same item. At startup a new instance sweeps and
# deletes tmp files older than the task timeout -- they are inert
# litter (never promoted by anyone) but worth reclaiming.
_live_children_lock = threading.Lock()
_live_children: set[int] = set()
def register_child(pid: int) -> None:
with _live_children_lock:
_live_children.add(pid)
def unregister_child(pid: int) -> None:
with _live_children_lock:
_live_children.discard(pid)
def live_children() -> set[int]:
"""Snapshot of currently-tracked child pids. Test/inspection hook."""
with _live_children_lock:
return set(_live_children)
def kill_all_children() -> None:
"""Kill the process group of every tracked live child. Safe to call
repeatedly / with a stale or empty registry -- a dead or nonexistent pid
just raises ProcessLookupError, which is swallowed. Each child was
started with start_new_session=True, so its pid is also its pgid."""
with _live_children_lock:
pids = list(_live_children)
_live_children.clear()
for pid in pids:
try:
os.killpg(pid, signal.SIGKILL)
except (ProcessLookupError, PermissionError, OSError):
pass
def install_signal_handlers() -> None:
"""Install SIGTERM/SIGINT handlers so a killed dispatcher takes its
in-flight `claude` children down with it rather than orphaning them.
Only meaningful in the main thread of a real (non-dryrun) dispatch run,
where children actually get spawned."""
def _handler(signum, frame):
kill_all_children()
sys.exit(1)
signal.signal(signal.SIGTERM, _handler)
signal.signal(signal.SIGINT, _handler)
# Belt-and-braces: even a normal unhandled-exception exit (no signal
# involved) should not leave children behind. atexit fires on interpreter
# shutdown from any cause except os._exit()/SIGKILL.
atexit.register(kill_all_children)
TMP_SUFFIX_RE = re.compile(r"\.tmp\.\d+$")
def tmp_output_path(canonical: Path, pid: int | None = None) -> Path:
"""Per-invocation-unique tmp path for a task's output. Defaults to this
process's pid (shared by all worker threads of one dispatcher run, so
every task in one invocation uses the same suffix); a pid can be passed
explicitly for testing or preview purposes."""
pid = pid if pid is not None else os.getpid()
return canonical.with_name(canonical.name + f".tmp.{pid}")
def find_tmp_files(repo: Path, dossiers: list) -> list:
"""Every leftover `*.tmp.<pid>` file directly under any configured
dossier directory (where canonical outputs and their tmp siblings live)."""
found = []
for dossier in dossiers:
d = repo / dossier
if not d.is_dir():
continue
for f in d.iterdir():
if f.is_file() and TMP_SUFFIX_RE.search(f.name):
found.append(f)
return found
def sweep_stale_tmp_files(repo: Path, dossiers: list, task_timeout: int, now: float | None = None) -> list:
"""Delete tmp files whose mtime is older than task_timeout seconds --
orphaned litter from a dispatcher that died mid-task (GAP 2b). Tmp files
younger than the timeout are left alone: they could belong to a live
orphan still finishing up, or to another concurrent dispatch cycle's
in-flight task. Since a tmp file is only ever promoted by the exact
run_task() call that created it, an untouched one is inert at worst --
never silently adopted by anyone else. Returns the paths actually
deleted, for logging by the caller."""
now = now if now is not None else time.time()
deleted = []
for f in find_tmp_files(repo, dossiers):
try:
mtime = f.stat().st_mtime
except OSError:
continue
if now - mtime > task_timeout:
try:
f.unlink()
deleted.append(f)
except OSError:
continue
return deleted
# === Real task execution === # === Real task execution ===
@@ -856,7 +987,7 @@ def run_task(repo: Path, config: dict, candidate: dict, max_turns: int, task_tim
slug = candidate["slug"] slug = candidate["slug"]
canonical = canonical_output_path(repo, candidate) canonical = canonical_output_path(repo, candidate)
tmp_path = canonical.with_name(canonical.name + ".tmp") tmp_path = tmp_output_path(canonical)
mapping = build_prompt_mapping(repo, dossier, slug, work_type, config, tmp_path) mapping = build_prompt_mapping(repo, dossier, slug, work_type, config, tmp_path)
prompt_text = render_prompt(work_type, mapping) prompt_text = render_prompt(work_type, mapping)
@@ -872,14 +1003,34 @@ def run_task(repo: Path, config: dict, candidate: dict, max_turns: int, task_tim
returncode = -1 returncode = -1
stderr = "" stderr = ""
try: try:
proc = subprocess.run( proc = subprocess.Popen(
argv, cwd=str(repo), env=env, input=prompt_text, argv, cwd=str(repo), env=env,
capture_output=True, text=True, timeout=task_timeout, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
text=True, start_new_session=True,
) )
except OSError as e:
return {"outcome": "transient", "candidate": candidate, "reason": f"failed to start claude subprocess: {e}"}
# GAP 2a: own process group + tracked pid, so a killed dispatcher can
# take this (and any grandchildren it spawns) down with it instead of
# orphaning it. See "Child process tracking" section above.
register_child(proc.pid)
try:
try:
_, stderr = proc.communicate(input=prompt_text, timeout=task_timeout)
returncode = proc.returncode returncode = proc.returncode
stderr = proc.stderr
except subprocess.TimeoutExpired: except subprocess.TimeoutExpired:
timed_out = True timed_out = True
try:
os.killpg(proc.pid, signal.SIGKILL)
except (ProcessLookupError, PermissionError, OSError):
pass
try:
proc.communicate(timeout=5)
except Exception:
pass
finally:
unregister_child(proc.pid)
failure = classify_failure(returncode, stderr, timed_out) failure = classify_failure(returncode, stderr, timed_out)
@@ -972,12 +1123,20 @@ def apply_failure_state(state: dict, candidate: dict, max_attempts: int) -> str
def run_dispatch(args, config: dict, repo: Path, state_path: Path) -> int: def run_dispatch(args, config: dict, repo: Path, state_path: Path) -> int:
rotate_log_if_needed(repo) rotate_log_if_needed(repo)
defaults = get_config_defaults(config)
lock = None lock = None
if not args.dryrun: if not args.dryrun:
lock = acquire_lock(repo) lock = acquire_lock(repo)
if lock is None: if lock is None:
log_event(repo, "lock held by another invocation; exiting") log_event(repo, "lock held by another invocation; exiting")
return 0 return 0
# Held for the full dispatch loop, including while children run (see
# module-level "Child process tracking" section) -- a second cron-fired
# invocation exits 0 immediately per the check above, it never races
# this instance even if this instance runs for hours.
install_signal_handlers()
for swept in sweep_stale_tmp_files(repo, config["dossiers"], defaults["task_timeout"]):
log_event(repo, f"swept stale tmp file (orphaned litter): {swept}")
try: try:
state = load_state(state_path, repo) state = load_state(state_path, repo)
@@ -985,7 +1144,6 @@ def run_dispatch(args, config: dict, repo: Path, state_path: Path) -> int:
log_event(repo, f"FATAL: state validation failed: {e}") log_event(repo, f"FATAL: state validation failed: {e}")
return 2 return 2
defaults = get_config_defaults(config)
parallel = args.parallel or config.get("parallel", DEFAULT_PARALLEL) parallel = args.parallel or config.get("parallel", DEFAULT_PARALLEL)
try: try:

View File

@@ -65,6 +65,14 @@ lock), `flock` (`fcntl.flock`, `LOCK_EX | LOCK_NB`) on `<repo>/.idle-draft.lock`
already locked by another invocation, log one line and exit **0** — a concurrent cron already locked by another invocation, log one line and exit **0** — a concurrent cron
tick is not a failure. tick is not a failure.
**Guarantee:** the lock is acquired before any other work and held for the *entire*
dispatch loop, including while dispatched children are running — not released until the
process is about to exit. A second cron-fired invocation therefore always exits 0
immediately for as long as the first is alive, however long that is (a low-usage
provider can let one instance run for hours "catching up"; the lock does not expire or
time out). This is the primary concurrency guard; §13 covers what happens when the
*lock-holding process itself* dies uncleanly rather than exiting normally.
### 2. Log rotation ### 2. Log rotation
At startup, if `<repo>/idle-draft.log` exceeds 5 MB, rename it to `idle-draft.log.1` At startup, if `<repo>/idle-draft.log` exceeds 5 MB, rename it to `idle-draft.log.1`
@@ -216,6 +224,11 @@ For the selected `(item_or_dossier, work_type, provider)`:
2. Render the template (`string.Template`, `$placeholder` substitution) with the 2. Render the template (`string.Template`, `$placeholder` substitution) with the
resolved paths for that item (overview, agent, research, draft as applicable), resolved paths for that item (overview, agent, research, draft as applicable),
dossier name, slug, style directory, source register path, and a temp output path. dossier name, slug, style directory, source register path, and a temp output path.
The temp output path is `<canonical>.tmp.<dispatcher-pid>` — unique per dispatcher
*invocation* (all tasks within one dispatch run share the same suffix, since it's
this process's own pid), not per task. This is what makes tmp paths collision-proof
across dispatcher instances even when one instance dies leaving orphaned children
behind — see §13.
3. Resolve the profile directory from `config["providers"][provider]["profile"]` 3. Resolve the profile directory from `config["providers"][provider]["profile"]`
(`~` expanded). Build the child environment: `CLAUDE_CONFIG_DIR=<profile>`, plus — (`~` expanded). Build the child environment: `CLAUDE_CONFIG_DIR=<profile>`, plus —
if `<profile>/provider.env` exists — `ANTHROPIC_BASE_URL` (if set), if `<profile>/provider.env` exists — `ANTHROPIC_BASE_URL` (if set),
@@ -237,7 +250,11 @@ For the selected `(item_or_dossier, work_type, provider)`:
5. Run the subprocess: `cwd=<repo>`, `env=<built env>`, prompt piped via **stdin** 5. Run the subprocess: `cwd=<repo>`, `env=<built env>`, prompt piped via **stdin**
(not as an argv element — avoids `ARG_MAX` on large rendered prompts, same lesson (not as an argv element — avoids `ARG_MAX` on large rendered prompts, same lesson
`claude-profile` already applies to its system-prompt injection), timeout = per-task `claude-profile` already applies to its system-prompt injection), timeout = per-task
timeout (1800s unless overridden). timeout (1800s unless overridden). The child is started in its own process group
(`start_new_session=True`) and its pid is tracked in a live-children registry for the
duration of the call — see §13. On timeout, the whole process group is sent
`SIGKILL` (not just the immediate `claude` process), so any grandchild it spawned
dies too, before the task is classified as a transient failure.
6. Classify the result: 6. Classify the result:
- **Timeout** → transient failure. Do not increment `attempts`. - **Timeout** → transient failure. Do not increment `attempts`.
- **Non-zero exit** whose stderr matches a retryable signature (`429`, `5xx`, - **Non-zero exit** whose stderr matches a retryable signature (`429`, `5xx`,
@@ -265,7 +282,10 @@ For the selected `(item_or_dossier, work_type, provider)`:
`<repo>/<dossier>/<NN-slug>.<work_type>.md.rejected` (kept for human inspection, `<repo>/<dossier>/<NN-slug>.<work_type>.md.rejected` (kept for human inspection,
never promoted, never committed to the canonical name; `topic_ideas` content never promoted, never committed to the canonical name; `topic_ideas` content
failures are simply discarded — nothing is appended, nothing is blocked, since failures are simply discarded — nothing is appended, nothing is blocked, since
`topic_ideas` has no per-item state entry to carry an attempt counter). `topic_ideas` has no per-item state entry to carry an attempt counter). The
`.rejected` path is derived from the canonical path, not the tmp path, so it is
unaffected by the `.tmp.<dispatcher-pid>` suffix (§13) — always
`<canonical>.rejected`, never `<canonical>.rejected.<pid>`.
10. On transient failure: temp file is discarded; no state change; no commit; the 10. On transient failure: temp file is discarded; no state change; no commit; the
candidate may be retried on a later cycle. candidate may be retried on a later cycle.
11. On success: write `idle-draft.state.json` atomically (temp file + `os.replace()` in 11. On success: write `idle-draft.state.json` atomically (temp file + `os.replace()` in
@@ -284,6 +304,53 @@ For the selected `(item_or_dossier, work_type, provider)`:
One line per event (gate decision, dispatch, completion, failure, commit) appended to One line per event (gate decision, dispatch, completion, failure, commit) appended to
`<repo>/idle-draft.log` with an ISO-8601 timestamp, mirrored to stderr. `<repo>/idle-draft.log` with an ISO-8601 timestamp, mirrored to stderr.
### 13. Concurrency protection beyond the lock
§1's flock is the primary guard and handles the common case: a second cron-fired
invocation always sees the lock held and exits 0. This section covers the gap the lock
alone cannot close — a dispatcher process that dies *without* releasing its children
cleanly (`SIGTERM`, `SIGKILL`, OOM kill, machine reboot). The flock itself is released
the instant the holding process's file descriptors close (on any death, clean or not),
but a `claude` child spawned via `subprocess` is a separate OS process and does **not**
die automatically when its parent does — left alone it becomes an orphan, reparented to
init, still writing its output. A fresh cron-fired instance starting immediately after
would then see the lock free, pick up the *same* ready item, and dispatch a *second*
`claude` process against the *same* deterministic output path — two writers racing on
the "same" canonical file. Two independent mechanisms close this:
**a. Children die with the dispatcher (the common death path).** Every `claude` child is
spawned with `start_new_session=True`, putting it (and any process it forks) in its own
process group with the child's pid as the group id. The dispatcher tracks each live
child's pid in an in-process registry for the duration of the call (`register_child` /
`unregister_child`, guarded by a lock since tasks run in worker threads). On `SIGTERM` or
`SIGINT`, an installed handler walks the registry and `os.killpg`s each group with
`SIGKILL`, then exits — no child outlives a dispatcher that receives a catchable signal.
The same cleanup also runs via `atexit`, so a normal unhandled-exception exit (no signal
involved) still cannot leave children behind. **This cannot catch `SIGKILL` of the
dispatcher itself** — no userspace handler can — which is exactly the case mechanism (b)
exists for.
**b. Tmp-path collisions are structurally impossible (the uncatchable-death path).** Each
task's temp output file is named `<canonical>.tmp.<dispatcher-pid>` (§11 step 2), not
the old bare `<canonical>.tmp`. Every dispatcher process has a distinct pid, so even an
orphaned child that outlives a `SIGKILL`ed dispatcher is writing to a filename no other
dispatcher instance — past, present, or future — will ever target. There is no race to
resolve: the collision that used to be structurally possible (two writers, one path) is
now structurally impossible (two writers, two paths).
**Startup sweep.** After acquiring the lock, before touching config-driven state, a
fresh dispatch instance globs each configured dossier directory for leftover
`*.tmp.<pid>` files. Any whose mtime is older than the per-task timeout (1800s unless
overridden) is deleted and logged as swept — it is orphaned litter from a dispatcher
that died mid-task; since it was never promoted (only the exact `run_task()` call that
created it ever calls `os.replace()` on it) it can never silently become the canonical
file no matter how long it sits there. Files younger than the timeout are left alone —
they may belong to a live orphan (mechanism (a) failed to reap it, e.g. it was itself
`SIGKILL`ed independently, or belongs to another concurrent dispatch cycle that legitimately
still has it in flight) still finishing up; touching them risks nothing since, again,
nothing ever silently adopts a tmp file it didn't create — at worst a stale one sits
unswept until the *next* startup sweep, once it ages past the timeout.
## Dryrun behaviour ## Dryrun behaviour
`--dryrun` runs the full probe (or reads `--probe-json`), computes all gates, builds `--dryrun` runs the full probe (or reads `--probe-json`), computes all gates, builds
@@ -346,7 +413,9 @@ a lighter-touch check); `draft` and `review` carry no `allowed_tools` key.
| Scenario | Handling | | Scenario | Handling |
|---|---| |---|---|
| Lock already held | Log one line, exit 0 (not an error — another cron tick is running) | | Lock already held | Log one line, exit 0 (not an error — another cron tick is running), regardless of how long the holder has been running |
| Dispatcher receives `SIGTERM`/`SIGINT` while children are running | Handler `os.killpg`s every live child's process group, then exits (§13a) |
| Dispatcher is `SIGKILL`ed (uncatchable) | Children orphaned, but each was writing to a `.tmp.<dispatcher-pid>` path unique to that dead instance — no live instance can ever collide with it; stale ones swept once they age past the task timeout (§13b) |
| `idle-draft.state.json` missing | Treated as `{"items": {}}`, not an error | | `idle-draft.state.json` missing | Treated as `{"items": {}}`, not an error |
| `idle-draft.state.json` present but invalid | Exit 2, loud message, run never starts | | `idle-draft.state.json` present but invalid | Exit 2, loud message, run never starts |
| Item's `.agent.md` missing | Item excluded from all dispatch (commissioning briefs exist today for every current item; this guards future additions) | | Item's `.agent.md` missing | Item excluded from all dispatch (commissioning briefs exist today for every current item; this guards future additions) |

View File

@@ -162,6 +162,50 @@ after_hash=$(cd "$FIXTURE_REPO" && git rev-parse HEAD)
[[ "$before_hash" == "$after_hash" ]] && pass "dryrun makes no git commits" || fail "dryrun makes no git commits" [[ "$before_hash" == "$after_hash" ]] && pass "dryrun makes no git commits" || fail "dryrun makes no git commits"
[[ ! -f "$FIXTURE_REPO/ai/03-topic-three.review.md" ]] && pass "dryrun writes no output file" || fail "dryrun writes no output file" [[ ! -f "$FIXTURE_REPO/ai/03-topic-three.review.md" ]] && pass "dryrun writes no output file" || fail "dryrun writes no output file"
echo "-- concurrency: second invocation exits 0 when lock is held (GAP 1) --"
# Holds .idle-draft.lock in a background helper (independent process, so the
# flock is real -- fcntl locks are per-process/per-fd, a second flock from
# this same bash process wouldn't contend). The helper touches a sentinel
# file only *after* it has the lock, so we never race the flock call itself.
LOCK_HELPER="$TMPDIR/lock_helper.py"
cat > "$LOCK_HELPER" <<'PYEOF'
import fcntl, sys, time
lock_path, sentinel_path = sys.argv[1], sys.argv[2]
f = open(lock_path, "w")
fcntl.flock(f.fileno(), fcntl.LOCK_EX)
with open(sentinel_path, "w") as s:
s.write("locked\n")
time.sleep(30)
PYEOF
LOCK_SENTINEL="$TMPDIR/lock-sentinel"
rm -f "$LOCK_SENTINEL"
python3 "$LOCK_HELPER" "$FIXTURE_REPO/.idle-draft.lock" "$LOCK_SENTINEL" &
HELPER_PID=$!
waited=0
while [[ ! -f "$LOCK_SENTINEL" && $waited -lt 50 ]]; do
sleep 0.1
waited=$((waited + 1))
done
if [[ ! -f "$LOCK_SENTINEL" ]]; then
fail "lock-contention test setup: helper never confirmed it holds the lock"
else
before_hash=$(cd "$FIXTURE_REPO" && git rev-parse HEAD)
# Real dispatch mode (no --dryrun): a fixture with plenty of idle capacity,
# so absent the lock this would dispatch item 03's review task for real.
output=$("$SCRIPT" --config "$FIXTURE_CONFIG" --repo "$FIXTURE_REPO" --probe-json "$FIXTURE_PROBE" --once 2>&1)
code=$?
assert_exit_code "$code" 0 "second invocation with lock held exits 0"
assert_contains "$output" "lock held" "log line reports lock held"
after_hash=$(cd "$FIXTURE_REPO" && git rev-parse HEAD)
[[ "$before_hash" == "$after_hash" ]] && pass "no commit occurred while lock held" || fail "no commit occurred while lock held"
[[ ! -f "$FIXTURE_REPO/ai/03-topic-three.review.md" ]] && pass "no output file written while lock held" || fail "no output file written while lock held"
fi
kill "$HELPER_PID" 2>/dev/null
wait "$HELPER_PID" 2>/dev/null
echo "-- state validation failure exits 2 --" echo "-- state validation failure exits 2 --"
BADSTATE="$TMPDIR/badstate-repo" BADSTATE="$TMPDIR/badstate-repo"
mkdir -p "$BADSTATE/ai" mkdir -p "$BADSTATE/ai"
@@ -547,6 +591,85 @@ except m.StateValidationError:
m.validate_state({"items": {"ai/01-topic-one": {"human_edit_done": True, "attempts": {"draft": 1}}}}, repo) m.validate_state({"items": {"ai/01-topic-one": {"human_edit_done": True, "attempts": {"draft": 1}}}}, repo)
check("state validation: well-formed state passes", True) check("state validation: well-formed state passes", True)
# --- GAP 2b: unique tmp path per invocation ---
import os as _os
import time as _time
canonical = repo / "ai" / "03-topic-three.review.md"
tmp_a = m.tmp_output_path(canonical, pid=11111)
tmp_b = m.tmp_output_path(canonical, pid=22222)
check("tmp path uniqueness: two different pids produce two different tmp paths",
tmp_a != tmp_b and tmp_a.name == "03-topic-three.review.md.tmp.11111"
and tmp_b.name == "03-topic-three.review.md.tmp.22222", (str(tmp_a), str(tmp_b)))
tmp_default = m.tmp_output_path(canonical)
check("tmp path uniqueness: default pid is os.getpid()",
tmp_default.name == f"03-topic-three.review.md.tmp.{_os.getpid()}", str(tmp_default))
rejected = m.rejected_output_path(canonical)
check("rejected-path derivation is unaffected by the new tmp naming (derived from canonical, not tmp)",
rejected.name == "03-topic-three.review.md.rejected", str(rejected))
# --- GAP 2b: startup sweep of stale tmp files ---
sweep_repo = tmpdir / "sweep-repo"
(sweep_repo / "ai").mkdir(parents=True, exist_ok=True)
old_tmp = sweep_repo / "ai" / "01-a.research.md.tmp.111"
fresh_tmp = sweep_repo / "ai" / "02-b.research.md.tmp.222"
old_tmp.write_text("stale orphan output\n")
fresh_tmp.write_text("still-in-progress output\n")
now = _time.time()
task_timeout = 1800
old_mtime = now - task_timeout - 100 # older than the timeout -> stale
fresh_mtime = now - 10 # well within the timeout -> live orphan candidate, keep
_os.utime(old_tmp, (old_mtime, old_mtime))
_os.utime(fresh_tmp, (fresh_mtime, fresh_mtime))
deleted = m.sweep_stale_tmp_files(sweep_repo, ["ai"], task_timeout, now=now)
check("stale sweep: deletes a tmp file older than the task timeout",
old_tmp in deleted and not old_tmp.exists(), deleted)
check("stale sweep: leaves a tmp file younger than the task timeout untouched",
fresh_tmp not in deleted and fresh_tmp.exists(), deleted)
# A non-.tmp.<pid> file (e.g. the canonical output itself) must never be swept.
canonical_lookalike = sweep_repo / "ai" / "01-a.research.md"
canonical_lookalike.write_text("# Research: a\n")
_os.utime(canonical_lookalike, (old_mtime, old_mtime))
deleted2 = m.sweep_stale_tmp_files(sweep_repo, ["ai"], task_timeout, now=now)
check("stale sweep: never touches a file without a .tmp.<pid> suffix, however old",
canonical_lookalike.exists() and canonical_lookalike not in deleted2, canonical_lookalike)
# --- GAP 2a: child process-group tracking / signal handling (smoke tests) ---
# No real subprocess is spawned here -- just the bookkeeping primitives that
# the signal handler and run_task() rely on.
fake_pid = 2**30 # astronomically unlikely to collide with a real pid/pgid
m.register_child(fake_pid)
check("child tracking: register_child adds the pid to the live set", fake_pid in m.live_children(), m.live_children())
m.unregister_child(fake_pid)
check("child tracking: unregister_child removes the pid", fake_pid not in m.live_children(), m.live_children())
m.register_child(fake_pid)
m.register_child(fake_pid + 1)
try:
m.kill_all_children()
kill_all_ok = True
except Exception as e:
kill_all_ok = False
check("child tracking: kill_all_children on nonexistent pids raises nothing (ProcessLookupError swallowed)",
kill_all_ok, "")
check("child tracking: kill_all_children clears the registry", m.live_children() == set(), m.live_children())
m.install_signal_handlers()
import signal as _signal
check("signal handling: install_signal_handlers installs a non-default SIGTERM handler",
_signal.getsignal(_signal.SIGTERM) not in (_signal.SIG_DFL, _signal.SIG_IGN, None), "")
check("signal handling: install_signal_handlers installs a non-default SIGINT handler",
_signal.getsignal(_signal.SIGINT) not in (_signal.SIG_DFL, _signal.SIG_IGN, None), "")
# Restore defaults so this subprocess (which is about to exit anyway) doesn't
# leave anything surprising behind for the interpreter teardown.
_signal.signal(_signal.SIGTERM, _signal.SIG_DFL)
_signal.signal(_signal.SIGINT, _signal.SIG_DFL)
for status, name, detail in results: for status, name, detail in results:
print(f"{status}: {name}", detail if detail else "") print(f"{status}: {name}", detail if detail else "")