mm-worktree-task: --gate re-runs the test suite itself, never trusts the agent's claim
Across ~12 supervised MiniMax units, more than half reported a green suite where a manual re-run found red (see memory/gotchas-minimax.md). Add --gate "<command>" (with --gate-timeout, default 900s): after the detached sub-session exits, mm-worktree-task itself re-runs the command in the worktree against the sub-session's LAST commit, in its own reapable process group (setsid + group kill, so a hung gate and everything it forked are bounded and cleaned up), and appends an unambiguous PASSED/FAILED verdict — plus an explicit MISMATCH callout when the agent's own report claimed success — to the result file. A missing command, non-zero exit, timeout, or dirty worktree are all failures, never a silent skip. mm-worktree-task's own exit code is unchanged: the launch already returns before the sub-session (and gate) run, detached. Claude-Session: https://claude.ai/code/session_014KNyfHmHpMbdMrHZXDRkAp
This commit is contained in:
Paul O'Reilly
@@ -153,6 +153,178 @@ UPLOAD_RC=$?
|
||||
assert_eq "upload without collect exits 1" "1" "$UPLOAD_RC"
|
||||
assert_contains "upload error message" "--upload requires --collect" "$UPLOAD_OUT"
|
||||
|
||||
echo
|
||||
echo "== mm-worktree-task: --gate supervisor re-run =="
|
||||
# Strategy: a fake `ask-minimax-run` on PATH stands in for the real sub-session
|
||||
# (it commits in the worktree, optionally writes a result file, optionally
|
||||
# leaves the worktree dirty) so these tests drive the REAL script end to end —
|
||||
# real worktree, real backgrounded subshell, real `timeout`/gate re-run —
|
||||
# without ever launching a real MiniMax session.
|
||||
|
||||
REPORT_ROOT="${XDG_RUNTIME_DIR:-/tmp}/mm-worktree-task/$(basename "$REPO_ROOT")"
|
||||
FAKE_BIN_DIR="$(mktemp -d)"
|
||||
cat > "$FAKE_BIN_DIR/ask-minimax-run" <<'FAKE_MM'
|
||||
#!/usr/bin/env bash
|
||||
# Stand-in for ask-minimax-run: commits in the worktree it is run from (cwd
|
||||
# is already the worktree — mm-worktree-task cd's there before invoking it),
|
||||
# optionally writes a result file, optionally leaves an uncommitted change.
|
||||
set -e
|
||||
[[ -n "${FAKE_MM_SLEEP:-}" ]] && sleep "$FAKE_MM_SLEEP"
|
||||
git add -A >/dev/null 2>&1 || true
|
||||
git -c user.email=fake-mm@test.invalid -c user.name="fake mm" \
|
||||
commit -m "fake mm commit" --allow-empty >/dev/null 2>&1
|
||||
if [[ -n "${FAKE_MM_RESULT_FILE:-}" ]]; then
|
||||
mkdir -p "$(dirname "$FAKE_MM_RESULT_FILE")"
|
||||
printf '%s\n' "${FAKE_MM_RESULT_CONTENT:-Summary: done}" > "$FAKE_MM_RESULT_FILE"
|
||||
fi
|
||||
[[ -n "${FAKE_MM_LEAVE_DIRTY:-}" ]] && echo "uncommitted" > uncommitted-change.txt
|
||||
exit 0
|
||||
FAKE_MM
|
||||
chmod +x "$FAKE_BIN_DIR/ask-minimax-run"
|
||||
|
||||
cleanup_slug() {
|
||||
local slug=$1
|
||||
git -C "$REPO_ROOT" worktree remove --force "$REPO_ROOT/.claude/worktrees/mm-${slug}" >/dev/null 2>&1 || true
|
||||
git -C "$REPO_ROOT" branch -D "mm/${slug}" >/dev/null 2>&1 || true
|
||||
rm -f "$REPORT_ROOT/${slug}-bundle.md" "$REPORT_ROOT/${slug}-prompt.txt" \
|
||||
"$REPORT_ROOT/${slug}-stdout.txt" "$REPORT_ROOT/${slug}-result.md"
|
||||
}
|
||||
|
||||
# Bounded poll (not a fixed sleep chain) for the backgrounded subshell —
|
||||
# ask-minimax-run + gate, per the current code path — to finish and append
|
||||
# its launch.log line.
|
||||
wait_for_launch() {
|
||||
local slug=$1 timeout_s=${2:-20} waited=0
|
||||
while (( waited < timeout_s * 5 )); do
|
||||
grep -q "^${slug} rc=" "$REPORT_ROOT/launch.log" 2>/dev/null && return 0
|
||||
sleep 0.2
|
||||
waited=$((waited + 1))
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
run_real_launch() {
|
||||
# args: slug [extra mm-worktree-task args...]
|
||||
local slug=$1; shift
|
||||
PATH="$FAKE_BIN_DIR:$PATH" \
|
||||
MM_WORKTREE_TASK_FAKE_ITEM="$FIXTURE_ITEM" \
|
||||
MM_WORKTREE_TASK_FAKE_DOCS_DIR="$FIXTURE_DOCS_DIR" \
|
||||
"$SCRIPT" "$TEST_UUID" --slug "$slug" "$@" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
export MM_WORKTREE_TASK_FAKE_ITEM="$FIXTURE_ITEM"
|
||||
export MM_WORKTREE_TASK_FAKE_DOCS_DIR="$FIXTURE_DOCS_DIR"
|
||||
|
||||
# --- gate passes ---
|
||||
SLUG="gate-pass-$$"
|
||||
cleanup_slug "$SLUG"
|
||||
FAKE_MM_RESULT_FILE="${REPORT_ROOT}/${SLUG}-result.md" \
|
||||
FAKE_MM_RESULT_CONTENT=$'Summary: implemented feature\nTests run: all tests passed' \
|
||||
run_real_launch "$SLUG" --gate "true"
|
||||
if wait_for_launch "$SLUG"; then
|
||||
RESULT="$(cat "${REPORT_ROOT}/${SLUG}-result.md" 2>/dev/null || echo "<missing>")"
|
||||
HEAD_SHA="$(git -C "$REPO_ROOT/.claude/worktrees/mm-${SLUG}" rev-parse HEAD 2>/dev/null || echo "<no-worktree>")"
|
||||
assert_contains "gate-pass: section present" "## Supervisor gate" "$RESULT"
|
||||
assert_contains "gate-pass: verdict PASSED" "Verdict: **PASSED**" "$RESULT"
|
||||
assert_contains "gate-pass: names the actual worktree HEAD sha" "$HEAD_SHA" "$RESULT"
|
||||
assert_not_contains "gate-pass: no MISMATCH raised" "MISMATCH" "$RESULT"
|
||||
else
|
||||
printf " ${RED}FAIL${NC} gate-pass: backgrounded launch did not complete in time\n"
|
||||
FAIL=$((FAIL + 1))
|
||||
fi
|
||||
cleanup_slug "$SLUG"
|
||||
|
||||
# --- gate fails, and the agent's own report claimed success (the mismatch case) ---
|
||||
SLUG="gate-fail-$$"
|
||||
cleanup_slug "$SLUG"
|
||||
FAKE_MM_RESULT_FILE="${REPORT_ROOT}/${SLUG}-result.md" \
|
||||
FAKE_MM_RESULT_CONTENT=$'Summary: implemented feature\nTests run: all tests passed, 0 failed' \
|
||||
run_real_launch "$SLUG" --gate "exit 1"
|
||||
if wait_for_launch "$SLUG"; then
|
||||
RESULT="$(cat "${REPORT_ROOT}/${SLUG}-result.md" 2>/dev/null || echo "<missing>")"
|
||||
HEAD_SHA="$(git -C "$REPO_ROOT/.claude/worktrees/mm-${SLUG}" rev-parse HEAD 2>/dev/null || echo "<no-worktree>")"
|
||||
assert_contains "gate-fail: verdict FAILED" "Verdict: **FAILED**" "$RESULT"
|
||||
assert_contains "gate-fail: names the actual worktree HEAD sha" "$HEAD_SHA" "$RESULT"
|
||||
assert_contains "gate-fail: exit code recorded" "Exit code: 1" "$RESULT"
|
||||
assert_contains "gate-fail: mismatch called out against agent's own claim" "MISMATCH" "$RESULT"
|
||||
else
|
||||
printf " ${RED}FAIL${NC} gate-fail: backgrounded launch did not complete in time\n"
|
||||
FAIL=$((FAIL + 1))
|
||||
fi
|
||||
cleanup_slug "$SLUG"
|
||||
|
||||
# --- gate command does not exist: failure, never a silent skip ---
|
||||
SLUG="gate-missing-$$"
|
||||
cleanup_slug "$SLUG"
|
||||
run_real_launch "$SLUG" --gate "this-command-truly-does-not-exist-xyz"
|
||||
if wait_for_launch "$SLUG"; then
|
||||
RESULT="$(cat "${REPORT_ROOT}/${SLUG}-result.md" 2>/dev/null || echo "<missing>")"
|
||||
assert_contains "gate-missing: verdict FAILED (not skipped)" "Verdict: **FAILED**" "$RESULT"
|
||||
assert_not_contains "gate-missing: never reads as a skip" "SKIP" "$RESULT"
|
||||
else
|
||||
printf " ${RED}FAIL${NC} gate-missing: backgrounded launch did not complete in time\n"
|
||||
FAIL=$((FAIL + 1))
|
||||
fi
|
||||
cleanup_slug "$SLUG"
|
||||
|
||||
# --- gate times out: bounded, reaped, recorded as failed ---
|
||||
# The sleep duration embeds this test run's own PID as a fractional-second
|
||||
# fingerprint (GNU sleep accepts arbitrary-precision fractions) so the
|
||||
# leftover-process check below can never false-positive on some unrelated
|
||||
# "sleep 30" already running on a shared box.
|
||||
SLUG="gate-timeout-$$"
|
||||
SLEEP_MARKER="30.$$"
|
||||
cleanup_slug "$SLUG"
|
||||
run_real_launch "$SLUG" --gate "sleep ${SLEEP_MARKER}" --gate-timeout 1
|
||||
if wait_for_launch "$SLUG" 15; then
|
||||
RESULT="$(cat "${REPORT_ROOT}/${SLUG}-result.md" 2>/dev/null || echo "<missing>")"
|
||||
assert_contains "gate-timeout: verdict FAILED" "Verdict: **FAILED**" "$RESULT"
|
||||
assert_contains "gate-timeout: TIMED OUT noted with the bound" "TIMED OUT after 1s" "$RESULT"
|
||||
# the marked sleep process must actually have been reaped, not left running
|
||||
sleep 1
|
||||
LEFTOVER="$(pgrep -f "sleep ${SLEEP_MARKER}" 2>/dev/null || true)"
|
||||
assert_eq "gate-timeout: no leftover marked sleep process" "" "$LEFTOVER"
|
||||
else
|
||||
printf " ${RED}FAIL${NC} gate-timeout: backgrounded launch did not complete within the bound\n"
|
||||
FAIL=$((FAIL + 1))
|
||||
fi
|
||||
cleanup_slug "$SLUG"
|
||||
|
||||
# --- dirty worktree after the sub-session: failed regardless of the gate command's own exit ---
|
||||
SLUG="gate-dirty-$$"
|
||||
cleanup_slug "$SLUG"
|
||||
FAKE_MM_LEAVE_DIRTY=1 run_real_launch "$SLUG" --gate "true"
|
||||
if wait_for_launch "$SLUG"; then
|
||||
RESULT="$(cat "${REPORT_ROOT}/${SLUG}-result.md" 2>/dev/null || echo "<missing>")"
|
||||
assert_contains "gate-dirty: verdict FAILED despite gate command exiting 0" "Verdict: **FAILED**" "$RESULT"
|
||||
assert_contains "gate-dirty: dirty worktree called out" "DIRTY" "$RESULT"
|
||||
else
|
||||
printf " ${RED}FAIL${NC} gate-dirty: backgrounded launch did not complete in time\n"
|
||||
FAIL=$((FAIL + 1))
|
||||
fi
|
||||
cleanup_slug "$SLUG"
|
||||
|
||||
# --- no --gate at all: result file shape is exactly what the sub-session wrote ---
|
||||
SLUG="no-gate-$$"
|
||||
cleanup_slug "$SLUG"
|
||||
FAKE_MM_RESULT_FILE="${REPORT_ROOT}/${SLUG}-result.md" \
|
||||
FAKE_MM_RESULT_CONTENT=$'Summary: implemented feature\nTests run: all tests passed' \
|
||||
run_real_launch "$SLUG"
|
||||
if wait_for_launch "$SLUG"; then
|
||||
RESULT="$(cat "${REPORT_ROOT}/${SLUG}-result.md" 2>/dev/null || echo "<missing>")"
|
||||
assert_not_contains "no-gate: never adds a Supervisor gate section" "Supervisor gate" "$RESULT"
|
||||
assert_eq "no-gate: report is byte-for-byte what the sub-session wrote" \
|
||||
$'Summary: implemented feature\nTests run: all tests passed' "$RESULT"
|
||||
else
|
||||
printf " ${RED}FAIL${NC} no-gate: backgrounded launch did not complete in time\n"
|
||||
FAIL=$((FAIL + 1))
|
||||
fi
|
||||
cleanup_slug "$SLUG"
|
||||
|
||||
rm -rf "$FAKE_BIN_DIR"
|
||||
unset MM_WORKTREE_TASK_FAKE_ITEM
|
||||
unset MM_WORKTREE_TASK_FAKE_DOCS_DIR
|
||||
|
||||
echo
|
||||
echo "=================================="
|
||||
echo " PASS: $PASS FAIL: $FAIL"
|
||||
|
||||
Reference in New Issue
Block a user