bdd10cbbc1
Optional config key (default false). When true, research candidates are offered even at max_unreviewed_research_per_dossier; research_sampled bookkeeping, mark sampled, and status flags are unchanged so unsampled items stay visible for later human verification. Claude-Session: https://claude.ai/code/session_01Lgv4Qn82boNFC1jn8QXSNw
1231 lines
62 KiB
Bash
Executable File
1231 lines
62 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Test script for idle-draft: exercises --dryrun / --probe-json against fixture
|
|
# dirs, plus direct unit tests of the pure logic functions (gates, stage
|
|
# derivation, prioritisation, credential parity, citation validation, state
|
|
# validation) by importing the script as a Python module.
|
|
#
|
|
# No API call, no `claude` invocation, and no real profile dir is ever touched.
|
|
set -uo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
SCRIPT="$REPO_ROOT/scripts/idle-draft"
|
|
|
|
RED='\033[31m'; GREEN='\033[32m'; RESET='\033[0m'
|
|
PASS=0
|
|
FAIL=0
|
|
|
|
pass() { printf " ${GREEN}PASS${RESET}: %s\n" "$1"; PASS=$((PASS + 1)); }
|
|
fail() { printf " ${RED}FAIL${RESET}: %s\n" "$1"; [[ -n "${2:-}" ]] && printf " %s\n" "$2"; FAIL=$((FAIL + 1)); }
|
|
|
|
assert_contains() {
|
|
local output="$1" expected="$2" label="$3"
|
|
if echo "$output" | grep -qF -- "$expected"; then pass "$label"; else fail "$label" "Expected to find: $expected"; fi
|
|
}
|
|
assert_not_contains() {
|
|
local output="$1" unexpected="$2" label="$3"
|
|
if echo "$output" | grep -qF -- "$unexpected"; then fail "$label" "Did not expect: $unexpected"; else pass "$label"; fi
|
|
}
|
|
assert_exit_code() {
|
|
local actual="$1" expected="$2" label="$3"
|
|
if [[ "$actual" -eq "$expected" ]]; then pass "$label"; else fail "$label" "Expected exit $expected, got $actual"; fi
|
|
}
|
|
|
|
TMPDIR=""
|
|
cleanup() { [[ -n "$TMPDIR" && -d "$TMPDIR" ]] && rm -rf "$TMPDIR"; }
|
|
trap cleanup EXIT
|
|
TMPDIR=$(mktemp -d)
|
|
|
|
FIXTURE_REPO="$TMPDIR/repo"
|
|
FIXTURE_CONFIG="$TMPDIR/idle-draft.config.json"
|
|
FIXTURE_PROBE="$TMPDIR/probe.json"
|
|
PROFILE_ANTHROPIC="$TMPDIR/profiles/anthropic"
|
|
PROFILE_MINIMAX="$TMPDIR/profiles/minimax"
|
|
|
|
# --- Build a fixture writing-repo tree ---------------------------------
|
|
mkdir -p "$FIXTURE_REPO/ai" "$FIXTURE_REPO/style" "$PROFILE_ANTHROPIC" "$PROFILE_MINIMAX"
|
|
(
|
|
cd "$FIXTURE_REPO" && git init -q && git config user.email t@example.com && git config user.name test
|
|
)
|
|
printf '# AGENTS root\n' > "$FIXTURE_REPO/AGENTS.md"
|
|
printf '# ai AGENTS\n' > "$FIXTURE_REPO/ai/AGENTS.md"
|
|
printf '# Source register\n' > "$FIXTURE_REPO/ai/SOURCE-REGISTER.md"
|
|
printf '# goes target voice\n' > "$FIXTURE_REPO/style/goes-target-voice.md"
|
|
printf '# review prompt\n' > "$FIXTURE_REPO/style/review-prompt.md"
|
|
|
|
# Item 01: only overview+agent -> next = research
|
|
printf '# Topic one\n\nPitch.\n' > "$FIXTURE_REPO/ai/01-topic-one.overview.md"
|
|
printf '# Commissioning brief: topic one\n' > "$FIXTURE_REPO/ai/01-topic-one.agent.md"
|
|
|
|
# Item 02: overview+agent+research -> next = draft
|
|
printf '# Topic two\n\nPitch.\n' > "$FIXTURE_REPO/ai/02-topic-two.overview.md"
|
|
printf '# Commissioning brief: topic two\n' > "$FIXTURE_REPO/ai/02-topic-two.agent.md"
|
|
printf '# Research: topic two\n\nEvidence.\n' > "$FIXTURE_REPO/ai/02-topic-two.research.md"
|
|
|
|
# Item 03: research+draft, human_edit_done -> next = review
|
|
printf '# Topic three\n\nPitch.\n' > "$FIXTURE_REPO/ai/03-topic-three.overview.md"
|
|
printf '# Commissioning brief: topic three\n' > "$FIXTURE_REPO/ai/03-topic-three.agent.md"
|
|
printf '# Research: topic three\n\nEvidence.\n' > "$FIXTURE_REPO/ai/03-topic-three.research.md"
|
|
printf '# Topic three draft\n\nBody.\n' > "$FIXTURE_REPO/ai/03-topic-three.draft.md"
|
|
|
|
cat > "$FIXTURE_REPO/idle-draft.state.json" <<'EOF'
|
|
{
|
|
"items": {
|
|
"ai/03-topic-three": { "human_edit_done": true }
|
|
}
|
|
}
|
|
EOF
|
|
|
|
(cd "$FIXTURE_REPO" && git add -A && git commit -q -m init)
|
|
|
|
cat > "$FIXTURE_CONFIG" <<EOF
|
|
{
|
|
"parallel": 2,
|
|
"providers": {
|
|
"anthropic": { "profile": "$PROFILE_ANTHROPIC", "threshold_pct": 80, "five_hour_ceiling": 50, "min_idle": 5 },
|
|
"minimax": { "profile": "$PROFILE_MINIMAX", "threshold_pct": 80, "five_hour_ceiling": 50, "min_idle": 5 }
|
|
},
|
|
"work_types": {
|
|
"review": { "providers": ["anthropic"] },
|
|
"draft": { "providers": ["anthropic"] },
|
|
"research": { "providers": ["anthropic", "minimax"], "allowed_tools": ["WebSearch", "WebFetch"] },
|
|
"topic_ideas": { "providers": ["anthropic", "minimax"], "allowed_tools": ["WebSearch", "WebFetch"] }
|
|
},
|
|
"dossiers": ["ai"],
|
|
"review_score_threshold": 11,
|
|
"max_unreviewed_research_per_dossier": 3,
|
|
"max_open_topic_proposals": 6,
|
|
"evidence_dirs": ["$TMPDIR/evidence"]
|
|
}
|
|
EOF
|
|
|
|
cat > "$FIXTURE_PROBE" <<'EOF'
|
|
{
|
|
"probed_at": "2026-08-02T00:00:00+00:00",
|
|
"providers": [
|
|
{"provider": "Anthropic", "available": true, "windows": {
|
|
"five_hour": {"utilization_pct": 10.0, "reset_at": null, "reset_in_seconds": 1000, "window_seconds": 18000, "elapsed_pct": 0.5},
|
|
"seven_day": {"utilization_pct": 5.0, "reset_at": null, "reset_in_seconds": 100000, "window_seconds": 604800, "elapsed_pct": 0.5}
|
|
}},
|
|
{"provider": "MiniMax", "available": true, "windows": {
|
|
"five_hour": {"utilization_pct": 10.0, "reset_at": null, "reset_in_seconds": 1000, "window_seconds": 18000, "elapsed_pct": 0.5},
|
|
"seven_day": {"utilization_pct": 5.0, "reset_at": null, "reset_in_seconds": 100000, "window_seconds": 604800, "elapsed_pct": 0.5}
|
|
}}
|
|
]
|
|
}
|
|
EOF
|
|
|
|
# ============================================================
|
|
echo "=== idle-draft: CLI / dryrun tests ==="
|
|
echo ""
|
|
|
|
echo "-- help --"
|
|
output=$("$SCRIPT" --help 2>&1); code=$?
|
|
assert_exit_code "$code" 0 "--help exits 0"
|
|
assert_contains "$output" "Usage:" "--help shows usage"
|
|
assert_contains "$output" "--dryrun" "--help mentions --dryrun"
|
|
assert_contains "$output" "mark" "--help mentions mark subcommand"
|
|
assert_contains "$output" "status" "--help mentions status subcommand"
|
|
|
|
echo "-- dryrun: highest-priority ready item wins (review > draft > research) --"
|
|
output=$("$SCRIPT" --config "$FIXTURE_CONFIG" --repo "$FIXTURE_REPO" --probe-json "$FIXTURE_PROBE" --dryrun --parallel 1 2>&1)
|
|
code=$?
|
|
assert_exit_code "$code" 0 "dryrun exits 0"
|
|
assert_contains "$output" "item: ai/03-topic-three" "dryrun picks item 03 (review-ready, highest stage rank)"
|
|
assert_contains "$output" "work_type: review" "dryrun names work_type review"
|
|
assert_contains "$output" "provider: anthropic" "dryrun names provider anthropic"
|
|
assert_not_contains "$output" "would run" "dryrun does not execute claude"
|
|
assert_not_contains "$output" "--allowedTools" "dryrun argv for review (no allowed_tools configured) omits the flag"
|
|
|
|
echo "-- dryrun: exact resolved claude argv for a research item --"
|
|
# Force only the research-ready item to be eligible by pointing --repo at a
|
|
# single-item fixture (item 01 only).
|
|
SOLO="$TMPDIR/solo-repo"
|
|
mkdir -p "$SOLO/ai" "$SOLO/style"
|
|
cp "$FIXTURE_REPO/ai/01-topic-one.overview.md" "$SOLO/ai/"
|
|
cp "$FIXTURE_REPO/ai/01-topic-one.agent.md" "$SOLO/ai/"
|
|
cp "$FIXTURE_REPO/AGENTS.md" "$SOLO/AGENTS.md"
|
|
cp -r "$FIXTURE_REPO/style" "$SOLO/style"
|
|
cp "$FIXTURE_REPO/ai/SOURCE-REGISTER.md" "$SOLO/ai/SOURCE-REGISTER.md"
|
|
(cd "$SOLO" && git init -q && git config user.email t@example.com && git config user.name test && git add -A && git commit -q -m init)
|
|
output=$("$SCRIPT" --config "$FIXTURE_CONFIG" --repo "$SOLO" --probe-json "$FIXTURE_PROBE" --dryrun --once 2>&1)
|
|
assert_contains "$output" "work_type: research" "dryrun (solo fixture) picks research"
|
|
assert_contains "$output" "argv: ['claude', '-p', '--max-turns', '25', '--add-dir'" "dryrun prints resolved claude argv with --add-dir for research"
|
|
assert_contains "$output" "$TMPDIR/evidence" "dryrun argv includes configured evidence dir"
|
|
assert_contains "$output" "--allowedTools" "dryrun argv for research includes --allowedTools (allowed_tools configured)"
|
|
assert_contains "$output" "WebSearch,WebFetch" "dryrun argv --allowedTools value is the comma-joined tool list"
|
|
|
|
echo "-- dryrun: work-type effort/model override profile model, appear in resolved argv --"
|
|
PROFILE_ANTHROPIC_MODEL="$TMPDIR/profiles/anthropic-with-model"
|
|
mkdir -p "$PROFILE_ANTHROPIC_MODEL"
|
|
cat > "$PROFILE_ANTHROPIC_MODEL/provider.env" <<'EOF'
|
|
MODEL_ID=profile-default-model
|
|
EOF
|
|
EFFORT_CONFIG="$TMPDIR/effort.config.json"
|
|
cat > "$EFFORT_CONFIG" <<EOF
|
|
{
|
|
"parallel": 2,
|
|
"providers": {
|
|
"anthropic": { "profile": "$PROFILE_ANTHROPIC_MODEL", "threshold_pct": 80, "five_hour_ceiling": 50, "min_idle": 5 },
|
|
"minimax": { "profile": "$PROFILE_MINIMAX", "threshold_pct": 80, "five_hour_ceiling": 50, "min_idle": 5 }
|
|
},
|
|
"work_types": {
|
|
"review": { "providers": ["anthropic"] },
|
|
"draft": { "providers": ["anthropic"] },
|
|
"research": { "providers": ["anthropic", "minimax"], "allowed_tools": ["WebSearch", "WebFetch"], "effort": "medium", "model": "worktype-override-model" },
|
|
"topic_ideas": { "providers": ["anthropic", "minimax"], "allowed_tools": ["WebSearch", "WebFetch"] }
|
|
},
|
|
"dossiers": ["ai"],
|
|
"review_score_threshold": 11,
|
|
"max_unreviewed_research_per_dossier": 3,
|
|
"max_open_topic_proposals": 6,
|
|
"evidence_dirs": ["$TMPDIR/evidence"]
|
|
}
|
|
EOF
|
|
output=$("$SCRIPT" --config "$EFFORT_CONFIG" --repo "$SOLO" --probe-json "$FIXTURE_PROBE" --dryrun --once 2>&1)
|
|
assert_contains "$output" "'--model', 'worktype-override-model'" "dryrun argv model uses work-type override, not profile MODEL_ID"
|
|
assert_not_contains "$output" "profile-default-model" "dryrun argv does not contain the profile's MODEL_ID once a work-type model overrides it"
|
|
assert_contains "$output" "'--effort', 'medium'" "dryrun argv includes --effort with the configured level"
|
|
model_flag_count=$(echo "$output" | grep -o -- "--model" | wc -l)
|
|
[[ "$model_flag_count" -eq 1 ]] && pass "resolved argv contains exactly one --model flag (no duplicate from profile)" || fail "resolved argv contains exactly one --model flag" "found $model_flag_count"
|
|
|
|
echo "-- dryrun: no effort/model configured -> flags omitted (unchanged behaviour) --"
|
|
output=$("$SCRIPT" --config "$FIXTURE_CONFIG" --repo "$SOLO" --probe-json "$FIXTURE_PROBE" --dryrun --once 2>&1)
|
|
assert_not_contains "$output" "--effort" "dryrun argv omits --effort when work_type has no effort key"
|
|
|
|
echo "-- dryrun: no mutation --"
|
|
before_hash=$(cd "$FIXTURE_REPO" && git rev-parse HEAD)
|
|
"$SCRIPT" --config "$FIXTURE_CONFIG" --repo "$FIXTURE_REPO" --probe-json "$FIXTURE_PROBE" --dryrun >/dev/null 2>&1
|
|
after_hash=$(cd "$FIXTURE_REPO" && git rev-parse HEAD)
|
|
[[ "$before_hash" == "$after_hash" ]] && pass "dryrun makes no git commits" || fail "dryrun makes no git commits"
|
|
[[ ! -f "$FIXTURE_REPO/ai/03-topic-three.review.md" ]] && pass "dryrun writes no output file" || fail "dryrun writes no output file"
|
|
|
|
echo "-- concurrency: second invocation exits 0 when lock is held (GAP 1) --"
|
|
# Holds .idle-draft.lock in a background helper (independent process, so the
|
|
# flock is real -- fcntl locks are per-process/per-fd, a second flock from
|
|
# this same bash process wouldn't contend). The helper touches a sentinel
|
|
# file only *after* it has the lock, so we never race the flock call itself.
|
|
LOCK_HELPER="$TMPDIR/lock_helper.py"
|
|
cat > "$LOCK_HELPER" <<'PYEOF'
|
|
import fcntl, sys, time
|
|
lock_path, sentinel_path = sys.argv[1], sys.argv[2]
|
|
f = open(lock_path, "w")
|
|
fcntl.flock(f.fileno(), fcntl.LOCK_EX)
|
|
with open(sentinel_path, "w") as s:
|
|
s.write("locked\n")
|
|
time.sleep(30)
|
|
PYEOF
|
|
LOCK_SENTINEL="$TMPDIR/lock-sentinel"
|
|
rm -f "$LOCK_SENTINEL"
|
|
python3 "$LOCK_HELPER" "$FIXTURE_REPO/.idle-draft.lock" "$LOCK_SENTINEL" &
|
|
HELPER_PID=$!
|
|
|
|
waited=0
|
|
while [[ ! -f "$LOCK_SENTINEL" && $waited -lt 50 ]]; do
|
|
sleep 0.1
|
|
waited=$((waited + 1))
|
|
done
|
|
|
|
if [[ ! -f "$LOCK_SENTINEL" ]]; then
|
|
fail "lock-contention test setup: helper never confirmed it holds the lock"
|
|
else
|
|
before_hash=$(cd "$FIXTURE_REPO" && git rev-parse HEAD)
|
|
# Real dispatch mode (no --dryrun): a fixture with plenty of idle capacity,
|
|
# so absent the lock this would dispatch item 03's review task for real.
|
|
output=$("$SCRIPT" --config "$FIXTURE_CONFIG" --repo "$FIXTURE_REPO" --probe-json "$FIXTURE_PROBE" --once 2>&1)
|
|
code=$?
|
|
assert_exit_code "$code" 0 "second invocation with lock held exits 0"
|
|
assert_contains "$output" "lock held" "log line reports lock held"
|
|
after_hash=$(cd "$FIXTURE_REPO" && git rev-parse HEAD)
|
|
[[ "$before_hash" == "$after_hash" ]] && pass "no commit occurred while lock held" || fail "no commit occurred while lock held"
|
|
[[ ! -f "$FIXTURE_REPO/ai/03-topic-three.review.md" ]] && pass "no output file written while lock held" || fail "no output file written while lock held"
|
|
fi
|
|
|
|
kill "$HELPER_PID" 2>/dev/null
|
|
wait "$HELPER_PID" 2>/dev/null
|
|
|
|
echo "-- state validation failure exits 2 --"
|
|
BADSTATE="$TMPDIR/badstate-repo"
|
|
mkdir -p "$BADSTATE/ai"
|
|
cp "$FIXTURE_REPO/ai/01-topic-one.overview.md" "$BADSTATE/ai/"
|
|
cp "$FIXTURE_REPO/ai/01-topic-one.agent.md" "$BADSTATE/ai/"
|
|
echo '{"items": {"ai/01-topic-one": {"unknown_field": true}}}' > "$BADSTATE/idle-draft.state.json"
|
|
output=$("$SCRIPT" --config "$FIXTURE_CONFIG" --repo "$BADSTATE" --probe-json "$FIXTURE_PROBE" --dryrun 2>&1)
|
|
code=$?
|
|
assert_exit_code "$code" 2 "unknown item field in state exits 2"
|
|
assert_contains "$output" "unknown key" "state error names the unknown key"
|
|
|
|
echo '{"items": {"ai/does-not-exist": {}}}' > "$BADSTATE/idle-draft.state.json"
|
|
output=$("$SCRIPT" --config "$FIXTURE_CONFIG" --repo "$BADSTATE" --probe-json "$FIXTURE_PROBE" --dryrun 2>&1)
|
|
code=$?
|
|
assert_exit_code "$code" 2 "state item not resolving to overview.md exits 2"
|
|
|
|
echo "-- config validation failure exits 2 --"
|
|
BADCONFIG="$TMPDIR/bad.config.json"
|
|
echo '{"parallel": 2}' > "$BADCONFIG"
|
|
output=$("$SCRIPT" --config "$BADCONFIG" --repo "$FIXTURE_REPO" --probe-json "$FIXTURE_PROBE" --dryrun 2>&1)
|
|
code=$?
|
|
assert_exit_code "$code" 2 "config missing required keys exits 2"
|
|
|
|
BADALLOWED="$TMPDIR/bad-allowed.config.json"
|
|
cat > "$BADALLOWED" <<EOF
|
|
{
|
|
"parallel": 2,
|
|
"providers": {
|
|
"anthropic": { "profile": "$PROFILE_ANTHROPIC", "threshold_pct": 80, "five_hour_ceiling": 50, "min_idle": 5 }
|
|
},
|
|
"work_types": {
|
|
"review": { "providers": ["anthropic"] },
|
|
"draft": { "providers": ["anthropic"] },
|
|
"research": { "providers": ["anthropic"], "allowed_tools": "WebSearch" },
|
|
"topic_ideas": { "providers": ["anthropic"] }
|
|
},
|
|
"dossiers": ["ai"],
|
|
"review_score_threshold": 11,
|
|
"max_unreviewed_research_per_dossier": 3,
|
|
"max_open_topic_proposals": 6,
|
|
"evidence_dirs": ["$TMPDIR/evidence"]
|
|
}
|
|
EOF
|
|
output=$("$SCRIPT" --config "$BADALLOWED" --repo "$FIXTURE_REPO" --probe-json "$FIXTURE_PROBE" --dryrun 2>&1)
|
|
code=$?
|
|
assert_exit_code "$code" 2 "allowed_tools not a list of strings exits 2"
|
|
assert_contains "$output" "allowed_tools" "allowed_tools config error names the offending key"
|
|
|
|
BADEFFORT="$TMPDIR/bad-effort.config.json"
|
|
cat > "$BADEFFORT" <<EOF
|
|
{
|
|
"parallel": 2,
|
|
"providers": {
|
|
"anthropic": { "profile": "$PROFILE_ANTHROPIC", "threshold_pct": 80, "five_hour_ceiling": 50, "min_idle": 5 }
|
|
},
|
|
"work_types": {
|
|
"review": { "providers": ["anthropic"] },
|
|
"draft": { "providers": ["anthropic"] },
|
|
"research": { "providers": ["anthropic"], "effort": "ultra-mega" },
|
|
"topic_ideas": { "providers": ["anthropic"] }
|
|
},
|
|
"dossiers": ["ai"],
|
|
"review_score_threshold": 11,
|
|
"max_unreviewed_research_per_dossier": 3,
|
|
"max_open_topic_proposals": 6,
|
|
"evidence_dirs": ["$TMPDIR/evidence"]
|
|
}
|
|
EOF
|
|
output=$("$SCRIPT" --config "$BADEFFORT" --repo "$FIXTURE_REPO" --probe-json "$FIXTURE_PROBE" --dryrun 2>&1)
|
|
code=$?
|
|
assert_exit_code "$code" 2 "invalid effort level exits 2"
|
|
assert_contains "$output" "effort" "invalid effort config error names the offending key"
|
|
|
|
BADMODEL="$TMPDIR/bad-model.config.json"
|
|
cat > "$BADMODEL" <<EOF
|
|
{
|
|
"parallel": 2,
|
|
"providers": {
|
|
"anthropic": { "profile": "$PROFILE_ANTHROPIC", "threshold_pct": 80, "five_hour_ceiling": 50, "min_idle": 5 }
|
|
},
|
|
"work_types": {
|
|
"review": { "providers": ["anthropic"] },
|
|
"draft": { "providers": ["anthropic"] },
|
|
"research": { "providers": ["anthropic"], "model": "" },
|
|
"topic_ideas": { "providers": ["anthropic"] }
|
|
},
|
|
"dossiers": ["ai"],
|
|
"review_score_threshold": 11,
|
|
"max_unreviewed_research_per_dossier": 3,
|
|
"max_open_topic_proposals": 6,
|
|
"evidence_dirs": ["$TMPDIR/evidence"]
|
|
}
|
|
EOF
|
|
output=$("$SCRIPT" --config "$BADMODEL" --repo "$FIXTURE_REPO" --probe-json "$FIXTURE_PROBE" --dryrun 2>&1)
|
|
code=$?
|
|
assert_exit_code "$code" 2 "empty model string exits 2"
|
|
assert_contains "$output" "model" "empty model config error names the offending key"
|
|
|
|
echo "-- config validation: per-provider effort/model overrides (providers-list entries) --"
|
|
BADPROVIDERNAME="$TMPDIR/bad-provider-name.config.json"
|
|
cat > "$BADPROVIDERNAME" <<EOF
|
|
{
|
|
"parallel": 2,
|
|
"providers": {
|
|
"anthropic": { "profile": "$PROFILE_ANTHROPIC", "threshold_pct": 80, "five_hour_ceiling": 50, "min_idle": 5 }
|
|
},
|
|
"work_types": {
|
|
"review": { "providers": ["anthropic"] },
|
|
"draft": { "providers": ["anthropic"] },
|
|
"research": { "providers": ["anthropic", {"provider": "nonexistent-provider", "effort": "high"}] },
|
|
"topic_ideas": { "providers": ["anthropic"] }
|
|
},
|
|
"dossiers": ["ai"],
|
|
"review_score_threshold": 11,
|
|
"max_unreviewed_research_per_dossier": 3,
|
|
"max_open_topic_proposals": 6,
|
|
"evidence_dirs": ["$TMPDIR/evidence"]
|
|
}
|
|
EOF
|
|
output=$("$SCRIPT" --config "$BADPROVIDERNAME" --repo "$FIXTURE_REPO" --probe-json "$FIXTURE_PROBE" --dryrun 2>&1)
|
|
code=$?
|
|
assert_exit_code "$code" 2 "providers-list entry naming an undefined provider exits 2"
|
|
assert_contains "$output" "unknown provider" "undefined provider config error names the problem"
|
|
|
|
BADPROVIDERKEY="$TMPDIR/bad-provider-key.config.json"
|
|
cat > "$BADPROVIDERKEY" <<EOF
|
|
{
|
|
"parallel": 2,
|
|
"providers": {
|
|
"anthropic": { "profile": "$PROFILE_ANTHROPIC", "threshold_pct": 80, "five_hour_ceiling": 50, "min_idle": 5 }
|
|
},
|
|
"work_types": {
|
|
"review": { "providers": ["anthropic"] },
|
|
"draft": { "providers": ["anthropic"] },
|
|
"research": { "providers": [{"provider": "anthropic", "priority": 1}] },
|
|
"topic_ideas": { "providers": ["anthropic"] }
|
|
},
|
|
"dossiers": ["ai"],
|
|
"review_score_threshold": 11,
|
|
"max_unreviewed_research_per_dossier": 3,
|
|
"max_open_topic_proposals": 6,
|
|
"evidence_dirs": ["$TMPDIR/evidence"]
|
|
}
|
|
EOF
|
|
output=$("$SCRIPT" --config "$BADPROVIDERKEY" --repo "$FIXTURE_REPO" --probe-json "$FIXTURE_PROBE" --dryrun 2>&1)
|
|
code=$?
|
|
assert_exit_code "$code" 2 "providers-list object entry with an unknown key exits 2"
|
|
assert_contains "$output" "unknown key" "unknown provider-entry key config error names the problem"
|
|
|
|
BADPROVIDEREFFORT="$TMPDIR/bad-provider-effort.config.json"
|
|
cat > "$BADPROVIDEREFFORT" <<EOF
|
|
{
|
|
"parallel": 2,
|
|
"providers": {
|
|
"anthropic": { "profile": "$PROFILE_ANTHROPIC", "threshold_pct": 80, "five_hour_ceiling": 50, "min_idle": 5 }
|
|
},
|
|
"work_types": {
|
|
"review": { "providers": ["anthropic"] },
|
|
"draft": { "providers": ["anthropic"] },
|
|
"research": { "providers": [{"provider": "anthropic", "effort": "super-duper"}] },
|
|
"topic_ideas": { "providers": ["anthropic"] }
|
|
},
|
|
"dossiers": ["ai"],
|
|
"review_score_threshold": 11,
|
|
"max_unreviewed_research_per_dossier": 3,
|
|
"max_open_topic_proposals": 6,
|
|
"evidence_dirs": ["$TMPDIR/evidence"]
|
|
}
|
|
EOF
|
|
output=$("$SCRIPT" --config "$BADPROVIDEREFFORT" --repo "$FIXTURE_REPO" --probe-json "$FIXTURE_PROBE" --dryrun 2>&1)
|
|
code=$?
|
|
assert_exit_code "$code" 2 "providers-list object entry with an invalid effort level exits 2"
|
|
assert_contains "$output" "effort" "invalid provider-entry effort config error names the problem"
|
|
|
|
echo "-- mark subcommand round-trip --"
|
|
MARKREPO="$TMPDIR/mark-repo"
|
|
mkdir -p "$MARKREPO/ai"
|
|
cp "$FIXTURE_REPO/ai/01-topic-one.overview.md" "$MARKREPO/ai/"
|
|
cp "$FIXTURE_REPO/ai/01-topic-one.agent.md" "$MARKREPO/ai/"
|
|
output=$("$SCRIPT" mark ai/01-topic-one edited --config "$FIXTURE_CONFIG" --repo "$MARKREPO" 2>&1)
|
|
code=$?
|
|
assert_exit_code "$code" 0 "mark edited exits 0"
|
|
assert_contains "$(cat "$MARKREPO/idle-draft.state.json")" '"human_edit_done": true' "mark edited sets human_edit_done"
|
|
|
|
"$SCRIPT" mark ai/01-topic-one sampled --config "$FIXTURE_CONFIG" --repo "$MARKREPO" >/dev/null 2>&1
|
|
assert_contains "$(cat "$MARKREPO/idle-draft.state.json")" '"research_sampled": true' "mark sampled sets research_sampled"
|
|
|
|
"$SCRIPT" mark ai/01-topic-one approved --config "$FIXTURE_CONFIG" --repo "$MARKREPO" >/dev/null 2>&1
|
|
assert_contains "$(cat "$MARKREPO/idle-draft.state.json")" '"approved": true' "mark approved sets approved"
|
|
|
|
output=$("$SCRIPT" mark ai/99-nope edited --config "$FIXTURE_CONFIG" --repo "$MARKREPO" 2>&1); code=$?
|
|
assert_exit_code "$code" 2 "mark on nonexistent item exits 2"
|
|
|
|
echo "-- status subcommand --"
|
|
output=$("$SCRIPT" status --config "$FIXTURE_CONFIG" --repo "$FIXTURE_REPO" --probe-json "$FIXTURE_PROBE" 2>&1)
|
|
code=$?
|
|
assert_exit_code "$code" 0 "status exits 0"
|
|
assert_contains "$output" "ai/01-topic-one" "status lists item 01"
|
|
assert_contains "$output" "next=research" "status shows item 01 next=research"
|
|
assert_contains "$output" "ai/03-topic-three" "status lists item 03"
|
|
assert_contains "$output" "next=review" "status shows item 03 next=review"
|
|
assert_contains "$output" "anthropic:" "status shows provider gate line"
|
|
|
|
echo ""
|
|
echo "=== idle-draft: pure-logic unit tests (imported module) ==="
|
|
echo ""
|
|
|
|
PYOUT=$(python3 - "$SCRIPT" "$TMPDIR" <<'PYEOF'
|
|
import sys, json, importlib.util, importlib.machinery
|
|
from pathlib import Path
|
|
|
|
script_path, tmpdir = sys.argv[1], Path(sys.argv[2])
|
|
loader = importlib.machinery.SourceFileLoader("idle_draft_under_test", script_path)
|
|
spec = importlib.util.spec_from_loader(loader.name, loader)
|
|
m = importlib.util.module_from_spec(spec)
|
|
loader.exec_module(m)
|
|
|
|
results = []
|
|
|
|
def check(name, cond, detail=""):
|
|
results.append(("PY-PASS" if cond else "PY-FAIL", name, detail))
|
|
|
|
# --- Gate math ---
|
|
pcfg = {"threshold_pct": 80, "min_idle": 5, "five_hour_ceiling": 50}
|
|
|
|
def probe(elapsed, usage_sd, usage_fh, available=True):
|
|
return {
|
|
"available": available,
|
|
"windows": {
|
|
"seven_day": {"elapsed_pct": elapsed, "utilization_pct": usage_sd},
|
|
"five_hour": {"utilization_pct": usage_fh},
|
|
},
|
|
}
|
|
|
|
g = m.compute_gate(pcfg, probe(0.5, 5.0, 10.0))
|
|
check("gate: eligible when idle_points > min_idle and 5h < ceiling",
|
|
g["eligible"] is True and abs(g["idle_points"] - 35.0) < 1e-9, g)
|
|
|
|
g = m.compute_gate(pcfg, probe(0.5, 75.0, 10.0))
|
|
check("gate: ineligible when idle_points <= min_idle",
|
|
g["eligible"] is False and g["idle_points"] is not None, g)
|
|
|
|
g = m.compute_gate(pcfg, probe(0.5, 5.0, 60.0))
|
|
check("gate: ineligible when five_hour >= ceiling", g["eligible"] is False, g)
|
|
|
|
g = m.compute_gate(pcfg, probe(None, 5.0, 10.0))
|
|
check("gate: null elapsed_pct -> ineligible, idle_points None, fail-closed",
|
|
g["eligible"] is False and g["idle_points"] is None and "cannot pace" in g["reason"], g)
|
|
|
|
g = m.compute_gate(pcfg, probe(0.5, None, 10.0))
|
|
check("gate: null seven_day utilization -> ineligible", g["eligible"] is False, g)
|
|
|
|
g = m.compute_gate(pcfg, probe(0.5, 5.0, None))
|
|
check("gate: null five_hour utilization -> ineligible", g["eligible"] is False, g)
|
|
|
|
g = m.compute_gate(pcfg, {"available": False, "error": "boom"})
|
|
check("gate: unavailable provider -> ineligible", g["eligible"] is False and "unavailable" in g["reason"], g)
|
|
|
|
g = m.compute_gate(pcfg, None)
|
|
check("gate: no probe data -> ineligible", g["eligible"] is False, g)
|
|
|
|
g = m.compute_gate(pcfg, probe(1.0, 5.0, 10.0))
|
|
check("gate: elapsed_pct=1.0 (fully elapsed) computes idle_points=threshold-usage",
|
|
abs(g["idle_points"] - 75.0) < 1e-9, g)
|
|
|
|
# --- Stage derivation ---
|
|
def files(agent=True, research=False, draft=False, review=False):
|
|
return {"overview": True, "agent": agent, "research": research, "draft": draft, "review": review}
|
|
|
|
wt, reason = m.next_work_type_for_item(files(agent=False), {})
|
|
check("stage: missing agent.md -> None", wt is None and "commissioning brief" in reason, reason)
|
|
|
|
wt, reason = m.next_work_type_for_item(files(agent=True), {})
|
|
check("stage: agent only -> research", wt == "research", (wt, reason))
|
|
|
|
wt, reason = m.next_work_type_for_item(files(agent=True, research=True), {})
|
|
check("stage: research done -> draft", wt == "draft", (wt, reason))
|
|
|
|
wt, reason = m.next_work_type_for_item(files(agent=True, research=True, draft=True), {"human_edit_done": False})
|
|
check("stage: draft done, not human-edited -> None (waiting on human)",
|
|
wt is None and "human" in reason, (wt, reason))
|
|
|
|
wt, reason = m.next_work_type_for_item(files(agent=True, research=True, draft=True), {"human_edit_done": True})
|
|
check("stage: draft done, human-edited -> review", wt == "review", (wt, reason))
|
|
|
|
wt, reason = m.next_work_type_for_item(files(agent=True, research=True, draft=True, review=True), {"human_edit_done": True})
|
|
check("stage: review.md exists -> None (awaiting human revise/approve)", wt is None, (wt, reason))
|
|
|
|
wt, reason = m.next_work_type_for_item(files(agent=True), {"blocked": "too many failures"})
|
|
check("stage: blocked item -> None regardless of files", wt is None and "blocked" in reason, (wt, reason))
|
|
|
|
wt, reason = m.next_work_type_for_item(files(agent=True), {"approved": True})
|
|
check("stage: approved item -> None (terminal)", wt is None and "approved" in reason, (wt, reason))
|
|
|
|
# --- Prioritisation / cold-start / topic_ideas via build_ready_queue on the fixture repo ---
|
|
repo = tmpdir / "repo"
|
|
config = json.loads((tmpdir / "idle-draft.config.json").read_text())
|
|
state = m.load_state(repo / "idle-draft.state.json", repo)
|
|
queue = m.build_ready_queue(repo, config, state, set())
|
|
work_types_in_order = [c["work_type"] for c in queue]
|
|
check("priority: review-ready item ranks before draft/research candidates",
|
|
work_types_in_order[0] == "review", work_types_in_order)
|
|
check("priority: full queue is review, draft, research (stage rank descending)",
|
|
work_types_in_order == ["review", "draft", "research"], work_types_in_order)
|
|
|
|
# Cold-start throttle: two dossiers' worth of unreviewed research under a low cap
|
|
throttle_repo = tmpdir / "throttle-repo"
|
|
(throttle_repo / "ai").mkdir(parents=True, exist_ok=True)
|
|
for n, slug in [("01", "a"), ("02", "b"), ("03", "c")]:
|
|
base = throttle_repo / "ai" / f"{n}-{slug}"
|
|
(base.with_suffix("")).parent.mkdir(exist_ok=True, parents=True)
|
|
(throttle_repo / "ai" / f"{n}-{slug}.overview.md").write_text(f"# {slug}\n")
|
|
(throttle_repo / "ai" / f"{n}-{slug}.agent.md").write_text("# brief\n")
|
|
(throttle_repo / "ai" / "01-a.research.md").write_text("# Research: a\n")
|
|
(throttle_repo / "ai" / "02-b.research.md").write_text("# Research: b\n")
|
|
throttle_config = dict(config)
|
|
throttle_config["max_unreviewed_research_per_dossier"] = 2
|
|
throttle_state = m.default_state()
|
|
tqueue = m.build_ready_queue(throttle_repo, throttle_config, throttle_state, set())
|
|
research_candidates = [c["slug"] for c in tqueue if c["work_type"] == "research"]
|
|
check("cold-start throttle: item 03's research is suppressed once 2 unreviewed research files exist",
|
|
"03-c" not in research_candidates, research_candidates)
|
|
draft_candidates = [c["slug"] for c in tqueue if c["work_type"] == "draft"]
|
|
check("cold-start throttle: draft candidates for 01/02 unaffected by the research throttle",
|
|
set(draft_candidates) == {"01-a", "02-b"}, draft_candidates)
|
|
|
|
# sample_override: throttle bypassed, sampling state untouched
|
|
override_config = dict(throttle_config)
|
|
override_config["sample_override"] = True
|
|
oqueue = m.build_ready_queue(throttle_repo, override_config, throttle_state, set())
|
|
override_research = [c["slug"] for c in oqueue if c["work_type"] == "research"]
|
|
check("sample_override: item 03's research is offered despite the dossier being at the cap",
|
|
"03-c" in override_research, override_research)
|
|
check("sample_override: research_sampled state is not mutated by queue building",
|
|
not any(m.get_item_state(throttle_state, f"ai/{s}")["research_sampled"]
|
|
for s in ("01-a", "02-b", "03-c")), throttle_state)
|
|
|
|
# topic_ideas: only offered when nothing else is eligible
|
|
idle_repo = tmpdir / "idle-repo"
|
|
(idle_repo / "ai").mkdir(parents=True, exist_ok=True)
|
|
(idle_repo / "ai" / "01-done.overview.md").write_text("# done\n")
|
|
(idle_repo / "ai" / "01-done.agent.md").write_text("# brief\n")
|
|
idle_state = {"items": {"ai/01-done": {"approved": True}}}
|
|
iqueue = m.build_ready_queue(idle_repo, config, idle_state, set())
|
|
check("topic_ideas: offered when no other work is eligible",
|
|
len(iqueue) == 1 and iqueue[0]["work_type"] == "topic_ideas", iqueue)
|
|
|
|
(idle_repo / "ai" / "TOPIC-PROPOSALS.md").write_text("\n".join(f"## Idea {i}" for i in range(6)))
|
|
capped_config = dict(config)
|
|
capped_config["max_open_topic_proposals"] = 6
|
|
iqueue2 = m.build_ready_queue(idle_repo, capped_config, idle_state, set())
|
|
check("topic_ideas: suppressed once max_open_topic_proposals reached",
|
|
len(iqueue2) == 0, iqueue2)
|
|
|
|
# --- Credential parity ---
|
|
ok, why = m._credential_parity_from_values("secret-abc", "secret-abc")
|
|
check("credential parity: matching values -> True", ok is True, why)
|
|
ok, why = m._credential_parity_from_values("secret-abc", "secret-xyz")
|
|
check("credential parity: mismatched values -> False", ok is False, why)
|
|
ok, why = m._credential_parity_from_values("", "secret-abc")
|
|
check("credential parity: empty profile value -> False", ok is False, why)
|
|
|
|
cred_repo = tmpdir / "cred"
|
|
cred_repo.mkdir(exist_ok=True)
|
|
profile_dir = cred_repo / "profile"
|
|
profile_dir.mkdir(exist_ok=True)
|
|
key_file = cred_repo / "key.txt"
|
|
key_file.write_text("shared-secret-value\n")
|
|
(profile_dir / "provider.env").write_text(f"ANTHROPIC_BASE_URL=https://example.invalid\nANTHROPIC_API_KEY_FILE={key_file}\n")
|
|
resolved = m.resolve_profile_credential_value("minimax", profile_dir)
|
|
check("credential parity: resolves ANTHROPIC_API_KEY_FILE from provider.env", resolved == "shared-secret-value", resolved)
|
|
|
|
ok, why = m.credential_parity("minimax", profile_dir, probe_credential_fn=lambda: "shared-secret-value")
|
|
check("credential parity: full check matches via injected probe reader", ok is True, why)
|
|
ok, why = m.credential_parity("minimax", profile_dir, probe_credential_fn=lambda: "different-secret")
|
|
check("credential parity: full check flags mismatch via injected probe reader", ok is False, why)
|
|
|
|
# --- provider.env parsing (replicates claude-profile) ---
|
|
parsed = m.parse_provider_env(profile_dir / "provider.env")
|
|
check("provider.env: base_url parsed", parsed["base_url"] == "https://example.invalid", parsed)
|
|
check("provider.env: api_key_file parsed", parsed["api_key_file"] == str(key_file), parsed)
|
|
|
|
extra_env_dir = cred_repo / "profile-extra"
|
|
extra_env_dir.mkdir(exist_ok=True)
|
|
(extra_env_dir / "provider.env").write_text("MODEL_ID=claude-haiku-4-5-20251001\nSOME_EXTRA_VAR=hello\n")
|
|
parsed2 = m.parse_provider_env(extra_env_dir / "provider.env")
|
|
check("provider.env: MODEL_ID recognised", parsed2["model_id"] == "claude-haiku-4-5-20251001", parsed2)
|
|
check("provider.env: unrecognised key falls into extra", parsed2["extra"].get("SOME_EXTRA_VAR") == "hello", parsed2)
|
|
|
|
env = m.build_child_env(profile_dir, base_env={})
|
|
check("build_child_env: CLAUDE_CONFIG_DIR set", env["CLAUDE_CONFIG_DIR"] == str(profile_dir), env)
|
|
check("build_child_env: ANTHROPIC_BASE_URL exported from provider.env", env.get("ANTHROPIC_BASE_URL") == "https://example.invalid", env)
|
|
check("build_child_env: ANTHROPIC_API_KEY read from key file", env.get("ANTHROPIC_API_KEY") == "shared-secret-value", env)
|
|
|
|
no_provider_env_dir = cred_repo / "profile-plain"
|
|
no_provider_env_dir.mkdir(exist_ok=True)
|
|
env2 = m.build_child_env(no_provider_env_dir, base_env={})
|
|
check("build_child_env: profile without provider.env gets only CLAUDE_CONFIG_DIR",
|
|
"ANTHROPIC_BASE_URL" not in env2 and "ANTHROPIC_API_KEY" not in env2 and env2["CLAUDE_CONFIG_DIR"] == str(no_provider_env_dir),
|
|
env2)
|
|
|
|
# --- allowed_tools / --allowedTools argv ---
|
|
argv = m.build_claude_argv("model-x", 25, ["/tmp/ev"], ["WebSearch", "WebFetch"])
|
|
check("build_claude_argv: appends --allowedTools with comma-joined list when provided",
|
|
"--allowedTools" in argv and argv[argv.index("--allowedTools") + 1] == "WebSearch,WebFetch", argv)
|
|
|
|
argv_empty = m.build_claude_argv("model-x", 25, ["/tmp/ev"], [])
|
|
check("build_claude_argv: omits --allowedTools when allowed_tools is an empty list", "--allowedTools" not in argv_empty, argv_empty)
|
|
|
|
argv_none = m.build_claude_argv("model-x", 25, ["/tmp/ev"], None)
|
|
check("build_claude_argv: omits --allowedTools when allowed_tools is None", "--allowedTools" not in argv_none, argv_none)
|
|
|
|
cfg_at = {
|
|
"work_types": {
|
|
"research": {"providers": ["anthropic"], "allowed_tools": ["WebSearch", "WebFetch"]},
|
|
"draft": {"providers": ["anthropic"]},
|
|
}
|
|
}
|
|
check("resolve_allowed_tools: returns configured list for research",
|
|
m.resolve_allowed_tools(cfg_at, "research") == ["WebSearch", "WebFetch"], m.resolve_allowed_tools(cfg_at, "research"))
|
|
check("resolve_allowed_tools: returns [] when work_type has no allowed_tools key",
|
|
m.resolve_allowed_tools(cfg_at, "draft") == [], m.resolve_allowed_tools(cfg_at, "draft"))
|
|
check("resolve_allowed_tools: returns [] for a work_type not present in config at all",
|
|
m.resolve_allowed_tools(cfg_at, "review") == [], m.resolve_allowed_tools(cfg_at, "review"))
|
|
|
|
bad_allowed_cfg_path = tmpdir / "bad-allowed-direct.config.json"
|
|
bad_allowed_cfg = json.loads((tmpdir / "idle-draft.config.json").read_text())
|
|
bad_allowed_cfg["work_types"]["research"]["allowed_tools"] = "WebSearch"
|
|
bad_allowed_cfg_path.write_text(json.dumps(bad_allowed_cfg))
|
|
try:
|
|
m.load_config(bad_allowed_cfg_path)
|
|
check("load_config: allowed_tools as a bare string (not a list) raises ConfigError", False)
|
|
except m.ConfigError as e:
|
|
check("load_config: allowed_tools as a bare string (not a list) raises ConfigError", "allowed_tools" in str(e), str(e))
|
|
|
|
bad_allowed_cfg2_path = tmpdir / "bad-allowed-direct2.config.json"
|
|
bad_allowed_cfg2 = json.loads((tmpdir / "idle-draft.config.json").read_text())
|
|
bad_allowed_cfg2["work_types"]["research"]["allowed_tools"] = ["WebSearch", 5]
|
|
bad_allowed_cfg2_path.write_text(json.dumps(bad_allowed_cfg2))
|
|
try:
|
|
m.load_config(bad_allowed_cfg2_path)
|
|
check("load_config: allowed_tools list containing a non-string raises ConfigError", False)
|
|
except m.ConfigError as e:
|
|
check("load_config: allowed_tools list containing a non-string raises ConfigError", "allowed_tools" in str(e), str(e))
|
|
|
|
good_allowed_cfg = m.load_config(tmpdir / "idle-draft.config.json")
|
|
check("load_config: well-formed allowed_tools (list of strings) loads without error",
|
|
good_allowed_cfg["work_types"]["research"]["allowed_tools"] == ["WebSearch", "WebFetch"],
|
|
good_allowed_cfg["work_types"]["research"])
|
|
|
|
# --- per-work-type effort / model: argv construction ---
|
|
argv_effort = m.build_claude_argv("model-x", 25, [], None, "high")
|
|
check("build_claude_argv: appends --effort when given",
|
|
"--effort" in argv_effort and argv_effort[argv_effort.index("--effort") + 1] == "high", argv_effort)
|
|
|
|
argv_no_effort = m.build_claude_argv("model-x", 25, [], None, None)
|
|
check("build_claude_argv: omits --effort when not given", "--effort" not in argv_no_effort, argv_no_effort)
|
|
|
|
argv_model_and_effort = m.build_claude_argv("worktype-model", 25, [], None, "xhigh")
|
|
check("build_claude_argv: --model reflects whatever single value the caller resolved "
|
|
"(precedence already applied upstream) -- exactly one --model flag",
|
|
argv_model_and_effort.count("--model") == 1
|
|
and argv_model_and_effort[argv_model_and_effort.index("--model") + 1] == "worktype-model",
|
|
argv_model_and_effort)
|
|
|
|
argv_neither = m.build_claude_argv(None, 25, [], None, None)
|
|
check("build_claude_argv: omits both --model and --effort when neither is given",
|
|
"--model" not in argv_neither and "--effort" not in argv_neither, argv_neither)
|
|
|
|
# --- per-work-type effort / model: precedence resolution ---
|
|
check("resolve_effective_model: work-type model overrides profile MODEL_ID",
|
|
m.resolve_effective_model("profile-model", "worktype-model") == "worktype-model", "")
|
|
check("resolve_effective_model: falls back to profile MODEL_ID when no work-type model set",
|
|
m.resolve_effective_model("profile-model", None) == "profile-model", "")
|
|
check("resolve_effective_model: both absent -> None",
|
|
m.resolve_effective_model(None, None) is None, "")
|
|
|
|
cfg_em = {"work_types": {"research": {"providers": ["anthropic"], "effort": "medium", "model": "wt-model"},
|
|
"draft": {"providers": ["anthropic"]}}}
|
|
check("resolve_work_type_effort: returns configured value", m.resolve_work_type_effort(cfg_em, "research") == "medium", "")
|
|
check("resolve_work_type_effort: returns None when absent", m.resolve_work_type_effort(cfg_em, "draft") is None, "")
|
|
check("resolve_work_type_model: returns configured value", m.resolve_work_type_model(cfg_em, "research") == "wt-model", "")
|
|
check("resolve_work_type_model: returns None when absent", m.resolve_work_type_model(cfg_em, "draft") is None, "")
|
|
|
|
# --- per-work-type effort / model: config validation ---
|
|
bad_effort_cfg_path = tmpdir / "bad-effort-direct.config.json"
|
|
bad_effort_cfg = json.loads((tmpdir / "idle-draft.config.json").read_text())
|
|
bad_effort_cfg["work_types"]["research"]["effort"] = "ultra-mega"
|
|
bad_effort_cfg_path.write_text(json.dumps(bad_effort_cfg))
|
|
try:
|
|
m.load_config(bad_effort_cfg_path)
|
|
check("load_config: effort value outside CLAUDE_EFFORT_LEVELS raises ConfigError", False)
|
|
except m.ConfigError as e:
|
|
check("load_config: effort value outside CLAUDE_EFFORT_LEVELS raises ConfigError", "effort" in str(e), str(e))
|
|
|
|
bad_effort_type_path = tmpdir / "bad-effort-type.config.json"
|
|
bad_effort_type_cfg = json.loads((tmpdir / "idle-draft.config.json").read_text())
|
|
bad_effort_type_cfg["work_types"]["research"]["effort"] = 5
|
|
bad_effort_type_path.write_text(json.dumps(bad_effort_type_cfg))
|
|
try:
|
|
m.load_config(bad_effort_type_path)
|
|
check("load_config: non-string effort raises ConfigError", False)
|
|
except m.ConfigError as e:
|
|
check("load_config: non-string effort raises ConfigError", "effort" in str(e), str(e))
|
|
|
|
bad_model_path = tmpdir / "bad-model-direct.config.json"
|
|
bad_model_cfg = json.loads((tmpdir / "idle-draft.config.json").read_text())
|
|
bad_model_cfg["work_types"]["research"]["model"] = ""
|
|
bad_model_path.write_text(json.dumps(bad_model_cfg))
|
|
try:
|
|
m.load_config(bad_model_path)
|
|
check("load_config: empty-string model raises ConfigError", False)
|
|
except m.ConfigError as e:
|
|
check("load_config: empty-string model raises ConfigError", "model" in str(e), str(e))
|
|
|
|
bad_model_type_path = tmpdir / "bad-model-type.config.json"
|
|
bad_model_type_cfg = json.loads((tmpdir / "idle-draft.config.json").read_text())
|
|
bad_model_type_cfg["work_types"]["research"]["model"] = 42
|
|
bad_model_type_path.write_text(json.dumps(bad_model_type_cfg))
|
|
try:
|
|
m.load_config(bad_model_type_path)
|
|
check("load_config: non-string model raises ConfigError", False)
|
|
except m.ConfigError as e:
|
|
check("load_config: non-string model raises ConfigError", "model" in str(e), str(e))
|
|
|
|
good_effort_model_cfg = json.loads((tmpdir / "idle-draft.config.json").read_text())
|
|
good_effort_model_cfg["work_types"]["research"]["effort"] = "high"
|
|
good_effort_model_cfg["work_types"]["research"]["model"] = "claude-example-model"
|
|
good_effort_model_path = tmpdir / "good-effort-model.config.json"
|
|
good_effort_model_path.write_text(json.dumps(good_effort_model_cfg))
|
|
loaded_good = m.load_config(good_effort_model_path)
|
|
check("load_config: well-formed effort + model (in the valid set) loads without error",
|
|
loaded_good["work_types"]["research"]["effort"] == "high"
|
|
and loaded_good["work_types"]["research"]["model"] == "claude-example-model",
|
|
loaded_good["work_types"]["research"])
|
|
|
|
for level in sorted(m.CLAUDE_EFFORT_LEVELS):
|
|
per_level_cfg = json.loads((tmpdir / "idle-draft.config.json").read_text())
|
|
per_level_cfg["work_types"]["research"]["effort"] = level
|
|
per_level_path = tmpdir / f"effort-{level}.config.json"
|
|
per_level_path.write_text(json.dumps(per_level_cfg))
|
|
loaded = m.load_config(per_level_path)
|
|
check(f"load_config: effort level {level!r} (member of CLAUDE_EFFORT_LEVELS) loads without error",
|
|
loaded["work_types"]["research"]["effort"] == level, loaded["work_types"]["research"])
|
|
|
|
# --- per-provider effort / model overrides (within a work type) ---
|
|
|
|
check("provider_entry_name: plain string entry returns itself",
|
|
m.provider_entry_name("anthropic") == "anthropic", "")
|
|
check("provider_entry_name: object entry returns its 'provider' value",
|
|
m.provider_entry_name({"provider": "minimax", "effort": "high"}) == "minimax", "")
|
|
|
|
cfg_pp = {
|
|
"providers": {"anthropic": {}, "minimax": {}},
|
|
"work_types": {
|
|
"research": {
|
|
"providers": [
|
|
"anthropic",
|
|
{"provider": "minimax", "model": "minimax-model", "effort": "high"},
|
|
],
|
|
"effort": "medium",
|
|
"model": "worktype-model",
|
|
},
|
|
"draft": {"providers": ["anthropic"], "effort": "low"},
|
|
},
|
|
}
|
|
|
|
entry_a = m.find_provider_entry(cfg_pp, "research", "anthropic")
|
|
check("find_provider_entry: returns the plain-string entry for a string-form provider",
|
|
entry_a == "anthropic", entry_a)
|
|
entry_m = m.find_provider_entry(cfg_pp, "research", "minimax")
|
|
check("find_provider_entry: returns the dict entry for an object-form provider",
|
|
isinstance(entry_m, dict) and entry_m["provider"] == "minimax", entry_m)
|
|
entry_missing = m.find_provider_entry(cfg_pp, "research", "nonexistent")
|
|
check("find_provider_entry: returns None when the provider isn't in this work_type's list",
|
|
entry_missing is None, entry_missing)
|
|
|
|
# Precedence: per-provider entry > work-type-level > (caller supplies profile fallback separately)
|
|
check("resolve_work_type_effort: plain-string provider entry falls through to work-type-level effort",
|
|
m.resolve_work_type_effort(cfg_pp, "research", "anthropic") == "medium", "")
|
|
check("resolve_work_type_model: plain-string provider entry falls through to work-type-level model",
|
|
m.resolve_work_type_model(cfg_pp, "research", "anthropic") == "worktype-model", "")
|
|
check("resolve_work_type_effort: object-form provider entry's own effort wins over work-type-level",
|
|
m.resolve_work_type_effort(cfg_pp, "research", "minimax") == "high", "")
|
|
check("resolve_work_type_model: object-form provider entry's own model wins over work-type-level",
|
|
m.resolve_work_type_model(cfg_pp, "research", "minimax") == "minimax-model", "")
|
|
check("resolve_work_type_effort: provider_name omitted -> work-type-level value only (back-compat)",
|
|
m.resolve_work_type_effort(cfg_pp, "research") == "medium", "")
|
|
check("resolve_work_type_model: provider_name omitted -> work-type-level value only (back-compat)",
|
|
m.resolve_work_type_model(cfg_pp, "research") == "worktype-model", "")
|
|
check("resolve_work_type_effort: work type with no per-work-type or per-provider effort -> None",
|
|
m.resolve_work_type_effort(cfg_pp, "draft", "anthropic") == "low", "")
|
|
|
|
# Full chain including the ultimate profile-MODEL_ID fallback (resolve_effective_model),
|
|
# for a provider with NO per-provider or work-type-level model at all.
|
|
cfg_pp_no_wt_model = {
|
|
"providers": {"anthropic": {}},
|
|
"work_types": {"review": {"providers": ["anthropic"], "effort": "high"}},
|
|
}
|
|
wt_model = m.resolve_work_type_model(cfg_pp_no_wt_model, "review", "anthropic")
|
|
check("resolve_work_type_model: no per-provider, no work-type-level -> None (falls to profile next)",
|
|
wt_model is None, wt_model)
|
|
check("resolve_effective_model: full chain falls back to profile MODEL_ID when neither override is set",
|
|
m.resolve_effective_model("profile-model-id", wt_model) == "profile-model-id", "")
|
|
|
|
# --- per-provider effort / model overrides: dispatch_preview reflects the resolution ---
|
|
# dispatch_preview() takes a candidate with "provider" already selected -- this exercises
|
|
# the exact same resolution path run_task() uses, without needing a real subprocess or a
|
|
# provider to actually be gate-eligible (credential/gate selection is select_provider()'s
|
|
# job, tested separately; dispatch_preview only needs a chosen provider name).
|
|
pp_repo = tmpdir / "pp-repo"
|
|
(pp_repo / "ai").mkdir(parents=True, exist_ok=True)
|
|
(pp_repo / "style").mkdir(parents=True, exist_ok=True)
|
|
(pp_repo / "ai" / "01-a.overview.md").write_text("# a\n")
|
|
(pp_repo / "ai" / "01-a.agent.md").write_text("# brief\n")
|
|
(pp_repo / "ai" / "SOURCE-REGISTER.md").write_text("# reg\n")
|
|
(pp_repo / "AGENTS.md").write_text("# root\n")
|
|
(pp_repo / "ai" / "AGENTS.md").write_text("# ai\n")
|
|
|
|
pp_profile_anthropic = tmpdir / "pp-profiles" / "anthropic"
|
|
pp_profile_minimax = tmpdir / "pp-profiles" / "minimax"
|
|
pp_profile_anthropic.mkdir(parents=True, exist_ok=True)
|
|
pp_profile_minimax.mkdir(parents=True, exist_ok=True)
|
|
(pp_profile_anthropic / "provider.env").write_text("MODEL_ID=anthropic-profile-model\n")
|
|
# minimax profile deliberately has no provider.env -- its --model must come entirely
|
|
# from the per-provider config override, not any profile fallback.
|
|
|
|
pp_config = {
|
|
"providers": {
|
|
"anthropic": {"profile": str(pp_profile_anthropic), "threshold_pct": 80,
|
|
"five_hour_ceiling": 50, "min_idle": 5},
|
|
"minimax": {"profile": str(pp_profile_minimax), "threshold_pct": 80,
|
|
"five_hour_ceiling": 50, "min_idle": 5},
|
|
},
|
|
"work_types": {
|
|
"research": {
|
|
"providers": [
|
|
"anthropic",
|
|
{"provider": "minimax", "model": "minimax-pinned-model", "effort": "xhigh"},
|
|
],
|
|
"effort": "medium",
|
|
"model": "worktype-fallback-model",
|
|
},
|
|
},
|
|
"dossiers": ["ai"],
|
|
"review_score_threshold": 11,
|
|
"max_unreviewed_research_per_dossier": 3,
|
|
"max_open_topic_proposals": 6,
|
|
"evidence_dirs": [],
|
|
}
|
|
|
|
candidate_anthropic = {"dossier": "ai", "slug": "01-a", "work_type": "research", "provider": "anthropic"}
|
|
preview_anthropic = m.dispatch_preview(pp_repo, pp_config, candidate_anthropic, 25)
|
|
check("dispatch_preview: plain-string provider entry (anthropic) uses work-type-level model, not its own profile MODEL_ID",
|
|
"worktype-fallback-model" in preview_anthropic["argv"]
|
|
and "anthropic-profile-model" not in preview_anthropic["argv"], preview_anthropic["argv"])
|
|
check("dispatch_preview: plain-string provider entry (anthropic) uses work-type-level effort",
|
|
"medium" in preview_anthropic["argv"] and preview_anthropic["argv"].count("--effort") == 1,
|
|
preview_anthropic["argv"])
|
|
|
|
candidate_minimax = {"dossier": "ai", "slug": "01-a", "work_type": "research", "provider": "minimax"}
|
|
preview_minimax = m.dispatch_preview(pp_repo, pp_config, candidate_minimax, 25)
|
|
check("dispatch_preview: object-form provider entry (minimax) uses its own pinned model over the work-type-level one",
|
|
"minimax-pinned-model" in preview_minimax["argv"]
|
|
and "worktype-fallback-model" not in preview_minimax["argv"], preview_minimax["argv"])
|
|
check("dispatch_preview: object-form provider entry (minimax) uses its own pinned effort over the work-type-level one",
|
|
"xhigh" in preview_minimax["argv"] and "medium" not in preview_minimax["argv"], preview_minimax["argv"])
|
|
check("dispatch_preview: exactly one --model flag regardless of which provider was selected (mixed list)",
|
|
preview_anthropic["argv"].count("--model") == 1 and preview_minimax["argv"].count("--model") == 1, "")
|
|
|
|
# --- per-provider effort / model overrides: config validation ---
|
|
|
|
def _base_pp_cfg():
|
|
c = json.loads((tmpdir / "idle-draft.config.json").read_text())
|
|
return c
|
|
|
|
bad_provider_name_str_path = tmpdir / "bad-provider-name-str.config.json"
|
|
c = _base_pp_cfg()
|
|
c["work_types"]["research"]["providers"] = ["anthropic", "nonexistent-provider"]
|
|
bad_provider_name_str_path.write_text(json.dumps(c))
|
|
try:
|
|
m.load_config(bad_provider_name_str_path)
|
|
check("load_config: string-form providers entry naming an unknown provider raises ConfigError", False)
|
|
except m.ConfigError as e:
|
|
check("load_config: string-form providers entry naming an unknown provider raises ConfigError",
|
|
"unknown provider" in str(e) and "nonexistent-provider" in str(e), str(e))
|
|
|
|
bad_provider_name_obj_path = tmpdir / "bad-provider-name-obj.config.json"
|
|
c = _base_pp_cfg()
|
|
c["work_types"]["research"]["providers"] = ["anthropic", {"provider": "nonexistent-provider"}]
|
|
bad_provider_name_obj_path.write_text(json.dumps(c))
|
|
try:
|
|
m.load_config(bad_provider_name_obj_path)
|
|
check("load_config: object-form providers entry naming an unknown provider raises ConfigError", False)
|
|
except m.ConfigError as e:
|
|
check("load_config: object-form providers entry naming an unknown provider raises ConfigError",
|
|
"unknown provider" in str(e) and "nonexistent-provider" in str(e), str(e))
|
|
|
|
missing_provider_key_path = tmpdir / "missing-provider-key.config.json"
|
|
c = _base_pp_cfg()
|
|
c["work_types"]["research"]["providers"] = ["anthropic", {"effort": "high"}]
|
|
missing_provider_key_path.write_text(json.dumps(c))
|
|
try:
|
|
m.load_config(missing_provider_key_path)
|
|
check("load_config: object-form providers entry missing 'provider' key raises ConfigError", False)
|
|
except m.ConfigError as e:
|
|
check("load_config: object-form providers entry missing 'provider' key raises ConfigError",
|
|
"provider" in str(e), str(e))
|
|
|
|
unknown_key_path = tmpdir / "unknown-provider-entry-key.config.json"
|
|
c = _base_pp_cfg()
|
|
c["work_types"]["research"]["providers"] = ["anthropic", {"provider": "minimax", "bogus_key": 1}]
|
|
unknown_key_path.write_text(json.dumps(c))
|
|
try:
|
|
m.load_config(unknown_key_path)
|
|
check("load_config: object-form providers entry with an unknown key raises ConfigError", False)
|
|
except m.ConfigError as e:
|
|
check("load_config: object-form providers entry with an unknown key raises ConfigError",
|
|
"unknown key" in str(e) and "bogus_key" in str(e), str(e))
|
|
|
|
bad_entry_effort_path = tmpdir / "bad-provider-entry-effort.config.json"
|
|
c = _base_pp_cfg()
|
|
c["work_types"]["research"]["providers"] = ["anthropic", {"provider": "minimax", "effort": "ultra-mega"}]
|
|
bad_entry_effort_path.write_text(json.dumps(c))
|
|
try:
|
|
m.load_config(bad_entry_effort_path)
|
|
check("load_config: object-form providers entry with invalid effort level raises ConfigError", False)
|
|
except m.ConfigError as e:
|
|
check("load_config: object-form providers entry with invalid effort level raises ConfigError",
|
|
"effort" in str(e) and "minimax" in str(e), str(e))
|
|
|
|
bad_entry_model_path = tmpdir / "bad-provider-entry-model.config.json"
|
|
c = _base_pp_cfg()
|
|
c["work_types"]["research"]["providers"] = ["anthropic", {"provider": "minimax", "model": ""}]
|
|
bad_entry_model_path.write_text(json.dumps(c))
|
|
try:
|
|
m.load_config(bad_entry_model_path)
|
|
check("load_config: object-form providers entry with empty model raises ConfigError", False)
|
|
except m.ConfigError as e:
|
|
check("load_config: object-form providers entry with empty model raises ConfigError",
|
|
"model" in str(e) and "minimax" in str(e), str(e))
|
|
|
|
bad_entry_type_path = tmpdir / "bad-provider-entry-type.config.json"
|
|
c = _base_pp_cfg()
|
|
c["work_types"]["research"]["providers"] = ["anthropic", 5]
|
|
bad_entry_type_path.write_text(json.dumps(c))
|
|
try:
|
|
m.load_config(bad_entry_type_path)
|
|
check("load_config: providers entry that is neither string nor object raises ConfigError", False)
|
|
except m.ConfigError as e:
|
|
check("load_config: providers entry that is neither string nor object raises ConfigError",
|
|
"string or object" in str(e), str(e))
|
|
|
|
good_mixed_path = tmpdir / "good-mixed-providers.config.json"
|
|
c = _base_pp_cfg()
|
|
c["work_types"]["research"]["providers"] = ["anthropic", {"provider": "minimax", "model": "m-model", "effort": "high"}]
|
|
good_mixed_path.write_text(json.dumps(c))
|
|
loaded_mixed = m.load_config(good_mixed_path)
|
|
check("load_config: well-formed mixed (string + object) providers list loads without error",
|
|
loaded_mixed["work_types"]["research"]["providers"][0] == "anthropic"
|
|
and loaded_mixed["work_types"]["research"]["providers"][1] == {"provider": "minimax", "model": "m-model", "effort": "high"},
|
|
loaded_mixed["work_types"]["research"]["providers"])
|
|
|
|
# The shipped example config itself demonstrates the mixed form -- confirm it validates
|
|
# and that select_provider() still walks the list in order via provider_entry_name().
|
|
example_cfg = m.load_config(m.REPO_SELF / "data" / "idle-draft" / "config.example.json")
|
|
example_research_providers = example_cfg["work_types"]["research"]["providers"]
|
|
check("config.example.json: loads without error (mixed providers list included)", True, "")
|
|
check("config.example.json: research providers mixes a plain string (anthropic) and an object (minimax)",
|
|
example_research_providers[0] == "anthropic"
|
|
and isinstance(example_research_providers[1], dict)
|
|
and example_research_providers[1]["provider"] == "minimax"
|
|
and example_research_providers[1]["effort"] == "medium",
|
|
example_research_providers)
|
|
check("provider_entry_name: walks config.example.json's mixed research providers list in order",
|
|
[m.provider_entry_name(e) for e in example_research_providers] == ["anthropic", "minimax"],
|
|
example_research_providers)
|
|
|
|
# --- Citation validation ---
|
|
real_path = cred_repo / "exists.txt"
|
|
real_path.write_text("x")
|
|
good_text = f"# Research: topic\n\nSee {real_path} for detail.\n"
|
|
ok, why = m.validate_output("research", good_text)
|
|
check("citation validation: accepts research output with an existing cited path", ok is True, why)
|
|
|
|
bad_text = "# Research: topic\n\nSee /home/nonexistent-user/definitely-not-here.md for detail.\n"
|
|
ok, why = m.validate_output("research", bad_text)
|
|
check("citation validation: rejects research output citing a dead path", ok is False, why)
|
|
|
|
ranged_good_text = f"# Research: topic\n\nSee {real_path}:12-18 for detail.\n"
|
|
ok, why = m.validate_output("research", ranged_good_text)
|
|
check("citation validation: accepts existing path cited with a :12-18 line range", ok is True, why)
|
|
|
|
bare_good_text = f"# Research: topic\n\nSee {real_path} for detail.\n"
|
|
ok, why = m.validate_output("research", bare_good_text)
|
|
check("citation validation: accepts existing path cited bare (no line range)", ok is True, why)
|
|
|
|
ranged_bad_text = "# Research: topic\n\nSee /home/nonexistent-user/definitely-not-here.md:1-20 for detail.\n"
|
|
ok, why = m.validate_output("research", ranged_bad_text)
|
|
check("citation validation: rejects nonexistent path cited with a :1-20 line range", ok is False, why)
|
|
|
|
bare_bad_text = "# Research: topic\n\nSee /home/nonexistent-user/definitely-not-here.md for detail.\n"
|
|
ok, why = m.validate_output("research", bare_bad_text)
|
|
check("citation validation: rejects nonexistent path cited bare (no line range)", ok is False, why)
|
|
|
|
noisy_text = f"# Research: topic\n\nSee ({real_path}), also `{real_path}`, and {real_path}, again.\n"
|
|
ok, why = m.validate_output("research", noisy_text)
|
|
check("citation validation: accepts existing path with trailing comma/backtick noise", ok is True, why)
|
|
|
|
ok, why = m.validate_output("research", "")
|
|
check("citation validation: rejects empty output", ok is False, why)
|
|
|
|
ok, why = m.validate_output("research", "no heading here\njust text\n")
|
|
check("citation validation: rejects output missing a top-level heading", ok is False, why)
|
|
|
|
ok, why = m.validate_output("topic_ideas", "## A proposal\n\nBody.\n")
|
|
check("citation validation: topic_ideas accepts ## as its top-level heading", ok is True, why)
|
|
|
|
ok, why = m.validate_output("topic_ideas", "# Wrong heading level\n")
|
|
check("citation validation: topic_ideas rejects a single # heading", ok is False, why)
|
|
|
|
# --- State validation ---
|
|
try:
|
|
m.validate_state({"items": {}, "bogus": 1}, repo)
|
|
check("state validation: unknown top-level key raises", False)
|
|
except m.StateValidationError:
|
|
check("state validation: unknown top-level key raises", True)
|
|
|
|
try:
|
|
m.validate_state({"items": {"ai/does-not-exist": {}}}, repo)
|
|
check("state validation: item not resolving to overview.md raises", False)
|
|
except m.StateValidationError:
|
|
check("state validation: item not resolving to overview.md raises", True)
|
|
|
|
try:
|
|
m.validate_state({"items": {"ai/01-topic-one": {"attempts": {"bogus_type": 1}}}}, repo)
|
|
check("state validation: unknown attempts work_type raises", False)
|
|
except m.StateValidationError:
|
|
check("state validation: unknown attempts work_type raises", True)
|
|
|
|
try:
|
|
m.validate_state({"items": {"ai/01-topic-one": {"human_edit_done": "yes"}}}, repo)
|
|
check("state validation: wrong-typed bool field raises", False)
|
|
except m.StateValidationError:
|
|
check("state validation: wrong-typed bool field raises", True)
|
|
|
|
# Valid state should not raise
|
|
m.validate_state({"items": {"ai/01-topic-one": {"human_edit_done": True, "attempts": {"draft": 1}}}}, repo)
|
|
check("state validation: well-formed state passes", True)
|
|
|
|
# --- GAP 2b: unique tmp path per invocation ---
|
|
import os as _os
|
|
import time as _time
|
|
|
|
canonical = repo / "ai" / "03-topic-three.review.md"
|
|
tmp_a = m.tmp_output_path(canonical, pid=11111)
|
|
tmp_b = m.tmp_output_path(canonical, pid=22222)
|
|
check("tmp path uniqueness: two different pids produce two different tmp paths",
|
|
tmp_a != tmp_b and tmp_a.name == "03-topic-three.review.md.tmp.11111"
|
|
and tmp_b.name == "03-topic-three.review.md.tmp.22222", (str(tmp_a), str(tmp_b)))
|
|
|
|
tmp_default = m.tmp_output_path(canonical)
|
|
check("tmp path uniqueness: default pid is os.getpid()",
|
|
tmp_default.name == f"03-topic-three.review.md.tmp.{_os.getpid()}", str(tmp_default))
|
|
|
|
rejected = m.rejected_output_path(canonical)
|
|
check("rejected-path derivation is unaffected by the new tmp naming (derived from canonical, not tmp)",
|
|
rejected.name == "03-topic-three.review.md.rejected", str(rejected))
|
|
|
|
# --- GAP 2b: startup sweep of stale tmp files ---
|
|
sweep_repo = tmpdir / "sweep-repo"
|
|
(sweep_repo / "ai").mkdir(parents=True, exist_ok=True)
|
|
old_tmp = sweep_repo / "ai" / "01-a.research.md.tmp.111"
|
|
fresh_tmp = sweep_repo / "ai" / "02-b.research.md.tmp.222"
|
|
old_tmp.write_text("stale orphan output\n")
|
|
fresh_tmp.write_text("still-in-progress output\n")
|
|
now = _time.time()
|
|
task_timeout = 1800
|
|
old_mtime = now - task_timeout - 100 # older than the timeout -> stale
|
|
fresh_mtime = now - 10 # well within the timeout -> live orphan candidate, keep
|
|
_os.utime(old_tmp, (old_mtime, old_mtime))
|
|
_os.utime(fresh_tmp, (fresh_mtime, fresh_mtime))
|
|
|
|
deleted = m.sweep_stale_tmp_files(sweep_repo, ["ai"], task_timeout, now=now)
|
|
check("stale sweep: deletes a tmp file older than the task timeout",
|
|
old_tmp in deleted and not old_tmp.exists(), deleted)
|
|
check("stale sweep: leaves a tmp file younger than the task timeout untouched",
|
|
fresh_tmp not in deleted and fresh_tmp.exists(), deleted)
|
|
|
|
# A non-.tmp.<pid> file (e.g. the canonical output itself) must never be swept.
|
|
canonical_lookalike = sweep_repo / "ai" / "01-a.research.md"
|
|
canonical_lookalike.write_text("# Research: a\n")
|
|
_os.utime(canonical_lookalike, (old_mtime, old_mtime))
|
|
deleted2 = m.sweep_stale_tmp_files(sweep_repo, ["ai"], task_timeout, now=now)
|
|
check("stale sweep: never touches a file without a .tmp.<pid> suffix, however old",
|
|
canonical_lookalike.exists() and canonical_lookalike not in deleted2, canonical_lookalike)
|
|
|
|
# --- GAP 2a: child process-group tracking / signal handling (smoke tests) ---
|
|
# No real subprocess is spawned here -- just the bookkeeping primitives that
|
|
# the signal handler and run_task() rely on.
|
|
fake_pid = 2**30 # astronomically unlikely to collide with a real pid/pgid
|
|
m.register_child(fake_pid)
|
|
check("child tracking: register_child adds the pid to the live set", fake_pid in m.live_children(), m.live_children())
|
|
|
|
m.unregister_child(fake_pid)
|
|
check("child tracking: unregister_child removes the pid", fake_pid not in m.live_children(), m.live_children())
|
|
|
|
m.register_child(fake_pid)
|
|
m.register_child(fake_pid + 1)
|
|
try:
|
|
m.kill_all_children()
|
|
kill_all_ok = True
|
|
except Exception as e:
|
|
kill_all_ok = False
|
|
check("child tracking: kill_all_children on nonexistent pids raises nothing (ProcessLookupError swallowed)",
|
|
kill_all_ok, "")
|
|
check("child tracking: kill_all_children clears the registry", m.live_children() == set(), m.live_children())
|
|
|
|
m.install_signal_handlers()
|
|
import signal as _signal
|
|
check("signal handling: install_signal_handlers installs a non-default SIGTERM handler",
|
|
_signal.getsignal(_signal.SIGTERM) not in (_signal.SIG_DFL, _signal.SIG_IGN, None), "")
|
|
check("signal handling: install_signal_handlers installs a non-default SIGINT handler",
|
|
_signal.getsignal(_signal.SIGINT) not in (_signal.SIG_DFL, _signal.SIG_IGN, None), "")
|
|
# Restore defaults so this subprocess (which is about to exit anyway) doesn't
|
|
# leave anything surprising behind for the interpreter teardown.
|
|
_signal.signal(_signal.SIGTERM, _signal.SIG_DFL)
|
|
_signal.signal(_signal.SIGINT, _signal.SIG_DFL)
|
|
|
|
# --- apply_success_state: clears attempts for the succeeded work type only ---
|
|
success_state = {"items": {"ai/01-topic-one": {"human_edit_done": True, "attempts": {"draft": 1, "research": 2}}}}
|
|
success_candidate = {"kind": "item", "item_key": "ai/01-topic-one", "work_type": "draft"}
|
|
m.apply_success_state(success_state, success_candidate)
|
|
entry_after = success_state["items"]["ai/01-topic-one"]
|
|
check("apply_success_state: clears the succeeded work type's attempts key",
|
|
"draft" not in entry_after.get("attempts", {}), entry_after)
|
|
check("apply_success_state: leaves an unrelated work type's attempts counter untouched",
|
|
entry_after.get("attempts", {}).get("research") == 2, entry_after)
|
|
check("apply_success_state: leaves other item fields (e.g. human_edit_done) untouched",
|
|
entry_after.get("human_edit_done") is True, entry_after)
|
|
|
|
# When the succeeded work type was the item's only attempts entry, the whole
|
|
# `attempts` key is dropped (not left behind as an empty {}).
|
|
success_state2 = {"items": {"ai/01-topic-one": {"attempts": {"draft": 1}}}}
|
|
m.apply_success_state(success_state2, success_candidate)
|
|
entry_after2 = success_state2["items"]["ai/01-topic-one"]
|
|
check("apply_success_state: drops the attempts key entirely once it's empty",
|
|
"attempts" not in entry_after2, entry_after2)
|
|
|
|
# No prior attempts entry at all -> no-op, no KeyError, no spurious key created.
|
|
success_state3 = {"items": {"ai/01-topic-one": {}}}
|
|
m.apply_success_state(success_state3, success_candidate)
|
|
check("apply_success_state: no-op when the item has no attempts history",
|
|
success_state3["items"]["ai/01-topic-one"] == {}, success_state3)
|
|
|
|
# Result is well-formed state per validate_state (schema still satisfied post-clear).
|
|
try:
|
|
m.validate_state(success_state, repo)
|
|
validate_ok = True
|
|
except m.StateValidationError:
|
|
validate_ok = False
|
|
check("apply_success_state: resulting state still passes validate_state", validate_ok, success_state)
|
|
|
|
for status, name, detail in results:
|
|
print(f"{status}: {name}", detail if detail else "")
|
|
|
|
n_fail = sum(1 for s, _, _ in results if s == "PY-FAIL")
|
|
sys.exit(1 if n_fail else 0)
|
|
PYEOF
|
|
)
|
|
PY_EXIT=$?
|
|
|
|
echo "$PYOUT"
|
|
py_pass=$(echo "$PYOUT" | grep -c '^PY-PASS' || true)
|
|
py_fail=$(echo "$PYOUT" | grep -c '^PY-FAIL' || true)
|
|
PASS=$((PASS + py_pass))
|
|
FAIL=$((FAIL + py_fail))
|
|
|
|
echo ""
|
|
echo "Results: $PASS passed, $FAIL failed"
|
|
if [[ $FAIL -gt 0 ]]; then
|
|
exit 1
|
|
fi
|
|
printf "${GREEN}All tests passed.${RESET}\n"
|