131486e0ff
Default run: 40 passed + SKIP. CR789_IMPL_READY=1: 10 red until implementation. Claude-Session: https://claude.ai/code/session_01G3SkMRBRWXyiGuMGs8WVas
119 lines
7.3 KiB
Bash
Executable File
119 lines
7.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# test-review-loop-run.sh — dryrun/behaviour tests for review-loop-run (no launches).
|
|
set -uo pipefail
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"; REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
SCRIPT="$REPO_ROOT/scripts/review-loop-run"
|
|
GREEN='\033[0;32m'; RED='\033[0;31m'; NC='\033[0m'; PASS=0; FAIL=0
|
|
assert_contains() { if [[ "$3" == *"$2"* ]]; then printf " ${GREEN}PASS${NC} %s\n" "$1"; PASS=$((PASS+1)); else printf " ${RED}FAIL${NC} %s\n expected: %s\n" "$1" "$2"; FAIL=$((FAIL+1)); fi; }
|
|
assert_exit() { if [[ "$2" -eq "$3" ]]; then printf " ${GREEN}PASS${NC} %s\n" "$1"; PASS=$((PASS+1)); else printf " ${RED}FAIL${NC} %s (exit %s, expected %s)\n" "$1" "$3" "$2"; FAIL=$((FAIL+1)); fi; }
|
|
|
|
TMP=$(mktemp -d); trap 'rm -rf "$TMP"' EXIT
|
|
export REVIEW_LOOP_RUN_DIR="$TMP/reports"; export REVIEW_LOOP_BP_DIR="$TMP/bp"; mkdir -p "$TMP/bp"
|
|
echo "# bp" > "$TMP/bp/spec-driven-development.md"
|
|
echo "diff" > "$TMP/spec.diff"; echo "# item" > "$TMP/item.md"
|
|
|
|
echo "review-loop-run — dryrun tests"
|
|
out=$("$SCRIPT" launch --kind spec --name spec-X --file "$TMP/spec.diff" --context "closes bug 1234" -n 2>&1); rc=$?
|
|
assert_exit "spec dryrun exits 0" 0 "$rc"
|
|
assert_contains "dryrun banner" "[dryrun]" "$out"
|
|
assert_contains "nothing launched" "nothing launched" "$out"
|
|
assert_contains "task text is the spec template" "OUT OF SCOPE: reporting that a symbol" "$out"
|
|
assert_contains "context included" "Context: closes bug 1234" "$out"
|
|
assert_contains "output path uses tag" "spec-X-r1-result.md" "$out"
|
|
assert_contains "present default listed" "$TMP/bp/spec-driven-development.md" "$out"
|
|
assert_contains "absent default skipped" "default file absent, skipped" "$out"
|
|
assert_contains "launch uses setsid-safe script with timeout" "timeout 1500 ask-minimax-run" "$out"
|
|
[[ -d "$REVIEW_LOOP_RUN_DIR" ]] && { printf " ${RED}FAIL${NC} dryrun wrote report dir\n"; FAIL=$((FAIL+1)); } || { printf " ${GREEN}PASS${NC} dryrun writes nothing\n"; PASS=$((PASS+1)); }
|
|
|
|
out=$("$SCRIPT" launch --kind verify --name verify-1 --file "$TMP/item.md" -n 2>&1); rc=$?
|
|
assert_exit "verify dryrun exits 0" 0 "$rc"
|
|
assert_contains "verify allowlist has git show" "Bash(git show *)" "$out"
|
|
assert_contains "verify verdict format" "Verdict:" "$out"
|
|
|
|
out=$("$SCRIPT" launch --kind tests --name t --file "$TMP/item.md" --round 2 -n 2>&1); rc=$?
|
|
assert_exit "round 2 without prev still exits 0" 0 "$rc"
|
|
assert_contains "round 2 warning" "no previous result" "$out"
|
|
assert_contains "tests allowlist has pytest" "Bash(python3 -m pytest *)" "$out"
|
|
|
|
mkdir -p "$REVIEW_LOOP_RUN_DIR"; printf '| CRITICAL | 0 |\n| HIGH | 2 |\n' > "$REVIEW_LOOP_RUN_DIR/t-r1-result.md"
|
|
out=$("$SCRIPT" launch --kind tests --name t --file "$TMP/item.md" --round 2 -n 2>&1)
|
|
assert_contains "round 2 picks up previous result" "t-r1-result.md" "$out"
|
|
assert_contains "adjudication paragraph" "do not re-raise" "$out"
|
|
|
|
out=$("$SCRIPT" launch --kind bogus --name x --file "$TMP/item.md" -n 2>&1); rc=$?
|
|
assert_exit "unknown kind exits 2" 2 "$rc"
|
|
out=$("$SCRIPT" launch --kind spec --name x --file "$TMP/nope" -n 2>&1); rc=$?
|
|
assert_exit "missing file exits 2" 2 "$rc"
|
|
out=$("$SCRIPT" launch --kind spec --name "bad name" --file "$TMP/item.md" -n 2>&1); rc=$?
|
|
assert_exit "bad name exits 2" 2 "$rc"
|
|
|
|
echo "review-loop-run — counts"
|
|
printf '| CRITICAL | 0 |\n| HIGH | 0 |\n| MEDIUM | 3 |\n' > "$TMP/clean.md"
|
|
out=$("$SCRIPT" counts "$TMP/clean.md"); rc=$?
|
|
assert_exit "clean result exits 0" 0 "$rc"; assert_contains "clean counts" "CRITICAL=0 HIGH=0 MEDIUM=3" "$out"
|
|
printf -- '- **CRITICAL:** 1 (C1)\n- **HIGH:** 2\n' > "$TMP/dirty.md"
|
|
out=$("$SCRIPT" counts "$TMP/dirty.md"); rc=$?
|
|
assert_exit "dirty result exits 4" 4 "$rc"; assert_contains "list-style counts" "CRITICAL=1 HIGH=2" "$out"
|
|
printf '## CRITICAL findings\n### C1 CRITICAL x\n### H1 HIGH y\n' > "$TMP/heads.md"
|
|
out=$("$SCRIPT" counts "$TMP/heads.md"); rc=$?
|
|
assert_exit "heading fallback exits 4" 4 "$rc"
|
|
|
|
FIXTURES="$REPO_ROOT/tests/fixtures/review-loop-run"
|
|
out=$("$SCRIPT" counts "$FIXTURES/bullet-list.md"); rc=$?
|
|
assert_exit "bullet-list block exits 4" 4 "$rc"
|
|
assert_contains "bullet-list counts" "CRITICAL=0 HIGH=1 MEDIUM=2 LOW=0" "$out"
|
|
|
|
out=$("$SCRIPT" counts "$FIXTURES/table.md"); rc=$?
|
|
assert_exit "table block exits 0" 0 "$rc"
|
|
assert_contains "table counts" "CRITICAL=0 HIGH=0 MEDIUM=1 LOW=3" "$out"
|
|
|
|
out=$("$SCRIPT" counts "$FIXTURES/inline-bold.md"); rc=$?
|
|
assert_exit "inline bold exits 4" 4 "$rc"
|
|
assert_contains "inline bold counts" "CRITICAL=0 HIGH=2 MEDIUM=0 LOW=0" "$out"
|
|
|
|
out=$("$SCRIPT" counts "$FIXTURES/heading-numbers.md"); rc=$?
|
|
assert_exit "heading with HTTP 404 exits 0" 0 "$rc"
|
|
assert_contains "heading-numbers ignores 404/42, uses own count block" "CRITICAL=0 HIGH=0 MEDIUM=1 LOW=0" "$out"
|
|
|
|
out=$("$SCRIPT" counts "$FIXTURES/no-count-block.md"); rc=$?
|
|
assert_exit "no count block derives, exits 4" 4 "$rc"
|
|
assert_contains "no count block counts" "CRITICAL=1 HIGH=2 MEDIUM=0 LOW=0" "$out"
|
|
assert_contains "no count block marked derived" "(derived)" "$out"
|
|
|
|
echo "review-loop-run — counts (CR-7/CR-8/CR-9, bug 5352fe72)"
|
|
if [[ "${CR789_IMPL_READY:-0}" != "1" ]]; then
|
|
echo " SKIP CR-7/CR-8/CR-9 counts tests (set CR789_IMPL_READY=1 once implemented)"
|
|
else
|
|
out=$("$SCRIPT" counts "$FIXTURES/annotated-table-cell.md"); rc=$?
|
|
assert_contains "counts annotated table cell" "CRITICAL=0 HIGH=1 MEDIUM=2 LOW=0" "$out"
|
|
out=$("$SCRIPT" counts "$FIXTURES/annotated-table-cell.md"); rc=$?
|
|
assert_exit "counts annotated cell exits 4" 4 "$rc"
|
|
|
|
out=$("$SCRIPT" counts "$FIXTURES/prose-historic-quote.md"); rc=$?
|
|
assert_contains "counts prose historic quote ignored" "CRITICAL=0 HIGH=2 MEDIUM=0 LOW=0" "$out"
|
|
assert_exit "counts prose historic quote ignored exit 4" 4 "$rc"
|
|
|
|
out=$("$SCRIPT" counts "$FIXTURES/suspect-derived.md"); rc=$?
|
|
assert_contains "counts suspect derived zero marks suspect" "CRITICAL=0 HIGH=0 MEDIUM=0 LOW=0 (suspect)" "$out"
|
|
# companion handle (not in spec matrix): asserts the negative -- no (derived) marker on a suspect result
|
|
[[ "$out" == *"(derived)"* ]] && { printf " ${RED}FAIL${NC} %s\n unexpected: (derived) present\n" "counts suspect derived zero not marked derived"; FAIL=$((FAIL+1)); } || { printf " ${GREEN}PASS${NC} %s\n" "counts suspect derived zero not marked derived"; PASS=$((PASS+1)); }
|
|
out=$("$SCRIPT" counts "$FIXTURES/suspect-derived.md"); rc=$?
|
|
assert_exit "counts suspect derived zero exits 4" 4 "$rc"
|
|
|
|
out=$("$SCRIPT" counts "$FIXTURES/suspect-derived-lower.md"); rc=$?
|
|
assert_contains "counts suspect derived zero case insensitive" "CRITICAL=0 HIGH=0 MEDIUM=0 LOW=0 (suspect)" "$out"
|
|
assert_exit "counts suspect derived zero case insensitive exit 4" 4 "$rc"
|
|
|
|
out=$("$SCRIPT" counts "$FIXTURES/declaration-with-annotation.md"); rc=$?
|
|
assert_contains "counts declaration line annotation rejected" "CRITICAL=0 HIGH=0 MEDIUM=0 LOW=0 (derived)" "$out"
|
|
# companion handle (not in spec matrix): exit-code check for the same declaration-line-annotation-rejected case
|
|
assert_exit "counts declaration line annotation rejected exits 0" 0 "$rc"
|
|
fi
|
|
|
|
echo "review-loop-run — status"
|
|
printf 'x\nexit=0\n' > "$REVIEW_LOOP_RUN_DIR/t-r1-stdout.txt"
|
|
out=$("$SCRIPT" status --name t 2>&1); rc=$?
|
|
assert_exit "status exits 0" 0 "$rc"; assert_contains "status shows done" "done exit=0" "$out"; assert_contains "status shows counts" "HIGH=2" "$out"
|
|
|
|
echo; echo "$PASS passed, $FAIL failed"; [[ $FAIL -eq 0 ]]
|