feat: review-minimax-run + ci-triage-minimax-run MiniMax offload wrappers
First-pass branch review and CI failure triage in MiniMax M3 sub-sessions. Both materialise inputs themselves (no Bash in the sub-session), gate results strictly, retry once, and follow the explore-minimax 0/1/2/3 exit contract. Specs with RM-n/CT-n IDs; offline tests via fake seams (36 assertions). Verified live: planted-issue branch -> 5 confirmed findings, clean branch -> NO FINDINGS, failed gac-scratch run -> correct step + honest low confidence. Claude-Session: https://claude.ai/code/session_01Ku87DDWwbJkTx6SuV3sQDc
This commit is contained in:
Paul O'Reilly
106
tests/test-ci-triage-minimax-run.sh
Executable file
106
tests/test-ci-triage-minimax-run.sh
Executable file
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env bash
|
||||
# test-ci-triage-minimax-run.sh — Verify ci-triage-minimax-run against its spec.
|
||||
#
|
||||
# Strategy: CI_TRIAGE_FAKE_LOGS bypasses gac, CI_TRIAGE_FAKE_JSON /
|
||||
# CI_TRIAGE_FAKE_TRIAGE drive the sub-session paths, so exits 0/2/3,
|
||||
# redaction, and the failed_step handling are all exercised offline.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
SCRIPT="$REPO_ROOT/scripts/ci-triage-minimax-run"
|
||||
|
||||
GREEN='\033[0;32m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m'
|
||||
PASS=0
|
||||
FAIL=0
|
||||
|
||||
assert_contains() {
|
||||
local label=$1 needle=$2 haystack=$3
|
||||
if [[ "$haystack" == *"$needle"* ]]; then
|
||||
printf " ${GREEN}PASS${NC} %s\n" "$label"; PASS=$((PASS + 1))
|
||||
else
|
||||
printf " ${RED}FAIL${NC} %s\n" "$label"
|
||||
printf " expected to contain: %s\n" "$needle"; FAIL=$((FAIL + 1))
|
||||
fi
|
||||
}
|
||||
|
||||
assert_eq() {
|
||||
local label=$1 expected=$2 actual=$3
|
||||
if [[ "$expected" == "$actual" ]]; then
|
||||
printf " ${GREEN}PASS${NC} %s\n" "$label"; PASS=$((PASS + 1))
|
||||
else
|
||||
printf " ${RED}FAIL${NC} %s\n" "$label"
|
||||
printf " expected: %s actual: %s\n" "$expected" "$actual"; FAIL=$((FAIL + 1))
|
||||
fi
|
||||
}
|
||||
|
||||
TMP=$(mktemp -d /tmp/test-ci-triage.XXXXXX)
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
# gac-shaped NDJSON envelope + log lines, including a credential-shaped URL
|
||||
FAKE_LOGS="$TMP/logs.ndjson"
|
||||
cat > "$FAKE_LOGS" <<'EOF'
|
||||
{"job_id":1,"conclusion":"failure","failed_step":"Run tests","sliced":true,"note":"","markers":[],"lines":["cloning https://x-access-token:ghp_abcdEFGH1234@gitea.example/r.git","FAILED tests/test_x.py::test_y - AssertionError","exit code 1"]}
|
||||
EOF
|
||||
FAKE_LOGS_NOSTEP="$TMP/logs-nostep.ndjson"
|
||||
echo '{"job_id":1,"conclusion":"cancelled","failed_step":null,"sliced":false,"note":"","markers":[],"lines":["cancelled"]}' > "$FAKE_LOGS_NOSTEP"
|
||||
|
||||
FAKE_OK_JSON="$TMP/ok.json"
|
||||
echo '{"is_error":false,"result":"done"}' > "$FAKE_OK_JSON"
|
||||
FAKE_ERR_JSON="$TMP/err.json"
|
||||
echo '{"is_error":true,"result":"boom"}' > "$FAKE_ERR_JSON"
|
||||
FAKE_TRIAGE="$TMP/triage.md"
|
||||
printf '## Failing step\nRun tests\n## Error excerpt\nFAILED tests/test_x.py\n## Probable cause\nAssertion failure in test_y.\n## Suggested next action\nRe-run that test.\n## Confidence\nhigh\n' > "$FAKE_TRIAGE"
|
||||
FAKE_TRIAGE_EMPTY_CAUSE="$TMP/triage-empty.md"
|
||||
printf '## Failing step\nRun tests\n## Probable cause\n## Confidence\nlow\n' > "$FAKE_TRIAGE_EMPTY_CAUSE"
|
||||
|
||||
echo "== ci-triage-minimax-run: --help =="
|
||||
HELP_OUT="$("$SCRIPT" --help 2>&1)"
|
||||
assert_contains "help shows usage" "--repo" "$HELP_OUT"
|
||||
|
||||
echo "== usage errors (exit 1) =="
|
||||
"$SCRIPT" >/dev/null 2>&1; assert_eq "no args exits 1" 1 $?
|
||||
"$SCRIPT" --repo badformat --run 1 >/dev/null 2>&1; assert_eq "bad repo format exits 1" 1 $?
|
||||
"$SCRIPT" --repo o/r --run abc >/dev/null 2>&1; assert_eq "non-integer run exits 1" 1 $?
|
||||
|
||||
echo "== dryrun =="
|
||||
DRY_OUT="$("$SCRIPT" --repo skynet/gac-scratch --run 42 --dryrun 2>&1)"
|
||||
assert_eq "dryrun exits 0" 0 $?
|
||||
assert_contains "dryrun shows gac command" "inspect skynet/gac-scratch --run 42" "$DRY_OUT"
|
||||
assert_contains "dryrun shows model pin" "MiniMax-M3[1m]" "$DRY_OUT"
|
||||
assert_contains "dryrun defers fetch" "not fetched in dryrun" "$DRY_OUT"
|
||||
assert_contains "dryrun shows job passthrough default" "job: auto" "$DRY_OUT"
|
||||
|
||||
echo "== fake logs: redaction + failing step in prompt, exit 0 =="
|
||||
OUT0="$TMP/out0.md"
|
||||
RUN_OUT="$(CI_TRIAGE_FAKE_LOGS="$FAKE_LOGS" CI_TRIAGE_FAKE_JSON="$FAKE_OK_JSON" CI_TRIAGE_FAKE_TRIAGE="$FAKE_TRIAGE" \
|
||||
"$SCRIPT" --repo o/r --run 7 --out "$OUT0" 2>&1)"
|
||||
assert_eq "happy path exits 0" 0 $?
|
||||
assert_contains "triage file written" "ok: triage written" "$RUN_OUT"
|
||||
|
||||
echo "== fake logs: null failed_step still succeeds =="
|
||||
CI_TRIAGE_FAKE_LOGS="$FAKE_LOGS_NOSTEP" CI_TRIAGE_FAKE_JSON="$FAKE_OK_JSON" CI_TRIAGE_FAKE_TRIAGE="$FAKE_TRIAGE" \
|
||||
"$SCRIPT" --repo o/r --run 8 --out "$TMP/out-nostep.md" >/dev/null 2>&1
|
||||
assert_eq "null failed_step exits 0" 0 $?
|
||||
|
||||
echo "== gate: empty probable cause -> exit 3 =="
|
||||
CI_TRIAGE_FAKE_LOGS="$FAKE_LOGS" CI_TRIAGE_FAKE_JSON="$FAKE_OK_JSON" CI_TRIAGE_FAKE_TRIAGE="$FAKE_TRIAGE_EMPTY_CAUSE" \
|
||||
"$SCRIPT" --repo o/r --run 9 --out "$TMP/out3.md" >/dev/null 2>&1
|
||||
assert_eq "empty probable-cause exits 3" 3 $?
|
||||
|
||||
echo "== gate: no triage file -> exit 3 =="
|
||||
CI_TRIAGE_FAKE_LOGS="$FAKE_LOGS" CI_TRIAGE_FAKE_JSON="$FAKE_OK_JSON" \
|
||||
"$SCRIPT" --repo o/r --run 10 --out "$TMP/out3b.md" >/dev/null 2>&1
|
||||
assert_eq "missing triage file exits 3" 3 $?
|
||||
|
||||
echo "== is_error envelope -> exit 2 =="
|
||||
CI_TRIAGE_FAKE_LOGS="$FAKE_LOGS" CI_TRIAGE_FAKE_JSON="$FAKE_ERR_JSON" \
|
||||
"$SCRIPT" --repo o/r --run 11 --out "$TMP/out2.md" >/dev/null 2>&1
|
||||
assert_eq "is_error envelope exits 2" 2 $?
|
||||
|
||||
echo ""
|
||||
echo "ci-triage-minimax-run: $PASS passed, $FAIL failed"
|
||||
[[ "$FAIL" -eq 0 ]]
|
||||
139
tests/test-review-minimax-run.sh
Executable file
139
tests/test-review-minimax-run.sh
Executable file
@@ -0,0 +1,139 @@
|
||||
#!/usr/bin/env bash
|
||||
# test-review-minimax-run.sh — Verify review-minimax-run against its spec.
|
||||
#
|
||||
# Strategy: a throwaway git repo with a scratch branch provides real diffs;
|
||||
# REVIEW_MINIMAX_FAKE_JSON drives the sub-session paths so exits 0/2/3 and
|
||||
# the result gate are all exercised offline. No live MiniMax calls.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
SCRIPT="$REPO_ROOT/scripts/review-minimax-run"
|
||||
|
||||
GREEN='\033[0;32m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m'
|
||||
PASS=0
|
||||
FAIL=0
|
||||
|
||||
assert_contains() {
|
||||
local label=$1 needle=$2 haystack=$3
|
||||
if [[ "$haystack" == *"$needle"* ]]; then
|
||||
printf " ${GREEN}PASS${NC} %s\n" "$label"; PASS=$((PASS + 1))
|
||||
else
|
||||
printf " ${RED}FAIL${NC} %s\n" "$label"
|
||||
printf " expected to contain: %s\n" "$needle"; FAIL=$((FAIL + 1))
|
||||
fi
|
||||
}
|
||||
|
||||
assert_eq() {
|
||||
local label=$1 expected=$2 actual=$3
|
||||
if [[ "$expected" == "$actual" ]]; then
|
||||
printf " ${GREEN}PASS${NC} %s\n" "$label"; PASS=$((PASS + 1))
|
||||
else
|
||||
printf " ${RED}FAIL${NC} %s\n" "$label"
|
||||
printf " expected: %s actual: %s\n" "$expected" "$actual"; FAIL=$((FAIL + 1))
|
||||
fi
|
||||
}
|
||||
|
||||
# --- fixture repo -------------------------------------------------------------
|
||||
TMP=$(mktemp -d /tmp/test-review-minimax.XXXXXX)
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
REPO="$TMP/repo"
|
||||
mkdir -p "$REPO"
|
||||
git -C "$REPO" -c init.defaultBranch=main init -q
|
||||
git -C "$REPO" -c user.email=t@t -c user.name=t commit -q --allow-empty -m init
|
||||
echo "safe line" > "$REPO/app.sh"
|
||||
git -C "$REPO" add -A
|
||||
git -C "$REPO" -c user.email=t@t -c user.name=t commit -qm base
|
||||
git -C "$REPO" checkout -qb feature
|
||||
echo 'eval "$user_input"' >> "$REPO/app.sh"
|
||||
git -C "$REPO" add -A
|
||||
git -C "$REPO" -c user.email=t@t -c user.name=t commit -qm change
|
||||
|
||||
FAKE_OK_JSON="$TMP/ok.json"
|
||||
echo '{"is_error":false,"result":"done","usage":{"input_tokens":1,"output_tokens":1}}' > "$FAKE_OK_JSON"
|
||||
|
||||
echo "== review-minimax-run: --help =="
|
||||
HELP_OUT="$("$SCRIPT" --help 2>&1)"
|
||||
assert_contains "help shows usage" "--worktree" "$HELP_OUT"
|
||||
|
||||
echo "== usage errors (exit 1) =="
|
||||
"$SCRIPT" >/dev/null 2>&1; assert_eq "no args exits 1" 1 $?
|
||||
"$SCRIPT" --worktree /nonexistent >/dev/null 2>&1; assert_eq "missing worktree exits 1" 1 $?
|
||||
"$SCRIPT" --worktree "$TMP" >/dev/null 2>&1; assert_eq "non-git worktree exits 1" 1 $?
|
||||
|
||||
echo "== dryrun =="
|
||||
DRY_OUT="$("$SCRIPT" --worktree "$REPO" --dryrun 2>&1)"
|
||||
assert_eq "dryrun exits 0" 0 $?
|
||||
assert_contains "dryrun shows branch" "branch: feature" "$DRY_OUT"
|
||||
assert_contains "dryrun shows model pin" "MiniMax-M3[1m]" "$DRY_OUT"
|
||||
assert_contains "dryrun shows deny list" "Bash,WebFetch,WebSearch" "$DRY_OUT"
|
||||
assert_contains "dryrun prompt has findings format" "NO FINDINGS" "$DRY_OUT"
|
||||
assert_contains "dryrun shows checklist" "shell=True" "$DRY_OUT"
|
||||
|
||||
echo "== profile =="
|
||||
PROF_OUT="$("$SCRIPT" --worktree "$REPO" --profile agent-runtimes --dryrun 2>&1)"
|
||||
assert_contains "agent-runtimes profile adds test-bug checklist" "/v1/ prefix" "$PROF_OUT"
|
||||
NOPROF_OUT="$("$SCRIPT" --worktree "$REPO" --dryrun 2>&1)"
|
||||
if [[ "$NOPROF_OUT" != *"/v1/ prefix"* ]]; then
|
||||
printf " ${GREEN}PASS${NC} no profile omits repo-specific checklist\n"; PASS=$((PASS + 1))
|
||||
else
|
||||
printf " ${RED}FAIL${NC} no profile omits repo-specific checklist\n"; FAIL=$((FAIL + 1))
|
||||
fi
|
||||
|
||||
echo "== diff gates (exit 1) =="
|
||||
"$SCRIPT" --worktree "$REPO" --max-diff-lines 1 --dryrun >/dev/null 2>&1
|
||||
assert_eq "oversize diff exits 1" 1 $?
|
||||
git -C "$REPO" checkout -q main
|
||||
git -C "$REPO" checkout -qb empty-branch
|
||||
"$SCRIPT" --worktree "$REPO" --dryrun >/dev/null 2>&1
|
||||
assert_eq "empty diff exits 1" 1 $?
|
||||
git -C "$REPO" checkout -q feature
|
||||
|
||||
echo "== credential-shaped diff (exit 1) =="
|
||||
git -C "$REPO" checkout -qb cred-branch main
|
||||
echo 'url = https://x-access-token:tok@example.com/r.git' > "$REPO/cfg"
|
||||
git -C "$REPO" add -A
|
||||
git -C "$REPO" -c user.email=t@t -c user.name=t commit -qm cred
|
||||
CRED_OUT="$("$SCRIPT" --worktree "$REPO" --dryrun 2>&1)"
|
||||
assert_eq "credential-shaped diff exits 1" 1 $?
|
||||
assert_contains "credential message" "credential-shaped" "$CRED_OUT"
|
||||
git -C "$REPO" checkout -q feature
|
||||
|
||||
echo "== fake seam: no findings file -> exit 3 after retry =="
|
||||
OUT3="$TMP/findings3.md"
|
||||
REVIEW_MINIMAX_FAKE_JSON="$FAKE_OK_JSON" "$SCRIPT" --worktree "$REPO" --out "$OUT3" >/dev/null 2>&1
|
||||
assert_eq "missing findings file exits 3" 3 $?
|
||||
|
||||
echo "== fake seam: uncited findings -> exit 3 =="
|
||||
FAKE_BAD="$TMP/fake-bad-findings.md"
|
||||
printf '### [NIT] vague worry\n- Location: made/up/file.py:12\n- Confidence: suspected\n' > "$FAKE_BAD"
|
||||
REVIEW_MINIMAX_FAKE_JSON="$FAKE_OK_JSON" REVIEW_MINIMAX_FAKE_FINDINGS="$FAKE_BAD" \
|
||||
"$SCRIPT" --worktree "$REPO" --out "$TMP/findings4.md" >/dev/null 2>&1
|
||||
assert_eq "uncited findings exits 3" 3 $?
|
||||
|
||||
echo "== fake seam: cited findings -> exit 0 =="
|
||||
FAKE_GOOD="$TMP/fake-good-findings.md"
|
||||
printf '### [BLOCKER] eval on user input\n- Location: app.sh:2\n- Category: Injection\n- Confidence: confirmed\n' > "$FAKE_GOOD"
|
||||
REVIEW_MINIMAX_FAKE_JSON="$FAKE_OK_JSON" REVIEW_MINIMAX_FAKE_FINDINGS="$FAKE_GOOD" \
|
||||
"$SCRIPT" --worktree "$REPO" --out "$TMP/findings0.md" >/dev/null 2>&1
|
||||
assert_eq "cited findings exits 0" 0 $?
|
||||
|
||||
echo "== fake seam: NO FINDINGS -> exit 0 =="
|
||||
FAKE_CLEAN="$TMP/fake-clean.md"
|
||||
echo 'NO FINDINGS' > "$FAKE_CLEAN"
|
||||
REVIEW_MINIMAX_FAKE_JSON="$FAKE_OK_JSON" REVIEW_MINIMAX_FAKE_FINDINGS="$FAKE_CLEAN" \
|
||||
"$SCRIPT" --worktree "$REPO" --out "$TMP/findings-clean.md" >/dev/null 2>&1
|
||||
assert_eq "NO FINDINGS exits 0" 0 $?
|
||||
|
||||
echo "== fake seam: is_error envelope -> exit 2 =="
|
||||
FAKE_ERR_JSON="$TMP/err.json"
|
||||
echo '{"is_error":true,"result":"boom"}' > "$FAKE_ERR_JSON"
|
||||
REVIEW_MINIMAX_FAKE_JSON="$FAKE_ERR_JSON" "$SCRIPT" --worktree "$REPO" --out "$TMP/f5.md" >/dev/null 2>&1
|
||||
assert_eq "is_error envelope exits 2" 2 $?
|
||||
|
||||
echo ""
|
||||
echo "review-minimax-run: $PASS passed, $FAIL failed"
|
||||
[[ "$FAIL" -eq 0 ]]
|
||||
Reference in New Issue
Block a user