16586541ea
- driver_model_id(): opus -> claude-opus-4-8, opus-1m -> claude-opus-4-8[1m] - driver_model_id(): add fable -> claude-fable-5 - error message updated: Valid: haiku sonnet opus opus-1m fable - spec: driver->model table updated (opus 4.8, fable added); frontmatter driver enum includes fable; note that opus-1m retained for [1m] convention - tests: add Test 13 (deep-fable dryrun asserting claude-fable-5); add Test 14 (unknown driver exits 1) — 47 assertions total Note: deep-fable mode file lands in the next commit
248 lines
8.1 KiB
Bash
Executable File
248 lines
8.1 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# test-claude-profile.sh — Verify claude-profile dryrun behaviour against the spec.
|
|
#
|
|
# Strategy: every assertion runs claude-profile --dryrun, so nothing is launched
|
|
# and no real profile state is touched. For preset testing, a fake profile
|
|
# directory is created under a tempdir-based HOME override.
|
|
|
|
set -uo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
CLAUDE_PROFILE="$REPO_ROOT/scripts/claude-profile"
|
|
|
|
GREEN='\033[0;32m'
|
|
RED='\033[0;31m'
|
|
NC='\033[0m'
|
|
|
|
PASS=0
|
|
FAIL=0
|
|
|
|
assert_contains() {
|
|
local label=$1 needle=$2 haystack=$3
|
|
if [[ "$haystack" == *"$needle"* ]]; then
|
|
printf " ${GREEN}PASS${NC} %s\n" "$label"
|
|
((PASS++))
|
|
else
|
|
printf " ${RED}FAIL${NC} %s\n" "$label"
|
|
printf " expected to contain: %s\n" "$needle"
|
|
printf " actual: %s\n" "$haystack"
|
|
((FAIL++))
|
|
fi
|
|
}
|
|
|
|
assert_not_contains() {
|
|
local label=$1 needle=$2 haystack=$3
|
|
if [[ "$haystack" != *"$needle"* ]]; then
|
|
printf " ${GREEN}PASS${NC} %s\n" "$label"
|
|
((PASS++))
|
|
else
|
|
printf " ${RED}FAIL${NC} %s\n" "$label"
|
|
printf " expected NOT to contain: %s\n" "$needle"
|
|
((FAIL++))
|
|
fi
|
|
}
|
|
|
|
assert_exit_code() {
|
|
local label=$1 expected=$2 actual=$3
|
|
if [[ "$expected" == "$actual" ]]; then
|
|
printf " ${GREEN}PASS${NC} %s (exit=%s)\n" "$label" "$actual"
|
|
((PASS++))
|
|
else
|
|
printf " ${RED}FAIL${NC} %s (expected exit=%s, got exit=%s)\n" "$label" "$expected" "$actual"
|
|
((FAIL++))
|
|
fi
|
|
}
|
|
|
|
run() {
|
|
"$CLAUDE_PROFILE" "$@" </dev/null 2>&1
|
|
}
|
|
|
|
# === Pre-flight ===
|
|
|
|
if [[ ! -x "$CLAUDE_PROFILE" ]]; then
|
|
echo "claude-profile not found or not executable at $CLAUDE_PROFILE" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# === Test 1: --help ===
|
|
echo "Test 1: --help"
|
|
out=$(run --help)
|
|
rc=$?
|
|
assert_exit_code "--help exits 0" 0 "$rc"
|
|
assert_contains "--help shows usage" "claude-profile" "$out"
|
|
assert_contains "--help shows --mode" "--mode" "$out"
|
|
assert_contains "--help shows --preset" "--preset" "$out"
|
|
|
|
# === Test 2: chat dryrun ===
|
|
echo
|
|
echo "Test 2: chat mode dryrun"
|
|
out=$(run --dryrun oreillyit-anthropic --mode chat)
|
|
rc=$?
|
|
assert_exit_code "chat dryrun exits 0" 0 "$rc"
|
|
assert_contains "chat shows haiku driver" "Driver: haiku" "$out"
|
|
assert_contains "chat shows model ID" "Model ID: claude-haiku-4-5-20251001" "$out"
|
|
assert_contains "chat shows mode tag" "Mode: chat" "$out"
|
|
assert_contains "chat says no changes" "No changes made" "$out"
|
|
assert_contains "chat shows model in exec command" "claude --model claude-haiku-4-5-20251001" "$out"
|
|
|
|
# === Test 3: deep dryrun with project flag ===
|
|
echo
|
|
echo "Test 3: deep mode dryrun"
|
|
out=$(run --dryrun oreillyit-anthropic --mode deep --project cluster-bootstrap)
|
|
rc=$?
|
|
assert_exit_code "deep dryrun exits 0" 0 "$rc"
|
|
assert_contains "deep shows sonnet driver" "Driver: sonnet" "$out"
|
|
assert_contains "deep shows model ID" "Model ID: claude-sonnet-4-6" "$out"
|
|
assert_contains "deep shows --model in command" "claude --model claude-sonnet-4-6" "$out"
|
|
assert_contains "deep records the project" "Project: cluster-bootstrap" "$out"
|
|
assert_contains "deep says async OK" "Async OK: yes" "$out"
|
|
assert_contains "deep says plan_mode_auto yes" "Plan mode auto: yes" "$out"
|
|
assert_contains "deep says spec_driven yes" "Spec driven: yes" "$out"
|
|
|
|
# === Test 4: orch dryrun preserves autoloop ===
|
|
echo
|
|
echo "Test 4: orch mode dryrun (autoloop preserved)"
|
|
out=$(run --dryrun oreillyit-anthropic --mode orch --project ai-image-gen)
|
|
rc=$?
|
|
assert_exit_code "orch dryrun exits 0" 0 "$rc"
|
|
assert_contains "orch preserves the slash command in autoloop" "/loop 2m /orchestrate" "$out"
|
|
assert_contains "orch driver is sonnet" "Driver: sonnet" "$out"
|
|
|
|
# === Test 5: hybrid dryrun ===
|
|
echo
|
|
echo "Test 5: hybrid mode dryrun"
|
|
out=$(run --dryrun oreillyit-anthropic --mode hybrid)
|
|
rc=$?
|
|
assert_exit_code "hybrid dryrun exits 0" 0 "$rc"
|
|
assert_contains "hybrid uses haiku driver" "Driver: haiku" "$out"
|
|
assert_contains "hybrid allows async" "Async OK: yes" "$out"
|
|
|
|
# === Test 6: quick dryrun ===
|
|
echo
|
|
echo "Test 6: quick mode dryrun"
|
|
out=$(run --dryrun oreillyit-anthropic --mode quick)
|
|
rc=$?
|
|
assert_exit_code "quick dryrun exits 0" 0 "$rc"
|
|
assert_contains "quick uses sonnet driver" "Driver: sonnet" "$out"
|
|
assert_contains "quick disables async" "Async OK: no" "$out"
|
|
|
|
# === Test 7: error — unknown mode ===
|
|
echo
|
|
echo "Test 7: error path — unknown mode"
|
|
out=$(run --dryrun oreillyit-anthropic --mode bogus)
|
|
rc=$?
|
|
assert_exit_code "unknown mode exits 1" 1 "$rc"
|
|
assert_contains "unknown mode error message" "Mode file not found" "$out"
|
|
|
|
# === Test 8: error — --preset and --mode together ===
|
|
echo
|
|
echo "Test 8: error path — --preset + --mode mutual exclusion"
|
|
out=$(run --dryrun --preset whatever --mode quick)
|
|
rc=$?
|
|
assert_exit_code "preset+mode exits 1" 1 "$rc"
|
|
assert_contains "preset+mode error message" "mutually exclusive" "$out"
|
|
|
|
# === Test 9: error — preset not found ===
|
|
echo
|
|
echo "Test 9: error path — preset not found"
|
|
out=$(run --dryrun --preset definitely-not-a-real-preset-name)
|
|
rc=$?
|
|
assert_exit_code "missing preset exits 1" 1 "$rc"
|
|
assert_contains "missing preset error message" "not found" "$out"
|
|
|
|
# === Test 10: pass-through args via -- ===
|
|
echo
|
|
echo "Test 10: pass-through args"
|
|
out=$(run --dryrun oreillyit-anthropic --mode chat -- --resume some-session)
|
|
rc=$?
|
|
assert_exit_code "pass-through exits 0" 0 "$rc"
|
|
assert_contains "pass-through preserves --resume" "--resume some-session" "$out"
|
|
|
|
# === Test 11: minimax profile picks up provider.env ===
|
|
echo
|
|
echo "Test 11: minimax profile uses provider.env"
|
|
out=$(run --dryrun oreillyit-minimax --mode quick)
|
|
rc=$?
|
|
assert_exit_code "minimax dryrun exits 0" 0 "$rc"
|
|
assert_contains "minimax shows MiniMax model ID" "Model ID: MiniMax-M2.7" "$out"
|
|
assert_contains "minimax shows provider env file" "Provider env:" "$out"
|
|
assert_contains "minimax shows model in exec command" "claude --model MiniMax-M2.7" "$out"
|
|
|
|
# === Test 12: error — unknown flag (--provider no longer valid) ===
|
|
echo
|
|
echo "Test 12: error path — unknown flag"
|
|
out=$(run --dryrun oreillyit-anthropic --mode quick --provider anthropic-sonnet)
|
|
rc=$?
|
|
assert_exit_code "unknown flag exits 1" 1 "$rc"
|
|
assert_contains "unknown flag error message" "Unknown flag" "$out"
|
|
|
|
# === Test 13: deep-fable dryrun (deep-fable mode file lands in the next commit) ===
|
|
echo
|
|
echo "Test 13: deep-fable mode dryrun"
|
|
out=$(run --dryrun oreillyit-anthropic --mode deep-fable)
|
|
rc=$?
|
|
assert_exit_code "deep-fable dryrun exits 0" 0 "$rc"
|
|
assert_contains "deep-fable uses fable driver" "Driver: fable" "$out"
|
|
assert_contains "deep-fable shows fable model ID" "Model ID: claude-fable-5" "$out"
|
|
assert_contains "deep-fable shows model in exec command" "claude --model claude-fable-5" "$out"
|
|
|
|
# === Test 14: error — unknown driver produces exit 1 ===
|
|
echo
|
|
echo "Test 14: error path — unknown driver"
|
|
# Create a temporary mode file with an unknown driver in the actual modes dir,
|
|
# run the test, then remove it immediately.
|
|
baddriver_file="$REPO_ROOT/data/claude-profile/modes/_test-baddriver.md"
|
|
cat > "$baddriver_file" <<'MEOF'
|
|
---
|
|
name: _test-baddriver
|
|
tag: bad
|
|
driver: notadriver
|
|
async_ok: no
|
|
autoloop: none
|
|
plan_mode_auto: no
|
|
spec_driven: no
|
|
escalates_to: none
|
|
---
|
|
|
|
# Mode: Bad Driver Test
|
|
## Purpose
|
|
Temporary test fixture — do not use.
|
|
## Driver constraint
|
|
Unused.
|
|
## Subagent policy
|
|
Unused.
|
|
## Reasoning posture
|
|
Unused.
|
|
## Async policy
|
|
Unused.
|
|
## Workflow stance
|
|
Unused.
|
|
## Auto-fire at session start
|
|
none
|
|
## Status line format
|
|
none
|
|
## Escalation triggers
|
|
none
|
|
## Out of scope
|
|
none
|
|
## On context wipe (/clear)
|
|
none
|
|
MEOF
|
|
out=$(run --dryrun oreillyit-anthropic --mode _test-baddriver 2>&1); rc=$?
|
|
rm -f "$baddriver_file"
|
|
assert_exit_code "unknown driver exits 1" 1 "$rc"
|
|
assert_contains "unknown driver error message" "Unknown driver" "$out"
|
|
|
|
# === Summary ===
|
|
echo
|
|
echo "================================"
|
|
TOTAL=$((PASS + FAIL))
|
|
if (( FAIL == 0 )); then
|
|
printf "${GREEN}All %d tests passed${NC}\n" "$TOTAL"
|
|
exit 0
|
|
else
|
|
printf "${RED}%d of %d tests failed${NC}\n" "$FAIL" "$TOTAL"
|
|
exit 1
|
|
fi
|