small-scripts/tests/test-bg-model-call.sh

#!/usr/bin/env bash
# test-bg-model-call.sh — Verify bg-model-call dryrun behaviour against the spec.
#
# Strategy: all assertions use --dryrun (-n), so no CP calls are made and
# no output files are written. Prompt files are created in a tempdir.

set -uo pipefail

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
BG_MODEL_CALL="$REPO_ROOT/scripts/bg-model-call"

GREEN='\033[0;32m'
RED='\033[0;31m'
NC='\033[0m'

PASS=0
FAIL=0

assert_contains() {
    local label=$1 needle=$2 haystack=$3
    if [[ "$haystack" == *"$needle"* ]]; then
        printf "  ${GREEN}PASS${NC}  %s\n" "$label"
        PASS=$((PASS + 1))
    else
        printf "  ${RED}FAIL${NC}  %s\n" "$label"
        printf "        expected to contain: %s\n" "$needle"
        printf "        actual output:\n"
        printf '%s\n' "$haystack" | sed 's/^/          /'
        FAIL=$((FAIL + 1))
    fi
}

assert_not_contains() {
    local label=$1 needle=$2 haystack=$3
    if [[ "$haystack" != *"$needle"* ]]; then
        printf "  ${GREEN}PASS${NC}  %s\n" "$label"
        PASS=$((PASS + 1))
    else
        printf "  ${RED}FAIL${NC}  %s\n" "$label"
        printf "        expected NOT to contain: %s\n" "$needle"
        FAIL=$((FAIL + 1))
    fi
}

assert_exit_code() {
    local label=$1 expected=$2 actual=$3
    if [[ "$expected" == "$actual" ]]; then
        printf "  ${GREEN}PASS${NC}  %s (exit=%s)\n" "$label" "$actual"
        PASS=$((PASS + 1))
    else
        printf "  ${RED}FAIL${NC}  %s (expected exit=%s, got exit=%s)\n" "$label" "$expected" "$actual"
        FAIL=$((FAIL + 1))
    fi
}

# Create a temp directory for prompt and output files
TMPDIR_TEST="$(mktemp -d)"
trap 'rm -rf "$TMPDIR_TEST"' EXIT

PROMPT_FILE="${TMPDIR_TEST}/prompt.txt"
OUTPUT_FILE="${TMPDIR_TEST}/result.txt"
echo "This is a test prompt for the spec review." > "$PROMPT_FILE"

run() {
    "$BG_MODEL_CALL" "$@" 2>&1
}

# === Pre-flight ===
if [[ ! -x "$BG_MODEL_CALL" ]]; then
    echo "bg-model-call not found or not executable at $BG_MODEL_CALL" >&2
    exit 1
fi

# === Test 1: --help ===
echo "Test 1: --help"
out=$(run --help)
rc=$?
assert_exit_code "--help exits 0" 0 "$rc"
assert_contains "--help shows script name" "bg-model-call" "$out"
assert_contains "--help shows --model" "--model" "$out"
assert_contains "--help shows --prompt" "--prompt" "$out"
assert_contains "--help shows --output" "--output" "$out"
assert_contains "--help shows --dryrun" "--dryrun" "$out"
assert_contains "--help shows --template" "--template" "$out"

# === Test 2: basic minimax dryrun ===
echo
echo "Test 2: minimax dryrun"
out=$(run --dryrun --model minimax --prompt "$PROMPT_FILE" --output "$OUTPUT_FILE")
rc=$?
assert_exit_code "minimax dryrun exits 0" 0 "$rc"
assert_contains "shows dryrun header" "[dryrun]" "$out"
assert_contains "shows model" "minimax" "$out"
assert_contains "shows resolved template" "minimax-planning" "$out"
assert_contains "shows CP URL default" "http://localhost:8100" "$out"
assert_contains "shows project ID default" "bg-model-call" "$out"
assert_contains "shows task name" "bg-model-call-minimax" "$out"
assert_contains "shows prompt file path" "$PROMPT_FILE" "$out"
assert_contains "shows output file path" "$OUTPUT_FILE" "$out"
assert_contains "shows sentinel path" "${OUTPUT_FILE}.done" "$out"
assert_contains "shows prompt size" "bytes" "$out"
assert_not_contains "no CP calls in dryrun" "Submitting task" "$out"

# === Test 3: sonnet dryrun — template mapping ===
echo
echo "Test 3: sonnet dryrun — template mapping"
out=$(run --dryrun --model sonnet --prompt "$PROMPT_FILE" --output "$OUTPUT_FILE")
rc=$?
assert_exit_code "sonnet dryrun exits 0" 0 "$rc"
assert_contains "sonnet maps to sonnet-code-repo" "sonnet-code-repo" "$out"
assert_contains "sonnet task name correct" "bg-model-call-sonnet" "$out"

# === Test 4: haiku dryrun — template mapping ===
echo
echo "Test 4: haiku dryrun — template mapping"
out=$(run --dryrun --model haiku --prompt "$PROMPT_FILE" --output "$OUTPUT_FILE")
rc=$?
assert_exit_code "haiku dryrun exits 0" 0 "$rc"
assert_contains "haiku maps to haiku-code-repo" "haiku-code-repo" "$out"

# === Test 5: opus dryrun — template mapping ===
echo
echo "Test 5: opus dryrun — template mapping"
out=$(run --dryrun --model opus --prompt "$PROMPT_FILE" --output "$OUTPUT_FILE")
rc=$?
assert_exit_code "opus dryrun exits 0" 0 "$rc"
assert_contains "opus maps to opus-code-repo" "opus-code-repo" "$out"

# === Test 6: --template override ===
echo
echo "Test 6: --template override"
out=$(run --dryrun --template minimax-planning --prompt "$PROMPT_FILE" --output "$OUTPUT_FILE")
rc=$?
assert_exit_code "--template override exits 0" 0 "$rc"
assert_contains "template override shown" "minimax-planning" "$out"
# No model line since only --template was given, not --model
assert_not_contains "no model line when using --template only" "Model:" "$out"

# === Test 7: custom --project and --task-name ===
echo
echo "Test 7: custom project and task name"
out=$(run --dryrun --model minimax --prompt "$PROMPT_FILE" --output "$OUTPUT_FILE" \
    --project my-project --task-name review-task)
rc=$?
assert_exit_code "custom project/task exits 0" 0 "$rc"
assert_contains "custom project shown" "my-project" "$out"
assert_contains "custom task name shown" "review-task" "$out"

# === Test 8: custom --url ===
echo
echo "Test 8: custom --url"
out=$(run --dryrun --model minimax --prompt "$PROMPT_FILE" --output "$OUTPUT_FILE" \
    --url "http://10.111.1.11:31800")
rc=$?
assert_exit_code "custom url exits 0" 0 "$rc"
assert_contains "custom URL shown" "http://10.111.1.11:31800" "$out"

# === Test 9: CP_URL env var ===
echo
echo "Test 9: CP_URL env var"
out=$(CP_URL="http://my-cp:9000" run --dryrun --model minimax --prompt "$PROMPT_FILE" --output "$OUTPUT_FILE")
rc=$?
assert_exit_code "CP_URL env var exits 0" 0 "$rc"
assert_contains "CP_URL env var used" "http://my-cp:9000" "$out"

# === Test 10: -n shorthand ===
echo
echo "Test 10: -n shorthand for dryrun"
out=$(run -n --model sonnet --prompt "$PROMPT_FILE" --output "$OUTPUT_FILE")
rc=$?
assert_exit_code "-n shorthand exits 0" 0 "$rc"
assert_contains "-n shows dryrun header" "[dryrun]" "$out"

# === Test 11: error — missing --model and --template ===
echo
echo "Test 11: error — missing --model and --template"
out=$(run --dryrun --prompt "$PROMPT_FILE" --output "$OUTPUT_FILE")
rc=$?
assert_exit_code "missing model exits 1" 1 "$rc"
assert_contains "missing model error message" "--model is required" "$out"

# === Test 12: error — missing --prompt ===
echo
echo "Test 12: error — missing --prompt"
out=$(run --dryrun --model minimax --output "$OUTPUT_FILE")
rc=$?
assert_exit_code "missing prompt exits 1" 1 "$rc"
assert_contains "missing prompt error" "--prompt is required" "$out"

# === Test 13: error — missing --output ===
echo
echo "Test 13: error — missing --output"
out=$(run --dryrun --model minimax --prompt "$PROMPT_FILE")
rc=$?
assert_exit_code "missing output exits 1" 1 "$rc"
assert_contains "missing output error" "--output is required" "$out"

# === Test 14: error — unknown model alias ===
echo
echo "Test 14: error — unknown model alias"
out=$(run --dryrun --model bogusmodel --prompt "$PROMPT_FILE" --output "$OUTPUT_FILE")
rc=$?
assert_exit_code "unknown model exits 1" 1 "$rc"
assert_contains "unknown model error" "Unknown model alias" "$out"
assert_contains "unknown model error lists valid values" "minimax" "$out"

# === Test 15: error — prompt file not found ===
echo
echo "Test 15: error — prompt file not found"
out=$(run --dryrun --model minimax --prompt "/tmp/definitely-does-not-exist-99999.txt" --output "$OUTPUT_FILE")
rc=$?
assert_exit_code "missing prompt file exits 1" 1 "$rc"
assert_contains "missing prompt file error" "Cannot read prompt file" "$out"

# === Test 16: error — output directory does not exist ===
echo
echo "Test 16: error — output directory does not exist"
out=$(run --dryrun --model minimax --prompt "$PROMPT_FILE" --output "/nonexistent/dir/result.txt")
rc=$?
assert_exit_code "missing output dir exits 1" 1 "$rc"
assert_contains "missing output dir error" "Output directory does not exist" "$out"

# === Test 17: error — invalid timeout ===
echo
echo "Test 17: error — invalid --timeout"
out=$(run --dryrun --model minimax --prompt "$PROMPT_FILE" --output "$OUTPUT_FILE" --timeout abc)
rc=$?
assert_exit_code "invalid timeout exits 1" 1 "$rc"
assert_contains "invalid timeout error" "--timeout must be a positive integer" "$out"

# === Test 18: error — invalid poll-interval ===
echo
echo "Test 18: error — invalid --poll-interval"
out=$(run --dryrun --model minimax --prompt "$PROMPT_FILE" --output "$OUTPUT_FILE" --poll-interval 0)
rc=$?
assert_exit_code "invalid poll-interval exits 1" 1 "$rc"
assert_contains "invalid poll-interval error" "--poll-interval must be a positive integer" "$out"

# === Test 19: custom timeout and poll-interval shown in dryrun ===
echo
echo "Test 19: custom timeout and poll-interval in dryrun"
out=$(run --dryrun --model minimax --prompt "$PROMPT_FILE" --output "$OUTPUT_FILE" \
    --timeout 3600 --poll-interval 30)
rc=$?
assert_exit_code "custom timeout/poll exits 0" 0 "$rc"
assert_contains "custom timeout shown" "3600s" "$out"
assert_contains "custom poll shown" "30s" "$out"

# === Test 20: unknown flag ===
echo
echo "Test 20: unknown flag"
out=$(run --dryrun --bogus-flag --model minimax --prompt "$PROMPT_FILE" --output "$OUTPUT_FILE")
rc=$?
assert_exit_code "unknown flag exits 1" 1 "$rc"
assert_contains "unknown flag error" "Unknown flag" "$out"

# === Summary ===
echo
echo "================================"
TOTAL=$((PASS + FAIL))
if [[ "$FAIL" -eq 0 ]]; then
    printf "${GREEN}All %d tests passed${NC}\n" "$TOTAL"
    exit 0
else
    printf "${RED}%d of %d tests failed${NC}\n" "$FAIL" "$TOTAL"
    exit 1
fi