#!/usr/bin/env bash # Test spec-revocation-check against throwaway git repos. # # Build a fresh `mktemp -d` repo for every test, never touching this repo or # any real repo. Use `git -c commit.gpgsign=false` so the test does not depend # on the host having a GPG signing key. set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" SCRIPT="$SCRIPT_DIR/../scripts/spec-revocation-check" RED='\033[0;31m' GREEN='\033[0;32m' NC='\033[0m' pass=0 fail=0 assert_exit() { local desc="$1" expected="$2" actual="$3" if [[ "$expected" == "$actual" ]]; then echo -e "${GREEN}PASS${NC}: $desc" pass=$((pass + 1)) else echo -e "${RED}FAIL${NC}: $desc (expected exit $expected, got $actual)" echo " stdout: $4" echo " stderr: $5" fail=$((fail + 1)) fi } assert_contains() { local desc="$1" pattern="$2" actual="$3" if [[ "$actual" == *"$pattern"* ]]; then echo -e "${GREEN}PASS${NC}: $desc" pass=$((pass + 1)) else echo -e "${RED}FAIL${NC}: $desc" echo " expected substring: $pattern" echo " actual: $actual" fail=$((fail + 1)) fi } # --- Fixture helpers (shell-safe, written via Python) ------------------- # Common glob pattern used in most tests. GLOB='spec/**/*.md' mk_repo() { local d="$1" mkdir -p "$d" git -C "$d" init -q git -C "$d" -c commit.gpgsign=false config user.email "t@t.invalid" git -C "$d" -c commit.gpgsign=false config user.name "t" } # write_file_via_py — write content as a file via # Python so embedded quotes / newlines / backslashes survive intact. write_file_via_py() { local d="$1" rel="$2" content="$3" python3 - "$d" "$rel" "$content" <<'PYEOF' import os, sys d, rel, content = sys.argv[1], sys.argv[2], sys.argv[3] full = os.path.join(d, rel) os.makedirs(os.path.dirname(full), exist_ok=True) with open(full, "w") as f: f.write(content) PYEOF } commit_all() { local d="$1" msg="${2:-x}" git -C "$d" -c commit.gpgsign=false add -A git -C "$d" -c commit.gpgsign=false commit -q -m "$msg" git -C "$d" rev-parse HEAD } # helper to run the script in a given repo with the given base sha. # usage: run_check [extra args...] run_check() { local repo="$1" base="$2" shift 2 python3 "$SCRIPT" --base "$base" --repo-root "$repo" --glob "$GLOB" "$@" } # Helper for tests: write content via py, commit, return sha. commit_file() { local d="$1" rel="$2" content="$3" msg="${4:-base}" write_file_via_py "$d" "$rel" "$content" commit_all "$d" "$msg" } # --- Tests --------------------------------------------------------------- SCRATCH="$(mktemp -d)" trap 'rm -rf "$SCRATCH"' EXIT # ---- Test 1: clean amendment (exit 0) ----------------------------------- echo "=== clean amendment ===" T="$SCRATCH/t1"; mk_repo "$T" BASE=$(commit_file "$T" "spec/foo.md" \ $'# Spec\n\nA line that stays.\n\nA line that gets revoked.\n\n') write_file_via_py "$T" "spec/foo.md" \ $'# Spec\n\nA line that stays.\n\nRevokes prior text (TEST): "A line that gets revoked."\n\n' out=$(run_check "$T" "$BASE" --tag TEST); rc=$? assert_exit "clean amendment exits 0" 0 "$rc" "$out" "" assert_contains "clean amendment says clean" "clean" "$out" # ---- Test 2: unquoted deletion (exit 1, names the missing line) --------- echo "=== unquoted deletion ===" T="$SCRATCH/t2"; mk_repo "$T" commit_file "$T" "spec/foo.md" \ $'# Spec\n\nA line that stays.\n\nA line that gets revoked.\n\n' >/dev/null BASE=$(git -C "$T" rev-parse HEAD) write_file_via_py "$T" "spec/foo.md" $'# Spec\n\nA line that stays.\n\n' out=$(run_check "$T" "$BASE" --tag TEST); rc=$? assert_exit "unquoted deletion exits 1" 1 "$rc" "$out" "" assert_contains "unquoted deletion names path" "spec/foo.md" "$out" assert_contains "unquoted deletion names text" "A line that gets revoked" "$out" assert_contains "unquoted deletion under correct header" "unquoted deletions" "$out" # ---- Test 3: fabricated quote (exit 1, quotes non-existent text) -------- echo "=== fabricated quote ===" T="$SCRATCH/t3"; mk_repo "$T" commit_file "$T" "spec/foo.md" \ $'# Spec\n\nA line that stays.\n\nA line that gets revoked.\n\n' >/dev/null BASE=$(git -C "$T" rev-parse HEAD) write_file_via_py "$T" "spec/foo.md" \ $'# Spec\n\nA line that stays.\n\nRevokes prior text (TEST): "Text that was never in the base."\n\n' out=$(run_check "$T" "$BASE" --tag TEST); rc=$? assert_exit "fabricated quote exits 1" 1 "$rc" "$out" "" assert_contains "fabricated quote under correct header" "fabricated quotes" "$out" assert_contains "fabricated quote names quote text" "Text that was never in the base" "$out" # ---- Test 4: quote with bold dropped (exit 1) --------------------------- echo "=== quote with bold dropped ===" T="$SCRATCH/t4"; mk_repo "$T" commit_file "$T" "spec/foo.md" \ $'# Spec\n\nA line that stays.\n\n**Important** line that gets revoked.\n\n' >/dev/null BASE=$(git -C "$T" rev-parse HEAD) write_file_via_py "$T" "spec/foo.md" \ $'# Spec\n\nA line that stays.\n\nRevokes prior text (TEST): "Important line that gets revoked."\n\n' out=$(run_check "$T" "$BASE" --tag TEST); rc=$? assert_exit "quote with bold dropped exits 1" 1 "$rc" "$out" "" assert_contains "fabricated quote names dropped-bold" "fabricated quotes" "$out" # ---- Test 5: quote containing " (exit 0, greedy match handles it) ------- echo "=== quote containing double-quote ===" T="$SCRATCH/t5"; mk_repo "$T" # Base text contains literal '"' inside a quoted phrase. BASE=$(commit_file "$T" "spec/foo.md" \ $'# Spec\n\nA line that has a "quoted" word.\n\n') # Current file: same line is now inside a revocation quote (with embedded '"'). write_file_via_py "$T" "spec/foo.md" \ $'# Spec\n\nRevokes prior text (TEST): "A line that has a "quoted" word."\n\n' out=$(run_check "$T" "$BASE" --tag TEST); rc=$? assert_exit "quote with embedded quote exits 0" 0 "$rc" "$out" "" assert_contains "quote with embedded quote says clean" "clean" "$out" # ---- Test 6: malformed marker (exit 1) ---------------------------------- echo "=== malformed marker ===" T="$SCRATCH/t6"; mk_repo "$T" commit_file "$T" "spec/foo.md" \ $'# Spec\n\nA line that stays.\n\nA line that gets revoked.\n\n' >/dev/null BASE=$(git -C "$T" rev-parse HEAD) # Marker present but no closing quote (so the line fails to parse). write_file_via_py "$T" "spec/foo.md" \ $'# Spec\n\nA line that stays.\n\nRevokes prior text (TEST) no quote\n\n' out=$(run_check "$T" "$BASE" --tag TEST); rc=$? assert_exit "malformed marker exits 1" 1 "$rc" "$out" "" assert_contains "malformed marker under correct header" "malformed markers" "$out" assert_contains "malformed marker names raw line" "no quote" "$out" # ---- Test 7: deleted file (exit 1) -------------------------------------- echo "=== deleted file ===" T="$SCRATCH/t7"; mk_repo "$T" commit_file "$T" "spec/foo.md" \ $'# Spec\n\nA line that stays.\n\nA line that gets revoked.\n\n' >/dev/null BASE=$(git -C "$T" rev-parse HEAD) # Delete the file entirely. rm "$T/spec/foo.md" out=$(run_check "$T" "$BASE" --tag TEST); rc=$? assert_exit "deleted file exits 1" 1 "$rc" "$out" "" assert_contains "deleted file under correct header" "deleted since base" "$out" assert_contains "deleted file names path" "spec/foo.md" "$out" # ---- Test 8: renamed file with quote (exit 0) --------------------------- echo "=== renamed file with quote ===" T="$SCRATCH/t8"; mk_repo "$T" commit_file "$T" "spec/foo.md" \ $'# Spec\n\nA line that stays.\n\nA line that gets revoked.\n\n' >/dev/null BASE=$(git -C "$T" rev-parse HEAD) # Rename + quote the old content in the new file. git -C "$T" mv "spec/foo.md" "spec/bar.md" >/dev/null write_file_via_py "$T" "spec/bar.md" \ $'# Spec\n\nA line that stays.\n\nRevokes prior text (TEST): "A line that gets revoked."\n\n' out=$(run_check "$T" "$BASE" --tag TEST); rc=$? assert_exit "renamed file with quote exits 0" 0 "$rc" "$out" "" assert_contains "renamed file with quote says clean" "clean" "$out" # ---- Test 9: short / markdown-only line removed (exit 0) ---------------- echo "=== short / markdown-only line removed ===" T="$SCRATCH/t9"; mk_repo "$T" # Base file has: a heading, a very short line (< 12 chars), a markdown table # (with separator row + a data row), a horizontal rule, and a long real # content line. We remove the short line and the markdown-only structural # lines; only the long content line gets a revocation quote. BASE=$(commit_file "$T" "spec/foo.md" \ $'# Spec\n\nshort\n\n| a | b |\n|---|---|\n| 1 | 2 |\n\n---\n\nReal content here that is plenty long enough.\n') write_file_via_py "$T" "spec/foo.md" \ $'# Spec\n\n| a | b |\n| 1 | 2 |\n\nRevokes prior text (TEST): "Real content here that is plenty long enough."\n' out=$(run_check "$T" "$BASE" --tag TEST); rc=$? assert_exit "short/markdown-only removed exits 0" 0 "$rc" "$out" "" assert_contains "short/markdown-only removed says clean" "clean" "$out" # ---- Test 10: untagged marker with no --tag (exit 0) ------------------- echo "=== untagged marker with no --tag ===" T="$SCRATCH/t10"; mk_repo "$T" commit_file "$T" "spec/foo.md" \ $'# Spec\n\nA line that stays.\n\nA line that gets revoked.\n\n' >/dev/null BASE=$(git -C "$T" rev-parse HEAD) # Use the untagged marker (no parens). write_file_via_py "$T" "spec/foo.md" \ $'# Spec\n\nA line that stays.\n\nRevokes prior text: "A line that gets revoked."\n\n' # No --tag → should accept untagged marker. out=$(run_check "$T" "$BASE"); rc=$? assert_exit "untagged marker no --tag exits 0" 0 "$rc" "$out" "" assert_contains "untagged marker no --tag says clean" "clean" "$out" # ---- Test 11: --tag filters other tags --------------------------------- # With --tag X, lines containing markers for tag Y (or untagged or free- # prose) are entirely ignored: not parsed, not reported, not used as # quotes. The base line that the foreign marker quoted is still in the # current file (inside the ignored marker text), so Check B still passes. echo "=== --tag filters other tags ===" T="$SCRATCH/t11"; mk_repo "$T" commit_file "$T" "spec/foo.md" \ $'# Spec\n\nA line that stays.\n\nA line that gets revoked.\n\n' >/dev/null BASE=$(git -C "$T" rev-parse HEAD) # Quote uses M33, but we run with --tag M99 — the M33 marker is ignored # entirely (not parsed as malformed). The base line text is still in the # current file (inside the ignored marker), so Check B still passes. write_file_via_py "$T" "spec/foo.md" \ $'# Spec\n\nA line that stays.\n\nRevokes prior text (M33): "A line that gets revoked."\n\n' out=$(run_check "$T" "$BASE" --tag M99); rc=$? assert_exit "--tag M99 ignores M33 marker exits 0" 0 "$rc" "$out" "" assert_contains "--tag M99 ignores M33 says clean" "clean" "$out" # Without --tag, the marker parses and the line is covered. out_notag=$(run_check "$T" "$BASE"); rc_notag=$? assert_exit "no --tag accepts M33 marker exits 0" 0 "$rc_notag" "$out_notag" "" # Now run --tag M99 with an M33 marker that quotes something OTHER than # a base line — the base line is then truly unquoted, and with --tag M99 # the M33 marker doesn't cover it → unquoted deletion → violation. write_file_via_py "$T" "spec/foo.md" \ $'# Spec\n\nA line that stays.\n\nRevokes prior text (M33): "Some other thing entirely."\n\n' out=$(run_check "$T" "$BASE" --tag M99); rc=$? assert_exit "--tag M99 + bogus M33 quote exits 1" 1 "$rc" "$out" "" assert_contains "--tag M99 + bogus M33 quote names unquoted" "unquoted deletions" "$out" # ---- Test 12: bad --base (exit 2) --------------------------------------- echo "=== bad --base ===" T="$SCRATCH/t12"; mk_repo "$T" commit_file "$T" "spec/foo.md" $'# Spec\n\nA line.\n' >/dev/null out=$(run_check "$T" "0000000000000000000000000000000000000000" --tag TEST 2>&1); rc=$? assert_exit "bad --base exits 2" 2 "$rc" "$out" "" assert_contains "bad --base names offending arg" "does not resolve" "$out" # ---- Test 13: not a git repo (exit 2) ----------------------------------- echo "=== not a git repo ===" T="$SCRATCH/t13"; mkdir -p "$T" write_file_via_py "$T" "spec/foo.md" $'# Spec\n\nA line.\n' out=$(run_check "$T" "HEAD" --tag TEST 2>&1); rc=$? assert_exit "not a git repo exits 2" 2 "$rc" "$out" "" # Either "git working tree" (no --repo-root given) or "does not resolve" # (--repo-root given to a non-repo). assert_contains "not a git repo error" "does not resolve" "$out" # ---- Test 14: --json parses (exit 1, valid JSON) ------------------------ echo "=== --json parses ===" T="$SCRATCH/t14"; mk_repo "$T" commit_file "$T" "spec/foo.md" \ $'# Spec\n\nA line that stays.\n\nA line that gets revoked.\n\n' >/dev/null BASE=$(git -C "$T" rev-parse HEAD) write_file_via_py "$T" "spec/foo.md" $'# Spec\n\nA line that stays.\n\n' out=$(run_check "$T" "$BASE" --tag TEST --json); rc=$? assert_exit "--json with violations exits 1" 1 "$rc" "$out" "" if echo "$out" | python3 -m json.tool >/dev/null 2>&1; then echo -e "${GREEN}PASS${NC}: --json parses with python3 -m json.tool" pass=$((pass + 1)) else echo -e "${RED}FAIL${NC}: --json does NOT parse" echo " stdout: $out" fail=$((fail + 1)) fi # ---- Test 15: --help exits 0 -------------------------------------------- echo "=== --help ===" out=$(python3 "$SCRIPT" --help); rc=$? assert_exit "--help exits 0" 0 "$rc" "$out" "" assert_contains "--help shows usage" "spec-revocation-check" "$out" assert_contains "--help mentions --base" "--base" "$out" assert_contains "--help mentions --strict" "--strict" "$out" # ---- Tests 16-19: real-case fixture (round 2) ------------------------- # One base file, one current file with four marker styles: # - (M33) tagged quote (valid) # - Older untagged mid-line marker (M32-era, M23 amendment note) # - (M23) tagged quote # - Free-prose `**Revokes prior text:**` line # Each marker covers a real base line. With --tag M33 the foreign markers # must be ignored; the free-prose line must be ignored unless --strict. echo "=== --tag scopes marker parsing to the literal tag ===" T="$SCRATCH/t16"; mk_repo "$T" # Base: a stable line + one line per marker style + a short line that # Check B skips (under min-line-chars=12). The short line is what the # free-prose marker refers to (so Check B won't flag its deletion). commit_file "$T" "spec/foo.md" \ $'# Spec\n\nA line that stays.\n\nLine revoked by M33.\n\nLine revoked by untagged mid-line.\n\nLine revoked by M23.\n\nshort text\n' >/dev/null BASE=$(git -C "$T" rev-parse HEAD) write_file_via_py "$T" "spec/foo.md" \ $'# Spec\n\nA line that stays.\n\nRevokes prior text (M33): "Line revoked by M33."\n\n*(Amended M23: ... — Revokes prior text: "Line revoked by untagged mid-line.".)*\n\nRevokes prior text (M23): "Line revoked by M23."\n\n> **Revokes prior text:** the short text\n' # Sub-case 1: --tag M33 — only the (M33) line is parsed; foreign markers # are ignored entirely. The base lines they refer to are still verbatim # in the current file (inside the ignored marker text), so Check B # passes. Clean. out=$(run_check "$T" "$BASE" --tag M33); rc=$? assert_exit "--tag M33 ignores foreign markers exits 0" 0 "$rc" "$out" "" assert_contains "--tag M33 ignores foreign says clean" "clean" "$out" # Sub-case 2: malformed (M33) line with --tag M33 — flagged. The other # foreign markers must STILL be ignored (not malformed). write_file_via_py "$T" "spec/foo.md" \ $'# Spec\n\nA line that stays.\n\nRevokes prior text (M33) no quote\n\n*(Amended M23: ... — Revokes prior text: "Line revoked by untagged mid-line.".)*\n\nRevokes prior text (M23): "Line revoked by M23."\n\n> **Revokes prior text:** the short text\n' out=$(run_check "$T" "$BASE" --tag M33); rc=$? assert_exit "malformed (M33) line exits 1" 1 "$rc" "$out" "" assert_contains "malformed (M33) under header" "malformed markers" "$out" assert_contains "malformed (M33) names the raw line" "no quote" "$out" # Restore the valid fixture for sub-cases 3 and 4. write_file_via_py "$T" "spec/foo.md" \ $'# Spec\n\nA line that stays.\n\nRevokes prior text (M33): "Line revoked by M33."\n\n*(Amended M23: ... — Revokes prior text: "Line revoked by untagged mid-line.".)*\n\nRevokes prior text (M23): "Line revoked by M23."\n\n> **Revokes prior text:** the short text\n' # Sub-case 3: no --tag — foreign markers parse (untagged branch + M23 # tagged branch); the free-prose line is silently ignored by default. # Output: zero violations but a summary line that counts the ignored # marker (NOT the early "clean — every line survives" message, since # ignored > 0). out=$(run_check "$T" "$BASE"); rc=$? assert_exit "no --tag, free-prose ignored, exits 0" 0 "$rc" "$out" "" assert_contains "no --tag, free-prose ignored, shows summary line" "Summary:" "$out" assert_contains "no --tag, free-prose ignored, summary shows ignored count" "1 ignored" "$out" # Sub-case 4: no --tag --strict — free-prose becomes malformed. out=$(run_check "$T" "$BASE" --strict); rc=$? assert_exit "no --tag --strict, free-prose malformed exits 1" 1 "$rc" "$out" "" assert_contains "no --tag --strict, free-prose under header" "malformed markers" "$out" # ---- Round-2 dedicated negative fixtures -------------------------------- # The negative-test proof loop below needs fixtures that exit 1 # consistently (not the ones above, whose state mutates between sub- # cases). t17 is a malformed-(M33) fixture, t19 is a free-prose line with # --strict; both should fail when run against the synthetic no-op. # ---- Test 17 (negative): malformed (M33) line with --tag M33 ------------ echo "=== negative: malformed (M33) line with --tag M33 ===" T="$SCRATCH/t17"; mk_repo "$T" commit_file "$T" "spec/foo.md" \ $'# Spec\n\nA line that stays.\n\nA line that gets revoked.\n\n' >/dev/null BASE=$(git -C "$T" rev-parse HEAD) write_file_via_py "$T" "spec/foo.md" \ $'# Spec\n\nA line that stays.\n\nRevokes prior text (M33) no quote\n\n' out=$(run_check "$T" "$BASE" --tag M33); rc=$? assert_exit "negative t17 malformed (M33) exits 1" 1 "$rc" "$out" "" assert_contains "negative t17 malformed (M33) under header" "malformed markers" "$out" # ---- Test 19 (negative): free-prose line with no --tag --strict --------- echo "=== negative: free-prose with no --tag --strict ===" T="$SCRATCH/t19"; mk_repo "$T" commit_file "$T" "spec/foo.md" \ $'# Spec\n\nA line that stays.\n\nshort text\n' >/dev/null BASE=$(git -C "$T" rev-parse HEAD) write_file_via_py "$T" "spec/foo.md" \ $'# Spec\n\nA line that stays.\n\n> **Revokes prior text:** the short text\n' out=$(run_check "$T" "$BASE" --strict); rc=$? assert_exit "negative t19 free-prose strict exits 1" 1 "$rc" "$out" "" assert_contains "negative t19 free-prose strict under header" "malformed markers" "$out" # ---- Negative-test verification (would each negative fail if disabled?) - # A "negative" test would PASS against a no-op script that always exits 0. # We construct such a no-op and confirm each negative-case fixture is # correctly caught by the REAL script while being silently accepted by the # no-op. This proves the real script is what detected each issue. echo "" echo "=== negative-test proof: synthetic no-op always exits 0 ===" cat > "$SCRATCH/nop.py" <<'PYEOF' #!/usr/bin/env python3 import json, os, sys out = { "repo_root": os.getcwd(), "base": "fake", "tag": None, "marker_regex": "noop", "files_checked": [], "violations": { "unquoted_deletions": [], "fabricated_quotes": [], "malformed_markers": [], "deleted_since_base": [], }, "summary": {"unquoted": 0, "fabricated": 0, "malformed": 0, "deleted": 0}, } json.dump(out, sys.stdout) sys.stdout.write("\n") sys.exit(0) PYEOF chmod +x "$SCRATCH/nop.py" for case in t2 t3 t4 t6 t7 t17 t19; do T="$SCRATCH/$case" BASE=$(git -C "$T" rev-parse HEAD 2>/dev/null || echo "HEAD") out_nop=$(python3 "$SCRATCH/nop.py" --base HEAD --repo-root "$T" --glob "spec/**/*.md" 2>&1); rc_nop=$? # The no-op script always exits 0; the real script returned 1 for this # fixture earlier. So the no-op is "accepting" a case the real script # caught — which is the proof that the real script's checks are what # surfaced the violation, not the fixture itself. if [ "$rc_nop" -eq 0 ]; then echo -e "${GREEN}PASS${NC}: negative-case $case: real=1, noop=0 (check is doing the work)" pass=$((pass + 1)) else echo -e "${RED}FAIL${NC}: noop unexpectedly returned $rc_nop on case $case" fail=$((fail + 1)) fi done # ---- Done --------------------------------------------------------------- echo "" echo "Results: $pass passed, $fail failed" [[ $fail -eq 0 ]]