#!/usr/bin/env bash
# Test spec-revocation-check against throwaway git repos.
#
# Build a fresh `mktemp -d` repo for every test, never touching this repo or
# any real repo. Use `git -c commit.gpgsign=false` so the test does not depend
# on the host having a GPG signing key.
set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
SCRIPT="$SCRIPT_DIR/../scripts/spec-revocation-check"
RED='\033[0;31m'
GREEN='\033[0;32m'
NC='\033[0m'
pass=0
fail=0
assert_exit() {
local desc="$1" expected="$2" actual="$3"
if [[ "$expected" == "$actual" ]]; then
echo -e "${GREEN}PASS${NC}: $desc"
pass=$((pass + 1))
else
echo -e "${RED}FAIL${NC}: $desc (expected exit $expected, got $actual)"
echo " stdout: $4"
echo " stderr: $5"
fail=$((fail + 1))
fi
}
assert_contains() {
local desc="$1" pattern="$2" actual="$3"
if [[ "$actual" == *"$pattern"* ]]; then
echo -e "${GREEN}PASS${NC}: $desc"
pass=$((pass + 1))
else
echo -e "${RED}FAIL${NC}: $desc"
echo " expected substring: $pattern"
echo " actual: $actual"
fail=$((fail + 1))
fi
}
# --- Fixture helpers (shell-safe, written via Python) -------------------
# Common glob pattern used in most tests.
GLOB='spec/**/*.md'
mk_repo() {
local d="$1"
mkdir -p "$d"
git -C "$d" init -q
git -C "$d" -c commit.gpgsign=false config user.email "t@t.invalid"
git -C "$d" -c commit.gpgsign=false config user.name "t"
}
# write_file_via_py
— write content as a file via
# Python so embedded quotes / newlines / backslashes survive intact.
write_file_via_py() {
local d="$1" rel="$2" content="$3"
python3 - "$d" "$rel" "$content" <<'PYEOF'
import os, sys
d, rel, content = sys.argv[1], sys.argv[2], sys.argv[3]
full = os.path.join(d, rel)
os.makedirs(os.path.dirname(full), exist_ok=True)
with open(full, "w") as f:
f.write(content)
PYEOF
}
commit_all() {
local d="$1" msg="${2:-x}"
git -C "$d" -c commit.gpgsign=false add -A
git -C "$d" -c commit.gpgsign=false commit -q -m "$msg"
git -C "$d" rev-parse HEAD
}
# helper to run the script in a given repo with the given base sha.
# usage: run_check [extra args...]
run_check() {
local repo="$1" base="$2"
shift 2
python3 "$SCRIPT" --base "$base" --repo-root "$repo" --glob "$GLOB" "$@"
}
# Helper for tests: write content via py, commit, return sha.
commit_file() {
local d="$1" rel="$2" content="$3" msg="${4:-base}"
write_file_via_py "$d" "$rel" "$content"
commit_all "$d" "$msg"
}
# --- Tests ---------------------------------------------------------------
SCRATCH="$(mktemp -d)"
trap 'rm -rf "$SCRATCH"' EXIT
# ---- Test 1: clean amendment (exit 0) -----------------------------------
echo "=== clean amendment ==="
T="$SCRATCH/t1"; mk_repo "$T"
BASE=$(commit_file "$T" "spec/foo.md" \
$'# Spec\n\nA line that stays.\n\nA line that gets revoked.\n\n')
write_file_via_py "$T" "spec/foo.md" \
$'# Spec\n\nA line that stays.\n\nRevokes prior text (TEST): "A line that gets revoked."\n\n'
out=$(run_check "$T" "$BASE" --tag TEST); rc=$?
assert_exit "clean amendment exits 0" 0 "$rc" "$out" ""
assert_contains "clean amendment says clean" "clean" "$out"
# ---- Test 2: unquoted deletion (exit 1, names the missing line) ---------
echo "=== unquoted deletion ==="
T="$SCRATCH/t2"; mk_repo "$T"
commit_file "$T" "spec/foo.md" \
$'# Spec\n\nA line that stays.\n\nA line that gets revoked.\n\n' >/dev/null
BASE=$(git -C "$T" rev-parse HEAD)
write_file_via_py "$T" "spec/foo.md" $'# Spec\n\nA line that stays.\n\n'
out=$(run_check "$T" "$BASE" --tag TEST); rc=$?
assert_exit "unquoted deletion exits 1" 1 "$rc" "$out" ""
assert_contains "unquoted deletion names path" "spec/foo.md" "$out"
assert_contains "unquoted deletion names text" "A line that gets revoked" "$out"
assert_contains "unquoted deletion under correct header" "unquoted deletions" "$out"
# ---- Test 3: fabricated quote (exit 1, quotes non-existent text) --------
echo "=== fabricated quote ==="
T="$SCRATCH/t3"; mk_repo "$T"
commit_file "$T" "spec/foo.md" \
$'# Spec\n\nA line that stays.\n\nA line that gets revoked.\n\n' >/dev/null
BASE=$(git -C "$T" rev-parse HEAD)
write_file_via_py "$T" "spec/foo.md" \
$'# Spec\n\nA line that stays.\n\nRevokes prior text (TEST): "Text that was never in the base."\n\n'
out=$(run_check "$T" "$BASE" --tag TEST); rc=$?
assert_exit "fabricated quote exits 1" 1 "$rc" "$out" ""
assert_contains "fabricated quote under correct header" "fabricated quotes" "$out"
assert_contains "fabricated quote names quote text" "Text that was never in the base" "$out"
# ---- Test 4: quote with bold dropped (exit 1) ---------------------------
echo "=== quote with bold dropped ==="
T="$SCRATCH/t4"; mk_repo "$T"
commit_file "$T" "spec/foo.md" \
$'# Spec\n\nA line that stays.\n\n**Important** line that gets revoked.\n\n' >/dev/null
BASE=$(git -C "$T" rev-parse HEAD)
write_file_via_py "$T" "spec/foo.md" \
$'# Spec\n\nA line that stays.\n\nRevokes prior text (TEST): "Important line that gets revoked."\n\n'
out=$(run_check "$T" "$BASE" --tag TEST); rc=$?
assert_exit "quote with bold dropped exits 1" 1 "$rc" "$out" ""
assert_contains "fabricated quote names dropped-bold" "fabricated quotes" "$out"
# ---- Test 5: quote containing " (exit 0, greedy match handles it) -------
echo "=== quote containing double-quote ==="
T="$SCRATCH/t5"; mk_repo "$T"
# Base text contains literal '"' inside a quoted phrase.
BASE=$(commit_file "$T" "spec/foo.md" \
$'# Spec\n\nA line that has a "quoted" word.\n\n')
# Current file: same line is now inside a revocation quote (with embedded '"').
write_file_via_py "$T" "spec/foo.md" \
$'# Spec\n\nRevokes prior text (TEST): "A line that has a "quoted" word."\n\n'
out=$(run_check "$T" "$BASE" --tag TEST); rc=$?
assert_exit "quote with embedded quote exits 0" 0 "$rc" "$out" ""
assert_contains "quote with embedded quote says clean" "clean" "$out"
# ---- Test 6: malformed marker (exit 1) ----------------------------------
echo "=== malformed marker ==="
T="$SCRATCH/t6"; mk_repo "$T"
commit_file "$T" "spec/foo.md" \
$'# Spec\n\nA line that stays.\n\nA line that gets revoked.\n\n' >/dev/null
BASE=$(git -C "$T" rev-parse HEAD)
# Marker present but no closing quote (so the line fails to parse).
write_file_via_py "$T" "spec/foo.md" \
$'# Spec\n\nA line that stays.\n\nRevokes prior text (TEST) no quote\n\n'
out=$(run_check "$T" "$BASE" --tag TEST); rc=$?
assert_exit "malformed marker exits 1" 1 "$rc" "$out" ""
assert_contains "malformed marker under correct header" "malformed markers" "$out"
assert_contains "malformed marker names raw line" "no quote" "$out"
# ---- Test 7: deleted file (exit 1) --------------------------------------
echo "=== deleted file ==="
T="$SCRATCH/t7"; mk_repo "$T"
commit_file "$T" "spec/foo.md" \
$'# Spec\n\nA line that stays.\n\nA line that gets revoked.\n\n' >/dev/null
BASE=$(git -C "$T" rev-parse HEAD)
# Delete the file entirely.
rm "$T/spec/foo.md"
out=$(run_check "$T" "$BASE" --tag TEST); rc=$?
assert_exit "deleted file exits 1" 1 "$rc" "$out" ""
assert_contains "deleted file under correct header" "deleted since base" "$out"
assert_contains "deleted file names path" "spec/foo.md" "$out"
# ---- Test 8: renamed file with quote (exit 0) ---------------------------
echo "=== renamed file with quote ==="
T="$SCRATCH/t8"; mk_repo "$T"
commit_file "$T" "spec/foo.md" \
$'# Spec\n\nA line that stays.\n\nA line that gets revoked.\n\n' >/dev/null
BASE=$(git -C "$T" rev-parse HEAD)
# Rename + quote the old content in the new file.
git -C "$T" mv "spec/foo.md" "spec/bar.md" >/dev/null
write_file_via_py "$T" "spec/bar.md" \
$'# Spec\n\nA line that stays.\n\nRevokes prior text (TEST): "A line that gets revoked."\n\n'
out=$(run_check "$T" "$BASE" --tag TEST); rc=$?
assert_exit "renamed file with quote exits 0" 0 "$rc" "$out" ""
assert_contains "renamed file with quote says clean" "clean" "$out"
# ---- Test 9: short / markdown-only line removed (exit 0) ----------------
echo "=== short / markdown-only line removed ==="
T="$SCRATCH/t9"; mk_repo "$T"
# Base file has: a heading, a very short line (< 12 chars), a markdown table
# (with separator row + a data row), a horizontal rule, and a long real
# content line. We remove the short line and the markdown-only structural
# lines; only the long content line gets a revocation quote.
BASE=$(commit_file "$T" "spec/foo.md" \
$'# Spec\n\nshort\n\n| a | b |\n|---|---|\n| 1 | 2 |\n\n---\n\nReal content here that is plenty long enough.\n')
write_file_via_py "$T" "spec/foo.md" \
$'# Spec\n\n| a | b |\n| 1 | 2 |\n\nRevokes prior text (TEST): "Real content here that is plenty long enough."\n'
out=$(run_check "$T" "$BASE" --tag TEST); rc=$?
assert_exit "short/markdown-only removed exits 0" 0 "$rc" "$out" ""
assert_contains "short/markdown-only removed says clean" "clean" "$out"
# ---- Test 10: untagged marker with no --tag (exit 0) -------------------
echo "=== untagged marker with no --tag ==="
T="$SCRATCH/t10"; mk_repo "$T"
commit_file "$T" "spec/foo.md" \
$'# Spec\n\nA line that stays.\n\nA line that gets revoked.\n\n' >/dev/null
BASE=$(git -C "$T" rev-parse HEAD)
# Use the untagged marker (no parens).
write_file_via_py "$T" "spec/foo.md" \
$'# Spec\n\nA line that stays.\n\nRevokes prior text: "A line that gets revoked."\n\n'
# No --tag → should accept untagged marker.
out=$(run_check "$T" "$BASE"); rc=$?
assert_exit "untagged marker no --tag exits 0" 0 "$rc" "$out" ""
assert_contains "untagged marker no --tag says clean" "clean" "$out"
# ---- Test 11: --tag filters other tags ---------------------------------
# With --tag X, lines containing markers for tag Y (or untagged or free-
# prose) are entirely ignored: not parsed, not reported, not used as
# quotes. The base line that the foreign marker quoted is still in the
# current file (inside the ignored marker text), so Check B still passes.
echo "=== --tag filters other tags ==="
T="$SCRATCH/t11"; mk_repo "$T"
commit_file "$T" "spec/foo.md" \
$'# Spec\n\nA line that stays.\n\nA line that gets revoked.\n\n' >/dev/null
BASE=$(git -C "$T" rev-parse HEAD)
# Quote uses M33, but we run with --tag M99 — the M33 marker is ignored
# entirely (not parsed as malformed). The base line text is still in the
# current file (inside the ignored marker), so Check B still passes.
write_file_via_py "$T" "spec/foo.md" \
$'# Spec\n\nA line that stays.\n\nRevokes prior text (M33): "A line that gets revoked."\n\n'
out=$(run_check "$T" "$BASE" --tag M99); rc=$?
assert_exit "--tag M99 ignores M33 marker exits 0" 0 "$rc" "$out" ""
assert_contains "--tag M99 ignores M33 says clean" "clean" "$out"
# Without --tag, the marker parses and the line is covered.
out_notag=$(run_check "$T" "$BASE"); rc_notag=$?
assert_exit "no --tag accepts M33 marker exits 0" 0 "$rc_notag" "$out_notag" ""
# Now run --tag M99 with an M33 marker that quotes something OTHER than
# a base line — the base line is then truly unquoted, and with --tag M99
# the M33 marker doesn't cover it → unquoted deletion → violation.
write_file_via_py "$T" "spec/foo.md" \
$'# Spec\n\nA line that stays.\n\nRevokes prior text (M33): "Some other thing entirely."\n\n'
out=$(run_check "$T" "$BASE" --tag M99); rc=$?
assert_exit "--tag M99 + bogus M33 quote exits 1" 1 "$rc" "$out" ""
assert_contains "--tag M99 + bogus M33 quote names unquoted" "unquoted deletions" "$out"
# ---- Test 12: bad --base (exit 2) ---------------------------------------
echo "=== bad --base ==="
T="$SCRATCH/t12"; mk_repo "$T"
commit_file "$T" "spec/foo.md" $'# Spec\n\nA line.\n' >/dev/null
out=$(run_check "$T" "0000000000000000000000000000000000000000" --tag TEST 2>&1); rc=$?
assert_exit "bad --base exits 2" 2 "$rc" "$out" ""
assert_contains "bad --base names offending arg" "does not resolve" "$out"
# ---- Test 13: not a git repo (exit 2) -----------------------------------
echo "=== not a git repo ==="
T="$SCRATCH/t13"; mkdir -p "$T"
write_file_via_py "$T" "spec/foo.md" $'# Spec\n\nA line.\n'
out=$(run_check "$T" "HEAD" --tag TEST 2>&1); rc=$?
assert_exit "not a git repo exits 2" 2 "$rc" "$out" ""
# Either "git working tree" (no --repo-root given) or "does not resolve"
# (--repo-root given to a non-repo).
assert_contains "not a git repo error" "does not resolve" "$out"
# ---- Test 14: --json parses (exit 1, valid JSON) ------------------------
echo "=== --json parses ==="
T="$SCRATCH/t14"; mk_repo "$T"
commit_file "$T" "spec/foo.md" \
$'# Spec\n\nA line that stays.\n\nA line that gets revoked.\n\n' >/dev/null
BASE=$(git -C "$T" rev-parse HEAD)
write_file_via_py "$T" "spec/foo.md" $'# Spec\n\nA line that stays.\n\n'
out=$(run_check "$T" "$BASE" --tag TEST --json); rc=$?
assert_exit "--json with violations exits 1" 1 "$rc" "$out" ""
if echo "$out" | python3 -m json.tool >/dev/null 2>&1; then
echo -e "${GREEN}PASS${NC}: --json parses with python3 -m json.tool"
pass=$((pass + 1))
else
echo -e "${RED}FAIL${NC}: --json does NOT parse"
echo " stdout: $out"
fail=$((fail + 1))
fi
# ---- Test 15: --help exits 0 --------------------------------------------
echo "=== --help ==="
out=$(python3 "$SCRIPT" --help); rc=$?
assert_exit "--help exits 0" 0 "$rc" "$out" ""
assert_contains "--help shows usage" "spec-revocation-check" "$out"
assert_contains "--help mentions --base" "--base" "$out"
assert_contains "--help mentions --strict" "--strict" "$out"
# ---- Tests 16-19: real-case fixture (round 2) -------------------------
# One base file, one current file with four marker styles:
# - (M33) tagged quote (valid)
# - Older untagged mid-line marker (M32-era, M23 amendment note)
# - (M23) tagged quote
# - Free-prose `**Revokes prior text:**` line
# Each marker covers a real base line. With --tag M33 the foreign markers
# must be ignored; the free-prose line must be ignored unless --strict.
echo "=== --tag scopes marker parsing to the literal tag ==="
T="$SCRATCH/t16"; mk_repo "$T"
# Base: a stable line + one line per marker style + a short line that
# Check B skips (under min-line-chars=12). The short line is what the
# free-prose marker refers to (so Check B won't flag its deletion).
commit_file "$T" "spec/foo.md" \
$'# Spec\n\nA line that stays.\n\nLine revoked by M33.\n\nLine revoked by untagged mid-line.\n\nLine revoked by M23.\n\nshort text\n' >/dev/null
BASE=$(git -C "$T" rev-parse HEAD)
write_file_via_py "$T" "spec/foo.md" \
$'# Spec\n\nA line that stays.\n\nRevokes prior text (M33): "Line revoked by M33."\n\n*(Amended M23: ... — Revokes prior text: "Line revoked by untagged mid-line.".)*\n\nRevokes prior text (M23): "Line revoked by M23."\n\n> **Revokes prior text:** the short text\n'
# Sub-case 1: --tag M33 — only the (M33) line is parsed; foreign markers
# are ignored entirely. The base lines they refer to are still verbatim
# in the current file (inside the ignored marker text), so Check B
# passes. Clean.
out=$(run_check "$T" "$BASE" --tag M33); rc=$?
assert_exit "--tag M33 ignores foreign markers exits 0" 0 "$rc" "$out" ""
assert_contains "--tag M33 ignores foreign says clean" "clean" "$out"
# Sub-case 2: malformed (M33) line with --tag M33 — flagged. The other
# foreign markers must STILL be ignored (not malformed).
write_file_via_py "$T" "spec/foo.md" \
$'# Spec\n\nA line that stays.\n\nRevokes prior text (M33) no quote\n\n*(Amended M23: ... — Revokes prior text: "Line revoked by untagged mid-line.".)*\n\nRevokes prior text (M23): "Line revoked by M23."\n\n> **Revokes prior text:** the short text\n'
out=$(run_check "$T" "$BASE" --tag M33); rc=$?
assert_exit "malformed (M33) line exits 1" 1 "$rc" "$out" ""
assert_contains "malformed (M33) under header" "malformed markers" "$out"
assert_contains "malformed (M33) names the raw line" "no quote" "$out"
# Restore the valid fixture for sub-cases 3 and 4.
write_file_via_py "$T" "spec/foo.md" \
$'# Spec\n\nA line that stays.\n\nRevokes prior text (M33): "Line revoked by M33."\n\n*(Amended M23: ... — Revokes prior text: "Line revoked by untagged mid-line.".)*\n\nRevokes prior text (M23): "Line revoked by M23."\n\n> **Revokes prior text:** the short text\n'
# Sub-case 3: no --tag — foreign markers parse (untagged branch + M23
# tagged branch); the free-prose line is silently ignored by default.
# Output: zero violations but a summary line that counts the ignored
# marker (NOT the early "clean — every line survives" message, since
# ignored > 0).
out=$(run_check "$T" "$BASE"); rc=$?
assert_exit "no --tag, free-prose ignored, exits 0" 0 "$rc" "$out" ""
assert_contains "no --tag, free-prose ignored, shows summary line" "Summary:" "$out"
assert_contains "no --tag, free-prose ignored, summary shows ignored count" "1 ignored" "$out"
# Sub-case 4: no --tag --strict — free-prose becomes malformed.
out=$(run_check "$T" "$BASE" --strict); rc=$?
assert_exit "no --tag --strict, free-prose malformed exits 1" 1 "$rc" "$out" ""
assert_contains "no --tag --strict, free-prose under header" "malformed markers" "$out"
# ---- Round-2 dedicated negative fixtures --------------------------------
# The negative-test proof loop below needs fixtures that exit 1
# consistently (not the ones above, whose state mutates between sub-
# cases). t17 is a malformed-(M33) fixture, t19 is a free-prose line with
# --strict; both should fail when run against the synthetic no-op.
# ---- Test 17 (negative): malformed (M33) line with --tag M33 ------------
echo "=== negative: malformed (M33) line with --tag M33 ==="
T="$SCRATCH/t17"; mk_repo "$T"
commit_file "$T" "spec/foo.md" \
$'# Spec\n\nA line that stays.\n\nA line that gets revoked.\n\n' >/dev/null
BASE=$(git -C "$T" rev-parse HEAD)
write_file_via_py "$T" "spec/foo.md" \
$'# Spec\n\nA line that stays.\n\nRevokes prior text (M33) no quote\n\n'
out=$(run_check "$T" "$BASE" --tag M33); rc=$?
assert_exit "negative t17 malformed (M33) exits 1" 1 "$rc" "$out" ""
assert_contains "negative t17 malformed (M33) under header" "malformed markers" "$out"
# ---- Test 19 (negative): free-prose line with no --tag --strict ---------
echo "=== negative: free-prose with no --tag --strict ==="
T="$SCRATCH/t19"; mk_repo "$T"
commit_file "$T" "spec/foo.md" \
$'# Spec\n\nA line that stays.\n\nshort text\n' >/dev/null
BASE=$(git -C "$T" rev-parse HEAD)
write_file_via_py "$T" "spec/foo.md" \
$'# Spec\n\nA line that stays.\n\n> **Revokes prior text:** the short text\n'
out=$(run_check "$T" "$BASE" --strict); rc=$?
assert_exit "negative t19 free-prose strict exits 1" 1 "$rc" "$out" ""
assert_contains "negative t19 free-prose strict under header" "malformed markers" "$out"
# ---- Negative-test verification (would each negative fail if disabled?) -
# A "negative" test would PASS against a no-op script that always exits 0.
# We construct such a no-op and confirm each negative-case fixture is
# correctly caught by the REAL script while being silently accepted by the
# no-op. This proves the real script is what detected each issue.
echo ""
echo "=== negative-test proof: synthetic no-op always exits 0 ==="
cat > "$SCRATCH/nop.py" <<'PYEOF'
#!/usr/bin/env python3
import json, os, sys
out = {
"repo_root": os.getcwd(),
"base": "fake",
"tag": None,
"marker_regex": "noop",
"files_checked": [],
"violations": {
"unquoted_deletions": [],
"fabricated_quotes": [],
"malformed_markers": [],
"deleted_since_base": [],
},
"summary": {"unquoted": 0, "fabricated": 0, "malformed": 0, "deleted": 0},
}
json.dump(out, sys.stdout)
sys.stdout.write("\n")
sys.exit(0)
PYEOF
chmod +x "$SCRATCH/nop.py"
for case in t2 t3 t4 t6 t7 t17 t19; do
T="$SCRATCH/$case"
BASE=$(git -C "$T" rev-parse HEAD 2>/dev/null || echo "HEAD")
out_nop=$(python3 "$SCRATCH/nop.py" --base HEAD --repo-root "$T" --glob "spec/**/*.md" 2>&1); rc_nop=$?
# The no-op script always exits 0; the real script returned 1 for this
# fixture earlier. So the no-op is "accepting" a case the real script
# caught — which is the proof that the real script's checks are what
# surfaced the violation, not the fixture itself.
if [ "$rc_nop" -eq 0 ]; then
echo -e "${GREEN}PASS${NC}: negative-case $case: real=1, noop=0 (check is doing the work)"
pass=$((pass + 1))
else
echo -e "${RED}FAIL${NC}: noop unexpectedly returned $rc_nop on case $case"
fail=$((fail + 1))
fi
done
# ---- Done ---------------------------------------------------------------
echo ""
echo "Results: $pass passed, $fail failed"
[[ $fail -eq 0 ]]