feat(spec-revocation-check): generic spec-amendment revocation gate (brief)
Catches two failure modes that each felled a previous spec-amendment attempt: a fabricated Revokes-prior-text quote (text not in the file at --base) and a normative line silently deleted with no quote at all. Generic: any --glob x --base <sha> against any repo, with optional --tag filter. Reuses the collapse() / quote_line_re() shape from the M33 reference, adds a Check D for deleted/renamed files (-M30 rename threshold), and four violation categories: unquoted deletions, fabricated quotes, malformed markers, deleted since base. Stdlib Python only, --json output, --dryrun prints resolved config and still runs the checks (read-only by nature).
This commit is contained in:
Paul O'Reilly
340
tests/test-spec-revocation-check.sh
Executable file
340
tests/test-spec-revocation-check.sh
Executable file
@@ -0,0 +1,340 @@
|
||||
#!/usr/bin/env bash
|
||||
# Test spec-revocation-check against throwaway git repos.
|
||||
#
|
||||
# Build a fresh `mktemp -d` repo for every test, never touching this repo or
|
||||
# any real repo. Use `git -c commit.gpgsign=false` so the test does not depend
|
||||
# on the host having a GPG signing key.
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
SCRIPT="$SCRIPT_DIR/../scripts/spec-revocation-check"
|
||||
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
NC='\033[0m'
|
||||
|
||||
pass=0
|
||||
fail=0
|
||||
|
||||
assert_exit() {
|
||||
local desc="$1" expected="$2" actual="$3"
|
||||
if [[ "$expected" == "$actual" ]]; then
|
||||
echo -e "${GREEN}PASS${NC}: $desc"
|
||||
pass=$((pass + 1))
|
||||
else
|
||||
echo -e "${RED}FAIL${NC}: $desc (expected exit $expected, got $actual)"
|
||||
echo " stdout: $4"
|
||||
echo " stderr: $5"
|
||||
fail=$((fail + 1))
|
||||
fi
|
||||
}
|
||||
|
||||
assert_contains() {
|
||||
local desc="$1" pattern="$2" actual="$3"
|
||||
if [[ "$actual" == *"$pattern"* ]]; then
|
||||
echo -e "${GREEN}PASS${NC}: $desc"
|
||||
pass=$((pass + 1))
|
||||
else
|
||||
echo -e "${RED}FAIL${NC}: $desc"
|
||||
echo " expected substring: $pattern"
|
||||
echo " actual: $actual"
|
||||
fail=$((fail + 1))
|
||||
fi
|
||||
}
|
||||
|
||||
# --- Fixture helpers (shell-safe, written via Python) -------------------
|
||||
|
||||
# Common glob pattern used in most tests.
|
||||
GLOB='spec/**/*.md'
|
||||
|
||||
mk_repo() {
|
||||
local d="$1"
|
||||
mkdir -p "$d"
|
||||
git -C "$d" init -q
|
||||
git -C "$d" -c commit.gpgsign=false config user.email "t@t.invalid"
|
||||
git -C "$d" -c commit.gpgsign=false config user.name "t"
|
||||
}
|
||||
|
||||
# write_file_via_py <dir> <rel> <content> — write content as a file via
|
||||
# Python so embedded quotes / newlines / backslashes survive intact.
|
||||
write_file_via_py() {
|
||||
local d="$1" rel="$2" content="$3"
|
||||
python3 - "$d" "$rel" "$content" <<'PYEOF'
|
||||
import os, sys
|
||||
d, rel, content = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
full = os.path.join(d, rel)
|
||||
os.makedirs(os.path.dirname(full), exist_ok=True)
|
||||
with open(full, "w") as f:
|
||||
f.write(content)
|
||||
PYEOF
|
||||
}
|
||||
|
||||
commit_all() {
|
||||
local d="$1" msg="${2:-x}"
|
||||
git -C "$d" -c commit.gpgsign=false add -A
|
||||
git -C "$d" -c commit.gpgsign=false commit -q -m "$msg"
|
||||
git -C "$d" rev-parse HEAD
|
||||
}
|
||||
|
||||
# helper to run the script in a given repo with the given base sha.
|
||||
# usage: run_check <repo> <base> [extra args...]
|
||||
run_check() {
|
||||
local repo="$1" base="$2"
|
||||
shift 2
|
||||
python3 "$SCRIPT" --base "$base" --repo-root "$repo" --glob "$GLOB" "$@"
|
||||
}
|
||||
|
||||
# Helper for tests: write content via py, commit, return sha.
|
||||
commit_file() {
|
||||
local d="$1" rel="$2" content="$3" msg="${4:-base}"
|
||||
write_file_via_py "$d" "$rel" "$content"
|
||||
commit_all "$d" "$msg"
|
||||
}
|
||||
|
||||
# --- Tests ---------------------------------------------------------------
|
||||
|
||||
SCRATCH="$(mktemp -d)"
|
||||
trap 'rm -rf "$SCRATCH"' EXIT
|
||||
|
||||
# ---- Test 1: clean amendment (exit 0) -----------------------------------
|
||||
echo "=== clean amendment ==="
|
||||
T="$SCRATCH/t1"; mk_repo "$T"
|
||||
BASE=$(commit_file "$T" "spec/foo.md" \
|
||||
$'# Spec\n\nA line that stays.\n\nA line that gets revoked.\n\n')
|
||||
write_file_via_py "$T" "spec/foo.md" \
|
||||
$'# Spec\n\nA line that stays.\n\nRevokes prior text (TEST): "A line that gets revoked."\n\n'
|
||||
out=$(run_check "$T" "$BASE" --tag TEST); rc=$?
|
||||
assert_exit "clean amendment exits 0" 0 "$rc" "$out" ""
|
||||
assert_contains "clean amendment says clean" "clean" "$out"
|
||||
|
||||
# ---- Test 2: unquoted deletion (exit 1, names the missing line) ---------
|
||||
echo "=== unquoted deletion ==="
|
||||
T="$SCRATCH/t2"; mk_repo "$T"
|
||||
commit_file "$T" "spec/foo.md" \
|
||||
$'# Spec\n\nA line that stays.\n\nA line that gets revoked.\n\n' >/dev/null
|
||||
BASE=$(git -C "$T" rev-parse HEAD)
|
||||
write_file_via_py "$T" "spec/foo.md" $'# Spec\n\nA line that stays.\n\n'
|
||||
out=$(run_check "$T" "$BASE" --tag TEST); rc=$?
|
||||
assert_exit "unquoted deletion exits 1" 1 "$rc" "$out" ""
|
||||
assert_contains "unquoted deletion names path" "spec/foo.md" "$out"
|
||||
assert_contains "unquoted deletion names text" "A line that gets revoked" "$out"
|
||||
assert_contains "unquoted deletion under correct header" "unquoted deletions" "$out"
|
||||
|
||||
# ---- Test 3: fabricated quote (exit 1, quotes non-existent text) --------
|
||||
echo "=== fabricated quote ==="
|
||||
T="$SCRATCH/t3"; mk_repo "$T"
|
||||
commit_file "$T" "spec/foo.md" \
|
||||
$'# Spec\n\nA line that stays.\n\nA line that gets revoked.\n\n' >/dev/null
|
||||
BASE=$(git -C "$T" rev-parse HEAD)
|
||||
write_file_via_py "$T" "spec/foo.md" \
|
||||
$'# Spec\n\nA line that stays.\n\nRevokes prior text (TEST): "Text that was never in the base."\n\n'
|
||||
out=$(run_check "$T" "$BASE" --tag TEST); rc=$?
|
||||
assert_exit "fabricated quote exits 1" 1 "$rc" "$out" ""
|
||||
assert_contains "fabricated quote under correct header" "fabricated quotes" "$out"
|
||||
assert_contains "fabricated quote names quote text" "Text that was never in the base" "$out"
|
||||
|
||||
# ---- Test 4: quote with bold dropped (exit 1) ---------------------------
|
||||
echo "=== quote with bold dropped ==="
|
||||
T="$SCRATCH/t4"; mk_repo "$T"
|
||||
commit_file "$T" "spec/foo.md" \
|
||||
$'# Spec\n\nA line that stays.\n\n**Important** line that gets revoked.\n\n' >/dev/null
|
||||
BASE=$(git -C "$T" rev-parse HEAD)
|
||||
write_file_via_py "$T" "spec/foo.md" \
|
||||
$'# Spec\n\nA line that stays.\n\nRevokes prior text (TEST): "Important line that gets revoked."\n\n'
|
||||
out=$(run_check "$T" "$BASE" --tag TEST); rc=$?
|
||||
assert_exit "quote with bold dropped exits 1" 1 "$rc" "$out" ""
|
||||
assert_contains "fabricated quote names dropped-bold" "fabricated quotes" "$out"
|
||||
|
||||
# ---- Test 5: quote containing " (exit 0, greedy match handles it) -------
|
||||
echo "=== quote containing double-quote ==="
|
||||
T="$SCRATCH/t5"; mk_repo "$T"
|
||||
# Base text contains literal '"' inside a quoted phrase.
|
||||
BASE=$(commit_file "$T" "spec/foo.md" \
|
||||
$'# Spec\n\nA line that has a "quoted" word.\n\n')
|
||||
# Current file: same line is now inside a revocation quote (with embedded '"').
|
||||
write_file_via_py "$T" "spec/foo.md" \
|
||||
$'# Spec\n\nRevokes prior text (TEST): "A line that has a "quoted" word."\n\n'
|
||||
out=$(run_check "$T" "$BASE" --tag TEST); rc=$?
|
||||
assert_exit "quote with embedded quote exits 0" 0 "$rc" "$out" ""
|
||||
assert_contains "quote with embedded quote says clean" "clean" "$out"
|
||||
|
||||
# ---- Test 6: malformed marker (exit 1) ----------------------------------
|
||||
echo "=== malformed marker ==="
|
||||
T="$SCRATCH/t6"; mk_repo "$T"
|
||||
commit_file "$T" "spec/foo.md" \
|
||||
$'# Spec\n\nA line that stays.\n\nA line that gets revoked.\n\n' >/dev/null
|
||||
BASE=$(git -C "$T" rev-parse HEAD)
|
||||
# Marker present but no closing quote (so the line fails to parse).
|
||||
write_file_via_py "$T" "spec/foo.md" \
|
||||
$'# Spec\n\nA line that stays.\n\nRevokes prior text (TEST) no quote\n\n'
|
||||
out=$(run_check "$T" "$BASE" --tag TEST); rc=$?
|
||||
assert_exit "malformed marker exits 1" 1 "$rc" "$out" ""
|
||||
assert_contains "malformed marker under correct header" "malformed markers" "$out"
|
||||
assert_contains "malformed marker names raw line" "no quote" "$out"
|
||||
|
||||
# ---- Test 7: deleted file (exit 1) --------------------------------------
|
||||
echo "=== deleted file ==="
|
||||
T="$SCRATCH/t7"; mk_repo "$T"
|
||||
commit_file "$T" "spec/foo.md" \
|
||||
$'# Spec\n\nA line that stays.\n\nA line that gets revoked.\n\n' >/dev/null
|
||||
BASE=$(git -C "$T" rev-parse HEAD)
|
||||
# Delete the file entirely.
|
||||
rm "$T/spec/foo.md"
|
||||
out=$(run_check "$T" "$BASE" --tag TEST); rc=$?
|
||||
assert_exit "deleted file exits 1" 1 "$rc" "$out" ""
|
||||
assert_contains "deleted file under correct header" "deleted since base" "$out"
|
||||
assert_contains "deleted file names path" "spec/foo.md" "$out"
|
||||
|
||||
# ---- Test 8: renamed file with quote (exit 0) ---------------------------
|
||||
echo "=== renamed file with quote ==="
|
||||
T="$SCRATCH/t8"; mk_repo "$T"
|
||||
commit_file "$T" "spec/foo.md" \
|
||||
$'# Spec\n\nA line that stays.\n\nA line that gets revoked.\n\n' >/dev/null
|
||||
BASE=$(git -C "$T" rev-parse HEAD)
|
||||
# Rename + quote the old content in the new file.
|
||||
git -C "$T" mv "spec/foo.md" "spec/bar.md" >/dev/null
|
||||
write_file_via_py "$T" "spec/bar.md" \
|
||||
$'# Spec\n\nA line that stays.\n\nRevokes prior text (TEST): "A line that gets revoked."\n\n'
|
||||
out=$(run_check "$T" "$BASE" --tag TEST); rc=$?
|
||||
assert_exit "renamed file with quote exits 0" 0 "$rc" "$out" ""
|
||||
assert_contains "renamed file with quote says clean" "clean" "$out"
|
||||
|
||||
# ---- Test 9: short / markdown-only line removed (exit 0) ----------------
|
||||
echo "=== short / markdown-only line removed ==="
|
||||
T="$SCRATCH/t9"; mk_repo "$T"
|
||||
# Base file has: a heading, a very short line (< 12 chars), a markdown table
|
||||
# (with separator row + a data row), a horizontal rule, and a long real
|
||||
# content line. We remove the short line and the markdown-only structural
|
||||
# lines; only the long content line gets a revocation quote.
|
||||
BASE=$(commit_file "$T" "spec/foo.md" \
|
||||
$'# Spec\n\nshort\n\n| a | b |\n|---|---|\n| 1 | 2 |\n\n---\n\nReal content here that is plenty long enough.\n')
|
||||
write_file_via_py "$T" "spec/foo.md" \
|
||||
$'# Spec\n\n| a | b |\n| 1 | 2 |\n\nRevokes prior text (TEST): "Real content here that is plenty long enough."\n'
|
||||
out=$(run_check "$T" "$BASE" --tag TEST); rc=$?
|
||||
assert_exit "short/markdown-only removed exits 0" 0 "$rc" "$out" ""
|
||||
assert_contains "short/markdown-only removed says clean" "clean" "$out"
|
||||
|
||||
# ---- Test 10: untagged marker with no --tag (exit 0) -------------------
|
||||
echo "=== untagged marker with no --tag ==="
|
||||
T="$SCRATCH/t10"; mk_repo "$T"
|
||||
commit_file "$T" "spec/foo.md" \
|
||||
$'# Spec\n\nA line that stays.\n\nA line that gets revoked.\n\n' >/dev/null
|
||||
BASE=$(git -C "$T" rev-parse HEAD)
|
||||
# Use the untagged marker (no parens).
|
||||
write_file_via_py "$T" "spec/foo.md" \
|
||||
$'# Spec\n\nA line that stays.\n\nRevokes prior text: "A line that gets revoked."\n\n'
|
||||
# No --tag → should accept untagged marker.
|
||||
out=$(run_check "$T" "$BASE"); rc=$?
|
||||
assert_exit "untagged marker no --tag exits 0" 0 "$rc" "$out" ""
|
||||
assert_contains "untagged marker no --tag says clean" "clean" "$out"
|
||||
|
||||
# ---- Test 11: --tag filters other tags ---------------------------------
|
||||
echo "=== --tag filters other tags ==="
|
||||
T="$SCRATCH/t11"; mk_repo "$T"
|
||||
commit_file "$T" "spec/foo.md" \
|
||||
$'# Spec\n\nA line that stays.\n\nA line that gets revoked.\n\n' >/dev/null
|
||||
BASE=$(git -C "$T" rev-parse HEAD)
|
||||
# Quote uses M33, but we run with --tag M99, so the marker is malformed
|
||||
# (no closing quote because the regex requires "M99").
|
||||
write_file_via_py "$T" "spec/foo.md" \
|
||||
$'# Spec\n\nA line that stays.\n\nRevokes prior text (M33): "A line that gets revoked."\n\n'
|
||||
out=$(run_check "$T" "$BASE" --tag M99); rc=$?
|
||||
assert_exit "--tag filters other tag exits 1" 1 "$rc" "$out" ""
|
||||
# Without --tag, the marker WOULD have parsed and the line would survive.
|
||||
out_notag=$(run_check "$T" "$BASE"); rc_notag=$?
|
||||
assert_exit "no --tag accepts M33 marker exits 0" 0 "$rc_notag" "$out_notag" ""
|
||||
|
||||
# ---- Test 12: bad --base (exit 2) ---------------------------------------
|
||||
echo "=== bad --base ==="
|
||||
T="$SCRATCH/t12"; mk_repo "$T"
|
||||
commit_file "$T" "spec/foo.md" $'# Spec\n\nA line.\n' >/dev/null
|
||||
out=$(run_check "$T" "0000000000000000000000000000000000000000" --tag TEST 2>&1); rc=$?
|
||||
assert_exit "bad --base exits 2" 2 "$rc" "$out" ""
|
||||
assert_contains "bad --base names offending arg" "does not resolve" "$out"
|
||||
|
||||
# ---- Test 13: not a git repo (exit 2) -----------------------------------
|
||||
echo "=== not a git repo ==="
|
||||
T="$SCRATCH/t13"; mkdir -p "$T"
|
||||
write_file_via_py "$T" "spec/foo.md" $'# Spec\n\nA line.\n'
|
||||
out=$(run_check "$T" "HEAD" --tag TEST 2>&1); rc=$?
|
||||
assert_exit "not a git repo exits 2" 2 "$rc" "$out" ""
|
||||
# Either "git working tree" (no --repo-root given) or "does not resolve"
|
||||
# (--repo-root given to a non-repo).
|
||||
assert_contains "not a git repo error" "does not resolve" "$out"
|
||||
|
||||
# ---- Test 14: --json parses (exit 1, valid JSON) ------------------------
|
||||
echo "=== --json parses ==="
|
||||
T="$SCRATCH/t14"; mk_repo "$T"
|
||||
commit_file "$T" "spec/foo.md" \
|
||||
$'# Spec\n\nA line that stays.\n\nA line that gets revoked.\n\n' >/dev/null
|
||||
BASE=$(git -C "$T" rev-parse HEAD)
|
||||
write_file_via_py "$T" "spec/foo.md" $'# Spec\n\nA line that stays.\n\n'
|
||||
out=$(run_check "$T" "$BASE" --tag TEST --json); rc=$?
|
||||
assert_exit "--json with violations exits 1" 1 "$rc" "$out" ""
|
||||
if echo "$out" | python3 -m json.tool >/dev/null 2>&1; then
|
||||
echo -e "${GREEN}PASS${NC}: --json parses with python3 -m json.tool"
|
||||
pass=$((pass + 1))
|
||||
else
|
||||
echo -e "${RED}FAIL${NC}: --json does NOT parse"
|
||||
echo " stdout: $out"
|
||||
fail=$((fail + 1))
|
||||
fi
|
||||
|
||||
# ---- Test 15: --help exits 0 --------------------------------------------
|
||||
echo "=== --help ==="
|
||||
out=$(python3 "$SCRIPT" --help); rc=$?
|
||||
assert_exit "--help exits 0" 0 "$rc" "$out" ""
|
||||
assert_contains "--help shows usage" "spec-revocation-check" "$out"
|
||||
assert_contains "--help mentions --base" "--base" "$out"
|
||||
|
||||
# ---- Negative-test verification (would each negative fail if disabled?) -
|
||||
# A "negative" test would PASS against a no-op script that always exits 0.
|
||||
# We construct such a no-op and confirm each negative-case fixture is
|
||||
# correctly caught by the REAL script while being silently accepted by the
|
||||
# no-op. This proves the real script is what detected each issue.
|
||||
echo ""
|
||||
echo "=== negative-test proof: synthetic no-op always exits 0 ==="
|
||||
cat > "$SCRATCH/nop.py" <<'PYEOF'
|
||||
#!/usr/bin/env python3
|
||||
import json, os, sys
|
||||
out = {
|
||||
"repo_root": os.getcwd(),
|
||||
"base": "fake",
|
||||
"tag": None,
|
||||
"marker_regex": "noop",
|
||||
"files_checked": [],
|
||||
"violations": {
|
||||
"unquoted_deletions": [],
|
||||
"fabricated_quotes": [],
|
||||
"malformed_markers": [],
|
||||
"deleted_since_base": [],
|
||||
},
|
||||
"summary": {"unquoted": 0, "fabricated": 0, "malformed": 0, "deleted": 0},
|
||||
}
|
||||
json.dump(out, sys.stdout)
|
||||
sys.stdout.write("\n")
|
||||
sys.exit(0)
|
||||
PYEOF
|
||||
chmod +x "$SCRATCH/nop.py"
|
||||
|
||||
for case in t2 t3 t4 t6 t7; do
|
||||
T="$SCRATCH/$case"
|
||||
BASE=$(git -C "$T" rev-parse HEAD 2>/dev/null || echo "HEAD")
|
||||
out_nop=$(python3 "$SCRATCH/nop.py" --base HEAD --repo-root "$T" --glob "spec/**/*.md" 2>&1); rc_nop=$?
|
||||
# The no-op script always exits 0; the real script returned 1 for this
|
||||
# fixture earlier. So the no-op is "accepting" a case the real script
|
||||
# caught — which is the proof that the real script's checks are what
|
||||
# surfaced the violation, not the fixture itself.
|
||||
if [ "$rc_nop" -eq 0 ]; then
|
||||
echo -e "${GREEN}PASS${NC}: negative-case $case: real=1, noop=0 (check is doing the work)"
|
||||
pass=$((pass + 1))
|
||||
else
|
||||
echo -e "${RED}FAIL${NC}: noop unexpectedly returned $rc_nop on case $case"
|
||||
fail=$((fail + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
# ---- Done ---------------------------------------------------------------
|
||||
echo ""
|
||||
echo "Results: $pass passed, $fail failed"
|
||||
[[ $fail -eq 0 ]]
|
||||
Reference in New Issue
Block a user