score-paragraphs: port GOES paragraph scorer as general-purpose script
Required --style, --json machine-readable output, mandatory --dryrun. Engine unchanged; known limitation documented: already-scored paragraphs are never re-examined (present in the original). Claude-Session: https://claude.ai/code/session_01YQDoWNM7XPPii28khFWoMc
This commit is contained in:
Paul O'Reilly
353
tests/test-score-paragraphs.sh
Executable file
353
tests/test-score-paragraphs.sh
Executable file
@@ -0,0 +1,353 @@
|
||||
#!/usr/bin/env bash
|
||||
# test-score-paragraphs.sh — Verify score-paragraphs dryrun behaviour against the spec.
|
||||
#
|
||||
# Strategy: all assertions use --dryrun (-n), so no `claude` subprocess is ever
|
||||
# invoked and no output files are written. Fixtures (a style guide + a sample
|
||||
# markdown file with pre-embedded score blocks) are generated in a tempdir.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
SCORE_PARAGRAPHS="$REPO_ROOT/scripts/score-paragraphs"
|
||||
|
||||
GREEN='\033[0;32m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m'
|
||||
|
||||
PASS=0
|
||||
FAIL=0
|
||||
|
||||
assert_contains() {
|
||||
local label=$1 needle=$2 haystack=$3
|
||||
if [[ "$haystack" == *"$needle"* ]]; then
|
||||
printf " ${GREEN}PASS${NC} %s\n" "$label"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
printf " ${RED}FAIL${NC} %s\n" "$label"
|
||||
printf " expected to contain: %s\n" "$needle"
|
||||
printf " actual output:\n"
|
||||
printf '%s\n' "$haystack" | sed 's/^/ /'
|
||||
FAIL=$((FAIL + 1))
|
||||
fi
|
||||
}
|
||||
|
||||
assert_not_contains() {
|
||||
local label=$1 needle=$2 haystack=$3
|
||||
if [[ "$haystack" != *"$needle"* ]]; then
|
||||
printf " ${GREEN}PASS${NC} %s\n" "$label"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
printf " ${RED}FAIL${NC} %s\n" "$label"
|
||||
printf " expected NOT to contain: %s\n" "$needle"
|
||||
FAIL=$((FAIL + 1))
|
||||
fi
|
||||
}
|
||||
|
||||
assert_exit_code() {
|
||||
local label=$1 expected=$2 actual=$3
|
||||
if [[ "$expected" == "$actual" ]]; then
|
||||
printf " ${GREEN}PASS${NC} %s (exit=%s)\n" "$label" "$actual"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
printf " ${RED}FAIL${NC} %s (expected exit=%s, got exit=%s)\n" "$label" "$expected" "$actual"
|
||||
FAIL=$((FAIL + 1))
|
||||
fi
|
||||
}
|
||||
|
||||
assert_eq() {
|
||||
local label=$1 expected=$2 actual=$3
|
||||
if [[ "$expected" == "$actual" ]]; then
|
||||
printf " ${GREEN}PASS${NC} %s\n" "$label"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
printf " ${RED}FAIL${NC} %s\n" "$label"
|
||||
printf " expected: %s\n" "$expected"
|
||||
printf " actual: %s\n" "$actual"
|
||||
FAIL=$((FAIL + 1))
|
||||
fi
|
||||
}
|
||||
|
||||
# === Pre-flight ===
|
||||
if [[ ! -x "$SCORE_PARAGRAPHS" ]]; then
|
||||
echo "score-paragraphs not found or not executable at $SCORE_PARAGRAPHS" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# === Fixture setup ===
|
||||
TMPDIR_TEST="$(mktemp -d)"
|
||||
trap 'rm -rf "$TMPDIR_TEST"' EXIT
|
||||
|
||||
STYLE_FILE="${TMPDIR_TEST}/style.md"
|
||||
SAMPLE_FILE="${TMPDIR_TEST}/sample.md"
|
||||
SAMPLE2_FILE="${TMPDIR_TEST}/sample2.md"
|
||||
JSON_OUT="${TMPDIR_TEST}/scores.json"
|
||||
|
||||
cat > "$STYLE_FILE" <<'EOF'
|
||||
# Test style guide
|
||||
|
||||
Direct, evidence-backed, practitioner voice. No hedging, no buzzwords.
|
||||
EOF
|
||||
|
||||
# PARA_ALREADY_SCORED: long enough to score (32 words), immediately followed
|
||||
# by a score block in the script's own write format (zero blank-line
|
||||
# separation) whose embedded hash matches it exactly. Per the ported
|
||||
# engine's verified behaviour (see "Known limitation" in the spec), a
|
||||
# paragraph in this exact shape is absorbed into the surrounding "skip"
|
||||
# region by the chunker and is never re-examined for scoring again -- not
|
||||
# because its hash matched, but because it is no longer seen as `content`
|
||||
# at all. This is intentionally NOT a "skip unchanged" case to assert on;
|
||||
# it is a "does not appear anywhere" case.
|
||||
PARA_ALREADY_SCORED="This paragraph is intentionally long enough to clear the twenty five word minimum threshold used by the scoring engine, discussing evidence and voice and rhythm at some length purely for fixture purposes."
|
||||
|
||||
# PARA_ALREADY_SCORED_STALE: same shape, but the attached score block's hash
|
||||
# deliberately does NOT match the paragraph text. Locks in that this makes
|
||||
# no difference: it is absorbed into "skip" exactly like the matching-hash
|
||||
# case above, so a *changed* already-scored paragraph is not re-scored
|
||||
# either -- this is the concrete, load-bearing part of the limitation.
|
||||
PARA_ALREADY_SCORED_STALE="Here is a second long paragraph included specifically to exercise the changed hash code path, since its embedded score block below will intentionally carry a hash that does not match this paragraph text at all today."
|
||||
|
||||
# PARA_NEW: long enough to score (>=25 words) and has no score block
|
||||
# attached at all -> the one paragraph in this fixture that genuinely can
|
||||
# be scored, i.e. "would score" under --dryrun.
|
||||
PARA_NEW="This is a genuinely new paragraph that has never been scored before and easily clears the minimum word count threshold required for scoring eligibility in this fixture file today."
|
||||
|
||||
cat > "$SAMPLE_FILE" <<EOF
|
||||
---
|
||||
title: Sample
|
||||
---
|
||||
|
||||
# Chapter One
|
||||
|
||||
${PARA_ALREADY_SCORED}
|
||||
> \`◈\` E:2 · J:2 · V:2 · R:2 · G:2 = **10/15** \`¶af539c2\`
|
||||
> ⚑ *hedge* · _tighten the opening sentence_
|
||||
|
||||
Too short.
|
||||
|
||||
${PARA_ALREADY_SCORED_STALE}
|
||||
> \`◈\` E:1 · J:1 · V:1 · R:1 · G:1 = **5/15** \`¶0000000\`
|
||||
|
||||
${PARA_NEW}
|
||||
|
||||
| a | b |
|
||||
|---|---|
|
||||
| 1 | 2 |
|
||||
|
||||
\`\`\`python
|
||||
print("this code block must never be scored")
|
||||
\`\`\`
|
||||
EOF
|
||||
|
||||
# A second, small file with one fresh (never-scored) eligible paragraph, for
|
||||
# multi-file dryrun coverage.
|
||||
PARA_D="This is a separate file containing a single paragraph that easily clears the minimum word count threshold and has never been scored before at all."
|
||||
cat > "$SAMPLE2_FILE" <<EOF
|
||||
${PARA_D}
|
||||
EOF
|
||||
|
||||
run() {
|
||||
"$SCORE_PARAGRAPHS" "$@" 2>&1
|
||||
}
|
||||
|
||||
ORIG_SAMPLE_CHECKSUM="$(sha256sum "$SAMPLE_FILE" | awk '{print $1}')"
|
||||
|
||||
echo "=== score-paragraphs tests ==="
|
||||
echo
|
||||
|
||||
# === Test 1: --help ===
|
||||
echo "Test 1: --help"
|
||||
out=$(run --help)
|
||||
rc=$?
|
||||
assert_exit_code "--help exits 0" 0 "$rc"
|
||||
assert_contains "--help shows script name" "score-paragraphs" "$out"
|
||||
assert_contains "--help shows --style" "--style" "$out"
|
||||
assert_contains "--help shows --dryrun" "--dryrun" "$out"
|
||||
assert_contains "--help shows --json" "--json" "$out"
|
||||
assert_contains "--help shows --force" "--force" "$out"
|
||||
assert_contains "--help shows --parallel" "--parallel" "$out"
|
||||
|
||||
# --book (GOES chapter/appendix discovery) was dropped from this generalised
|
||||
# port -- it must not be a recognised flag.
|
||||
book_out=$(run --style "$STYLE_FILE" --dryrun --book "$SAMPLE_FILE")
|
||||
book_rc=$?
|
||||
assert_exit_code "--book is rejected (unrecognized argument)" 2 "$book_rc"
|
||||
assert_contains "--book rejection mentions unrecognized argument" "unrecognized argument" "$book_out"
|
||||
|
||||
# === Test 2: basic dryrun over the fixture ===
|
||||
echo
|
||||
echo "Test 2: basic dryrun"
|
||||
out=$(run --style "$STYLE_FILE" --dryrun "$SAMPLE_FILE")
|
||||
rc=$?
|
||||
assert_exit_code "dryrun exits 0" 0 "$rc"
|
||||
assert_contains "shows dryrun header" "[dryrun]" "$out"
|
||||
assert_contains "shows style path" "$STYLE_FILE" "$out"
|
||||
assert_contains "reports would-score paragraph (the fresh one)" "[would score]" "$out"
|
||||
assert_contains "reports too-short paragraph" "[skip too-short]" "$out"
|
||||
assert_contains "shows the would-be claude invocation" "would run: claude -p" "$out"
|
||||
assert_contains "shows run total" "Total:" "$out"
|
||||
assert_not_contains "dryrun never calls claude" "Scoring error" "$out"
|
||||
# Only PARA_NEW is eligible; the two already-scored paragraphs (matching AND
|
||||
# stale hash) are absorbed by the chunker and never reach the paragraph walk
|
||||
# at all -- see "Known limitation" in specs/score-paragraphs.spec.md.
|
||||
assert_contains "reports exactly 1 would-be-scored paragraph" "1 would be scored" "$out"
|
||||
assert_contains "reports 0 unchanged (hash-skip is unreachable)" "0 unchanged" "$out"
|
||||
assert_not_contains "already-scored paragraph hash never appears" "af539c2" "$out"
|
||||
assert_not_contains "already-scored (stale) paragraph never resurfaces" "0000000" "$out"
|
||||
|
||||
# === Test 3: dryrun makes no filesystem changes ===
|
||||
echo
|
||||
echo "Test 3: dryrun writes nothing"
|
||||
run --style "$STYLE_FILE" --dryrun "$SAMPLE_FILE" >/dev/null 2>&1
|
||||
NEW_CHECKSUM="$(sha256sum "$SAMPLE_FILE" | awk '{print $1}')"
|
||||
assert_eq "input file untouched by dryrun" "$ORIG_SAMPLE_CHECKSUM" "$NEW_CHECKSUM"
|
||||
DEFAULT_OUTPUT="${TMPDIR_TEST}/sample-scored.md"
|
||||
if [[ -e "$DEFAULT_OUTPUT" ]]; then
|
||||
printf " ${RED}FAIL${NC} dryrun must not create the default output file\n"
|
||||
FAIL=$((FAIL + 1))
|
||||
else
|
||||
printf " ${GREEN}PASS${NC} dryrun did not create the default output file\n"
|
||||
PASS=$((PASS + 1))
|
||||
fi
|
||||
|
||||
# === Test 4: dryrun works with no claude binary on PATH at all ===
|
||||
echo
|
||||
echo "Test 4: dryrun does not require claude on PATH"
|
||||
out=$(PATH="/usr/bin:/bin" "$SCORE_PARAGRAPHS" --style "$STYLE_FILE" --dryrun "$SAMPLE_FILE" 2>&1)
|
||||
rc=$?
|
||||
assert_exit_code "dryrun without claude on PATH still exits 0" 0 "$rc"
|
||||
assert_not_contains "no 'claude not found' error under dryrun" "claude' not found" "$out"
|
||||
|
||||
# === Test 5: real (non-dryrun) run requires claude on PATH, fails before touching files ===
|
||||
echo
|
||||
echo "Test 5: non-dryrun run with claude missing fails closed"
|
||||
out=$(PATH="/usr/bin:/bin" "$SCORE_PARAGRAPHS" --style "$STYLE_FILE" "$SAMPLE_FILE" 2>&1)
|
||||
rc=$?
|
||||
assert_exit_code "missing claude exits 1" 1 "$rc"
|
||||
assert_contains "missing claude error message" "'claude' not found on PATH" "$out"
|
||||
NEW_CHECKSUM2="$(sha256sum "$SAMPLE_FILE" | awk '{print $1}')"
|
||||
assert_eq "input file untouched when claude missing" "$ORIG_SAMPLE_CHECKSUM" "$NEW_CHECKSUM2"
|
||||
|
||||
# === Test 6: --force does NOT resurrect already-scored paragraphs ===
|
||||
# This locks in the verified (if surprising) real behaviour: --force only
|
||||
# widens the hash-comparison inside _jobs_for_file, but that function only
|
||||
# ever sees `content`-classified chunks, and already-scored paragraphs are
|
||||
# not `content` chunks by the time --force would matter. See "Known
|
||||
# limitation" in specs/score-paragraphs.spec.md.
|
||||
echo
|
||||
echo "Test 6: --force does not resurrect already-scored paragraphs"
|
||||
out=$(run --style "$STYLE_FILE" --dryrun --force "$SAMPLE_FILE")
|
||||
rc=$?
|
||||
assert_exit_code "force dryrun exits 0" 0 "$rc"
|
||||
assert_contains "still exactly 1 would-be-scored paragraph with --force" "1 would be scored" "$out"
|
||||
assert_not_contains "already-scored paragraph still invisible under --force" "af539c2" "$out"
|
||||
assert_not_contains "stale already-scored paragraph still invisible under --force" "0000000" "$out"
|
||||
|
||||
# === Test 7: -n shorthand ===
|
||||
echo
|
||||
echo "Test 7: -n shorthand"
|
||||
out=$(run --style "$STYLE_FILE" -n "$SAMPLE_FILE")
|
||||
rc=$?
|
||||
assert_exit_code "-n shorthand exits 0" 0 "$rc"
|
||||
assert_contains "-n shows dryrun header" "[dryrun]" "$out"
|
||||
|
||||
# === Test 8: dryrun + --json - emits valid JSON to stdout only ===
|
||||
echo
|
||||
echo "Test 8: dryrun + --json - (stdout)"
|
||||
json_out=$("$SCORE_PARAGRAPHS" --style "$STYLE_FILE" --dryrun --json - "$SAMPLE_FILE" 2>/dev/null)
|
||||
rc=$?
|
||||
assert_exit_code "json-to-stdout dryrun exits 0" 0 "$rc"
|
||||
if printf '%s' "$json_out" | python3 -c 'import json,sys; json.load(sys.stdin)' 2>/dev/null; then
|
||||
printf " ${GREEN}PASS${NC} stdout is valid JSON when --json -\n"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
printf " ${RED}FAIL${NC} stdout is not valid JSON when --json -\n"
|
||||
printf '%s\n' "$json_out" | sed 's/^/ /'
|
||||
FAIL=$((FAIL + 1))
|
||||
fi
|
||||
assert_contains "JSON marks dryrun true" '"dryrun": true' "$json_out"
|
||||
assert_contains "JSON has a would_score action" '"action": "would_score"' "$json_out"
|
||||
assert_contains "JSON would_score entries have null scores" '"scores": null' "$json_out"
|
||||
assert_contains "JSON paragraphs_scored counts only the fresh paragraph" '"paragraphs_scored": 1' "$json_out"
|
||||
# "reused" is defined in the schema but unreachable via this fixture (or any
|
||||
# file this script itself wrote) -- see "Known limitation" in the spec.
|
||||
assert_not_contains "no reused action for this fixture (hash-skip is unreachable)" '"action": "reused"' "$json_out"
|
||||
|
||||
# Progress lines must NOT pollute stdout when --json - is used.
|
||||
stderr_out=$("$SCORE_PARAGRAPHS" --style "$STYLE_FILE" --dryrun --json - "$SAMPLE_FILE" 2>&1 1>/dev/null)
|
||||
assert_contains "progress lines move to stderr with --json -" "[dryrun]" "$stderr_out"
|
||||
|
||||
# === Test 9: dryrun + --json <file> does not write the file ===
|
||||
echo
|
||||
echo "Test 9: dryrun + --json FILE (no write)"
|
||||
out=$(run --style "$STYLE_FILE" --dryrun --json "$JSON_OUT" "$SAMPLE_FILE")
|
||||
rc=$?
|
||||
assert_exit_code "dryrun json-to-file exits 0" 0 "$rc"
|
||||
assert_contains "announces it would write the JSON file" "would write JSON results to" "$out"
|
||||
if [[ -e "$JSON_OUT" ]]; then
|
||||
printf " ${RED}FAIL${NC} --json FILE must not be written under --dryrun\n"
|
||||
FAIL=$((FAIL + 1))
|
||||
else
|
||||
printf " ${GREEN}PASS${NC} --json FILE not written under --dryrun\n"
|
||||
PASS=$((PASS + 1))
|
||||
fi
|
||||
|
||||
# === Test 10: multiple input files in one dryrun invocation ===
|
||||
echo
|
||||
echo "Test 10: multiple input files"
|
||||
out=$(run --style "$STYLE_FILE" --dryrun "$SAMPLE_FILE" "$SAMPLE2_FILE")
|
||||
rc=$?
|
||||
assert_exit_code "multi-file dryrun exits 0" 0 "$rc"
|
||||
assert_contains "mentions first file" "$SAMPLE_FILE" "$out"
|
||||
assert_contains "mentions second file" "$SAMPLE2_FILE" "$out"
|
||||
|
||||
# === Test 11: error — --output with multiple input files ===
|
||||
echo
|
||||
echo "Test 11: error — --output with multiple files"
|
||||
out=$(run --style "$STYLE_FILE" --dryrun --output "${TMPDIR_TEST}/out.md" "$SAMPLE_FILE" "$SAMPLE2_FILE")
|
||||
rc=$?
|
||||
assert_exit_code "multi-file --output exits 1" 1 "$rc"
|
||||
assert_contains "multi-file --output error" "only be used with a single input file" "$out"
|
||||
|
||||
# === Test 12: error — input file not found ===
|
||||
echo
|
||||
echo "Test 12: error — input file not found"
|
||||
out=$(run --style "$STYLE_FILE" --dryrun "${TMPDIR_TEST}/does-not-exist.md")
|
||||
rc=$?
|
||||
assert_exit_code "missing input file exits 1" 1 "$rc"
|
||||
assert_contains "missing input file error" "file not found" "$out"
|
||||
|
||||
# === Test 13: error — style guide not found ===
|
||||
echo
|
||||
echo "Test 13: error — style guide not found"
|
||||
out=$(run --style "${TMPDIR_TEST}/no-such-style.md" --dryrun "$SAMPLE_FILE")
|
||||
rc=$?
|
||||
assert_exit_code "missing style file exits 1" 1 "$rc"
|
||||
assert_contains "missing style file error" "style guide not found" "$out"
|
||||
|
||||
# === Test 14: error — --style is required ===
|
||||
echo
|
||||
echo "Test 14: error — --style missing entirely"
|
||||
out=$(run --dryrun "$SAMPLE_FILE")
|
||||
rc=$?
|
||||
assert_exit_code "missing --style exits 2 (argparse required-arg error)" 2 "$rc"
|
||||
assert_contains "missing --style mentions the flag" "--style" "$out"
|
||||
|
||||
# === Test 15: error — no input files given ===
|
||||
echo
|
||||
echo "Test 15: error — no positional files"
|
||||
out=$(run --style "$STYLE_FILE" --dryrun)
|
||||
rc=$?
|
||||
assert_exit_code "no files exits 2 (argparse required-arg error)" 2 "$rc"
|
||||
|
||||
# === Summary ===
|
||||
echo
|
||||
echo "================================"
|
||||
TOTAL=$((PASS + FAIL))
|
||||
if [[ "$FAIL" -eq 0 ]]; then
|
||||
printf "${GREEN}All %d tests passed${NC}\n" "$TOTAL"
|
||||
exit 0
|
||||
else
|
||||
printf "${RED}%d of %d tests failed${NC}\n" "$FAIL" "$TOTAL"
|
||||
exit 1
|
||||
fi
|
||||
Reference in New Issue
Block a user