skynet/small-scripts
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

score-paragraphs: port GOES paragraph scorer as general-purpose script

Browse Source 
Required --style, --json machine-readable output, mandatory --dryrun.
Engine unchanged; known limitation documented: already-scored paragraphs
are never re-examined (present in the original).

Claude-Session: https://claude.ai/code/session_01YQDoWNM7XPPii28khFWoMc
This commit is contained in:
Paul O'Reilly
2026-08-02 21:18:02 +12:00
parent 52eb484ef7
commit f78d292f05
3 changed files with 1409 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

733
scripts/score-paragraphs Executable file
View File

@@ -0,0 +1,733 @@
#!/usr/bin/env python3
"""Score markdown paragraphs against a style guide via `claude -p`.
Reads one or more markdown files, identifies paragraphs that need scoring
(new or changed), scores them in parallel via `claude -p` subprocesses, and
writes *-scored.md files with compact score blocks inserted after each
paragraph. Optionally also emits the results as machine-readable JSON.
Usage:
score-paragraphs --style style/voice.md book/chapter-01.md [chapter-02.md ...]
score-paragraphs --style style/voice.md book/chapter-01.md -o book/chapter-01-scored.md
score-paragraphs --style style/voice.md book/chapter-01.md --json /tmp/scores.json
score-paragraphs --style style/voice.md book/chapter-01.md --dryrun
Scoring dimensions (each 0–3):
E Evidence — specific claims backed by named sources / research links
J Judgment — direct language; no hedging where evidence supports a claim
V Voice — free of LLM tells, buzzwords, corporate softening
R Rhythm — sentence and paragraph length variety; not uniform blocks
G Register — appropriate to the target genre (not flat framework, not academic)
Score block format (inserted after each scored paragraph):
> `◈` E:3 · J:2 · V:3 · R:2 · G:3 = **13/15** `¶a3f5b2`
> ⚑ *"potentially" (s2) — evidence supports direct claim*
The `¶xxxxxxx` suffix is the first 7 chars of the SHA-256 of the paragraph text.
On re-runs, paragraphs whose hash matches their existing score block are skipped
automatically. Changed paragraphs are re-scored and their stale block replaced.
Use --force to re-score everything regardless of hash.
This is a generalised port of the GOES-repo score-paragraphs.py: --style
replaces a repo-relative default style guide, --book (GOES chapter/appendix
discovery) is gone, and --dryrun / --json are new. See
specs/score-paragraphs.spec.md for the full spec.
Requires: `claude` CLI on PATH (Claude Code) for any non-dryrun run.
"""
import argparse
import hashlib
import json
import re
import shutil
import subprocess
import sys
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from typing import Optional
# Paragraphs shorter than this (word count) are not scored
MIN_WORDS = 25
DIMENSIONS = ("evidence", "judgment", "voice", "rhythm", "register")
# ── Score block parsing / detection ──────────────────────────────────
SCORE_MARKER = "`◈`"
HASH_RE = re.compile(r"`¶([0-9a-f]{7})`")
SCORE_LINE_RE = re.compile(
r"E:(\d+)\s*·\s*J:(\d+)\s*·\s*V:(\d+)\s*·\s*R:(\d+)\s*·\s*G:(\d+)"
)
FLAG_ITEM_RE = re.compile(r"^⚑ \*(.*)\*$")
NOTE_ITEM_RE = re.compile(r"^_(.*)_$")
def para_hash(text: str) -> str:
"""Return first 7 hex chars of SHA-256 of the paragraph text."""
return hashlib.sha256(text.strip().encode()).hexdigest()[:7]
def is_score_block(line: str) -> bool:
return line.startswith("> ") and SCORE_MARKER in line
def extract_hash_from_score_block(block_text: str) -> Optional[str]:
"""Extract the embedded paragraph hash from a score block, or None."""
m = HASH_RE.search(block_text)
return m.group(1) if m else None
def parse_score_block(block_text: str) -> dict:
"""Reconstruct a scores dict from an already-written score block.
Used to recover the scores for a paragraph that was skipped this run
because its hash matched (--json needs its score too, but no `claude`
call is made to get it — it's read back out of the block text itself).
"""
lines = block_text.splitlines()
m = SCORE_LINE_RE.search(lines[0]) if lines else None
if not m:
return _error_score("unparseable score block")
e, j, v, r, g = (int(x) for x in m.groups())
flags = []
note = ""
if len(lines) > 1:
detail = lines[1]
if detail.startswith("> "):
detail = detail[2:]
for part in detail.split(" · "):
part = part.strip()
fm = FLAG_ITEM_RE.match(part)
if fm:
flags.append(fm.group(1))
continue
nm = NOTE_ITEM_RE.match(part)
if nm:
note = nm.group(1)
return {"evidence": e, "judgment": j, "voice": v, "rhythm": r,
"register": g, "flags": flags, "note": note}
# ── Markdown paragraph splitting ─────────────────────────────────────
def split_into_chunks(text: str) -> list:
"""Split markdown into chunks tagged as 'content' or 'skip'.
'skip': YAML frontmatter, fenced code blocks, headings, tables,
horizontal rules, empty lines, existing score blocks.
'content': substantive prose paragraphs worth scoring.
"""
lines = text.splitlines(keepends=True)
chunks = []
in_frontmatter = lines and lines[0].strip() == "---"
current_lines = []
def flush(kind):
if current_lines:
chunks.append({"kind": kind, "text": "".join(current_lines)})
current_lines.clear()
i = 0
while i < len(lines):
line = lines[i]
stripped = line.strip()
# YAML frontmatter
if in_frontmatter:
current_lines.append(line)
if stripped == "---" and i > 0:
in_frontmatter = False
flush("skip")
i += 1
continue
# Fenced code blocks
if stripped.startswith("```") or stripped.startswith("~~~"):
flush("skip")
current_lines.append(line)
fence = stripped[:3]
i += 1
while i < len(lines):
current_lines.append(lines[i])
if lines[i].strip().startswith(fence):
i += 1
break
i += 1
flush("skip")
continue
# Existing score blocks
#
# NOTE (verified during the port, not fixed — see "Known limitation"
# in specs/score-paragraphs.spec.md): this flush("skip") absorbs
# whatever is *currently accumulating* in current_lines, not just the
# score-block line itself. Since format_score_block() is written with
# zero blank-line separation from its paragraph, a paragraph that was
# already scored is still accumulating here when this branch fires —
# so it gets swallowed into the same "skip" chunk as its own score
# block, on every subsequent parse. That paragraph is no longer
# `kind == "content"` at all, so the hash-comparison branch below in
# `_jobs_for_file` (and even --force) never gets a chance to run
# against it. This makes that hash-comparison branch structurally
# unreachable for any file this script itself wrote — confirmed
# against real GOES output. The logic is kept as-is (unchanged
# engine, per the port's scope) rather than reworked here.
if is_score_block(line):
flush("skip")
current_lines.append(line)
i += 1
while i < len(lines) and lines[i].startswith("> "):
current_lines.append(lines[i])
i += 1
flush("skip")
continue
# Empty line — paragraph boundary
if not stripped:
flush("content")
current_lines.append(line)
flush("skip")
i += 1
continue
# Headings
if stripped.startswith("#"):
flush("content")
current_lines.append(line)
flush("skip")
i += 1
continue
# Tables
if stripped.startswith("|"):
flush("content")
current_lines.append(line)
i += 1
while i < len(lines) and lines[i].strip().startswith("|"):
current_lines.append(lines[i])
i += 1
flush("skip")
continue
# Horizontal rules
if re.match(r"^[-*_]{3,}$", stripped):
flush("content")
current_lines.append(line)
flush("skip")
i += 1
continue
current_lines.append(line)
i += 1
flush("content")
return chunks
def word_count(text: str) -> int:
return len(text.split())
# ── Claude scoring via `claude -p` ───────────────────────────────────
PROMPT_TEMPLATE = """\
You are a voice-quality reviewer. Score the paragraph at the end of this prompt \
against the style guide below.
{style_guide}
---
## Scoring rubric
Score the paragraph on five dimensions (each 0–3).
**E — Evidence (0–3)**
- 3: Specific claims backed by named research links or named adopters
- 2: Most claims grounded; one or two assertions lack explicit sourcing
- 1: Claims present but mostly asserted without named backing
- 0: Generic assertions with no evidence anchoring
**J — Judgment (0–3)**
- 3: Direct, committed language; no hedging where evidence supports a claim
- 2: Mostly direct; one unnecessary hedge (*could*, *might*, *may*, *potentially*)
- 1: Noticeable hedging; evidence would support stronger claims
- 0: Pervasive hedging throughout
**V — Voice (0–3)**
- 3: Practitioner register; free of LLM tells and burn-list buzzwords
- 2: Mostly clean; one slip into corporate softening or LLM phrasing
- 1: Several buzzwords (*leverage*, *empower*, *unlock*, *robust*, *transformative*, etc.)
- 0: Reads like an LLM or marketing copy
**R — Rhythm (0–3)**
- 3: Deliberate sentence and paragraph length variety
- 2: Mostly varied; slight tendency toward uniform medium-length sentences
- 1: Noticeably uniform sentence length throughout
- 0: Wall-of-text or robotic uniformity
**G — Register (0–3)**
- 3: Correctly matches the expected genre register (authored-prose, framework, blog, etc.)
- 2: Mostly correct; slight drift
- 1: Wrong register for the genre
- 0: Completely mismatched
## Smell-test flags
Report any present:
- Burn-list buzzword (*delve*, *journey*, *leverage*, *robust*, *foster*, *streamline*, \
*transformative*, *unlock*, *harness*, *pivotal*, *multifaceted*, *empower*, *synergize*)
- "It's important to note that" / "It's worth noting that"
- "In conclusion" / "Ultimately" / "In essence" as opener or closer
- Rule-of-three triplets in consecutive sentences
- Hedging where evidence supports a direct claim
- Throat-clearing opener ("In today's...", "In the ever-evolving...")
- Specific statistic without a research link
- Signposted conclusion that recaps and motivates
## Output format
Respond with ONLY a JSON object — no markdown fences, no preamble:
{{"evidence": <0-3>, "judgment": <0-3>, "voice": <0-3>, "rhythm": <0-3>, \
"register": <0-3>, "flags": ["<flag>", ...], "note": "<one actionable sentence or empty>"}}
If the text is too short or not scoreable prose, return all zeros and an empty note.
---
## Paragraph to score:
{paragraph}"""
def build_prompt(style_guide: str, paragraph: str) -> str:
return PROMPT_TEMPLATE.format(
style_guide=style_guide,
paragraph=paragraph.strip(),
)
def score_paragraph(style_guide: str, paragraph: str) -> dict:
"""Call `claude -p` to score one paragraph. Returns a score dict."""
prompt = build_prompt(style_guide, paragraph)
try:
result = subprocess.run(
["claude", "-p", prompt],
capture_output=True,
text=True,
timeout=120,
)
raw = result.stdout.strip()
# Strip markdown code fences if Claude wraps the JSON
raw = re.sub(r"^```(?:json)?\s*", "", raw)
raw = re.sub(r"\s*```\s*$", "", raw)
return json.loads(raw)
except subprocess.TimeoutExpired:
return _error_score("timeout")
except json.JSONDecodeError as e:
return _error_score(f"json parse error: {e}")
except Exception as e:
return _error_score(str(e))
def _error_score(reason: str) -> dict:
return {"evidence": 0, "judgment": 0, "voice": 0, "rhythm": 0,
"register": 0, "flags": [f"scoring error: {reason}"], "note": ""}
def format_score_block(scores: dict, phash: str) -> str:
"""Format a score dict as one or two blockquote lines, embedding the paragraph hash."""
e = scores.get("evidence", 0)
j = scores.get("judgment", 0)
v = scores.get("voice", 0)
r = scores.get("rhythm", 0)
g = scores.get("register", 0)
total = e + j + v + r + g
flags = scores.get("flags", [])
note = (scores.get("note") or "").strip()
line1 = f"> `◈` E:{e} · J:{j} · V:{v} · R:{r} · G:{g} = **{total}/15** `¶{phash}`"
lines = [line1]
parts = [f"⚑ *{f}*" for f in flags]
if note and note not in " ".join(flags):
parts.append(f"_{note}_")
if parts:
lines.append("> " + " · ".join(parts))
return "\n".join(lines)
def _json_paragraph_entry(index: int, phash: str, action: str, scores: dict) -> dict:
e = scores.get("evidence", 0)
j = scores.get("judgment", 0)
v = scores.get("voice", 0)
r = scores.get("rhythm", 0)
g = scores.get("register", 0)
return {
"index": index,
"hash": phash,
"action": action,
"scores": {"evidence": e, "judgment": j, "voice": v, "rhythm": r, "register": g},
"total": e + j + v + r + g,
"flags": scores.get("flags", []),
"note": (scores.get("note") or "").strip(),
}
def _summarize(paragraphs: list, paragraphs_skipped: int) -> dict:
totals = [p["total"] for p in paragraphs if p.get("total") is not None]
return {
"mean": round(sum(totals) / len(totals), 2) if totals else None,
"min": min(totals) if totals else None,
"paragraphs_scored": len(paragraphs),
"paragraphs_skipped": paragraphs_skipped,
}
# ── File processing ───────────────────────────────────────────────────
def _jobs_for_file(chunks: list, force: bool) -> list:
"""Return list of (chunk_idx, para_text, phash, stale_block_follows) for paragraphs
that need scoring."""
jobs = []
for idx, chunk in enumerate(chunks):
if chunk["kind"] != "content":
continue
para_text = chunk["text"]
if word_count(para_text) < MIN_WORDS:
continue
phash = para_hash(para_text)
stale_follows = False
if not force:
next_chunk = chunks[idx + 1] if idx + 1 < len(chunks) else None
if next_chunk:
first_line = next_chunk["text"].splitlines()[0] if next_chunk["text"] else ""
if next_chunk["kind"] == "skip" and is_score_block(first_line):
existing_hash = extract_hash_from_score_block(next_chunk["text"])
if existing_hash == phash:
continue # Unchanged — skip
stale_follows = True # Changed — stale block needs dropping
jobs.append((idx, para_text, phash, stale_follows))
return jobs
def dryrun_file(chunks: list, jobs: list, log) -> dict:
"""Report what would happen for one file, without calling claude or writing anything.
Returns a dict shaped like process_file's return value so the caller can
print the same summary lines and (optionally) assemble a preview JSON.
"""
job_idxs = {idx for idx, *_ in jobs}
total_eligible = sum(
1 for c in chunks
if c["kind"] == "content" and word_count(c["text"]) >= MIN_WORDS
)
total_short = sum(
1 for c in chunks
if c["kind"] == "content" and word_count(c["text"]) < MIN_WORDS
)
needs_scoring = len(jobs)
already_ok = total_eligible - needs_scoring
log(f" {needs_scoring} would be scored, {already_ok} unchanged "
f"(hash match, would skip), {total_short} too short (skipped)")
paragraphs = []
ordinal = 0
for idx, chunk in enumerate(chunks):
if chunk["kind"] != "content":
continue
wc = word_count(chunk["text"])
if wc < MIN_WORDS:
log(f" [skip too-short] paragraph ~{wc} words")
continue
ordinal += 1
phash = para_hash(chunk["text"])
if idx in job_idxs:
log(f" [would score] ¶{phash} (#{ordinal}, {wc} words) — "
f"would run: claude -p <scoring prompt>")
# Score is unknown pre-run — no claude call made in dryrun.
paragraphs.append({
"index": ordinal,
"hash": phash,
"action": "would_score",
"scores": None,
"total": None,
"flags": [],
"note": "",
})
else:
next_chunk = chunks[idx + 1] if idx + 1 < len(chunks) else None
reused_scores = None
if next_chunk and next_chunk["kind"] == "skip":
first_line = next_chunk["text"].splitlines()[0] if next_chunk["text"] else ""
if is_score_block(first_line):
reused_scores = parse_score_block(next_chunk["text"])
if reused_scores is None:
reused_scores = _error_score("could not reuse existing score block")
log(f" [skip unchanged] ¶{phash} (#{ordinal}) — "
f"hash matches existing score block")
paragraphs.append(_json_paragraph_entry(ordinal, phash, "reused", reused_scores))
return {
"scored": needs_scoring,
"skipped": already_ok,
"paragraphs": paragraphs,
"summary": _summarize(paragraphs, total_short),
}
def process_file(
input_path: Path,
output_path: Path,
style_guide: str,
max_workers: int,
force: bool,
log,
) -> dict:
"""Score paragraphs in input_path in parallel, write to output_path.
Returns a dict: {"scored", "skipped", "paragraphs", "summary"}.
"""
text = input_path.read_text(encoding="utf-8")
chunks = split_into_chunks(text)
jobs = _jobs_for_file(chunks, force)
total_eligible = sum(
1 for c in chunks
if c["kind"] == "content" and word_count(c["text"]) >= MIN_WORDS
)
total_short = sum(
1 for c in chunks
if c["kind"] == "content" and word_count(c["text"]) < MIN_WORDS
)
needs_scoring = len(jobs)
already_ok = total_eligible - needs_scoring
log(f" {input_path.name}: {needs_scoring} to score, "
f"{already_ok} unchanged (skipping)")
# Score all jobs in parallel
results = {} # chunk_idx -> (scores, phash)
stale = set() # chunk indices whose following score block should be dropped
if jobs:
completed = 0
with ThreadPoolExecutor(max_workers=max_workers) as executor:
future_map = {
executor.submit(score_paragraph, style_guide, para): (idx, phash, stale_follows)
for idx, para, phash, stale_follows in jobs
}
for future in as_completed(future_map):
idx, phash, stale_follows = future_map[future]
scores = future.result()
results[idx] = (scores, phash)
if stale_follows:
stale.add(idx)
completed += 1
total = sum(scores.get(k, 0) for k in DIMENSIONS)
log(f" [{completed}/{needs_scoring}] ¶{phash} {total}/15")
# Assemble output in chunk order, and the JSON paragraph list alongside it
output_parts = []
drop_next_score_block = False
json_paragraphs = []
ordinal = 0
for idx, chunk in enumerate(chunks):
if chunk["kind"] != "content":
first_line = chunk["text"].splitlines()[0] if chunk["text"] else ""
if drop_next_score_block and is_score_block(first_line):
drop_next_score_block = False
continue # Drop stale score block
drop_next_score_block = False
output_parts.append(chunk["text"])
continue
output_parts.append(chunk["text"])
wc = word_count(chunk["text"])
if wc < MIN_WORDS:
continue # Too short — never scored, no block, not in JSON
ordinal += 1
if idx in results:
scores, phash = results[idx]
output_parts.append(format_score_block(scores, phash) + "\n\n")
if idx in stale:
drop_next_score_block = True # Remove the now-replaced stale block
json_paragraphs.append(_json_paragraph_entry(ordinal, phash, "scored", scores))
else:
# Unchanged — hash matched, no claude call made. Recover its score
# from the existing block (already present in output_parts via
# the chunk text loop below, since we haven't touched that chunk).
phash = para_hash(chunk["text"])
next_chunk = chunks[idx + 1] if idx + 1 < len(chunks) else None
reused_scores = None
if next_chunk and next_chunk["kind"] == "skip":
first_line = next_chunk["text"].splitlines()[0] if next_chunk["text"] else ""
if is_score_block(first_line):
reused_scores = parse_score_block(next_chunk["text"])
if reused_scores is None:
reused_scores = _error_score("could not reuse existing score block")
json_paragraphs.append(_json_paragraph_entry(ordinal, phash, "reused", reused_scores))
output_path.write_text("".join(output_parts), encoding="utf-8")
return {
"scored": needs_scoring,
"skipped": already_ok,
"paragraphs": json_paragraphs,
"summary": _summarize(json_paragraphs, total_short),
}
# ── CLI ───────────────────────────────────────────────────────────────
def build_arg_parser() -> argparse.ArgumentParser:
ap = argparse.ArgumentParser(
prog="score-paragraphs",
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
ap.add_argument("files", nargs="+", help="Markdown files to score")
ap.add_argument("--style", required=True, metavar="FILE",
help="Style guide markdown file to score paragraphs against")
ap.add_argument("-o", "--output",
help="Output file (only valid with a single input file)")
ap.add_argument("--json", metavar="FILE",
help="Also emit machine-readable JSON results to FILE, or - for stdout")
ap.add_argument("--parallel", type=int, default=4, metavar="N",
help="Number of parallel claude -p calls (default: 4)")
ap.add_argument("--force", action="store_true",
help="Re-score all paragraphs, ignoring existing hash matches")
ap.add_argument("--dryrun", "-n", action="store_true",
help="Preview what would be scored/skipped; makes no claude calls, "
"writes no files")
return ap
def main(argv=None) -> int:
ap = build_arg_parser()
args = ap.parse_args(argv if argv is not None else sys.argv[1:])
json_to_stdout = args.json == "-"
def log(msg: str = "") -> None:
print(msg, file=sys.stderr if json_to_stdout else sys.stdout, flush=True)
# Check claude is available (skip in dryrun — no calls are made)
if not args.dryrun and not shutil.which("claude"):
print("Error: 'claude' not found on PATH. Install Claude Code to continue.",
file=sys.stderr)
return 1
# Resolve input files
input_paths = []
for f in args.files:
p = Path(f)
if not p.is_absolute():
p = Path.cwd() / p
input_paths.append(p)
if args.output and len(input_paths) > 1:
print("Error: --output can only be used with a single input file", file=sys.stderr)
return 1
# Resolve output paths
io_pairs = []
for inp in input_paths:
if args.output:
out = Path(args.output)
else:
stem = inp.stem
if not stem.endswith("-scored"):
stem += "-scored"
out = inp.parent / (stem + inp.suffix)
io_pairs.append((inp, out))
for inp, _ in io_pairs:
if not inp.exists():
print(f"Error: file not found: {inp}", file=sys.stderr)
return 1
# Load style guide
style_path = Path(args.style)
if not style_path.is_absolute():
style_path = Path.cwd() / style_path
if not style_path.exists():
print(f"Error: style guide not found: {style_path}", file=sys.stderr)
return 1
style_guide = style_path.read_text(encoding="utf-8")
if args.dryrun:
log("[dryrun] score-paragraphs — no claude calls will be made, no files will be written")
log(f"Style: {style_path}")
log(f"Parallel: {args.parallel} workers")
total_scored = total_skipped = 0
json_files = []
for inp, out in io_pairs:
log(f"\nScoring: {inp} → {out}")
if args.dryrun:
chunks = split_into_chunks(inp.read_text(encoding="utf-8"))
jobs = _jobs_for_file(chunks, args.force)
result = dryrun_file(chunks, jobs, log)
else:
result = process_file(inp, out, style_guide, args.parallel, args.force, log)
total_scored += result["scored"]
total_skipped += result["skipped"]
if args.dryrun:
log(f" Would score: {result['scored']}, unchanged: {result['skipped']}")
else:
log(f" Done: {result['scored']} scored, {result['skipped']} unchanged")
json_files.append({
"input": str(inp),
"output": str(out),
"paragraphs": result["paragraphs"],
"summary": result["summary"],
})
if args.dryrun:
log(f"\nTotal: {total_scored} would be scored, {total_skipped} unchanged.")
else:
log(f"\nTotal: {total_scored} paragraphs scored, {total_skipped} unchanged.")
# --- JSON output -------------------------------------------------------
if args.json:
payload = {
"style": str(style_path),
"dryrun": args.dryrun,
"files": json_files,
}
json_text = json.dumps(payload, indent=2)
if args.json == "-":
# Stdout is not a filesystem change — honoured even under --dryrun.
print(json_text)
elif args.dryrun:
log(f"[dryrun] would write JSON results to {args.json}")
else:
Path(args.json).write_text(json_text + "\n", encoding="utf-8")
log(f"JSON results written to {args.json}")
return 0
if __name__ == "__main__":
sys.exit(main())

323
specs/score-paragraphs.spec.md Normal file
View File

@@ -0,0 +1,323 @@
# score-paragraphs
## Purpose
Score markdown paragraphs against a style guide via `claude -p`, inserting a
compact per-paragraph score block after each scored paragraph and, on request,
emitting the same results as machine-readable JSON for downstream tooling.
This is a generalised port of `~/dev/claude/octopus/goes/scripts/score-paragraphs.py`
(the GOES book's voice-scoring engine). The scoring engine — chunking, hashing,
the five-dimension rubric, and the `claude -p` prompt — is unchanged. What
changed is the coupling to the GOES repo layout: the style guide is now an
explicit `--style` argument instead of a repo-relative default, and `--book`
(GOES chapter/appendix discovery) is gone.
## Usage
```
score-paragraphs --style FILE [OPTIONS] FILE [FILE ...]
```
### Required
| Argument | Description |
|---|---|
| `FILE ...` (positional) | One or more markdown files to score. At least one required. |
| `--style FILE` | Style guide markdown file to score paragraphs against. No default — the caller always states which voice the paragraphs are graded against. |
### Options
| Flag | Default | Description |
|---|---|---|
| `-o, --output FILE` | `<input>-scored.md` next to the input | Output file. Only valid with a single input `FILE`. |
| `--json FILE\|-` | (none) | Also emit machine-readable results (see [JSON output](#json-output)). `-` writes to stdout. |
| `--parallel N` | `4` | Number of parallel `claude -p` calls per file. |
| `--force` | off | Re-score all eligible paragraphs, ignoring existing hash matches. |
| `--dryrun`, `-n` | off | Preview which paragraphs would be scored/skipped and which `claude -p` calls would run. Makes no `claude` calls and writes no files (see [Dryrun behaviour](#dryrun-behaviour)). |
| `--help`, `-h` | — | Show usage and exit 0. |
Requires the `claude` CLI on PATH for any non-dryrun run — the script exits 1
immediately if it is missing, before touching any input.
## Scoring dimensions (each 0–3, unchanged from the source engine)
| Dim | Name | What it measures |
|---|---|---|
| E | Evidence | Specific claims backed by named sources / research links |
| J | Judgment | Direct language; no hedging where evidence supports a claim |
| V | Voice | Free of LLM tells, buzzwords, corporate softening |
| R | Rhythm | Sentence and paragraph length variety; not uniform blocks |
| G | Register | Appropriate to the target genre (not flat framework, not academic, etc.) |
Total is `E+J+V+R+G`, out of 15.
## Score block format
Inserted after each scored paragraph, unchanged from the source engine:
```
> `◈` E:3 · J:2 · V:3 · R:2 · G:3 = **13/15** `¶a3f5b2`
> ⚑ *"potentially" (s2) — evidence supports direct claim*
```
The `¶xxxxxxx` suffix is the first 7 hex characters of the SHA-256 hash of the
paragraph's stripped text. The engine is designed so that, on re-runs, a
paragraph whose hash matches the hash embedded in its existing score block is
skipped (no `claude` call) and the block is left in place, while a paragraph
whose text changed gets re-scored and its stale block replaced; `--force` is
designed to re-score every eligible paragraph regardless of hash. **In
practice this hash-comparison path never executes** — see
[Known limitation](#known-limitation-hash-comparison-is-unreachable) below.
What re-runs actually do: a paragraph that already has a score block attached
(in the exact adjacent format this script itself writes — no blank line
between the paragraph and its `` `◈` `` line) is not re-parsed as a scoreable
paragraph at all on the next run, with or without `--force`. Only paragraphs
with **no** score block attached get (re-)scored.
## Behaviour
1. Unless `--dryrun`, verify `claude` is on PATH; exit 1 with an error if not.
2. Resolve input paths (relative paths resolve against the current working
directory). Exit 1 if `--output` is given with more than one input file.
Exit 1 if any input file does not exist.
3. Resolve each input's output path: `--output` if given (single-file only),
else `<input-stem>-scored<suffix>` next to the input.
4. Load `--style` and exit 1 if it does not exist.
5. For each input file:
a. Split the markdown into chunks tagged `content` (substantive prose
paragraphs) or `skip` (YAML frontmatter, fenced code blocks, headings,
tables, horizontal rules, blank lines, existing score blocks).
b. Paragraphs under `MIN_WORDS` (25) words are never scored. Paragraphs
that already have a score block attached in the script's own output
format are **not classified as `content` at all** by the chunker (see
[Known limitation](#known-limitation-hash-comparison-is-unreachable)) —
they fall out of scoring consideration entirely, silently, forever.
c. For each remaining `content` paragraph, compute its hash. If `--force`
is not set and the immediately following chunk is an existing score
block whose embedded hash matches, skip it (unchanged) — this branch
is designed-for but, per the limitation above, structurally
unreachable. Otherwise queue it for scoring; if a stale score block
immediately follows, mark it for removal once the new block is written
(also unreachable for the same reason).
d. Score all queued paragraphs in parallel (`--parallel` workers) by
calling `claude -p <prompt>` per paragraph, where the prompt embeds the
style guide text and the paragraph text, and asks for a JSON object
`{"evidence":0-3,"judgment":0-3,"voice":0-3,"rhythm":0-3,"register":0-3,"flags":[...],"note":"..."}`.
A timeout, a JSON parse failure, or any other subprocess error produces
an all-zero score with a `scoring error: ...` flag rather than aborting
the run.
e. Write the output file: original text with a score block inserted after
each newly-scored or unchanged-and-already-scored paragraph; stale
blocks for changed paragraphs are dropped.
6. Print a per-file summary line and a run-total summary line.
7. If `--json` was given (and this is not a dryrun — see below), assemble and
write the JSON payload.
## JSON output
`--json FILE` (or `--json -` for stdout) emits, after scoring completes:
```json
{
"style": "/abs/path/to/style-guide.md",
"dryrun": false,
"files": [
{
"input": "/abs/path/to/chapter-01.md",
"output": "/abs/path/to/chapter-01-scored.md",
"paragraphs": [
{
"index": 1,
"hash": "a3f5b2c",
"action": "scored",
"scores": {"evidence": 3, "judgment": 2, "voice": 3, "rhythm": 2, "register": 3},
"total": 13,
"flags": ["hedge: potentially"],
"note": "evidence supports a direct claim here"
},
{
"index": 2,
"hash": "9c1d0ef",
"action": "reused",
"scores": {"evidence": 2, "judgment": 2, "voice": 2, "rhythm": 2, "register": 2},
"total": 10,
"flags": [],
"note": ""
}
],
"summary": {
"mean": 11.5,
"min": 10,
"paragraphs_scored": 2,
"paragraphs_skipped": 1
}
}
]
}
```
- `paragraphs` lists every paragraph the chunker still recognises as
`content` and that clears `MIN_WORDS`, in document order, 1-indexed via
`index`. `action` is `"scored"` (freshly scored this run) or `"reused"`
(hash matched an existing block; the score is parsed back out of that
block with no `claude` call). **`"reused"` is defined for completeness but
is not reachable through this script's own output format** — see
[Known limitation](#known-limitation-hash-comparison-is-unreachable).
Paragraphs below `MIN_WORDS`, and paragraphs that already carry an
attached score block, never appear in this list at all — the latter are
not "reused with a null diff", they are simply absent.
- `summary.paragraphs_scored` is `len(paragraphs)`; `summary.paragraphs_skipped`
is the count of too-short `content` paragraphs excluded from `paragraphs`.
It does **not** count already-scored paragraphs, since those are not
`content` chunks at all by the time this runs (they contribute to neither
`paragraphs_scored` nor `paragraphs_skipped` — they are invisible to this
accounting, not merely uncounted). `summary.mean`/`summary.min` are
computed over `paragraphs[*].total`; both are `null` if the list is empty.
- This is the interface a future `review` work type would consume to find
paragraphs below a threshold on the `/15` scale (see `IDLE-DRAFT-PLAN.md`,
`review_score_threshold`) — **but only on a first, from-scratch scoring
pass.** On any re-run of an already-scored file, `paragraphs` (and
therefore the JSON) covers only paragraphs that had no score block at all
going in; it is not a complete, current picture of every paragraph's score.
A caller that wants a complete `/15` picture of a fully-scored file must
keep its own copy of prior JSON output and merge it with each incremental
run's output, or parse the score blocks out of the `-scored.md` file
directly.
## Dryrun behaviour
`--dryrun`/`-n` makes no `claude -p` calls and writes no files. For each
`content`-classified paragraph it reports one of:
- **would score** — eligible, no matching existing block found for it at
parse time; shows the hash it would be scored under and that a
`claude -p` call would be made.
- **skip (unchanged)** — eligible, hash matches an existing score block
found immediately after it; no call would be made. Defined for
completeness — see [Known limitation](#known-limitation-hash-comparison-is-unreachable);
in practice this line is never printed by a real run.
- **skip (too short)** — under `MIN_WORDS` words; never scored.
Paragraphs that already have a score block attached are not `content` at
all by dryrun time, so they produce **no line whatsoever** — they are not
"unchanged", they are invisible to the paragraph walk, exactly as in a real
run.
Followed by a per-file and run-total count, mirroring the real-run summary
line shape.
```
[dryrun] score-paragraphs — no claude calls will be made, no files will be written
Style: /home/user/style/voice.md
Parallel: 4 workers
Scoring: /home/user/book/chapter-01.md → /home/user/book/chapter-01-scored.md
1 would be scored, 0 unchanged (hash match, would skip), 1 too short (skipped)
[would score] ¶7f0e412 (#1, 33 words) — would run: claude -p <scoring prompt>
[skip too-short] paragraph ~6 words
Total: 1 would be scored, 0 unchanged.
```
(Two other paragraphs in `chapter-01.md` already carry score blocks from a
prior run and simply do not appear above at all — see
[Known limitation](#known-limitation-hash-comparison-is-unreachable).)
### Dryrun + `--json`
`--json -` (stdout) is honoured under `--dryrun`: no filesystem writes occur
either way, so the preview JSON is printed. Progress lines above move to
stderr in this case, keeping stdout pure JSON.
`--json FILE` under `--dryrun` is **not** written (writing a file is a change,
which `--dryrun` promises not to make); instead the script logs
`[dryrun] would write JSON results to FILE`.
In both cases the payload shape matches the real-run schema with `"dryrun":
true` and one difference: paragraphs with `action: "would_score"` have
`scores: null`, `total: null`, `flags: []`, `note: ""` (the score is not
known without calling `claude`). Any `action: "reused"` entries (see the
[Known limitation](#known-limitation-hash-comparison-is-unreachable) — not
reachable via this script's own output format, but defined in case a
hand-edited file makes it reachable) still carry real scores, parsed back
out of the existing score block — no `claude` call needed. `summary.mean`/
`summary.min` are computed only over paragraphs with a non-null `total`.
## Edge cases
| Case | Handling |
|---|---|
| `claude` not on PATH, not `--dryrun` | Error to stderr, exit 1, before touching any input file |
| `--output` with multiple input files | Error: `--output can only be used with a single input file`, exit 1 |
| Input file does not exist | Error: `file not found: <path>`, exit 1 |
| `--style` file does not exist | Error: `style guide not found: <path>`, exit 1 |
| No positional files given | argparse usage error, exit 2 |
| Paragraph scoring subprocess errors (timeout, bad JSON, non-zero exit) | All-zero score, `scoring error: <reason>` flag; run continues |
| A file with zero eligible paragraphs | Output is a byte-identical copy (skip chunks only); JSON `paragraphs: []`, `summary.mean/min: null` |
| `--force` with `--json` | Every *`content`-classified* paragraph gets `action: "scored"`; already-scored paragraphs are still invisible regardless — `--force` does not resurrect them (see [Known limitation](#known-limitation-hash-comparison-is-unreachable)) |
| Re-running on an already-scored file | Paragraphs with an attached score block are silently excluded from processing and from `paragraphs_scored`/`paragraphs_skipped`/JSON entirely — the run only touches paragraphs with no block yet. This is the practical, verified behaviour, not a hash-based decision |
| `--json -` (stdout) without `--dryrun` | Progress lines still go to stdout normally *before* the final JSON print — the caller is expected to take the last JSON blob, or redirect and parse only after the run-total line. `--json -` combined with multiple files is fine (one JSON payload with a `files` array). |
## Known limitation: hash comparison is unreachable
Verified during the port (not introduced by it — present identically in the
source `score-paragraphs.py`, confirmed against real GOES production output,
`book/chapter-01-the-amplifier-test-scored.md`):
The chunker (`split_into_chunks`) and the writer (`format_score_block` /
`process_file`) disagree about spacing. The writer inserts a score block with
**zero** blank-line separation from its paragraph. The chunker's score-block
branch calls `flush("skip")`, which flushes *whatever text is currently
accumulating* — not just the score-block line. Because there is no blank
line, the paragraph itself is still accumulating when that branch fires, so
the paragraph is absorbed into the same `"skip"` chunk as its score block.
It is no longer `kind == "content"` on any subsequent parse.
Consequence: `_jobs_for_file`'s hash-comparison branch (compare an existing
block's embedded hash against the paragraph's current hash to decide
skip-vs-stale) requires a `content` chunk immediately followed by a `skip`
chunk whose first line is a score block. That combination cannot occur for
any file this script wrote — inserting a blank line to *avoid* the swallow
doesn't help either, because then the chunk immediately following the
paragraph is the blank line, not the score block (off by one). Checked
exhaustively against a fresh two-paragraph file, a hand-edited
blank-line-separated file, and the real GOES chapter file: **0 paragraphs
out of 15+ already-scored ones were ever re-examined, with or without
`--force`.**
Net effect, stated plainly: this script cannot currently detect that an
already-scored paragraph's text changed, and cannot force a re-score of an
already-scored paragraph. What it *can* do reliably: score paragraphs that
have never been scored before, leaving previously-scored paragraphs
untouched (which happens to look like "hash-skip working correctly" for the
common "add new paragraphs, don't touch old ones" case — the two are
behaviourally indistinguishable until someone edits an already-scored
paragraph and expects a re-score).
This is a scoring-engine bug, not a generalisation concern, so it is out of
scope for this port to fix (`scripts/score-paragraphs` keeps the chunker and
writer exactly as ported, with a code comment pointing here). Anyone relying
on this script for edit-detection — notably `IDLE-DRAFT-PLAN.md`'s `review`
work type, which assumes `--json` gives a complete, current `/15` picture on
every run — should design around this rather than assume the docstring's
aspirational description.
## Examples
```bash
# Single file, default output path
score-paragraphs --style style/voice.md book/chapter-01.md
# Explicit output path
score-paragraphs --style style/voice.md book/chapter-01.md -o /tmp/ch01-scored.md
# Multiple files, plus a JSON results file for a downstream review pass
score-paragraphs --style style/voice.md book/chapter-*.md --json /tmp/scores.json
# Preview only — no claude calls, no files written
score-paragraphs --style style/voice.md book/chapter-01.md --dryrun
# Force re-score everything, parallelism of 8
score-paragraphs --style style/voice.md book/chapter-01.md --force --parallel 8
```

353
tests/test-score-paragraphs.sh Executable file
View File

@@ -0,0 +1,353 @@
#!/usr/bin/env bash
# test-score-paragraphs.sh — Verify score-paragraphs dryrun behaviour against the spec.
#
# Strategy: all assertions use --dryrun (-n), so no `claude` subprocess is ever
# invoked and no output files are written. Fixtures (a style guide + a sample
# markdown file with pre-embedded score blocks) are generated in a tempdir.
set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
SCORE_PARAGRAPHS="$REPO_ROOT/scripts/score-paragraphs"
GREEN='\033[0;32m'
RED='\033[0;31m'
NC='\033[0m'
PASS=0
FAIL=0
assert_contains() {
local label=$1 needle=$2 haystack=$3
if [[ "$haystack" == *"$needle"* ]]; then
printf " ${GREEN}PASS${NC} %s\n" "$label"
PASS=$((PASS + 1))
else
printf " ${RED}FAIL${NC} %s\n" "$label"
printf " expected to contain: %s\n" "$needle"
printf " actual output:\n"
printf '%s\n' "$haystack" | sed 's/^/ /'
FAIL=$((FAIL + 1))
fi
}
assert_not_contains() {
local label=$1 needle=$2 haystack=$3
if [[ "$haystack" != *"$needle"* ]]; then
printf " ${GREEN}PASS${NC} %s\n" "$label"
PASS=$((PASS + 1))
else
printf " ${RED}FAIL${NC} %s\n" "$label"
printf " expected NOT to contain: %s\n" "$needle"
FAIL=$((FAIL + 1))
fi
}
assert_exit_code() {
local label=$1 expected=$2 actual=$3
if [[ "$expected" == "$actual" ]]; then
printf " ${GREEN}PASS${NC} %s (exit=%s)\n" "$label" "$actual"
PASS=$((PASS + 1))
else
printf " ${RED}FAIL${NC} %s (expected exit=%s, got exit=%s)\n" "$label" "$expected" "$actual"
FAIL=$((FAIL + 1))
fi
}
assert_eq() {
local label=$1 expected=$2 actual=$3
if [[ "$expected" == "$actual" ]]; then
printf " ${GREEN}PASS${NC} %s\n" "$label"
PASS=$((PASS + 1))
else
printf " ${RED}FAIL${NC} %s\n" "$label"
printf " expected: %s\n" "$expected"
printf " actual: %s\n" "$actual"
FAIL=$((FAIL + 1))
fi
}
# === Pre-flight ===
if [[ ! -x "$SCORE_PARAGRAPHS" ]]; then
echo "score-paragraphs not found or not executable at $SCORE_PARAGRAPHS" >&2
exit 1
fi
# === Fixture setup ===
TMPDIR_TEST="$(mktemp -d)"
trap 'rm -rf "$TMPDIR_TEST"' EXIT
STYLE_FILE="${TMPDIR_TEST}/style.md"
SAMPLE_FILE="${TMPDIR_TEST}/sample.md"
SAMPLE2_FILE="${TMPDIR_TEST}/sample2.md"
JSON_OUT="${TMPDIR_TEST}/scores.json"
cat > "$STYLE_FILE" <<'EOF'
# Test style guide
Direct, evidence-backed, practitioner voice. No hedging, no buzzwords.
EOF
# PARA_ALREADY_SCORED: long enough to score (32 words), immediately followed
# by a score block in the script's own write format (zero blank-line
# separation) whose embedded hash matches it exactly. Per the ported
# engine's verified behaviour (see "Known limitation" in the spec), a
# paragraph in this exact shape is absorbed into the surrounding "skip"
# region by the chunker and is never re-examined for scoring again -- not
# because its hash matched, but because it is no longer seen as `content`
# at all. This is intentionally NOT a "skip unchanged" case to assert on;
# it is a "does not appear anywhere" case.
PARA_ALREADY_SCORED="This paragraph is intentionally long enough to clear the twenty five word minimum threshold used by the scoring engine, discussing evidence and voice and rhythm at some length purely for fixture purposes."
# PARA_ALREADY_SCORED_STALE: same shape, but the attached score block's hash
# deliberately does NOT match the paragraph text. Locks in that this makes
# no difference: it is absorbed into "skip" exactly like the matching-hash
# case above, so a *changed* already-scored paragraph is not re-scored
# either -- this is the concrete, load-bearing part of the limitation.
PARA_ALREADY_SCORED_STALE="Here is a second long paragraph included specifically to exercise the changed hash code path, since its embedded score block below will intentionally carry a hash that does not match this paragraph text at all today."
# PARA_NEW: long enough to score (>=25 words) and has no score block
# attached at all -> the one paragraph in this fixture that genuinely can
# be scored, i.e. "would score" under --dryrun.
PARA_NEW="This is a genuinely new paragraph that has never been scored before and easily clears the minimum word count threshold required for scoring eligibility in this fixture file today."
cat > "$SAMPLE_FILE" <<EOF
---
title: Sample
---
# Chapter One
${PARA_ALREADY_SCORED}
> \`◈\` E:2 · J:2 · V:2 · R:2 · G:2 = **10/15** \`¶af539c2\`
> ⚑ *hedge* · _tighten the opening sentence_
Too short.
${PARA_ALREADY_SCORED_STALE}
> \`◈\` E:1 · J:1 · V:1 · R:1 · G:1 = **5/15** \`¶0000000\`
${PARA_NEW}
| a | b |
|---|---|
| 1 | 2 |
\`\`\`python
print("this code block must never be scored")
\`\`\`
EOF
# A second, small file with one fresh (never-scored) eligible paragraph, for
# multi-file dryrun coverage.
PARA_D="This is a separate file containing a single paragraph that easily clears the minimum word count threshold and has never been scored before at all."
cat > "$SAMPLE2_FILE" <<EOF
${PARA_D}
EOF
run() {
"$SCORE_PARAGRAPHS" "$@" 2>&1
}
ORIG_SAMPLE_CHECKSUM="$(sha256sum "$SAMPLE_FILE" | awk '{print $1}')"
echo "=== score-paragraphs tests ==="
echo
# === Test 1: --help ===
echo "Test 1: --help"
out=$(run --help)
rc=$?
assert_exit_code "--help exits 0" 0 "$rc"
assert_contains "--help shows script name" "score-paragraphs" "$out"
assert_contains "--help shows --style" "--style" "$out"
assert_contains "--help shows --dryrun" "--dryrun" "$out"
assert_contains "--help shows --json" "--json" "$out"
assert_contains "--help shows --force" "--force" "$out"
assert_contains "--help shows --parallel" "--parallel" "$out"
# --book (GOES chapter/appendix discovery) was dropped from this generalised
# port -- it must not be a recognised flag.
book_out=$(run --style "$STYLE_FILE" --dryrun --book "$SAMPLE_FILE")
book_rc=$?
assert_exit_code "--book is rejected (unrecognized argument)" 2 "$book_rc"
assert_contains "--book rejection mentions unrecognized argument" "unrecognized argument" "$book_out"
# === Test 2: basic dryrun over the fixture ===
echo
echo "Test 2: basic dryrun"
out=$(run --style "$STYLE_FILE" --dryrun "$SAMPLE_FILE")
rc=$?
assert_exit_code "dryrun exits 0" 0 "$rc"
assert_contains "shows dryrun header" "[dryrun]" "$out"
assert_contains "shows style path" "$STYLE_FILE" "$out"
assert_contains "reports would-score paragraph (the fresh one)" "[would score]" "$out"
assert_contains "reports too-short paragraph" "[skip too-short]" "$out"
assert_contains "shows the would-be claude invocation" "would run: claude -p" "$out"
assert_contains "shows run total" "Total:" "$out"
assert_not_contains "dryrun never calls claude" "Scoring error" "$out"
# Only PARA_NEW is eligible; the two already-scored paragraphs (matching AND
# stale hash) are absorbed by the chunker and never reach the paragraph walk
# at all -- see "Known limitation" in specs/score-paragraphs.spec.md.
assert_contains "reports exactly 1 would-be-scored paragraph" "1 would be scored" "$out"
assert_contains "reports 0 unchanged (hash-skip is unreachable)" "0 unchanged" "$out"
assert_not_contains "already-scored paragraph hash never appears" "af539c2" "$out"
assert_not_contains "already-scored (stale) paragraph never resurfaces" "0000000" "$out"
# === Test 3: dryrun makes no filesystem changes ===
echo
echo "Test 3: dryrun writes nothing"
run --style "$STYLE_FILE" --dryrun "$SAMPLE_FILE" >/dev/null 2>&1
NEW_CHECKSUM="$(sha256sum "$SAMPLE_FILE" | awk '{print $1}')"
assert_eq "input file untouched by dryrun" "$ORIG_SAMPLE_CHECKSUM" "$NEW_CHECKSUM"
DEFAULT_OUTPUT="${TMPDIR_TEST}/sample-scored.md"
if [[ -e "$DEFAULT_OUTPUT" ]]; then
printf " ${RED}FAIL${NC} dryrun must not create the default output file\n"
FAIL=$((FAIL + 1))
else
printf " ${GREEN}PASS${NC} dryrun did not create the default output file\n"
PASS=$((PASS + 1))
fi
# === Test 4: dryrun works with no claude binary on PATH at all ===
echo
echo "Test 4: dryrun does not require claude on PATH"
out=$(PATH="/usr/bin:/bin" "$SCORE_PARAGRAPHS" --style "$STYLE_FILE" --dryrun "$SAMPLE_FILE" 2>&1)
rc=$?
assert_exit_code "dryrun without claude on PATH still exits 0" 0 "$rc"
assert_not_contains "no 'claude not found' error under dryrun" "claude' not found" "$out"
# === Test 5: real (non-dryrun) run requires claude on PATH, fails before touching files ===
echo
echo "Test 5: non-dryrun run with claude missing fails closed"
out=$(PATH="/usr/bin:/bin" "$SCORE_PARAGRAPHS" --style "$STYLE_FILE" "$SAMPLE_FILE" 2>&1)
rc=$?
assert_exit_code "missing claude exits 1" 1 "$rc"
assert_contains "missing claude error message" "'claude' not found on PATH" "$out"
NEW_CHECKSUM2="$(sha256sum "$SAMPLE_FILE" | awk '{print $1}')"
assert_eq "input file untouched when claude missing" "$ORIG_SAMPLE_CHECKSUM" "$NEW_CHECKSUM2"
# === Test 6: --force does NOT resurrect already-scored paragraphs ===
# This locks in the verified (if surprising) real behaviour: --force only
# widens the hash-comparison inside _jobs_for_file, but that function only
# ever sees `content`-classified chunks, and already-scored paragraphs are
# not `content` chunks by the time --force would matter. See "Known
# limitation" in specs/score-paragraphs.spec.md.
echo
echo "Test 6: --force does not resurrect already-scored paragraphs"
out=$(run --style "$STYLE_FILE" --dryrun --force "$SAMPLE_FILE")
rc=$?
assert_exit_code "force dryrun exits 0" 0 "$rc"
assert_contains "still exactly 1 would-be-scored paragraph with --force" "1 would be scored" "$out"
assert_not_contains "already-scored paragraph still invisible under --force" "af539c2" "$out"
assert_not_contains "stale already-scored paragraph still invisible under --force" "0000000" "$out"
# === Test 7: -n shorthand ===
echo
echo "Test 7: -n shorthand"
out=$(run --style "$STYLE_FILE" -n "$SAMPLE_FILE")
rc=$?
assert_exit_code "-n shorthand exits 0" 0 "$rc"
assert_contains "-n shows dryrun header" "[dryrun]" "$out"
# === Test 8: dryrun + --json - emits valid JSON to stdout only ===
echo
echo "Test 8: dryrun + --json - (stdout)"
json_out=$("$SCORE_PARAGRAPHS" --style "$STYLE_FILE" --dryrun --json - "$SAMPLE_FILE" 2>/dev/null)
rc=$?
assert_exit_code "json-to-stdout dryrun exits 0" 0 "$rc"
if printf '%s' "$json_out" | python3 -c 'import json,sys; json.load(sys.stdin)' 2>/dev/null; then
printf " ${GREEN}PASS${NC} stdout is valid JSON when --json -\n"
PASS=$((PASS + 1))
else
printf " ${RED}FAIL${NC} stdout is not valid JSON when --json -\n"
printf '%s\n' "$json_out" | sed 's/^/ /'
FAIL=$((FAIL + 1))
fi
assert_contains "JSON marks dryrun true" '"dryrun": true' "$json_out"
assert_contains "JSON has a would_score action" '"action": "would_score"' "$json_out"
assert_contains "JSON would_score entries have null scores" '"scores": null' "$json_out"
assert_contains "JSON paragraphs_scored counts only the fresh paragraph" '"paragraphs_scored": 1' "$json_out"
# "reused" is defined in the schema but unreachable via this fixture (or any
# file this script itself wrote) -- see "Known limitation" in the spec.
assert_not_contains "no reused action for this fixture (hash-skip is unreachable)" '"action": "reused"' "$json_out"
# Progress lines must NOT pollute stdout when --json - is used.
stderr_out=$("$SCORE_PARAGRAPHS" --style "$STYLE_FILE" --dryrun --json - "$SAMPLE_FILE" 2>&1 1>/dev/null)
assert_contains "progress lines move to stderr with --json -" "[dryrun]" "$stderr_out"
# === Test 9: dryrun + --json <file> does not write the file ===
echo
echo "Test 9: dryrun + --json FILE (no write)"
out=$(run --style "$STYLE_FILE" --dryrun --json "$JSON_OUT" "$SAMPLE_FILE")
rc=$?
assert_exit_code "dryrun json-to-file exits 0" 0 "$rc"
assert_contains "announces it would write the JSON file" "would write JSON results to" "$out"
if [[ -e "$JSON_OUT" ]]; then
printf " ${RED}FAIL${NC} --json FILE must not be written under --dryrun\n"
FAIL=$((FAIL + 1))
else
printf " ${GREEN}PASS${NC} --json FILE not written under --dryrun\n"
PASS=$((PASS + 1))
fi
# === Test 10: multiple input files in one dryrun invocation ===
echo
echo "Test 10: multiple input files"
out=$(run --style "$STYLE_FILE" --dryrun "$SAMPLE_FILE" "$SAMPLE2_FILE")
rc=$?
assert_exit_code "multi-file dryrun exits 0" 0 "$rc"
assert_contains "mentions first file" "$SAMPLE_FILE" "$out"
assert_contains "mentions second file" "$SAMPLE2_FILE" "$out"
# === Test 11: error — --output with multiple input files ===
echo
echo "Test 11: error — --output with multiple files"
out=$(run --style "$STYLE_FILE" --dryrun --output "${TMPDIR_TEST}/out.md" "$SAMPLE_FILE" "$SAMPLE2_FILE")
rc=$?
assert_exit_code "multi-file --output exits 1" 1 "$rc"
assert_contains "multi-file --output error" "only be used with a single input file" "$out"
# === Test 12: error — input file not found ===
echo
echo "Test 12: error — input file not found"
out=$(run --style "$STYLE_FILE" --dryrun "${TMPDIR_TEST}/does-not-exist.md")
rc=$?
assert_exit_code "missing input file exits 1" 1 "$rc"
assert_contains "missing input file error" "file not found" "$out"
# === Test 13: error — style guide not found ===
echo
echo "Test 13: error — style guide not found"
out=$(run --style "${TMPDIR_TEST}/no-such-style.md" --dryrun "$SAMPLE_FILE")
rc=$?
assert_exit_code "missing style file exits 1" 1 "$rc"
assert_contains "missing style file error" "style guide not found" "$out"
# === Test 14: error — --style is required ===
echo
echo "Test 14: error — --style missing entirely"
out=$(run --dryrun "$SAMPLE_FILE")
rc=$?
assert_exit_code "missing --style exits 2 (argparse required-arg error)" 2 "$rc"
assert_contains "missing --style mentions the flag" "--style" "$out"
# === Test 15: error — no input files given ===
echo
echo "Test 15: error — no positional files"
out=$(run --style "$STYLE_FILE" --dryrun)
rc=$?
assert_exit_code "no files exits 2 (argparse required-arg error)" 2 "$rc"
# === Summary ===
echo
echo "================================"
TOTAL=$((PASS + FAIL))
if [[ "$FAIL" -eq 0 ]]; then
printf "${GREEN}All %d tests passed${NC}\n" "$TOTAL"
exit 0
else
printf "${RED}%d of %d tests failed${NC}\n" "$FAIL" "$TOTAL"
exit 1
fi