From f78d292f05e52c72662cfc8d1d767b3e1f707552 Mon Sep 17 00:00:00 2001 From: Paul O'Reilly Date: Sun, 2 Aug 2026 21:18:02 +1200 Subject: [PATCH] score-paragraphs: port GOES paragraph scorer as general-purpose script Required --style, --json machine-readable output, mandatory --dryrun. Engine unchanged; known limitation documented: already-scored paragraphs are never re-examined (present in the original). Claude-Session: https://claude.ai/code/session_01YQDoWNM7XPPii28khFWoMc --- scripts/score-paragraphs | 733 +++++++++++++++++++++++++++++++++ specs/score-paragraphs.spec.md | 323 +++++++++++++++ tests/test-score-paragraphs.sh | 353 ++++++++++++++++ 3 files changed, 1409 insertions(+) create mode 100755 scripts/score-paragraphs create mode 100644 specs/score-paragraphs.spec.md create mode 100755 tests/test-score-paragraphs.sh diff --git a/scripts/score-paragraphs b/scripts/score-paragraphs new file mode 100755 index 0000000..6a65561 --- /dev/null +++ b/scripts/score-paragraphs @@ -0,0 +1,733 @@ +#!/usr/bin/env python3 +"""Score markdown paragraphs against a style guide via `claude -p`. + +Reads one or more markdown files, identifies paragraphs that need scoring +(new or changed), scores them in parallel via `claude -p` subprocesses, and +writes *-scored.md files with compact score blocks inserted after each +paragraph. Optionally also emits the results as machine-readable JSON. + +Usage: + score-paragraphs --style style/voice.md book/chapter-01.md [chapter-02.md ...] + score-paragraphs --style style/voice.md book/chapter-01.md -o book/chapter-01-scored.md + score-paragraphs --style style/voice.md book/chapter-01.md --json /tmp/scores.json + score-paragraphs --style style/voice.md book/chapter-01.md --dryrun + +Scoring dimensions (each 0–3): + E Evidence — specific claims backed by named sources / research links + J Judgment — direct language; no hedging where evidence supports a claim + V Voice — free of LLM tells, buzzwords, corporate softening + R Rhythm — sentence and paragraph length variety; not uniform blocks + G Register — appropriate to the target genre (not flat framework, not academic) + +Score block format (inserted after each scored paragraph): + > `◈` E:3 · J:2 · V:3 · R:2 · G:3 = **13/15** `¶a3f5b2` + > ⚑ *"potentially" (s2) — evidence supports direct claim* + +The `¶xxxxxxx` suffix is the first 7 chars of the SHA-256 of the paragraph text. +On re-runs, paragraphs whose hash matches their existing score block are skipped +automatically. Changed paragraphs are re-scored and their stale block replaced. +Use --force to re-score everything regardless of hash. + +This is a generalised port of the GOES-repo score-paragraphs.py: --style +replaces a repo-relative default style guide, --book (GOES chapter/appendix +discovery) is gone, and --dryrun / --json are new. See +specs/score-paragraphs.spec.md for the full spec. + +Requires: `claude` CLI on PATH (Claude Code) for any non-dryrun run. +""" + +import argparse +import hashlib +import json +import re +import shutil +import subprocess +import sys +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path +from typing import Optional + +# Paragraphs shorter than this (word count) are not scored +MIN_WORDS = 25 + +DIMENSIONS = ("evidence", "judgment", "voice", "rhythm", "register") + + +# ── Score block parsing / detection ────────────────────────────────── + +SCORE_MARKER = "`◈`" +HASH_RE = re.compile(r"`¶([0-9a-f]{7})`") +SCORE_LINE_RE = re.compile( + r"E:(\d+)\s*·\s*J:(\d+)\s*·\s*V:(\d+)\s*·\s*R:(\d+)\s*·\s*G:(\d+)" +) +FLAG_ITEM_RE = re.compile(r"^⚑ \*(.*)\*$") +NOTE_ITEM_RE = re.compile(r"^_(.*)_$") + + +def para_hash(text: str) -> str: + """Return first 7 hex chars of SHA-256 of the paragraph text.""" + return hashlib.sha256(text.strip().encode()).hexdigest()[:7] + + +def is_score_block(line: str) -> bool: + return line.startswith("> ") and SCORE_MARKER in line + + +def extract_hash_from_score_block(block_text: str) -> Optional[str]: + """Extract the embedded paragraph hash from a score block, or None.""" + m = HASH_RE.search(block_text) + return m.group(1) if m else None + + +def parse_score_block(block_text: str) -> dict: + """Reconstruct a scores dict from an already-written score block. + + Used to recover the scores for a paragraph that was skipped this run + because its hash matched (--json needs its score too, but no `claude` + call is made to get it — it's read back out of the block text itself). + """ + lines = block_text.splitlines() + m = SCORE_LINE_RE.search(lines[0]) if lines else None + if not m: + return _error_score("unparseable score block") + e, j, v, r, g = (int(x) for x in m.groups()) + + flags = [] + note = "" + if len(lines) > 1: + detail = lines[1] + if detail.startswith("> "): + detail = detail[2:] + for part in detail.split(" · "): + part = part.strip() + fm = FLAG_ITEM_RE.match(part) + if fm: + flags.append(fm.group(1)) + continue + nm = NOTE_ITEM_RE.match(part) + if nm: + note = nm.group(1) + + return {"evidence": e, "judgment": j, "voice": v, "rhythm": r, + "register": g, "flags": flags, "note": note} + + +# ── Markdown paragraph splitting ───────────────────────────────────── + +def split_into_chunks(text: str) -> list: + """Split markdown into chunks tagged as 'content' or 'skip'. + + 'skip': YAML frontmatter, fenced code blocks, headings, tables, + horizontal rules, empty lines, existing score blocks. + 'content': substantive prose paragraphs worth scoring. + """ + lines = text.splitlines(keepends=True) + chunks = [] + in_frontmatter = lines and lines[0].strip() == "---" + current_lines = [] + + def flush(kind): + if current_lines: + chunks.append({"kind": kind, "text": "".join(current_lines)}) + current_lines.clear() + + i = 0 + while i < len(lines): + line = lines[i] + stripped = line.strip() + + # YAML frontmatter + if in_frontmatter: + current_lines.append(line) + if stripped == "---" and i > 0: + in_frontmatter = False + flush("skip") + i += 1 + continue + + # Fenced code blocks + if stripped.startswith("```") or stripped.startswith("~~~"): + flush("skip") + current_lines.append(line) + fence = stripped[:3] + i += 1 + while i < len(lines): + current_lines.append(lines[i]) + if lines[i].strip().startswith(fence): + i += 1 + break + i += 1 + flush("skip") + continue + + # Existing score blocks + # + # NOTE (verified during the port, not fixed — see "Known limitation" + # in specs/score-paragraphs.spec.md): this flush("skip") absorbs + # whatever is *currently accumulating* in current_lines, not just the + # score-block line itself. Since format_score_block() is written with + # zero blank-line separation from its paragraph, a paragraph that was + # already scored is still accumulating here when this branch fires — + # so it gets swallowed into the same "skip" chunk as its own score + # block, on every subsequent parse. That paragraph is no longer + # `kind == "content"` at all, so the hash-comparison branch below in + # `_jobs_for_file` (and even --force) never gets a chance to run + # against it. This makes that hash-comparison branch structurally + # unreachable for any file this script itself wrote — confirmed + # against real GOES output. The logic is kept as-is (unchanged + # engine, per the port's scope) rather than reworked here. + if is_score_block(line): + flush("skip") + current_lines.append(line) + i += 1 + while i < len(lines) and lines[i].startswith("> "): + current_lines.append(lines[i]) + i += 1 + flush("skip") + continue + + # Empty line — paragraph boundary + if not stripped: + flush("content") + current_lines.append(line) + flush("skip") + i += 1 + continue + + # Headings + if stripped.startswith("#"): + flush("content") + current_lines.append(line) + flush("skip") + i += 1 + continue + + # Tables + if stripped.startswith("|"): + flush("content") + current_lines.append(line) + i += 1 + while i < len(lines) and lines[i].strip().startswith("|"): + current_lines.append(lines[i]) + i += 1 + flush("skip") + continue + + # Horizontal rules + if re.match(r"^[-*_]{3,}$", stripped): + flush("content") + current_lines.append(line) + flush("skip") + i += 1 + continue + + current_lines.append(line) + i += 1 + + flush("content") + return chunks + + +def word_count(text: str) -> int: + return len(text.split()) + + +# ── Claude scoring via `claude -p` ─────────────────────────────────── + +PROMPT_TEMPLATE = """\ +You are a voice-quality reviewer. Score the paragraph at the end of this prompt \ +against the style guide below. + +{style_guide} + +--- + +## Scoring rubric + +Score the paragraph on five dimensions (each 0–3). + +**E — Evidence (0–3)** +- 3: Specific claims backed by named research links or named adopters +- 2: Most claims grounded; one or two assertions lack explicit sourcing +- 1: Claims present but mostly asserted without named backing +- 0: Generic assertions with no evidence anchoring + +**J — Judgment (0–3)** +- 3: Direct, committed language; no hedging where evidence supports a claim +- 2: Mostly direct; one unnecessary hedge (*could*, *might*, *may*, *potentially*) +- 1: Noticeable hedging; evidence would support stronger claims +- 0: Pervasive hedging throughout + +**V — Voice (0–3)** +- 3: Practitioner register; free of LLM tells and burn-list buzzwords +- 2: Mostly clean; one slip into corporate softening or LLM phrasing +- 1: Several buzzwords (*leverage*, *empower*, *unlock*, *robust*, *transformative*, etc.) +- 0: Reads like an LLM or marketing copy + +**R — Rhythm (0–3)** +- 3: Deliberate sentence and paragraph length variety +- 2: Mostly varied; slight tendency toward uniform medium-length sentences +- 1: Noticeably uniform sentence length throughout +- 0: Wall-of-text or robotic uniformity + +**G — Register (0–3)** +- 3: Correctly matches the expected genre register (authored-prose, framework, blog, etc.) +- 2: Mostly correct; slight drift +- 1: Wrong register for the genre +- 0: Completely mismatched + +## Smell-test flags + +Report any present: +- Burn-list buzzword (*delve*, *journey*, *leverage*, *robust*, *foster*, *streamline*, \ +*transformative*, *unlock*, *harness*, *pivotal*, *multifaceted*, *empower*, *synergize*) +- "It's important to note that" / "It's worth noting that" +- "In conclusion" / "Ultimately" / "In essence" as opener or closer +- Rule-of-three triplets in consecutive sentences +- Hedging where evidence supports a direct claim +- Throat-clearing opener ("In today's...", "In the ever-evolving...") +- Specific statistic without a research link +- Signposted conclusion that recaps and motivates + +## Output format + +Respond with ONLY a JSON object — no markdown fences, no preamble: +{{"evidence": <0-3>, "judgment": <0-3>, "voice": <0-3>, "rhythm": <0-3>, \ +"register": <0-3>, "flags": ["", ...], "note": ""}} + +If the text is too short or not scoreable prose, return all zeros and an empty note. + +--- + +## Paragraph to score: + +{paragraph}""" + + +def build_prompt(style_guide: str, paragraph: str) -> str: + return PROMPT_TEMPLATE.format( + style_guide=style_guide, + paragraph=paragraph.strip(), + ) + + +def score_paragraph(style_guide: str, paragraph: str) -> dict: + """Call `claude -p` to score one paragraph. Returns a score dict.""" + prompt = build_prompt(style_guide, paragraph) + try: + result = subprocess.run( + ["claude", "-p", prompt], + capture_output=True, + text=True, + timeout=120, + ) + raw = result.stdout.strip() + # Strip markdown code fences if Claude wraps the JSON + raw = re.sub(r"^```(?:json)?\s*", "", raw) + raw = re.sub(r"\s*```\s*$", "", raw) + return json.loads(raw) + except subprocess.TimeoutExpired: + return _error_score("timeout") + except json.JSONDecodeError as e: + return _error_score(f"json parse error: {e}") + except Exception as e: + return _error_score(str(e)) + + +def _error_score(reason: str) -> dict: + return {"evidence": 0, "judgment": 0, "voice": 0, "rhythm": 0, + "register": 0, "flags": [f"scoring error: {reason}"], "note": ""} + + +def format_score_block(scores: dict, phash: str) -> str: + """Format a score dict as one or two blockquote lines, embedding the paragraph hash.""" + e = scores.get("evidence", 0) + j = scores.get("judgment", 0) + v = scores.get("voice", 0) + r = scores.get("rhythm", 0) + g = scores.get("register", 0) + total = e + j + v + r + g + flags = scores.get("flags", []) + note = (scores.get("note") or "").strip() + + line1 = f"> `◈` E:{e} · J:{j} · V:{v} · R:{r} · G:{g} = **{total}/15** `¶{phash}`" + lines = [line1] + + parts = [f"⚑ *{f}*" for f in flags] + if note and note not in " ".join(flags): + parts.append(f"_{note}_") + if parts: + lines.append("> " + " · ".join(parts)) + + return "\n".join(lines) + + +def _json_paragraph_entry(index: int, phash: str, action: str, scores: dict) -> dict: + e = scores.get("evidence", 0) + j = scores.get("judgment", 0) + v = scores.get("voice", 0) + r = scores.get("rhythm", 0) + g = scores.get("register", 0) + return { + "index": index, + "hash": phash, + "action": action, + "scores": {"evidence": e, "judgment": j, "voice": v, "rhythm": r, "register": g}, + "total": e + j + v + r + g, + "flags": scores.get("flags", []), + "note": (scores.get("note") or "").strip(), + } + + +def _summarize(paragraphs: list, paragraphs_skipped: int) -> dict: + totals = [p["total"] for p in paragraphs if p.get("total") is not None] + return { + "mean": round(sum(totals) / len(totals), 2) if totals else None, + "min": min(totals) if totals else None, + "paragraphs_scored": len(paragraphs), + "paragraphs_skipped": paragraphs_skipped, + } + + +# ── File processing ─────────────────────────────────────────────────── + +def _jobs_for_file(chunks: list, force: bool) -> list: + """Return list of (chunk_idx, para_text, phash, stale_block_follows) for paragraphs + that need scoring.""" + jobs = [] + for idx, chunk in enumerate(chunks): + if chunk["kind"] != "content": + continue + para_text = chunk["text"] + if word_count(para_text) < MIN_WORDS: + continue + + phash = para_hash(para_text) + stale_follows = False + + if not force: + next_chunk = chunks[idx + 1] if idx + 1 < len(chunks) else None + if next_chunk: + first_line = next_chunk["text"].splitlines()[0] if next_chunk["text"] else "" + if next_chunk["kind"] == "skip" and is_score_block(first_line): + existing_hash = extract_hash_from_score_block(next_chunk["text"]) + if existing_hash == phash: + continue # Unchanged — skip + stale_follows = True # Changed — stale block needs dropping + + jobs.append((idx, para_text, phash, stale_follows)) + return jobs + + +def dryrun_file(chunks: list, jobs: list, log) -> dict: + """Report what would happen for one file, without calling claude or writing anything. + + Returns a dict shaped like process_file's return value so the caller can + print the same summary lines and (optionally) assemble a preview JSON. + """ + job_idxs = {idx for idx, *_ in jobs} + total_eligible = sum( + 1 for c in chunks + if c["kind"] == "content" and word_count(c["text"]) >= MIN_WORDS + ) + total_short = sum( + 1 for c in chunks + if c["kind"] == "content" and word_count(c["text"]) < MIN_WORDS + ) + needs_scoring = len(jobs) + already_ok = total_eligible - needs_scoring + + log(f" {needs_scoring} would be scored, {already_ok} unchanged " + f"(hash match, would skip), {total_short} too short (skipped)") + + paragraphs = [] + ordinal = 0 + for idx, chunk in enumerate(chunks): + if chunk["kind"] != "content": + continue + wc = word_count(chunk["text"]) + if wc < MIN_WORDS: + log(f" [skip too-short] paragraph ~{wc} words") + continue + + ordinal += 1 + phash = para_hash(chunk["text"]) + + if idx in job_idxs: + log(f" [would score] ¶{phash} (#{ordinal}, {wc} words) — " + f"would run: claude -p ") + # Score is unknown pre-run — no claude call made in dryrun. + paragraphs.append({ + "index": ordinal, + "hash": phash, + "action": "would_score", + "scores": None, + "total": None, + "flags": [], + "note": "", + }) + else: + next_chunk = chunks[idx + 1] if idx + 1 < len(chunks) else None + reused_scores = None + if next_chunk and next_chunk["kind"] == "skip": + first_line = next_chunk["text"].splitlines()[0] if next_chunk["text"] else "" + if is_score_block(first_line): + reused_scores = parse_score_block(next_chunk["text"]) + if reused_scores is None: + reused_scores = _error_score("could not reuse existing score block") + log(f" [skip unchanged] ¶{phash} (#{ordinal}) — " + f"hash matches existing score block") + paragraphs.append(_json_paragraph_entry(ordinal, phash, "reused", reused_scores)) + + return { + "scored": needs_scoring, + "skipped": already_ok, + "paragraphs": paragraphs, + "summary": _summarize(paragraphs, total_short), + } + + +def process_file( + input_path: Path, + output_path: Path, + style_guide: str, + max_workers: int, + force: bool, + log, +) -> dict: + """Score paragraphs in input_path in parallel, write to output_path. + + Returns a dict: {"scored", "skipped", "paragraphs", "summary"}. + """ + text = input_path.read_text(encoding="utf-8") + chunks = split_into_chunks(text) + jobs = _jobs_for_file(chunks, force) + + total_eligible = sum( + 1 for c in chunks + if c["kind"] == "content" and word_count(c["text"]) >= MIN_WORDS + ) + total_short = sum( + 1 for c in chunks + if c["kind"] == "content" and word_count(c["text"]) < MIN_WORDS + ) + needs_scoring = len(jobs) + already_ok = total_eligible - needs_scoring + + log(f" {input_path.name}: {needs_scoring} to score, " + f"{already_ok} unchanged (skipping)") + + # Score all jobs in parallel + results = {} # chunk_idx -> (scores, phash) + stale = set() # chunk indices whose following score block should be dropped + + if jobs: + completed = 0 + with ThreadPoolExecutor(max_workers=max_workers) as executor: + future_map = { + executor.submit(score_paragraph, style_guide, para): (idx, phash, stale_follows) + for idx, para, phash, stale_follows in jobs + } + for future in as_completed(future_map): + idx, phash, stale_follows = future_map[future] + scores = future.result() + results[idx] = (scores, phash) + if stale_follows: + stale.add(idx) + completed += 1 + total = sum(scores.get(k, 0) for k in DIMENSIONS) + log(f" [{completed}/{needs_scoring}] ¶{phash} {total}/15") + + # Assemble output in chunk order, and the JSON paragraph list alongside it + output_parts = [] + drop_next_score_block = False + json_paragraphs = [] + ordinal = 0 + + for idx, chunk in enumerate(chunks): + if chunk["kind"] != "content": + first_line = chunk["text"].splitlines()[0] if chunk["text"] else "" + if drop_next_score_block and is_score_block(first_line): + drop_next_score_block = False + continue # Drop stale score block + drop_next_score_block = False + output_parts.append(chunk["text"]) + continue + + output_parts.append(chunk["text"]) + + wc = word_count(chunk["text"]) + if wc < MIN_WORDS: + continue # Too short — never scored, no block, not in JSON + + ordinal += 1 + + if idx in results: + scores, phash = results[idx] + output_parts.append(format_score_block(scores, phash) + "\n\n") + if idx in stale: + drop_next_score_block = True # Remove the now-replaced stale block + json_paragraphs.append(_json_paragraph_entry(ordinal, phash, "scored", scores)) + else: + # Unchanged — hash matched, no claude call made. Recover its score + # from the existing block (already present in output_parts via + # the chunk text loop below, since we haven't touched that chunk). + phash = para_hash(chunk["text"]) + next_chunk = chunks[idx + 1] if idx + 1 < len(chunks) else None + reused_scores = None + if next_chunk and next_chunk["kind"] == "skip": + first_line = next_chunk["text"].splitlines()[0] if next_chunk["text"] else "" + if is_score_block(first_line): + reused_scores = parse_score_block(next_chunk["text"]) + if reused_scores is None: + reused_scores = _error_score("could not reuse existing score block") + json_paragraphs.append(_json_paragraph_entry(ordinal, phash, "reused", reused_scores)) + + output_path.write_text("".join(output_parts), encoding="utf-8") + + return { + "scored": needs_scoring, + "skipped": already_ok, + "paragraphs": json_paragraphs, + "summary": _summarize(json_paragraphs, total_short), + } + + +# ── CLI ─────────────────────────────────────────────────────────────── + +def build_arg_parser() -> argparse.ArgumentParser: + ap = argparse.ArgumentParser( + prog="score-paragraphs", + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + ap.add_argument("files", nargs="+", help="Markdown files to score") + ap.add_argument("--style", required=True, metavar="FILE", + help="Style guide markdown file to score paragraphs against") + ap.add_argument("-o", "--output", + help="Output file (only valid with a single input file)") + ap.add_argument("--json", metavar="FILE", + help="Also emit machine-readable JSON results to FILE, or - for stdout") + ap.add_argument("--parallel", type=int, default=4, metavar="N", + help="Number of parallel claude -p calls (default: 4)") + ap.add_argument("--force", action="store_true", + help="Re-score all paragraphs, ignoring existing hash matches") + ap.add_argument("--dryrun", "-n", action="store_true", + help="Preview what would be scored/skipped; makes no claude calls, " + "writes no files") + return ap + + +def main(argv=None) -> int: + ap = build_arg_parser() + args = ap.parse_args(argv if argv is not None else sys.argv[1:]) + + json_to_stdout = args.json == "-" + + def log(msg: str = "") -> None: + print(msg, file=sys.stderr if json_to_stdout else sys.stdout, flush=True) + + # Check claude is available (skip in dryrun — no calls are made) + if not args.dryrun and not shutil.which("claude"): + print("Error: 'claude' not found on PATH. Install Claude Code to continue.", + file=sys.stderr) + return 1 + + # Resolve input files + input_paths = [] + for f in args.files: + p = Path(f) + if not p.is_absolute(): + p = Path.cwd() / p + input_paths.append(p) + + if args.output and len(input_paths) > 1: + print("Error: --output can only be used with a single input file", file=sys.stderr) + return 1 + + # Resolve output paths + io_pairs = [] + for inp in input_paths: + if args.output: + out = Path(args.output) + else: + stem = inp.stem + if not stem.endswith("-scored"): + stem += "-scored" + out = inp.parent / (stem + inp.suffix) + io_pairs.append((inp, out)) + + for inp, _ in io_pairs: + if not inp.exists(): + print(f"Error: file not found: {inp}", file=sys.stderr) + return 1 + + # Load style guide + style_path = Path(args.style) + if not style_path.is_absolute(): + style_path = Path.cwd() / style_path + if not style_path.exists(): + print(f"Error: style guide not found: {style_path}", file=sys.stderr) + return 1 + style_guide = style_path.read_text(encoding="utf-8") + + if args.dryrun: + log("[dryrun] score-paragraphs — no claude calls will be made, no files will be written") + log(f"Style: {style_path}") + log(f"Parallel: {args.parallel} workers") + + total_scored = total_skipped = 0 + json_files = [] + for inp, out in io_pairs: + log(f"\nScoring: {inp} → {out}") + + if args.dryrun: + chunks = split_into_chunks(inp.read_text(encoding="utf-8")) + jobs = _jobs_for_file(chunks, args.force) + result = dryrun_file(chunks, jobs, log) + else: + result = process_file(inp, out, style_guide, args.parallel, args.force, log) + + total_scored += result["scored"] + total_skipped += result["skipped"] + + if args.dryrun: + log(f" Would score: {result['scored']}, unchanged: {result['skipped']}") + else: + log(f" Done: {result['scored']} scored, {result['skipped']} unchanged") + + json_files.append({ + "input": str(inp), + "output": str(out), + "paragraphs": result["paragraphs"], + "summary": result["summary"], + }) + + if args.dryrun: + log(f"\nTotal: {total_scored} would be scored, {total_skipped} unchanged.") + else: + log(f"\nTotal: {total_scored} paragraphs scored, {total_skipped} unchanged.") + + # --- JSON output ------------------------------------------------------- + if args.json: + payload = { + "style": str(style_path), + "dryrun": args.dryrun, + "files": json_files, + } + json_text = json.dumps(payload, indent=2) + + if args.json == "-": + # Stdout is not a filesystem change — honoured even under --dryrun. + print(json_text) + elif args.dryrun: + log(f"[dryrun] would write JSON results to {args.json}") + else: + Path(args.json).write_text(json_text + "\n", encoding="utf-8") + log(f"JSON results written to {args.json}") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/specs/score-paragraphs.spec.md b/specs/score-paragraphs.spec.md new file mode 100644 index 0000000..29a9407 --- /dev/null +++ b/specs/score-paragraphs.spec.md @@ -0,0 +1,323 @@ +# score-paragraphs + +## Purpose + +Score markdown paragraphs against a style guide via `claude -p`, inserting a +compact per-paragraph score block after each scored paragraph and, on request, +emitting the same results as machine-readable JSON for downstream tooling. + +This is a generalised port of `~/dev/claude/octopus/goes/scripts/score-paragraphs.py` +(the GOES book's voice-scoring engine). The scoring engine — chunking, hashing, +the five-dimension rubric, and the `claude -p` prompt — is unchanged. What +changed is the coupling to the GOES repo layout: the style guide is now an +explicit `--style` argument instead of a repo-relative default, and `--book` +(GOES chapter/appendix discovery) is gone. + +## Usage + +``` +score-paragraphs --style FILE [OPTIONS] FILE [FILE ...] +``` + +### Required + +| Argument | Description | +|---|---| +| `FILE ...` (positional) | One or more markdown files to score. At least one required. | +| `--style FILE` | Style guide markdown file to score paragraphs against. No default — the caller always states which voice the paragraphs are graded against. | + +### Options + +| Flag | Default | Description | +|---|---|---| +| `-o, --output FILE` | `-scored.md` next to the input | Output file. Only valid with a single input `FILE`. | +| `--json FILE\|-` | (none) | Also emit machine-readable results (see [JSON output](#json-output)). `-` writes to stdout. | +| `--parallel N` | `4` | Number of parallel `claude -p` calls per file. | +| `--force` | off | Re-score all eligible paragraphs, ignoring existing hash matches. | +| `--dryrun`, `-n` | off | Preview which paragraphs would be scored/skipped and which `claude -p` calls would run. Makes no `claude` calls and writes no files (see [Dryrun behaviour](#dryrun-behaviour)). | +| `--help`, `-h` | — | Show usage and exit 0. | + +Requires the `claude` CLI on PATH for any non-dryrun run — the script exits 1 +immediately if it is missing, before touching any input. + +## Scoring dimensions (each 0–3, unchanged from the source engine) + +| Dim | Name | What it measures | +|---|---|---| +| E | Evidence | Specific claims backed by named sources / research links | +| J | Judgment | Direct language; no hedging where evidence supports a claim | +| V | Voice | Free of LLM tells, buzzwords, corporate softening | +| R | Rhythm | Sentence and paragraph length variety; not uniform blocks | +| G | Register | Appropriate to the target genre (not flat framework, not academic, etc.) | + +Total is `E+J+V+R+G`, out of 15. + +## Score block format + +Inserted after each scored paragraph, unchanged from the source engine: + +``` +> `◈` E:3 · J:2 · V:3 · R:2 · G:3 = **13/15** `¶a3f5b2` +> ⚑ *"potentially" (s2) — evidence supports direct claim* +``` + +The `¶xxxxxxx` suffix is the first 7 hex characters of the SHA-256 hash of the +paragraph's stripped text. The engine is designed so that, on re-runs, a +paragraph whose hash matches the hash embedded in its existing score block is +skipped (no `claude` call) and the block is left in place, while a paragraph +whose text changed gets re-scored and its stale block replaced; `--force` is +designed to re-score every eligible paragraph regardless of hash. **In +practice this hash-comparison path never executes** — see +[Known limitation](#known-limitation-hash-comparison-is-unreachable) below. +What re-runs actually do: a paragraph that already has a score block attached +(in the exact adjacent format this script itself writes — no blank line +between the paragraph and its `` `◈` `` line) is not re-parsed as a scoreable +paragraph at all on the next run, with or without `--force`. Only paragraphs +with **no** score block attached get (re-)scored. + +## Behaviour + +1. Unless `--dryrun`, verify `claude` is on PATH; exit 1 with an error if not. +2. Resolve input paths (relative paths resolve against the current working + directory). Exit 1 if `--output` is given with more than one input file. + Exit 1 if any input file does not exist. +3. Resolve each input's output path: `--output` if given (single-file only), + else `-scored` next to the input. +4. Load `--style` and exit 1 if it does not exist. +5. For each input file: + a. Split the markdown into chunks tagged `content` (substantive prose + paragraphs) or `skip` (YAML frontmatter, fenced code blocks, headings, + tables, horizontal rules, blank lines, existing score blocks). + b. Paragraphs under `MIN_WORDS` (25) words are never scored. Paragraphs + that already have a score block attached in the script's own output + format are **not classified as `content` at all** by the chunker (see + [Known limitation](#known-limitation-hash-comparison-is-unreachable)) — + they fall out of scoring consideration entirely, silently, forever. + c. For each remaining `content` paragraph, compute its hash. If `--force` + is not set and the immediately following chunk is an existing score + block whose embedded hash matches, skip it (unchanged) — this branch + is designed-for but, per the limitation above, structurally + unreachable. Otherwise queue it for scoring; if a stale score block + immediately follows, mark it for removal once the new block is written + (also unreachable for the same reason). + d. Score all queued paragraphs in parallel (`--parallel` workers) by + calling `claude -p ` per paragraph, where the prompt embeds the + style guide text and the paragraph text, and asks for a JSON object + `{"evidence":0-3,"judgment":0-3,"voice":0-3,"rhythm":0-3,"register":0-3,"flags":[...],"note":"..."}`. + A timeout, a JSON parse failure, or any other subprocess error produces + an all-zero score with a `scoring error: ...` flag rather than aborting + the run. + e. Write the output file: original text with a score block inserted after + each newly-scored or unchanged-and-already-scored paragraph; stale + blocks for changed paragraphs are dropped. +6. Print a per-file summary line and a run-total summary line. +7. If `--json` was given (and this is not a dryrun — see below), assemble and + write the JSON payload. + +## JSON output + +`--json FILE` (or `--json -` for stdout) emits, after scoring completes: + +```json +{ + "style": "/abs/path/to/style-guide.md", + "dryrun": false, + "files": [ + { + "input": "/abs/path/to/chapter-01.md", + "output": "/abs/path/to/chapter-01-scored.md", + "paragraphs": [ + { + "index": 1, + "hash": "a3f5b2c", + "action": "scored", + "scores": {"evidence": 3, "judgment": 2, "voice": 3, "rhythm": 2, "register": 3}, + "total": 13, + "flags": ["hedge: potentially"], + "note": "evidence supports a direct claim here" + }, + { + "index": 2, + "hash": "9c1d0ef", + "action": "reused", + "scores": {"evidence": 2, "judgment": 2, "voice": 2, "rhythm": 2, "register": 2}, + "total": 10, + "flags": [], + "note": "" + } + ], + "summary": { + "mean": 11.5, + "min": 10, + "paragraphs_scored": 2, + "paragraphs_skipped": 1 + } + } + ] +} +``` + +- `paragraphs` lists every paragraph the chunker still recognises as + `content` and that clears `MIN_WORDS`, in document order, 1-indexed via + `index`. `action` is `"scored"` (freshly scored this run) or `"reused"` + (hash matched an existing block; the score is parsed back out of that + block with no `claude` call). **`"reused"` is defined for completeness but + is not reachable through this script's own output format** — see + [Known limitation](#known-limitation-hash-comparison-is-unreachable). + Paragraphs below `MIN_WORDS`, and paragraphs that already carry an + attached score block, never appear in this list at all — the latter are + not "reused with a null diff", they are simply absent. +- `summary.paragraphs_scored` is `len(paragraphs)`; `summary.paragraphs_skipped` + is the count of too-short `content` paragraphs excluded from `paragraphs`. + It does **not** count already-scored paragraphs, since those are not + `content` chunks at all by the time this runs (they contribute to neither + `paragraphs_scored` nor `paragraphs_skipped` — they are invisible to this + accounting, not merely uncounted). `summary.mean`/`summary.min` are + computed over `paragraphs[*].total`; both are `null` if the list is empty. +- This is the interface a future `review` work type would consume to find + paragraphs below a threshold on the `/15` scale (see `IDLE-DRAFT-PLAN.md`, + `review_score_threshold`) — **but only on a first, from-scratch scoring + pass.** On any re-run of an already-scored file, `paragraphs` (and + therefore the JSON) covers only paragraphs that had no score block at all + going in; it is not a complete, current picture of every paragraph's score. + A caller that wants a complete `/15` picture of a fully-scored file must + keep its own copy of prior JSON output and merge it with each incremental + run's output, or parse the score blocks out of the `-scored.md` file + directly. + +## Dryrun behaviour + +`--dryrun`/`-n` makes no `claude -p` calls and writes no files. For each +`content`-classified paragraph it reports one of: + +- **would score** — eligible, no matching existing block found for it at + parse time; shows the hash it would be scored under and that a + `claude -p` call would be made. +- **skip (unchanged)** — eligible, hash matches an existing score block + found immediately after it; no call would be made. Defined for + completeness — see [Known limitation](#known-limitation-hash-comparison-is-unreachable); + in practice this line is never printed by a real run. +- **skip (too short)** — under `MIN_WORDS` words; never scored. + +Paragraphs that already have a score block attached are not `content` at +all by dryrun time, so they produce **no line whatsoever** — they are not +"unchanged", they are invisible to the paragraph walk, exactly as in a real +run. + +Followed by a per-file and run-total count, mirroring the real-run summary +line shape. + +``` +[dryrun] score-paragraphs — no claude calls will be made, no files will be written +Style: /home/user/style/voice.md +Parallel: 4 workers + +Scoring: /home/user/book/chapter-01.md → /home/user/book/chapter-01-scored.md + 1 would be scored, 0 unchanged (hash match, would skip), 1 too short (skipped) + [would score] ¶7f0e412 (#1, 33 words) — would run: claude -p + [skip too-short] paragraph ~6 words + +Total: 1 would be scored, 0 unchanged. +``` + +(Two other paragraphs in `chapter-01.md` already carry score blocks from a +prior run and simply do not appear above at all — see +[Known limitation](#known-limitation-hash-comparison-is-unreachable).) + +### Dryrun + `--json` + +`--json -` (stdout) is honoured under `--dryrun`: no filesystem writes occur +either way, so the preview JSON is printed. Progress lines above move to +stderr in this case, keeping stdout pure JSON. + +`--json FILE` under `--dryrun` is **not** written (writing a file is a change, +which `--dryrun` promises not to make); instead the script logs +`[dryrun] would write JSON results to FILE`. + +In both cases the payload shape matches the real-run schema with `"dryrun": +true` and one difference: paragraphs with `action: "would_score"` have +`scores: null`, `total: null`, `flags: []`, `note: ""` (the score is not +known without calling `claude`). Any `action: "reused"` entries (see the +[Known limitation](#known-limitation-hash-comparison-is-unreachable) — not +reachable via this script's own output format, but defined in case a +hand-edited file makes it reachable) still carry real scores, parsed back +out of the existing score block — no `claude` call needed. `summary.mean`/ +`summary.min` are computed only over paragraphs with a non-null `total`. + +## Edge cases + +| Case | Handling | +|---|---| +| `claude` not on PATH, not `--dryrun` | Error to stderr, exit 1, before touching any input file | +| `--output` with multiple input files | Error: `--output can only be used with a single input file`, exit 1 | +| Input file does not exist | Error: `file not found: `, exit 1 | +| `--style` file does not exist | Error: `style guide not found: `, exit 1 | +| No positional files given | argparse usage error, exit 2 | +| Paragraph scoring subprocess errors (timeout, bad JSON, non-zero exit) | All-zero score, `scoring error: ` flag; run continues | +| A file with zero eligible paragraphs | Output is a byte-identical copy (skip chunks only); JSON `paragraphs: []`, `summary.mean/min: null` | +| `--force` with `--json` | Every *`content`-classified* paragraph gets `action: "scored"`; already-scored paragraphs are still invisible regardless — `--force` does not resurrect them (see [Known limitation](#known-limitation-hash-comparison-is-unreachable)) | +| Re-running on an already-scored file | Paragraphs with an attached score block are silently excluded from processing and from `paragraphs_scored`/`paragraphs_skipped`/JSON entirely — the run only touches paragraphs with no block yet. This is the practical, verified behaviour, not a hash-based decision | +| `--json -` (stdout) without `--dryrun` | Progress lines still go to stdout normally *before* the final JSON print — the caller is expected to take the last JSON blob, or redirect and parse only after the run-total line. `--json -` combined with multiple files is fine (one JSON payload with a `files` array). | + +## Known limitation: hash comparison is unreachable + +Verified during the port (not introduced by it — present identically in the +source `score-paragraphs.py`, confirmed against real GOES production output, +`book/chapter-01-the-amplifier-test-scored.md`): + +The chunker (`split_into_chunks`) and the writer (`format_score_block` / +`process_file`) disagree about spacing. The writer inserts a score block with +**zero** blank-line separation from its paragraph. The chunker's score-block +branch calls `flush("skip")`, which flushes *whatever text is currently +accumulating* — not just the score-block line. Because there is no blank +line, the paragraph itself is still accumulating when that branch fires, so +the paragraph is absorbed into the same `"skip"` chunk as its score block. +It is no longer `kind == "content"` on any subsequent parse. + +Consequence: `_jobs_for_file`'s hash-comparison branch (compare an existing +block's embedded hash against the paragraph's current hash to decide +skip-vs-stale) requires a `content` chunk immediately followed by a `skip` +chunk whose first line is a score block. That combination cannot occur for +any file this script wrote — inserting a blank line to *avoid* the swallow +doesn't help either, because then the chunk immediately following the +paragraph is the blank line, not the score block (off by one). Checked +exhaustively against a fresh two-paragraph file, a hand-edited +blank-line-separated file, and the real GOES chapter file: **0 paragraphs +out of 15+ already-scored ones were ever re-examined, with or without +`--force`.** + +Net effect, stated plainly: this script cannot currently detect that an +already-scored paragraph's text changed, and cannot force a re-score of an +already-scored paragraph. What it *can* do reliably: score paragraphs that +have never been scored before, leaving previously-scored paragraphs +untouched (which happens to look like "hash-skip working correctly" for the +common "add new paragraphs, don't touch old ones" case — the two are +behaviourally indistinguishable until someone edits an already-scored +paragraph and expects a re-score). + +This is a scoring-engine bug, not a generalisation concern, so it is out of +scope for this port to fix (`scripts/score-paragraphs` keeps the chunker and +writer exactly as ported, with a code comment pointing here). Anyone relying +on this script for edit-detection — notably `IDLE-DRAFT-PLAN.md`'s `review` +work type, which assumes `--json` gives a complete, current `/15` picture on +every run — should design around this rather than assume the docstring's +aspirational description. + +## Examples + +```bash +# Single file, default output path +score-paragraphs --style style/voice.md book/chapter-01.md + +# Explicit output path +score-paragraphs --style style/voice.md book/chapter-01.md -o /tmp/ch01-scored.md + +# Multiple files, plus a JSON results file for a downstream review pass +score-paragraphs --style style/voice.md book/chapter-*.md --json /tmp/scores.json + +# Preview only — no claude calls, no files written +score-paragraphs --style style/voice.md book/chapter-01.md --dryrun + +# Force re-score everything, parallelism of 8 +score-paragraphs --style style/voice.md book/chapter-01.md --force --parallel 8 +``` diff --git a/tests/test-score-paragraphs.sh b/tests/test-score-paragraphs.sh new file mode 100755 index 0000000..9c39ec8 --- /dev/null +++ b/tests/test-score-paragraphs.sh @@ -0,0 +1,353 @@ +#!/usr/bin/env bash +# test-score-paragraphs.sh — Verify score-paragraphs dryrun behaviour against the spec. +# +# Strategy: all assertions use --dryrun (-n), so no `claude` subprocess is ever +# invoked and no output files are written. Fixtures (a style guide + a sample +# markdown file with pre-embedded score blocks) are generated in a tempdir. + +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +SCORE_PARAGRAPHS="$REPO_ROOT/scripts/score-paragraphs" + +GREEN='\033[0;32m' +RED='\033[0;31m' +NC='\033[0m' + +PASS=0 +FAIL=0 + +assert_contains() { + local label=$1 needle=$2 haystack=$3 + if [[ "$haystack" == *"$needle"* ]]; then + printf " ${GREEN}PASS${NC} %s\n" "$label" + PASS=$((PASS + 1)) + else + printf " ${RED}FAIL${NC} %s\n" "$label" + printf " expected to contain: %s\n" "$needle" + printf " actual output:\n" + printf '%s\n' "$haystack" | sed 's/^/ /' + FAIL=$((FAIL + 1)) + fi +} + +assert_not_contains() { + local label=$1 needle=$2 haystack=$3 + if [[ "$haystack" != *"$needle"* ]]; then + printf " ${GREEN}PASS${NC} %s\n" "$label" + PASS=$((PASS + 1)) + else + printf " ${RED}FAIL${NC} %s\n" "$label" + printf " expected NOT to contain: %s\n" "$needle" + FAIL=$((FAIL + 1)) + fi +} + +assert_exit_code() { + local label=$1 expected=$2 actual=$3 + if [[ "$expected" == "$actual" ]]; then + printf " ${GREEN}PASS${NC} %s (exit=%s)\n" "$label" "$actual" + PASS=$((PASS + 1)) + else + printf " ${RED}FAIL${NC} %s (expected exit=%s, got exit=%s)\n" "$label" "$expected" "$actual" + FAIL=$((FAIL + 1)) + fi +} + +assert_eq() { + local label=$1 expected=$2 actual=$3 + if [[ "$expected" == "$actual" ]]; then + printf " ${GREEN}PASS${NC} %s\n" "$label" + PASS=$((PASS + 1)) + else + printf " ${RED}FAIL${NC} %s\n" "$label" + printf " expected: %s\n" "$expected" + printf " actual: %s\n" "$actual" + FAIL=$((FAIL + 1)) + fi +} + +# === Pre-flight === +if [[ ! -x "$SCORE_PARAGRAPHS" ]]; then + echo "score-paragraphs not found or not executable at $SCORE_PARAGRAPHS" >&2 + exit 1 +fi + +# === Fixture setup === +TMPDIR_TEST="$(mktemp -d)" +trap 'rm -rf "$TMPDIR_TEST"' EXIT + +STYLE_FILE="${TMPDIR_TEST}/style.md" +SAMPLE_FILE="${TMPDIR_TEST}/sample.md" +SAMPLE2_FILE="${TMPDIR_TEST}/sample2.md" +JSON_OUT="${TMPDIR_TEST}/scores.json" + +cat > "$STYLE_FILE" <<'EOF' +# Test style guide + +Direct, evidence-backed, practitioner voice. No hedging, no buzzwords. +EOF + +# PARA_ALREADY_SCORED: long enough to score (32 words), immediately followed +# by a score block in the script's own write format (zero blank-line +# separation) whose embedded hash matches it exactly. Per the ported +# engine's verified behaviour (see "Known limitation" in the spec), a +# paragraph in this exact shape is absorbed into the surrounding "skip" +# region by the chunker and is never re-examined for scoring again -- not +# because its hash matched, but because it is no longer seen as `content` +# at all. This is intentionally NOT a "skip unchanged" case to assert on; +# it is a "does not appear anywhere" case. +PARA_ALREADY_SCORED="This paragraph is intentionally long enough to clear the twenty five word minimum threshold used by the scoring engine, discussing evidence and voice and rhythm at some length purely for fixture purposes." + +# PARA_ALREADY_SCORED_STALE: same shape, but the attached score block's hash +# deliberately does NOT match the paragraph text. Locks in that this makes +# no difference: it is absorbed into "skip" exactly like the matching-hash +# case above, so a *changed* already-scored paragraph is not re-scored +# either -- this is the concrete, load-bearing part of the limitation. +PARA_ALREADY_SCORED_STALE="Here is a second long paragraph included specifically to exercise the changed hash code path, since its embedded score block below will intentionally carry a hash that does not match this paragraph text at all today." + +# PARA_NEW: long enough to score (>=25 words) and has no score block +# attached at all -> the one paragraph in this fixture that genuinely can +# be scored, i.e. "would score" under --dryrun. +PARA_NEW="This is a genuinely new paragraph that has never been scored before and easily clears the minimum word count threshold required for scoring eligibility in this fixture file today." + +cat > "$SAMPLE_FILE" < \`◈\` E:2 · J:2 · V:2 · R:2 · G:2 = **10/15** \`¶af539c2\` +> ⚑ *hedge* · _tighten the opening sentence_ + +Too short. + +${PARA_ALREADY_SCORED_STALE} +> \`◈\` E:1 · J:1 · V:1 · R:1 · G:1 = **5/15** \`¶0000000\` + +${PARA_NEW} + +| a | b | +|---|---| +| 1 | 2 | + +\`\`\`python +print("this code block must never be scored") +\`\`\` +EOF + +# A second, small file with one fresh (never-scored) eligible paragraph, for +# multi-file dryrun coverage. +PARA_D="This is a separate file containing a single paragraph that easily clears the minimum word count threshold and has never been scored before at all." +cat > "$SAMPLE2_FILE" <&1 +} + +ORIG_SAMPLE_CHECKSUM="$(sha256sum "$SAMPLE_FILE" | awk '{print $1}')" + +echo "=== score-paragraphs tests ===" +echo + +# === Test 1: --help === +echo "Test 1: --help" +out=$(run --help) +rc=$? +assert_exit_code "--help exits 0" 0 "$rc" +assert_contains "--help shows script name" "score-paragraphs" "$out" +assert_contains "--help shows --style" "--style" "$out" +assert_contains "--help shows --dryrun" "--dryrun" "$out" +assert_contains "--help shows --json" "--json" "$out" +assert_contains "--help shows --force" "--force" "$out" +assert_contains "--help shows --parallel" "--parallel" "$out" + +# --book (GOES chapter/appendix discovery) was dropped from this generalised +# port -- it must not be a recognised flag. +book_out=$(run --style "$STYLE_FILE" --dryrun --book "$SAMPLE_FILE") +book_rc=$? +assert_exit_code "--book is rejected (unrecognized argument)" 2 "$book_rc" +assert_contains "--book rejection mentions unrecognized argument" "unrecognized argument" "$book_out" + +# === Test 2: basic dryrun over the fixture === +echo +echo "Test 2: basic dryrun" +out=$(run --style "$STYLE_FILE" --dryrun "$SAMPLE_FILE") +rc=$? +assert_exit_code "dryrun exits 0" 0 "$rc" +assert_contains "shows dryrun header" "[dryrun]" "$out" +assert_contains "shows style path" "$STYLE_FILE" "$out" +assert_contains "reports would-score paragraph (the fresh one)" "[would score]" "$out" +assert_contains "reports too-short paragraph" "[skip too-short]" "$out" +assert_contains "shows the would-be claude invocation" "would run: claude -p" "$out" +assert_contains "shows run total" "Total:" "$out" +assert_not_contains "dryrun never calls claude" "Scoring error" "$out" +# Only PARA_NEW is eligible; the two already-scored paragraphs (matching AND +# stale hash) are absorbed by the chunker and never reach the paragraph walk +# at all -- see "Known limitation" in specs/score-paragraphs.spec.md. +assert_contains "reports exactly 1 would-be-scored paragraph" "1 would be scored" "$out" +assert_contains "reports 0 unchanged (hash-skip is unreachable)" "0 unchanged" "$out" +assert_not_contains "already-scored paragraph hash never appears" "af539c2" "$out" +assert_not_contains "already-scored (stale) paragraph never resurfaces" "0000000" "$out" + +# === Test 3: dryrun makes no filesystem changes === +echo +echo "Test 3: dryrun writes nothing" +run --style "$STYLE_FILE" --dryrun "$SAMPLE_FILE" >/dev/null 2>&1 +NEW_CHECKSUM="$(sha256sum "$SAMPLE_FILE" | awk '{print $1}')" +assert_eq "input file untouched by dryrun" "$ORIG_SAMPLE_CHECKSUM" "$NEW_CHECKSUM" +DEFAULT_OUTPUT="${TMPDIR_TEST}/sample-scored.md" +if [[ -e "$DEFAULT_OUTPUT" ]]; then + printf " ${RED}FAIL${NC} dryrun must not create the default output file\n" + FAIL=$((FAIL + 1)) +else + printf " ${GREEN}PASS${NC} dryrun did not create the default output file\n" + PASS=$((PASS + 1)) +fi + +# === Test 4: dryrun works with no claude binary on PATH at all === +echo +echo "Test 4: dryrun does not require claude on PATH" +out=$(PATH="/usr/bin:/bin" "$SCORE_PARAGRAPHS" --style "$STYLE_FILE" --dryrun "$SAMPLE_FILE" 2>&1) +rc=$? +assert_exit_code "dryrun without claude on PATH still exits 0" 0 "$rc" +assert_not_contains "no 'claude not found' error under dryrun" "claude' not found" "$out" + +# === Test 5: real (non-dryrun) run requires claude on PATH, fails before touching files === +echo +echo "Test 5: non-dryrun run with claude missing fails closed" +out=$(PATH="/usr/bin:/bin" "$SCORE_PARAGRAPHS" --style "$STYLE_FILE" "$SAMPLE_FILE" 2>&1) +rc=$? +assert_exit_code "missing claude exits 1" 1 "$rc" +assert_contains "missing claude error message" "'claude' not found on PATH" "$out" +NEW_CHECKSUM2="$(sha256sum "$SAMPLE_FILE" | awk '{print $1}')" +assert_eq "input file untouched when claude missing" "$ORIG_SAMPLE_CHECKSUM" "$NEW_CHECKSUM2" + +# === Test 6: --force does NOT resurrect already-scored paragraphs === +# This locks in the verified (if surprising) real behaviour: --force only +# widens the hash-comparison inside _jobs_for_file, but that function only +# ever sees `content`-classified chunks, and already-scored paragraphs are +# not `content` chunks by the time --force would matter. See "Known +# limitation" in specs/score-paragraphs.spec.md. +echo +echo "Test 6: --force does not resurrect already-scored paragraphs" +out=$(run --style "$STYLE_FILE" --dryrun --force "$SAMPLE_FILE") +rc=$? +assert_exit_code "force dryrun exits 0" 0 "$rc" +assert_contains "still exactly 1 would-be-scored paragraph with --force" "1 would be scored" "$out" +assert_not_contains "already-scored paragraph still invisible under --force" "af539c2" "$out" +assert_not_contains "stale already-scored paragraph still invisible under --force" "0000000" "$out" + +# === Test 7: -n shorthand === +echo +echo "Test 7: -n shorthand" +out=$(run --style "$STYLE_FILE" -n "$SAMPLE_FILE") +rc=$? +assert_exit_code "-n shorthand exits 0" 0 "$rc" +assert_contains "-n shows dryrun header" "[dryrun]" "$out" + +# === Test 8: dryrun + --json - emits valid JSON to stdout only === +echo +echo "Test 8: dryrun + --json - (stdout)" +json_out=$("$SCORE_PARAGRAPHS" --style "$STYLE_FILE" --dryrun --json - "$SAMPLE_FILE" 2>/dev/null) +rc=$? +assert_exit_code "json-to-stdout dryrun exits 0" 0 "$rc" +if printf '%s' "$json_out" | python3 -c 'import json,sys; json.load(sys.stdin)' 2>/dev/null; then + printf " ${GREEN}PASS${NC} stdout is valid JSON when --json -\n" + PASS=$((PASS + 1)) +else + printf " ${RED}FAIL${NC} stdout is not valid JSON when --json -\n" + printf '%s\n' "$json_out" | sed 's/^/ /' + FAIL=$((FAIL + 1)) +fi +assert_contains "JSON marks dryrun true" '"dryrun": true' "$json_out" +assert_contains "JSON has a would_score action" '"action": "would_score"' "$json_out" +assert_contains "JSON would_score entries have null scores" '"scores": null' "$json_out" +assert_contains "JSON paragraphs_scored counts only the fresh paragraph" '"paragraphs_scored": 1' "$json_out" +# "reused" is defined in the schema but unreachable via this fixture (or any +# file this script itself wrote) -- see "Known limitation" in the spec. +assert_not_contains "no reused action for this fixture (hash-skip is unreachable)" '"action": "reused"' "$json_out" + +# Progress lines must NOT pollute stdout when --json - is used. +stderr_out=$("$SCORE_PARAGRAPHS" --style "$STYLE_FILE" --dryrun --json - "$SAMPLE_FILE" 2>&1 1>/dev/null) +assert_contains "progress lines move to stderr with --json -" "[dryrun]" "$stderr_out" + +# === Test 9: dryrun + --json does not write the file === +echo +echo "Test 9: dryrun + --json FILE (no write)" +out=$(run --style "$STYLE_FILE" --dryrun --json "$JSON_OUT" "$SAMPLE_FILE") +rc=$? +assert_exit_code "dryrun json-to-file exits 0" 0 "$rc" +assert_contains "announces it would write the JSON file" "would write JSON results to" "$out" +if [[ -e "$JSON_OUT" ]]; then + printf " ${RED}FAIL${NC} --json FILE must not be written under --dryrun\n" + FAIL=$((FAIL + 1)) +else + printf " ${GREEN}PASS${NC} --json FILE not written under --dryrun\n" + PASS=$((PASS + 1)) +fi + +# === Test 10: multiple input files in one dryrun invocation === +echo +echo "Test 10: multiple input files" +out=$(run --style "$STYLE_FILE" --dryrun "$SAMPLE_FILE" "$SAMPLE2_FILE") +rc=$? +assert_exit_code "multi-file dryrun exits 0" 0 "$rc" +assert_contains "mentions first file" "$SAMPLE_FILE" "$out" +assert_contains "mentions second file" "$SAMPLE2_FILE" "$out" + +# === Test 11: error — --output with multiple input files === +echo +echo "Test 11: error — --output with multiple files" +out=$(run --style "$STYLE_FILE" --dryrun --output "${TMPDIR_TEST}/out.md" "$SAMPLE_FILE" "$SAMPLE2_FILE") +rc=$? +assert_exit_code "multi-file --output exits 1" 1 "$rc" +assert_contains "multi-file --output error" "only be used with a single input file" "$out" + +# === Test 12: error — input file not found === +echo +echo "Test 12: error — input file not found" +out=$(run --style "$STYLE_FILE" --dryrun "${TMPDIR_TEST}/does-not-exist.md") +rc=$? +assert_exit_code "missing input file exits 1" 1 "$rc" +assert_contains "missing input file error" "file not found" "$out" + +# === Test 13: error — style guide not found === +echo +echo "Test 13: error — style guide not found" +out=$(run --style "${TMPDIR_TEST}/no-such-style.md" --dryrun "$SAMPLE_FILE") +rc=$? +assert_exit_code "missing style file exits 1" 1 "$rc" +assert_contains "missing style file error" "style guide not found" "$out" + +# === Test 14: error — --style is required === +echo +echo "Test 14: error — --style missing entirely" +out=$(run --dryrun "$SAMPLE_FILE") +rc=$? +assert_exit_code "missing --style exits 2 (argparse required-arg error)" 2 "$rc" +assert_contains "missing --style mentions the flag" "--style" "$out" + +# === Test 15: error — no input files given === +echo +echo "Test 15: error — no positional files" +out=$(run --style "$STYLE_FILE" --dryrun) +rc=$? +assert_exit_code "no files exits 2 (argparse required-arg error)" 2 "$rc" + +# === Summary === +echo +echo "================================" +TOTAL=$((PASS + FAIL)) +if [[ "$FAIL" -eq 0 ]]; then + printf "${GREEN}All %d tests passed${NC}\n" "$TOTAL" + exit 0 +else + printf "${RED}%d of %d tests failed${NC}\n" "$FAIL" "$TOTAL" + exit 1 +fi