diff --git a/data/idle-draft/config.example.json b/data/idle-draft/config.example.json index 0d2afbd..f4b4b72 100644 --- a/data/idle-draft/config.example.json +++ b/data/idle-draft/config.example.json @@ -7,10 +7,11 @@ "five_hour_ceiling": 50, "min_idle": 5 } }, "work_types": { - "review": { "providers": ["anthropic"] }, - "draft": { "providers": ["anthropic"] }, - "research": { "providers": ["anthropic", "minimax"], "allowed_tools": ["WebSearch", "WebFetch"] }, - "topic_ideas": { "providers": ["anthropic", "minimax"], "allowed_tools": ["WebSearch", "WebFetch"] } + "review": { "providers": ["anthropic"], "effort": "high" }, + "draft": { "providers": ["anthropic"], "effort": "high" }, + "research": { "providers": ["anthropic", { "provider": "minimax", "effort": "medium" }], + "allowed_tools": ["WebSearch", "WebFetch"], "effort": "medium" }, + "topic_ideas": { "providers": ["anthropic", "minimax"], "allowed_tools": ["WebSearch", "WebFetch"], "effort": "low" } }, "dossiers": ["ai", "ai-technical", "devops-2020", "devops-2020-technical"], "review_score_threshold": 11, diff --git a/scripts/idle-draft b/scripts/idle-draft index a346449..7b94723 100755 --- a/scripts/idle-draft +++ b/scripts/idle-draft @@ -62,6 +62,17 @@ ALLOWED_ITEM_KEYS = { } ALLOWED_ATTEMPT_WORK_TYPES = {"research", "draft", "review"} +# Effort levels accepted by `claude --effort `. Derived empirically on +# 2026-08-02 by running `claude --effort obviously-bogus-level -p ""` (no API +# call -- an invalid flag value is rejected before any network activity) and +# reading the CLI's own error message, which enumerated the valid set +# verbatim: "Warning: Unknown --effort value 'obviously-bogus-level' — +# ignoring it and using the default effort. Valid values: low, medium, high, +# xhigh, max." Cross-checked against `claude --help`'s `--effort ` +# description, which lists the same five values. If a future CLI version +# changes this set, re-run the same probe and update this constant. +CLAUDE_EFFORT_LEVELS = frozenset({"low", "medium", "high", "xhigh", "max"}) + DEFAULT_MAX_TURNS = 25 DEFAULT_TASK_TIMEOUT_SECONDS = 1800 DEFAULT_MAX_ATTEMPTS = 2 @@ -224,13 +235,77 @@ def load_config(path: Path) -> dict: for key in ("profile", "threshold_pct", "five_hour_ceiling", "min_idle"): if key not in pcfg: raise ConfigError(f"provider '{pname}' missing key '{key}'") + known_providers = set(data["providers"].keys()) for wtype, wcfg in data["work_types"].items(): if "providers" not in wcfg or not isinstance(wcfg["providers"], list): raise ConfigError(f"work_type '{wtype}' missing 'providers' list") + for entry in wcfg["providers"]: + # A providers-list entry is either the legacy bare provider-name + # string, or an object carrying a per-provider model/effort + # override: {"provider": "", "model": "...", "effort": "..."}. + # Both forms may appear in the same list (list order still encodes + # preference for select_provider()). + if isinstance(entry, str): + pname_ref = entry + if pname_ref not in known_providers: + raise ConfigError( + f"work_type '{wtype}' providers entry names unknown provider '{pname_ref}'" + ) + elif isinstance(entry, dict): + unknown_keys = set(entry.keys()) - {"provider", "model", "effort"} + if unknown_keys: + raise ConfigError( + f"work_type '{wtype}' providers entry has unknown key(s): {sorted(unknown_keys)}" + ) + pname_ref = entry.get("provider") + if not isinstance(pname_ref, str) or not pname_ref: + raise ConfigError( + f"work_type '{wtype}' providers entry (object form) missing non-empty 'provider' key" + ) + if pname_ref not in known_providers: + raise ConfigError( + f"work_type '{wtype}' providers entry names unknown provider '{pname_ref}'" + ) + if "model" in entry: + entry_model = entry["model"] + if not isinstance(entry_model, str) or not entry_model: + raise ConfigError( + f"work_type '{wtype}' providers entry for '{pname_ref}' 'model' must be " + f"a non-empty string" + ) + if "effort" in entry: + entry_effort = entry["effort"] + if not isinstance(entry_effort, str) or not entry_effort: + raise ConfigError( + f"work_type '{wtype}' providers entry for '{pname_ref}' 'effort' must be " + f"a non-empty string" + ) + if entry_effort not in CLAUDE_EFFORT_LEVELS: + raise ConfigError( + f"work_type '{wtype}' providers entry for '{pname_ref}' 'effort' must be one " + f"of {sorted(CLAUDE_EFFORT_LEVELS)}, got {entry_effort!r}" + ) + else: + raise ConfigError( + f"work_type '{wtype}' providers entry must be a string or object, " + f"got {type(entry).__name__}" + ) if "allowed_tools" in wcfg: at = wcfg["allowed_tools"] if not isinstance(at, list) or not all(isinstance(x, str) for x in at): raise ConfigError(f"work_type '{wtype}' 'allowed_tools' must be a list of strings") + if "effort" in wcfg: + effort = wcfg["effort"] + if not isinstance(effort, str) or not effort: + raise ConfigError(f"work_type '{wtype}' 'effort' must be a non-empty string") + if effort not in CLAUDE_EFFORT_LEVELS: + raise ConfigError( + f"work_type '{wtype}' 'effort' must be one of {sorted(CLAUDE_EFFORT_LEVELS)}, got {effort!r}" + ) + if "model" in wcfg: + model = wcfg["model"] + if not isinstance(model, str) or not model: + raise ConfigError(f"work_type '{wtype}' 'model' must be a non-empty string") return data @@ -656,9 +731,19 @@ def build_ready_queue(repo: Path, config: dict, state: dict, in_flight: set) -> return candidates +def provider_entry_name(entry) -> str: + """A providers-list entry is either a plain provider-name string (the + legacy form) or an object {"provider": name, "model": ..., "effort": ...}. + This extracts just the name -- the only field select_provider() itself + needs; the model/effort overrides are resolved later, once a provider has + actually been selected, by resolve_work_type_model/_effort below.""" + return entry if isinstance(entry, str) else entry["provider"] + + def select_provider(work_type: str, config: dict, gates: dict, credential_ok: dict) -> tuple[str | None, str]: providers = config["work_types"].get(work_type, {}).get("providers", []) - for pname in providers: + for entry in providers: + pname = provider_entry_name(entry) gate = gates.get(pname, {}) if gate.get("eligible") and credential_ok.get(pname, (False, ""))[0]: return pname, "eligible" @@ -758,11 +843,20 @@ def validate_output(work_type: str, text: str) -> tuple[bool, str]: def build_claude_argv( - model_id: str | None, max_turns: int, add_dirs: list[str], allowed_tools: list[str] | None = None + model_id: str | None, + max_turns: int, + add_dirs: list[str], + allowed_tools: list[str] | None = None, + effort: str | None = None, ) -> list[str]: + """`model_id` is the already-resolved effective model (see + resolve_effective_model: work-type "model" beats the profile's MODEL_ID) -- + this function emits at most one --model flag, never two.""" argv = ["claude", "-p", "--max-turns", str(max_turns)] if model_id: argv += ["--model", model_id] + if effort: + argv += ["--effort", effort] for d in add_dirs: argv += ["--add-dir", d] if allowed_tools: @@ -821,16 +915,70 @@ def resolve_allowed_tools(config: dict, work_type: str) -> list[str]: return list(config["work_types"].get(work_type, {}).get("allowed_tools") or []) +def find_provider_entry(config: dict, work_type: str, provider_name: str): + """The raw providers-list entry (str or dict) for provider_name within + work_type's providers list, or None if not present. Defensive lookup -- + callers only ever pass a provider name that select_provider() just chose + from this same list, so a miss shouldn't happen, but resolve_work_type_* + below treat it as "no per-provider override" rather than raising.""" + for entry in config["work_types"].get(work_type, {}).get("providers", []): + if provider_entry_name(entry) == provider_name: + return entry + return None + + +def resolve_work_type_effort(config: dict, work_type: str, provider_name: str | None = None) -> str | None: + """Effort level for (work_type, provider_name). Precedence: + per-provider entry "effort" > work-type-level "effort" > None (CLI + default). `provider_name` is optional -- omit it to get just the + work-type-level value (e.g. before a provider has been selected). Both + layers are already validated (non-empty string, member of + CLAUDE_EFFORT_LEVELS) at config-load time -- this is a plain lookup.""" + wcfg = config["work_types"].get(work_type, {}) + if provider_name is not None: + entry = find_provider_entry(config, work_type, provider_name) + per_provider = entry.get("effort") if isinstance(entry, dict) else None + if per_provider: + return per_provider + return wcfg.get("effort") + + +def resolve_work_type_model(config: dict, work_type: str, provider_name: str | None = None) -> str | None: + """Model override for (work_type, provider_name). Precedence: + per-provider entry "model" > work-type-level "model" > None (falls back + to the profile's MODEL_ID via resolve_effective_model). `provider_name` + is optional -- omit it to get just the work-type-level value. Both + layers are already type-checked (non-empty string) at config-load time -- + this is a plain lookup.""" + wcfg = config["work_types"].get(work_type, {}) + if provider_name is not None: + entry = find_provider_entry(config, work_type, provider_name) + per_provider = entry.get("model") if isinstance(entry, dict) else None + if per_provider: + return per_provider + return wcfg.get("model") + + +def resolve_effective_model(profile_model_id: str | None, work_type_model: str | None) -> str | None: + """Precedence: a work-type "model" override always wins over the + profile's provider.env MODEL_ID. Pure function so the precedence rule is + independently testable without touching a profile dir.""" + return work_type_model or profile_model_id + + def dispatch_preview(repo: Path, config: dict, candidate: dict, max_turns: int) -> dict: """Compute everything a dryrun needs to print, without executing anything.""" dossier = candidate["dossier"] work_type = candidate["work_type"] provider = candidate["provider"] profile_dir = expand(config["providers"][provider]["profile"]) - model_id = resolve_model_id(profile_dir) + model_id = resolve_effective_model( + resolve_model_id(profile_dir), resolve_work_type_model(config, work_type, provider) + ) + effort = resolve_work_type_effort(config, work_type, provider) add_dirs = resolve_add_dirs(config, work_type) allowed_tools = resolve_allowed_tools(config, work_type) - argv = build_claude_argv(model_id, max_turns, add_dirs, allowed_tools) + argv = build_claude_argv(model_id, max_turns, add_dirs, allowed_tools, effort) canonical = canonical_output_path(repo, candidate) tmp_path = tmp_output_path(canonical) parsed_env = parse_provider_env(profile_dir / "provider.env") @@ -993,10 +1141,13 @@ def run_task(repo: Path, config: dict, candidate: dict, max_turns: int, task_tim prompt_text = render_prompt(work_type, mapping) profile_dir = expand(config["providers"][provider]["profile"]) - model_id = resolve_model_id(profile_dir) + model_id = resolve_effective_model( + resolve_model_id(profile_dir), resolve_work_type_model(config, work_type, provider) + ) + effort = resolve_work_type_effort(config, work_type, provider) add_dirs = resolve_add_dirs(config, work_type) allowed_tools = resolve_allowed_tools(config, work_type) - argv = build_claude_argv(model_id, max_turns, add_dirs, allowed_tools) + argv = build_claude_argv(model_id, max_turns, add_dirs, allowed_tools, effort) env = build_child_env(profile_dir) timed_out = False diff --git a/specs/idle-draft.spec.md b/specs/idle-draft.spec.md index e19e829..3717088 100644 --- a/specs/idle-draft.spec.md +++ b/specs/idle-draft.spec.md @@ -239,14 +239,21 @@ For the selected `(item_or_dossier, work_type, provider)`: invoked** (it has interactive pickers and terminal theming unsuitable for headless cron use). 4. Build the argv: `claude -p --max-turns ` (`N` = 25 unless overridden), plus - `--model ` if `provider.env` sets `MODEL_ID`, plus `--add-dir ` for each - entry in `config["evidence_dirs"]` **only** for `research` and `topic_ideas` work - types (the only ones that cite external evidence), plus `--allowedTools + `--model ` and `--effort ` resolved through the full precedence chain — + per-provider override on the selected provider's `providers`-list entry, then + work-type-level `model`/`effort`, then (for `model` only) `provider.env`'s + `MODEL_ID`; the flag is omitted at each layer where nothing resolves (see + "Per-work-type effort and model" and "Per-provider effort and model overrides" + below) — plus `--add-dir ` for each entry in `config["evidence_dirs"]` + **only** for `research` and `topic_ideas` + work types (the only ones that cite external evidence), plus `--allowedTools ""` if `config["work_types"][work_type]["allowed_tools"]` is present and non-empty (see Config schema below). Headless `claude -p` runs cannot answer permission prompts, so any tool beyond the CLI's defaults must be granted - this way or the child run stalls/fails; when the key is absent or an empty list, - the flag is omitted entirely (unchanged pre-existing behaviour). + this way or the child run stalls/fails; when the `allowed_tools` key is absent or an + empty list, that flag is omitted entirely (unchanged pre-existing behaviour). Same for + `effort`/`model`: absent key -> no corresponding flag, argv unchanged from before + these keys existed. 5. Run the subprocess: `cwd=`, `env=`, prompt piped via **stdin** (not as an argv element — avoids `ARG_MAX` on large rendered prompts, same lesson `claude-profile` already applies to its system-prompt injection), timeout = per-task @@ -401,6 +408,87 @@ config validation failure — exit **2** at config load, same as a missing requi before writing its Subjective stories section; the topic-ideas template requires one as a lighter-touch check); `draft` and `review` carry no `allowed_tools` key. +### Per-work-type effort and model + +Each entry in `work_types` may also carry two further optional keys, both consumed by +§11 step 4 when building the child's argv: + +- **`effort`** (string): appends `--effort ` to the child argv. Validated at + config-load time against the fixed set of levels the `claude` CLI itself accepts: + `low`, `medium`, `high`, `xhigh`, `max`. This set was determined empirically on + 2026-08-02 by running `claude --effort obviously-bogus-level -p ""` — an invalid + `--effort` value is rejected by the CLI's own argument parsing before any network + call is made, so the probe is safe to run offline — and reading the resulting error, + which enumerated the valid values verbatim (cross-checked against `claude --help`'s + `--effort ` description, which lists the same five). The set is hard-coded as + `CLAUDE_EFFORT_LEVELS` in `scripts/idle-draft`; if a future CLI version changes it, + re-run the same probe and update the constant (with a fresh date in the comment). + A value that is not a non-empty string, or not a member of this set, is a config + validation failure — exit **2**, same as a malformed `allowed_tools`. +- **`model`** (string): appends `--model ` to the child argv, **replacing** — + not adding to — the `--model ` that would otherwise come from the provider + profile's `provider.env` `MODEL_ID` (§11 step 4). Only type-checked at config-load + time (must be a non-empty string) — model identifiers are not enumerated, since new + ones ship independently of this tool and hard-coding a set would go stale. A + non-string or empty value is a config validation failure — exit **2**. + +Both keys are optional and independent of each other and of `allowed_tools`. Absent on +a given work type -> the corresponding flag is omitted / the profile's model is used +unchanged, matching the tool's behaviour before these keys existed. + +### Per-provider effort and model overrides (within a work type) + +`model`/`effort` can be pinned even more narrowly than per-work-type: per +`(work_type, provider)` pair. Each entry in a `work_types[*].providers` list may be +**either**: + +- a plain provider-name string (the original, unchanged form) — no per-provider + override, resolution falls through to the work-type-level `model`/`effort` (above); + or +- an object `{"provider": "", "model": "", "effort": ""}` — + `model`/`effort` here apply only when *this* provider is the one actually selected + (§9) for a dispatch of this work type. + +Both forms may be mixed freely within the same `providers` list; list order still +encodes provider preference exactly as before (§9) — the object form does not change +*which* provider is tried first, only what argv that provider gets if chosen. +`config.example.json` demonstrates this: `research`'s `providers` is +`["anthropic", {"provider": "minimax", "effort": "medium"}]` — `anthropic` stays a +plain string (falls through to `research`'s work-type-level `effort`), `minimax` pins +its own `effort` via the object form. + +**Full precedence chain**, evaluated once a `(work_type, provider)` pair has been +selected (§9), for each of `model` and `effort` independently: + +1. The per-provider override on the matching `providers`-list entry for that provider + (object form only; a plain-string entry has none). +2. The work-type-level `model`/`effort` (the keys directly under + `work_types[work_type]`, previous section). +3. Ultimate fallback: for `model`, the profile's `provider.env` `MODEL_ID` (or no + `--model` flag if unset); for `effort`, no `--effort` flag (the CLI's own default). + +**Validation** (config-load time, exit **2** on any failure, same style as every other +config check): + +- Every `providers`-list entry — string **or** object form — must name a provider that + exists as a key in the top-level `providers` map. An entry (of either form) naming an + unrecognised provider is rejected; this closes a gap that predates the object form — + a typo'd plain-string provider name was previously silently ineligible (never + selected, no error) rather than rejected at load time. +- An object-form entry must have a `"provider"` key whose value is a non-empty string. +- An object-form entry's keys are restricted to `provider`, `model`, `effort`; any other + key is rejected. +- An object-form entry's `model`, if present, must be a non-empty string (same rule as + the work-type-level `model`). +- An object-form entry's `effort`, if present, must be a non-empty string and a member + of `CLAUDE_EFFORT_LEVELS` (same rule as the work-type-level `effort`). +- A `providers`-list entry that is neither a string nor an object is rejected. + +`--dryrun` prints the fully resolved argv (§11 step 4's `--model`/`--effort` included, +after the full precedence chain above), so a preview reveals exactly which provider, +model, and effort level a given work type would route to before anything is dispatched +for real. + ## Failure classes | Class | Examples | `attempts` effect | Item outcome | @@ -438,6 +526,16 @@ a lighter-touch check); `draft` and `review` carry no `allowed_tools` key. | `--once` with no eligible work | Exits 0 immediately, no task dispatched | | Config missing a required key | Exit 2 | | `work_types[*].allowed_tools` present but not a list of strings | Exit 2 | +| `work_types[*].effort` present but not one of `low`/`medium`/`high`/`xhigh`/`max` | Exit 2 | +| `work_types[*].effort` present but empty string / not a string | Exit 2 | +| `work_types[*].model` present but empty string / not a string | Exit 2 | +| `work_types[*].model` set alongside a profile whose `provider.env` also sets `MODEL_ID` | Work-type `model` wins (absent a per-provider override); argv carries exactly one `--model` flag | +| `work_types[*].providers[*]` (string or object form) names a provider not present in the top-level `providers` map | Exit 2 | +| `work_types[*].providers[*]` object form missing/empty `"provider"` key | Exit 2 | +| `work_types[*].providers[*]` object form has an unknown key (anything besides `provider`/`model`/`effort`) | Exit 2 | +| `work_types[*].providers[*]` object form `model`/`effort` invalid (same rules as the work-type-level keys) | Exit 2 | +| `work_types[*].providers[*]` entry is neither a string nor an object | Exit 2 | +| A provider's `providers`-list entry sets its own `model`/`effort`, and the work type also sets a work-type-level `model`/`effort` | Per-provider entry wins for that provider; a sibling provider in the same list with a plain-string entry still falls through to the work-type-level value | | `agent-subscriptions` subprocess fails entirely | Exit 1, loud log line, no dispatch attempted | ## Examples diff --git a/tests/test-idle-draft.sh b/tests/test-idle-draft.sh index 6a1a7f4..f805fa4 100755 --- a/tests/test-idle-draft.sh +++ b/tests/test-idle-draft.sh @@ -155,6 +155,44 @@ assert_contains "$output" "$TMPDIR/evidence" "dryrun argv includes configured ev assert_contains "$output" "--allowedTools" "dryrun argv for research includes --allowedTools (allowed_tools configured)" assert_contains "$output" "WebSearch,WebFetch" "dryrun argv --allowedTools value is the comma-joined tool list" +echo "-- dryrun: work-type effort/model override profile model, appear in resolved argv --" +PROFILE_ANTHROPIC_MODEL="$TMPDIR/profiles/anthropic-with-model" +mkdir -p "$PROFILE_ANTHROPIC_MODEL" +cat > "$PROFILE_ANTHROPIC_MODEL/provider.env" <<'EOF' +MODEL_ID=profile-default-model +EOF +EFFORT_CONFIG="$TMPDIR/effort.config.json" +cat > "$EFFORT_CONFIG" <&1) +assert_contains "$output" "'--model', 'worktype-override-model'" "dryrun argv model uses work-type override, not profile MODEL_ID" +assert_not_contains "$output" "profile-default-model" "dryrun argv does not contain the profile's MODEL_ID once a work-type model overrides it" +assert_contains "$output" "'--effort', 'medium'" "dryrun argv includes --effort with the configured level" +model_flag_count=$(echo "$output" | grep -o -- "--model" | wc -l) +[[ "$model_flag_count" -eq 1 ]] && pass "resolved argv contains exactly one --model flag (no duplicate from profile)" || fail "resolved argv contains exactly one --model flag" "found $model_flag_count" + +echo "-- dryrun: no effort/model configured -> flags omitted (unchanged behaviour) --" +output=$("$SCRIPT" --config "$FIXTURE_CONFIG" --repo "$SOLO" --probe-json "$FIXTURE_PROBE" --dryrun --once 2>&1) +assert_not_contains "$output" "--effort" "dryrun argv omits --effort when work_type has no effort key" + echo "-- dryrun: no mutation --" before_hash=$(cd "$FIXTURE_REPO" && git rev-parse HEAD) "$SCRIPT" --config "$FIXTURE_CONFIG" --repo "$FIXTURE_REPO" --probe-json "$FIXTURE_PROBE" --dryrun >/dev/null 2>&1 @@ -254,6 +292,132 @@ code=$? assert_exit_code "$code" 2 "allowed_tools not a list of strings exits 2" assert_contains "$output" "allowed_tools" "allowed_tools config error names the offending key" +BADEFFORT="$TMPDIR/bad-effort.config.json" +cat > "$BADEFFORT" <&1) +code=$? +assert_exit_code "$code" 2 "invalid effort level exits 2" +assert_contains "$output" "effort" "invalid effort config error names the offending key" + +BADMODEL="$TMPDIR/bad-model.config.json" +cat > "$BADMODEL" <&1) +code=$? +assert_exit_code "$code" 2 "empty model string exits 2" +assert_contains "$output" "model" "empty model config error names the offending key" + +echo "-- config validation: per-provider effort/model overrides (providers-list entries) --" +BADPROVIDERNAME="$TMPDIR/bad-provider-name.config.json" +cat > "$BADPROVIDERNAME" <&1) +code=$? +assert_exit_code "$code" 2 "providers-list entry naming an undefined provider exits 2" +assert_contains "$output" "unknown provider" "undefined provider config error names the problem" + +BADPROVIDERKEY="$TMPDIR/bad-provider-key.config.json" +cat > "$BADPROVIDERKEY" <&1) +code=$? +assert_exit_code "$code" 2 "providers-list object entry with an unknown key exits 2" +assert_contains "$output" "unknown key" "unknown provider-entry key config error names the problem" + +BADPROVIDEREFFORT="$TMPDIR/bad-provider-effort.config.json" +cat > "$BADPROVIDEREFFORT" <&1) +code=$? +assert_exit_code "$code" 2 "providers-list object entry with an invalid effort level exits 2" +assert_contains "$output" "effort" "invalid provider-entry effort config error names the problem" + echo "-- mark subcommand round-trip --" MARKREPO="$TMPDIR/mark-repo" mkdir -p "$MARKREPO/ai" @@ -519,6 +683,334 @@ check("load_config: well-formed allowed_tools (list of strings) loads without er good_allowed_cfg["work_types"]["research"]["allowed_tools"] == ["WebSearch", "WebFetch"], good_allowed_cfg["work_types"]["research"]) +# --- per-work-type effort / model: argv construction --- +argv_effort = m.build_claude_argv("model-x", 25, [], None, "high") +check("build_claude_argv: appends --effort when given", + "--effort" in argv_effort and argv_effort[argv_effort.index("--effort") + 1] == "high", argv_effort) + +argv_no_effort = m.build_claude_argv("model-x", 25, [], None, None) +check("build_claude_argv: omits --effort when not given", "--effort" not in argv_no_effort, argv_no_effort) + +argv_model_and_effort = m.build_claude_argv("worktype-model", 25, [], None, "xhigh") +check("build_claude_argv: --model reflects whatever single value the caller resolved " + "(precedence already applied upstream) -- exactly one --model flag", + argv_model_and_effort.count("--model") == 1 + and argv_model_and_effort[argv_model_and_effort.index("--model") + 1] == "worktype-model", + argv_model_and_effort) + +argv_neither = m.build_claude_argv(None, 25, [], None, None) +check("build_claude_argv: omits both --model and --effort when neither is given", + "--model" not in argv_neither and "--effort" not in argv_neither, argv_neither) + +# --- per-work-type effort / model: precedence resolution --- +check("resolve_effective_model: work-type model overrides profile MODEL_ID", + m.resolve_effective_model("profile-model", "worktype-model") == "worktype-model", "") +check("resolve_effective_model: falls back to profile MODEL_ID when no work-type model set", + m.resolve_effective_model("profile-model", None) == "profile-model", "") +check("resolve_effective_model: both absent -> None", + m.resolve_effective_model(None, None) is None, "") + +cfg_em = {"work_types": {"research": {"providers": ["anthropic"], "effort": "medium", "model": "wt-model"}, + "draft": {"providers": ["anthropic"]}}} +check("resolve_work_type_effort: returns configured value", m.resolve_work_type_effort(cfg_em, "research") == "medium", "") +check("resolve_work_type_effort: returns None when absent", m.resolve_work_type_effort(cfg_em, "draft") is None, "") +check("resolve_work_type_model: returns configured value", m.resolve_work_type_model(cfg_em, "research") == "wt-model", "") +check("resolve_work_type_model: returns None when absent", m.resolve_work_type_model(cfg_em, "draft") is None, "") + +# --- per-work-type effort / model: config validation --- +bad_effort_cfg_path = tmpdir / "bad-effort-direct.config.json" +bad_effort_cfg = json.loads((tmpdir / "idle-draft.config.json").read_text()) +bad_effort_cfg["work_types"]["research"]["effort"] = "ultra-mega" +bad_effort_cfg_path.write_text(json.dumps(bad_effort_cfg)) +try: + m.load_config(bad_effort_cfg_path) + check("load_config: effort value outside CLAUDE_EFFORT_LEVELS raises ConfigError", False) +except m.ConfigError as e: + check("load_config: effort value outside CLAUDE_EFFORT_LEVELS raises ConfigError", "effort" in str(e), str(e)) + +bad_effort_type_path = tmpdir / "bad-effort-type.config.json" +bad_effort_type_cfg = json.loads((tmpdir / "idle-draft.config.json").read_text()) +bad_effort_type_cfg["work_types"]["research"]["effort"] = 5 +bad_effort_type_path.write_text(json.dumps(bad_effort_type_cfg)) +try: + m.load_config(bad_effort_type_path) + check("load_config: non-string effort raises ConfigError", False) +except m.ConfigError as e: + check("load_config: non-string effort raises ConfigError", "effort" in str(e), str(e)) + +bad_model_path = tmpdir / "bad-model-direct.config.json" +bad_model_cfg = json.loads((tmpdir / "idle-draft.config.json").read_text()) +bad_model_cfg["work_types"]["research"]["model"] = "" +bad_model_path.write_text(json.dumps(bad_model_cfg)) +try: + m.load_config(bad_model_path) + check("load_config: empty-string model raises ConfigError", False) +except m.ConfigError as e: + check("load_config: empty-string model raises ConfigError", "model" in str(e), str(e)) + +bad_model_type_path = tmpdir / "bad-model-type.config.json" +bad_model_type_cfg = json.loads((tmpdir / "idle-draft.config.json").read_text()) +bad_model_type_cfg["work_types"]["research"]["model"] = 42 +bad_model_type_path.write_text(json.dumps(bad_model_type_cfg)) +try: + m.load_config(bad_model_type_path) + check("load_config: non-string model raises ConfigError", False) +except m.ConfigError as e: + check("load_config: non-string model raises ConfigError", "model" in str(e), str(e)) + +good_effort_model_cfg = json.loads((tmpdir / "idle-draft.config.json").read_text()) +good_effort_model_cfg["work_types"]["research"]["effort"] = "high" +good_effort_model_cfg["work_types"]["research"]["model"] = "claude-example-model" +good_effort_model_path = tmpdir / "good-effort-model.config.json" +good_effort_model_path.write_text(json.dumps(good_effort_model_cfg)) +loaded_good = m.load_config(good_effort_model_path) +check("load_config: well-formed effort + model (in the valid set) loads without error", + loaded_good["work_types"]["research"]["effort"] == "high" + and loaded_good["work_types"]["research"]["model"] == "claude-example-model", + loaded_good["work_types"]["research"]) + +for level in sorted(m.CLAUDE_EFFORT_LEVELS): + per_level_cfg = json.loads((tmpdir / "idle-draft.config.json").read_text()) + per_level_cfg["work_types"]["research"]["effort"] = level + per_level_path = tmpdir / f"effort-{level}.config.json" + per_level_path.write_text(json.dumps(per_level_cfg)) + loaded = m.load_config(per_level_path) + check(f"load_config: effort level {level!r} (member of CLAUDE_EFFORT_LEVELS) loads without error", + loaded["work_types"]["research"]["effort"] == level, loaded["work_types"]["research"]) + +# --- per-provider effort / model overrides (within a work type) --- + +check("provider_entry_name: plain string entry returns itself", + m.provider_entry_name("anthropic") == "anthropic", "") +check("provider_entry_name: object entry returns its 'provider' value", + m.provider_entry_name({"provider": "minimax", "effort": "high"}) == "minimax", "") + +cfg_pp = { + "providers": {"anthropic": {}, "minimax": {}}, + "work_types": { + "research": { + "providers": [ + "anthropic", + {"provider": "minimax", "model": "minimax-model", "effort": "high"}, + ], + "effort": "medium", + "model": "worktype-model", + }, + "draft": {"providers": ["anthropic"], "effort": "low"}, + }, +} + +entry_a = m.find_provider_entry(cfg_pp, "research", "anthropic") +check("find_provider_entry: returns the plain-string entry for a string-form provider", + entry_a == "anthropic", entry_a) +entry_m = m.find_provider_entry(cfg_pp, "research", "minimax") +check("find_provider_entry: returns the dict entry for an object-form provider", + isinstance(entry_m, dict) and entry_m["provider"] == "minimax", entry_m) +entry_missing = m.find_provider_entry(cfg_pp, "research", "nonexistent") +check("find_provider_entry: returns None when the provider isn't in this work_type's list", + entry_missing is None, entry_missing) + +# Precedence: per-provider entry > work-type-level > (caller supplies profile fallback separately) +check("resolve_work_type_effort: plain-string provider entry falls through to work-type-level effort", + m.resolve_work_type_effort(cfg_pp, "research", "anthropic") == "medium", "") +check("resolve_work_type_model: plain-string provider entry falls through to work-type-level model", + m.resolve_work_type_model(cfg_pp, "research", "anthropic") == "worktype-model", "") +check("resolve_work_type_effort: object-form provider entry's own effort wins over work-type-level", + m.resolve_work_type_effort(cfg_pp, "research", "minimax") == "high", "") +check("resolve_work_type_model: object-form provider entry's own model wins over work-type-level", + m.resolve_work_type_model(cfg_pp, "research", "minimax") == "minimax-model", "") +check("resolve_work_type_effort: provider_name omitted -> work-type-level value only (back-compat)", + m.resolve_work_type_effort(cfg_pp, "research") == "medium", "") +check("resolve_work_type_model: provider_name omitted -> work-type-level value only (back-compat)", + m.resolve_work_type_model(cfg_pp, "research") == "worktype-model", "") +check("resolve_work_type_effort: work type with no per-work-type or per-provider effort -> None", + m.resolve_work_type_effort(cfg_pp, "draft", "anthropic") == "low", "") + +# Full chain including the ultimate profile-MODEL_ID fallback (resolve_effective_model), +# for a provider with NO per-provider or work-type-level model at all. +cfg_pp_no_wt_model = { + "providers": {"anthropic": {}}, + "work_types": {"review": {"providers": ["anthropic"], "effort": "high"}}, +} +wt_model = m.resolve_work_type_model(cfg_pp_no_wt_model, "review", "anthropic") +check("resolve_work_type_model: no per-provider, no work-type-level -> None (falls to profile next)", + wt_model is None, wt_model) +check("resolve_effective_model: full chain falls back to profile MODEL_ID when neither override is set", + m.resolve_effective_model("profile-model-id", wt_model) == "profile-model-id", "") + +# --- per-provider effort / model overrides: dispatch_preview reflects the resolution --- +# dispatch_preview() takes a candidate with "provider" already selected -- this exercises +# the exact same resolution path run_task() uses, without needing a real subprocess or a +# provider to actually be gate-eligible (credential/gate selection is select_provider()'s +# job, tested separately; dispatch_preview only needs a chosen provider name). +pp_repo = tmpdir / "pp-repo" +(pp_repo / "ai").mkdir(parents=True, exist_ok=True) +(pp_repo / "style").mkdir(parents=True, exist_ok=True) +(pp_repo / "ai" / "01-a.overview.md").write_text("# a\n") +(pp_repo / "ai" / "01-a.agent.md").write_text("# brief\n") +(pp_repo / "ai" / "SOURCE-REGISTER.md").write_text("# reg\n") +(pp_repo / "AGENTS.md").write_text("# root\n") +(pp_repo / "ai" / "AGENTS.md").write_text("# ai\n") + +pp_profile_anthropic = tmpdir / "pp-profiles" / "anthropic" +pp_profile_minimax = tmpdir / "pp-profiles" / "minimax" +pp_profile_anthropic.mkdir(parents=True, exist_ok=True) +pp_profile_minimax.mkdir(parents=True, exist_ok=True) +(pp_profile_anthropic / "provider.env").write_text("MODEL_ID=anthropic-profile-model\n") +# minimax profile deliberately has no provider.env -- its --model must come entirely +# from the per-provider config override, not any profile fallback. + +pp_config = { + "providers": { + "anthropic": {"profile": str(pp_profile_anthropic), "threshold_pct": 80, + "five_hour_ceiling": 50, "min_idle": 5}, + "minimax": {"profile": str(pp_profile_minimax), "threshold_pct": 80, + "five_hour_ceiling": 50, "min_idle": 5}, + }, + "work_types": { + "research": { + "providers": [ + "anthropic", + {"provider": "minimax", "model": "minimax-pinned-model", "effort": "xhigh"}, + ], + "effort": "medium", + "model": "worktype-fallback-model", + }, + }, + "dossiers": ["ai"], + "review_score_threshold": 11, + "max_unreviewed_research_per_dossier": 3, + "max_open_topic_proposals": 6, + "evidence_dirs": [], +} + +candidate_anthropic = {"dossier": "ai", "slug": "01-a", "work_type": "research", "provider": "anthropic"} +preview_anthropic = m.dispatch_preview(pp_repo, pp_config, candidate_anthropic, 25) +check("dispatch_preview: plain-string provider entry (anthropic) uses work-type-level model, not its own profile MODEL_ID", + "worktype-fallback-model" in preview_anthropic["argv"] + and "anthropic-profile-model" not in preview_anthropic["argv"], preview_anthropic["argv"]) +check("dispatch_preview: plain-string provider entry (anthropic) uses work-type-level effort", + "medium" in preview_anthropic["argv"] and preview_anthropic["argv"].count("--effort") == 1, + preview_anthropic["argv"]) + +candidate_minimax = {"dossier": "ai", "slug": "01-a", "work_type": "research", "provider": "minimax"} +preview_minimax = m.dispatch_preview(pp_repo, pp_config, candidate_minimax, 25) +check("dispatch_preview: object-form provider entry (minimax) uses its own pinned model over the work-type-level one", + "minimax-pinned-model" in preview_minimax["argv"] + and "worktype-fallback-model" not in preview_minimax["argv"], preview_minimax["argv"]) +check("dispatch_preview: object-form provider entry (minimax) uses its own pinned effort over the work-type-level one", + "xhigh" in preview_minimax["argv"] and "medium" not in preview_minimax["argv"], preview_minimax["argv"]) +check("dispatch_preview: exactly one --model flag regardless of which provider was selected (mixed list)", + preview_anthropic["argv"].count("--model") == 1 and preview_minimax["argv"].count("--model") == 1, "") + +# --- per-provider effort / model overrides: config validation --- + +def _base_pp_cfg(): + c = json.loads((tmpdir / "idle-draft.config.json").read_text()) + return c + +bad_provider_name_str_path = tmpdir / "bad-provider-name-str.config.json" +c = _base_pp_cfg() +c["work_types"]["research"]["providers"] = ["anthropic", "nonexistent-provider"] +bad_provider_name_str_path.write_text(json.dumps(c)) +try: + m.load_config(bad_provider_name_str_path) + check("load_config: string-form providers entry naming an unknown provider raises ConfigError", False) +except m.ConfigError as e: + check("load_config: string-form providers entry naming an unknown provider raises ConfigError", + "unknown provider" in str(e) and "nonexistent-provider" in str(e), str(e)) + +bad_provider_name_obj_path = tmpdir / "bad-provider-name-obj.config.json" +c = _base_pp_cfg() +c["work_types"]["research"]["providers"] = ["anthropic", {"provider": "nonexistent-provider"}] +bad_provider_name_obj_path.write_text(json.dumps(c)) +try: + m.load_config(bad_provider_name_obj_path) + check("load_config: object-form providers entry naming an unknown provider raises ConfigError", False) +except m.ConfigError as e: + check("load_config: object-form providers entry naming an unknown provider raises ConfigError", + "unknown provider" in str(e) and "nonexistent-provider" in str(e), str(e)) + +missing_provider_key_path = tmpdir / "missing-provider-key.config.json" +c = _base_pp_cfg() +c["work_types"]["research"]["providers"] = ["anthropic", {"effort": "high"}] +missing_provider_key_path.write_text(json.dumps(c)) +try: + m.load_config(missing_provider_key_path) + check("load_config: object-form providers entry missing 'provider' key raises ConfigError", False) +except m.ConfigError as e: + check("load_config: object-form providers entry missing 'provider' key raises ConfigError", + "provider" in str(e), str(e)) + +unknown_key_path = tmpdir / "unknown-provider-entry-key.config.json" +c = _base_pp_cfg() +c["work_types"]["research"]["providers"] = ["anthropic", {"provider": "minimax", "bogus_key": 1}] +unknown_key_path.write_text(json.dumps(c)) +try: + m.load_config(unknown_key_path) + check("load_config: object-form providers entry with an unknown key raises ConfigError", False) +except m.ConfigError as e: + check("load_config: object-form providers entry with an unknown key raises ConfigError", + "unknown key" in str(e) and "bogus_key" in str(e), str(e)) + +bad_entry_effort_path = tmpdir / "bad-provider-entry-effort.config.json" +c = _base_pp_cfg() +c["work_types"]["research"]["providers"] = ["anthropic", {"provider": "minimax", "effort": "ultra-mega"}] +bad_entry_effort_path.write_text(json.dumps(c)) +try: + m.load_config(bad_entry_effort_path) + check("load_config: object-form providers entry with invalid effort level raises ConfigError", False) +except m.ConfigError as e: + check("load_config: object-form providers entry with invalid effort level raises ConfigError", + "effort" in str(e) and "minimax" in str(e), str(e)) + +bad_entry_model_path = tmpdir / "bad-provider-entry-model.config.json" +c = _base_pp_cfg() +c["work_types"]["research"]["providers"] = ["anthropic", {"provider": "minimax", "model": ""}] +bad_entry_model_path.write_text(json.dumps(c)) +try: + m.load_config(bad_entry_model_path) + check("load_config: object-form providers entry with empty model raises ConfigError", False) +except m.ConfigError as e: + check("load_config: object-form providers entry with empty model raises ConfigError", + "model" in str(e) and "minimax" in str(e), str(e)) + +bad_entry_type_path = tmpdir / "bad-provider-entry-type.config.json" +c = _base_pp_cfg() +c["work_types"]["research"]["providers"] = ["anthropic", 5] +bad_entry_type_path.write_text(json.dumps(c)) +try: + m.load_config(bad_entry_type_path) + check("load_config: providers entry that is neither string nor object raises ConfigError", False) +except m.ConfigError as e: + check("load_config: providers entry that is neither string nor object raises ConfigError", + "string or object" in str(e), str(e)) + +good_mixed_path = tmpdir / "good-mixed-providers.config.json" +c = _base_pp_cfg() +c["work_types"]["research"]["providers"] = ["anthropic", {"provider": "minimax", "model": "m-model", "effort": "high"}] +good_mixed_path.write_text(json.dumps(c)) +loaded_mixed = m.load_config(good_mixed_path) +check("load_config: well-formed mixed (string + object) providers list loads without error", + loaded_mixed["work_types"]["research"]["providers"][0] == "anthropic" + and loaded_mixed["work_types"]["research"]["providers"][1] == {"provider": "minimax", "model": "m-model", "effort": "high"}, + loaded_mixed["work_types"]["research"]["providers"]) + +# The shipped example config itself demonstrates the mixed form -- confirm it validates +# and that select_provider() still walks the list in order via provider_entry_name(). +example_cfg = m.load_config(m.REPO_SELF / "data" / "idle-draft" / "config.example.json") +example_research_providers = example_cfg["work_types"]["research"]["providers"] +check("config.example.json: loads without error (mixed providers list included)", True, "") +check("config.example.json: research providers mixes a plain string (anthropic) and an object (minimax)", + example_research_providers[0] == "anthropic" + and isinstance(example_research_providers[1], dict) + and example_research_providers[1]["provider"] == "minimax" + and example_research_providers[1]["effort"] == "medium", + example_research_providers) +check("provider_entry_name: walks config.example.json's mixed research providers list in order", + [m.provider_entry_name(e) for e in example_research_providers] == ["anthropic", "minimax"], + example_research_providers) + # --- Citation validation --- real_path = cred_repo / "exists.txt" real_path.write_text("x")