idle-draft: per-work-type and per-provider model/effort settings
--effort/--model on child argv; provider entries accept object form
{provider, model, effort}; precedence entry > work-type > profile.
Effort levels validated against the CLI's enumerated set.
Claude-Session: https://claude.ai/code/session_01YQDoWNM7XPPii28khFWoMc
This commit is contained in:
Paul O'Reilly
@@ -7,10 +7,11 @@
|
||||
"five_hour_ceiling": 50, "min_idle": 5 }
|
||||
},
|
||||
"work_types": {
|
||||
"review": { "providers": ["anthropic"] },
|
||||
"draft": { "providers": ["anthropic"] },
|
||||
"research": { "providers": ["anthropic", "minimax"], "allowed_tools": ["WebSearch", "WebFetch"] },
|
||||
"topic_ideas": { "providers": ["anthropic", "minimax"], "allowed_tools": ["WebSearch", "WebFetch"] }
|
||||
"review": { "providers": ["anthropic"], "effort": "high" },
|
||||
"draft": { "providers": ["anthropic"], "effort": "high" },
|
||||
"research": { "providers": ["anthropic", { "provider": "minimax", "effort": "medium" }],
|
||||
"allowed_tools": ["WebSearch", "WebFetch"], "effort": "medium" },
|
||||
"topic_ideas": { "providers": ["anthropic", "minimax"], "allowed_tools": ["WebSearch", "WebFetch"], "effort": "low" }
|
||||
},
|
||||
"dossiers": ["ai", "ai-technical", "devops-2020", "devops-2020-technical"],
|
||||
"review_score_threshold": 11,
|
||||
|
||||
@@ -62,6 +62,17 @@ ALLOWED_ITEM_KEYS = {
|
||||
}
|
||||
ALLOWED_ATTEMPT_WORK_TYPES = {"research", "draft", "review"}
|
||||
|
||||
# Effort levels accepted by `claude --effort <level>`. Derived empirically on
|
||||
# 2026-08-02 by running `claude --effort obviously-bogus-level -p ""` (no API
|
||||
# call -- an invalid flag value is rejected before any network activity) and
|
||||
# reading the CLI's own error message, which enumerated the valid set
|
||||
# verbatim: "Warning: Unknown --effort value 'obviously-bogus-level' —
|
||||
# ignoring it and using the default effort. Valid values: low, medium, high,
|
||||
# xhigh, max." Cross-checked against `claude --help`'s `--effort <level>`
|
||||
# description, which lists the same five values. If a future CLI version
|
||||
# changes this set, re-run the same probe and update this constant.
|
||||
CLAUDE_EFFORT_LEVELS = frozenset({"low", "medium", "high", "xhigh", "max"})
|
||||
|
||||
DEFAULT_MAX_TURNS = 25
|
||||
DEFAULT_TASK_TIMEOUT_SECONDS = 1800
|
||||
DEFAULT_MAX_ATTEMPTS = 2
|
||||
@@ -224,13 +235,77 @@ def load_config(path: Path) -> dict:
|
||||
for key in ("profile", "threshold_pct", "five_hour_ceiling", "min_idle"):
|
||||
if key not in pcfg:
|
||||
raise ConfigError(f"provider '{pname}' missing key '{key}'")
|
||||
known_providers = set(data["providers"].keys())
|
||||
for wtype, wcfg in data["work_types"].items():
|
||||
if "providers" not in wcfg or not isinstance(wcfg["providers"], list):
|
||||
raise ConfigError(f"work_type '{wtype}' missing 'providers' list")
|
||||
for entry in wcfg["providers"]:
|
||||
# A providers-list entry is either the legacy bare provider-name
|
||||
# string, or an object carrying a per-provider model/effort
|
||||
# override: {"provider": "<name>", "model": "...", "effort": "..."}.
|
||||
# Both forms may appear in the same list (list order still encodes
|
||||
# preference for select_provider()).
|
||||
if isinstance(entry, str):
|
||||
pname_ref = entry
|
||||
if pname_ref not in known_providers:
|
||||
raise ConfigError(
|
||||
f"work_type '{wtype}' providers entry names unknown provider '{pname_ref}'"
|
||||
)
|
||||
elif isinstance(entry, dict):
|
||||
unknown_keys = set(entry.keys()) - {"provider", "model", "effort"}
|
||||
if unknown_keys:
|
||||
raise ConfigError(
|
||||
f"work_type '{wtype}' providers entry has unknown key(s): {sorted(unknown_keys)}"
|
||||
)
|
||||
pname_ref = entry.get("provider")
|
||||
if not isinstance(pname_ref, str) or not pname_ref:
|
||||
raise ConfigError(
|
||||
f"work_type '{wtype}' providers entry (object form) missing non-empty 'provider' key"
|
||||
)
|
||||
if pname_ref not in known_providers:
|
||||
raise ConfigError(
|
||||
f"work_type '{wtype}' providers entry names unknown provider '{pname_ref}'"
|
||||
)
|
||||
if "model" in entry:
|
||||
entry_model = entry["model"]
|
||||
if not isinstance(entry_model, str) or not entry_model:
|
||||
raise ConfigError(
|
||||
f"work_type '{wtype}' providers entry for '{pname_ref}' 'model' must be "
|
||||
f"a non-empty string"
|
||||
)
|
||||
if "effort" in entry:
|
||||
entry_effort = entry["effort"]
|
||||
if not isinstance(entry_effort, str) or not entry_effort:
|
||||
raise ConfigError(
|
||||
f"work_type '{wtype}' providers entry for '{pname_ref}' 'effort' must be "
|
||||
f"a non-empty string"
|
||||
)
|
||||
if entry_effort not in CLAUDE_EFFORT_LEVELS:
|
||||
raise ConfigError(
|
||||
f"work_type '{wtype}' providers entry for '{pname_ref}' 'effort' must be one "
|
||||
f"of {sorted(CLAUDE_EFFORT_LEVELS)}, got {entry_effort!r}"
|
||||
)
|
||||
else:
|
||||
raise ConfigError(
|
||||
f"work_type '{wtype}' providers entry must be a string or object, "
|
||||
f"got {type(entry).__name__}"
|
||||
)
|
||||
if "allowed_tools" in wcfg:
|
||||
at = wcfg["allowed_tools"]
|
||||
if not isinstance(at, list) or not all(isinstance(x, str) for x in at):
|
||||
raise ConfigError(f"work_type '{wtype}' 'allowed_tools' must be a list of strings")
|
||||
if "effort" in wcfg:
|
||||
effort = wcfg["effort"]
|
||||
if not isinstance(effort, str) or not effort:
|
||||
raise ConfigError(f"work_type '{wtype}' 'effort' must be a non-empty string")
|
||||
if effort not in CLAUDE_EFFORT_LEVELS:
|
||||
raise ConfigError(
|
||||
f"work_type '{wtype}' 'effort' must be one of {sorted(CLAUDE_EFFORT_LEVELS)}, got {effort!r}"
|
||||
)
|
||||
if "model" in wcfg:
|
||||
model = wcfg["model"]
|
||||
if not isinstance(model, str) or not model:
|
||||
raise ConfigError(f"work_type '{wtype}' 'model' must be a non-empty string")
|
||||
return data
|
||||
|
||||
|
||||
@@ -656,9 +731,19 @@ def build_ready_queue(repo: Path, config: dict, state: dict, in_flight: set) ->
|
||||
return candidates
|
||||
|
||||
|
||||
def provider_entry_name(entry) -> str:
|
||||
"""A providers-list entry is either a plain provider-name string (the
|
||||
legacy form) or an object {"provider": name, "model": ..., "effort": ...}.
|
||||
This extracts just the name -- the only field select_provider() itself
|
||||
needs; the model/effort overrides are resolved later, once a provider has
|
||||
actually been selected, by resolve_work_type_model/_effort below."""
|
||||
return entry if isinstance(entry, str) else entry["provider"]
|
||||
|
||||
|
||||
def select_provider(work_type: str, config: dict, gates: dict, credential_ok: dict) -> tuple[str | None, str]:
|
||||
providers = config["work_types"].get(work_type, {}).get("providers", [])
|
||||
for pname in providers:
|
||||
for entry in providers:
|
||||
pname = provider_entry_name(entry)
|
||||
gate = gates.get(pname, {})
|
||||
if gate.get("eligible") and credential_ok.get(pname, (False, ""))[0]:
|
||||
return pname, "eligible"
|
||||
@@ -758,11 +843,20 @@ def validate_output(work_type: str, text: str) -> tuple[bool, str]:
|
||||
|
||||
|
||||
def build_claude_argv(
|
||||
model_id: str | None, max_turns: int, add_dirs: list[str], allowed_tools: list[str] | None = None
|
||||
model_id: str | None,
|
||||
max_turns: int,
|
||||
add_dirs: list[str],
|
||||
allowed_tools: list[str] | None = None,
|
||||
effort: str | None = None,
|
||||
) -> list[str]:
|
||||
"""`model_id` is the already-resolved effective model (see
|
||||
resolve_effective_model: work-type "model" beats the profile's MODEL_ID) --
|
||||
this function emits at most one --model flag, never two."""
|
||||
argv = ["claude", "-p", "--max-turns", str(max_turns)]
|
||||
if model_id:
|
||||
argv += ["--model", model_id]
|
||||
if effort:
|
||||
argv += ["--effort", effort]
|
||||
for d in add_dirs:
|
||||
argv += ["--add-dir", d]
|
||||
if allowed_tools:
|
||||
@@ -821,16 +915,70 @@ def resolve_allowed_tools(config: dict, work_type: str) -> list[str]:
|
||||
return list(config["work_types"].get(work_type, {}).get("allowed_tools") or [])
|
||||
|
||||
|
||||
def find_provider_entry(config: dict, work_type: str, provider_name: str):
|
||||
"""The raw providers-list entry (str or dict) for provider_name within
|
||||
work_type's providers list, or None if not present. Defensive lookup --
|
||||
callers only ever pass a provider name that select_provider() just chose
|
||||
from this same list, so a miss shouldn't happen, but resolve_work_type_*
|
||||
below treat it as "no per-provider override" rather than raising."""
|
||||
for entry in config["work_types"].get(work_type, {}).get("providers", []):
|
||||
if provider_entry_name(entry) == provider_name:
|
||||
return entry
|
||||
return None
|
||||
|
||||
|
||||
def resolve_work_type_effort(config: dict, work_type: str, provider_name: str | None = None) -> str | None:
|
||||
"""Effort level for (work_type, provider_name). Precedence:
|
||||
per-provider entry "effort" > work-type-level "effort" > None (CLI
|
||||
default). `provider_name` is optional -- omit it to get just the
|
||||
work-type-level value (e.g. before a provider has been selected). Both
|
||||
layers are already validated (non-empty string, member of
|
||||
CLAUDE_EFFORT_LEVELS) at config-load time -- this is a plain lookup."""
|
||||
wcfg = config["work_types"].get(work_type, {})
|
||||
if provider_name is not None:
|
||||
entry = find_provider_entry(config, work_type, provider_name)
|
||||
per_provider = entry.get("effort") if isinstance(entry, dict) else None
|
||||
if per_provider:
|
||||
return per_provider
|
||||
return wcfg.get("effort")
|
||||
|
||||
|
||||
def resolve_work_type_model(config: dict, work_type: str, provider_name: str | None = None) -> str | None:
|
||||
"""Model override for (work_type, provider_name). Precedence:
|
||||
per-provider entry "model" > work-type-level "model" > None (falls back
|
||||
to the profile's MODEL_ID via resolve_effective_model). `provider_name`
|
||||
is optional -- omit it to get just the work-type-level value. Both
|
||||
layers are already type-checked (non-empty string) at config-load time --
|
||||
this is a plain lookup."""
|
||||
wcfg = config["work_types"].get(work_type, {})
|
||||
if provider_name is not None:
|
||||
entry = find_provider_entry(config, work_type, provider_name)
|
||||
per_provider = entry.get("model") if isinstance(entry, dict) else None
|
||||
if per_provider:
|
||||
return per_provider
|
||||
return wcfg.get("model")
|
||||
|
||||
|
||||
def resolve_effective_model(profile_model_id: str | None, work_type_model: str | None) -> str | None:
|
||||
"""Precedence: a work-type "model" override always wins over the
|
||||
profile's provider.env MODEL_ID. Pure function so the precedence rule is
|
||||
independently testable without touching a profile dir."""
|
||||
return work_type_model or profile_model_id
|
||||
|
||||
|
||||
def dispatch_preview(repo: Path, config: dict, candidate: dict, max_turns: int) -> dict:
|
||||
"""Compute everything a dryrun needs to print, without executing anything."""
|
||||
dossier = candidate["dossier"]
|
||||
work_type = candidate["work_type"]
|
||||
provider = candidate["provider"]
|
||||
profile_dir = expand(config["providers"][provider]["profile"])
|
||||
model_id = resolve_model_id(profile_dir)
|
||||
model_id = resolve_effective_model(
|
||||
resolve_model_id(profile_dir), resolve_work_type_model(config, work_type, provider)
|
||||
)
|
||||
effort = resolve_work_type_effort(config, work_type, provider)
|
||||
add_dirs = resolve_add_dirs(config, work_type)
|
||||
allowed_tools = resolve_allowed_tools(config, work_type)
|
||||
argv = build_claude_argv(model_id, max_turns, add_dirs, allowed_tools)
|
||||
argv = build_claude_argv(model_id, max_turns, add_dirs, allowed_tools, effort)
|
||||
canonical = canonical_output_path(repo, candidate)
|
||||
tmp_path = tmp_output_path(canonical)
|
||||
parsed_env = parse_provider_env(profile_dir / "provider.env")
|
||||
@@ -993,10 +1141,13 @@ def run_task(repo: Path, config: dict, candidate: dict, max_turns: int, task_tim
|
||||
prompt_text = render_prompt(work_type, mapping)
|
||||
|
||||
profile_dir = expand(config["providers"][provider]["profile"])
|
||||
model_id = resolve_model_id(profile_dir)
|
||||
model_id = resolve_effective_model(
|
||||
resolve_model_id(profile_dir), resolve_work_type_model(config, work_type, provider)
|
||||
)
|
||||
effort = resolve_work_type_effort(config, work_type, provider)
|
||||
add_dirs = resolve_add_dirs(config, work_type)
|
||||
allowed_tools = resolve_allowed_tools(config, work_type)
|
||||
argv = build_claude_argv(model_id, max_turns, add_dirs, allowed_tools)
|
||||
argv = build_claude_argv(model_id, max_turns, add_dirs, allowed_tools, effort)
|
||||
env = build_child_env(profile_dir)
|
||||
|
||||
timed_out = False
|
||||
|
||||
@@ -239,14 +239,21 @@ For the selected `(item_or_dossier, work_type, provider)`:
|
||||
invoked** (it has interactive pickers and terminal theming unsuitable for headless
|
||||
cron use).
|
||||
4. Build the argv: `claude -p --max-turns <N>` (`N` = 25 unless overridden), plus
|
||||
`--model <id>` if `provider.env` sets `MODEL_ID`, plus `--add-dir <dir>` for each
|
||||
entry in `config["evidence_dirs"]` **only** for `research` and `topic_ideas` work
|
||||
types (the only ones that cite external evidence), plus `--allowedTools
|
||||
`--model <id>` and `--effort <level>` resolved through the full precedence chain —
|
||||
per-provider override on the selected provider's `providers`-list entry, then
|
||||
work-type-level `model`/`effort`, then (for `model` only) `provider.env`'s
|
||||
`MODEL_ID`; the flag is omitted at each layer where nothing resolves (see
|
||||
"Per-work-type effort and model" and "Per-provider effort and model overrides"
|
||||
below) — plus `--add-dir <dir>` for each entry in `config["evidence_dirs"]`
|
||||
**only** for `research` and `topic_ideas`
|
||||
work types (the only ones that cite external evidence), plus `--allowedTools
|
||||
"<comma-joined list>"` if `config["work_types"][work_type]["allowed_tools"]` is
|
||||
present and non-empty (see Config schema below). Headless `claude -p` runs cannot
|
||||
answer permission prompts, so any tool beyond the CLI's defaults must be granted
|
||||
this way or the child run stalls/fails; when the key is absent or an empty list,
|
||||
the flag is omitted entirely (unchanged pre-existing behaviour).
|
||||
this way or the child run stalls/fails; when the `allowed_tools` key is absent or an
|
||||
empty list, that flag is omitted entirely (unchanged pre-existing behaviour). Same for
|
||||
`effort`/`model`: absent key -> no corresponding flag, argv unchanged from before
|
||||
these keys existed.
|
||||
5. Run the subprocess: `cwd=<repo>`, `env=<built env>`, prompt piped via **stdin**
|
||||
(not as an argv element — avoids `ARG_MAX` on large rendered prompts, same lesson
|
||||
`claude-profile` already applies to its system-prompt injection), timeout = per-task
|
||||
@@ -401,6 +408,87 @@ config validation failure — exit **2** at config load, same as a missing requi
|
||||
before writing its Subjective stories section; the topic-ideas template requires one as
|
||||
a lighter-touch check); `draft` and `review` carry no `allowed_tools` key.
|
||||
|
||||
### Per-work-type effort and model
|
||||
|
||||
Each entry in `work_types` may also carry two further optional keys, both consumed by
|
||||
§11 step 4 when building the child's argv:
|
||||
|
||||
- **`effort`** (string): appends `--effort <value>` to the child argv. Validated at
|
||||
config-load time against the fixed set of levels the `claude` CLI itself accepts:
|
||||
`low`, `medium`, `high`, `xhigh`, `max`. This set was determined empirically on
|
||||
2026-08-02 by running `claude --effort obviously-bogus-level -p ""` — an invalid
|
||||
`--effort` value is rejected by the CLI's own argument parsing before any network
|
||||
call is made, so the probe is safe to run offline — and reading the resulting error,
|
||||
which enumerated the valid values verbatim (cross-checked against `claude --help`'s
|
||||
`--effort <level>` description, which lists the same five). The set is hard-coded as
|
||||
`CLAUDE_EFFORT_LEVELS` in `scripts/idle-draft`; if a future CLI version changes it,
|
||||
re-run the same probe and update the constant (with a fresh date in the comment).
|
||||
A value that is not a non-empty string, or not a member of this set, is a config
|
||||
validation failure — exit **2**, same as a malformed `allowed_tools`.
|
||||
- **`model`** (string): appends `--model <value>` to the child argv, **replacing** —
|
||||
not adding to — the `--model <id>` that would otherwise come from the provider
|
||||
profile's `provider.env` `MODEL_ID` (§11 step 4). Only type-checked at config-load
|
||||
time (must be a non-empty string) — model identifiers are not enumerated, since new
|
||||
ones ship independently of this tool and hard-coding a set would go stale. A
|
||||
non-string or empty value is a config validation failure — exit **2**.
|
||||
|
||||
Both keys are optional and independent of each other and of `allowed_tools`. Absent on
|
||||
a given work type -> the corresponding flag is omitted / the profile's model is used
|
||||
unchanged, matching the tool's behaviour before these keys existed.
|
||||
|
||||
### Per-provider effort and model overrides (within a work type)
|
||||
|
||||
`model`/`effort` can be pinned even more narrowly than per-work-type: per
|
||||
`(work_type, provider)` pair. Each entry in a `work_types[*].providers` list may be
|
||||
**either**:
|
||||
|
||||
- a plain provider-name string (the original, unchanged form) — no per-provider
|
||||
override, resolution falls through to the work-type-level `model`/`effort` (above);
|
||||
or
|
||||
- an object `{"provider": "<name>", "model": "<optional>", "effort": "<optional>"}` —
|
||||
`model`/`effort` here apply only when *this* provider is the one actually selected
|
||||
(§9) for a dispatch of this work type.
|
||||
|
||||
Both forms may be mixed freely within the same `providers` list; list order still
|
||||
encodes provider preference exactly as before (§9) — the object form does not change
|
||||
*which* provider is tried first, only what argv that provider gets if chosen.
|
||||
`config.example.json` demonstrates this: `research`'s `providers` is
|
||||
`["anthropic", {"provider": "minimax", "effort": "medium"}]` — `anthropic` stays a
|
||||
plain string (falls through to `research`'s work-type-level `effort`), `minimax` pins
|
||||
its own `effort` via the object form.
|
||||
|
||||
**Full precedence chain**, evaluated once a `(work_type, provider)` pair has been
|
||||
selected (§9), for each of `model` and `effort` independently:
|
||||
|
||||
1. The per-provider override on the matching `providers`-list entry for that provider
|
||||
(object form only; a plain-string entry has none).
|
||||
2. The work-type-level `model`/`effort` (the keys directly under
|
||||
`work_types[work_type]`, previous section).
|
||||
3. Ultimate fallback: for `model`, the profile's `provider.env` `MODEL_ID` (or no
|
||||
`--model` flag if unset); for `effort`, no `--effort` flag (the CLI's own default).
|
||||
|
||||
**Validation** (config-load time, exit **2** on any failure, same style as every other
|
||||
config check):
|
||||
|
||||
- Every `providers`-list entry — string **or** object form — must name a provider that
|
||||
exists as a key in the top-level `providers` map. An entry (of either form) naming an
|
||||
unrecognised provider is rejected; this closes a gap that predates the object form —
|
||||
a typo'd plain-string provider name was previously silently ineligible (never
|
||||
selected, no error) rather than rejected at load time.
|
||||
- An object-form entry must have a `"provider"` key whose value is a non-empty string.
|
||||
- An object-form entry's keys are restricted to `provider`, `model`, `effort`; any other
|
||||
key is rejected.
|
||||
- An object-form entry's `model`, if present, must be a non-empty string (same rule as
|
||||
the work-type-level `model`).
|
||||
- An object-form entry's `effort`, if present, must be a non-empty string and a member
|
||||
of `CLAUDE_EFFORT_LEVELS` (same rule as the work-type-level `effort`).
|
||||
- A `providers`-list entry that is neither a string nor an object is rejected.
|
||||
|
||||
`--dryrun` prints the fully resolved argv (§11 step 4's `--model`/`--effort` included,
|
||||
after the full precedence chain above), so a preview reveals exactly which provider,
|
||||
model, and effort level a given work type would route to before anything is dispatched
|
||||
for real.
|
||||
|
||||
## Failure classes
|
||||
|
||||
| Class | Examples | `attempts` effect | Item outcome |
|
||||
@@ -438,6 +526,16 @@ a lighter-touch check); `draft` and `review` carry no `allowed_tools` key.
|
||||
| `--once` with no eligible work | Exits 0 immediately, no task dispatched |
|
||||
| Config missing a required key | Exit 2 |
|
||||
| `work_types[*].allowed_tools` present but not a list of strings | Exit 2 |
|
||||
| `work_types[*].effort` present but not one of `low`/`medium`/`high`/`xhigh`/`max` | Exit 2 |
|
||||
| `work_types[*].effort` present but empty string / not a string | Exit 2 |
|
||||
| `work_types[*].model` present but empty string / not a string | Exit 2 |
|
||||
| `work_types[*].model` set alongside a profile whose `provider.env` also sets `MODEL_ID` | Work-type `model` wins (absent a per-provider override); argv carries exactly one `--model` flag |
|
||||
| `work_types[*].providers[*]` (string or object form) names a provider not present in the top-level `providers` map | Exit 2 |
|
||||
| `work_types[*].providers[*]` object form missing/empty `"provider"` key | Exit 2 |
|
||||
| `work_types[*].providers[*]` object form has an unknown key (anything besides `provider`/`model`/`effort`) | Exit 2 |
|
||||
| `work_types[*].providers[*]` object form `model`/`effort` invalid (same rules as the work-type-level keys) | Exit 2 |
|
||||
| `work_types[*].providers[*]` entry is neither a string nor an object | Exit 2 |
|
||||
| A provider's `providers`-list entry sets its own `model`/`effort`, and the work type also sets a work-type-level `model`/`effort` | Per-provider entry wins for that provider; a sibling provider in the same list with a plain-string entry still falls through to the work-type-level value |
|
||||
| `agent-subscriptions` subprocess fails entirely | Exit 1, loud log line, no dispatch attempted |
|
||||
|
||||
## Examples
|
||||
|
||||
@@ -155,6 +155,44 @@ assert_contains "$output" "$TMPDIR/evidence" "dryrun argv includes configured ev
|
||||
assert_contains "$output" "--allowedTools" "dryrun argv for research includes --allowedTools (allowed_tools configured)"
|
||||
assert_contains "$output" "WebSearch,WebFetch" "dryrun argv --allowedTools value is the comma-joined tool list"
|
||||
|
||||
echo "-- dryrun: work-type effort/model override profile model, appear in resolved argv --"
|
||||
PROFILE_ANTHROPIC_MODEL="$TMPDIR/profiles/anthropic-with-model"
|
||||
mkdir -p "$PROFILE_ANTHROPIC_MODEL"
|
||||
cat > "$PROFILE_ANTHROPIC_MODEL/provider.env" <<'EOF'
|
||||
MODEL_ID=profile-default-model
|
||||
EOF
|
||||
EFFORT_CONFIG="$TMPDIR/effort.config.json"
|
||||
cat > "$EFFORT_CONFIG" <<EOF
|
||||
{
|
||||
"parallel": 2,
|
||||
"providers": {
|
||||
"anthropic": { "profile": "$PROFILE_ANTHROPIC_MODEL", "threshold_pct": 80, "five_hour_ceiling": 50, "min_idle": 5 },
|
||||
"minimax": { "profile": "$PROFILE_MINIMAX", "threshold_pct": 80, "five_hour_ceiling": 50, "min_idle": 5 }
|
||||
},
|
||||
"work_types": {
|
||||
"review": { "providers": ["anthropic"] },
|
||||
"draft": { "providers": ["anthropic"] },
|
||||
"research": { "providers": ["anthropic", "minimax"], "allowed_tools": ["WebSearch", "WebFetch"], "effort": "medium", "model": "worktype-override-model" },
|
||||
"topic_ideas": { "providers": ["anthropic", "minimax"], "allowed_tools": ["WebSearch", "WebFetch"] }
|
||||
},
|
||||
"dossiers": ["ai"],
|
||||
"review_score_threshold": 11,
|
||||
"max_unreviewed_research_per_dossier": 3,
|
||||
"max_open_topic_proposals": 6,
|
||||
"evidence_dirs": ["$TMPDIR/evidence"]
|
||||
}
|
||||
EOF
|
||||
output=$("$SCRIPT" --config "$EFFORT_CONFIG" --repo "$SOLO" --probe-json "$FIXTURE_PROBE" --dryrun --once 2>&1)
|
||||
assert_contains "$output" "'--model', 'worktype-override-model'" "dryrun argv model uses work-type override, not profile MODEL_ID"
|
||||
assert_not_contains "$output" "profile-default-model" "dryrun argv does not contain the profile's MODEL_ID once a work-type model overrides it"
|
||||
assert_contains "$output" "'--effort', 'medium'" "dryrun argv includes --effort with the configured level"
|
||||
model_flag_count=$(echo "$output" | grep -o -- "--model" | wc -l)
|
||||
[[ "$model_flag_count" -eq 1 ]] && pass "resolved argv contains exactly one --model flag (no duplicate from profile)" || fail "resolved argv contains exactly one --model flag" "found $model_flag_count"
|
||||
|
||||
echo "-- dryrun: no effort/model configured -> flags omitted (unchanged behaviour) --"
|
||||
output=$("$SCRIPT" --config "$FIXTURE_CONFIG" --repo "$SOLO" --probe-json "$FIXTURE_PROBE" --dryrun --once 2>&1)
|
||||
assert_not_contains "$output" "--effort" "dryrun argv omits --effort when work_type has no effort key"
|
||||
|
||||
echo "-- dryrun: no mutation --"
|
||||
before_hash=$(cd "$FIXTURE_REPO" && git rev-parse HEAD)
|
||||
"$SCRIPT" --config "$FIXTURE_CONFIG" --repo "$FIXTURE_REPO" --probe-json "$FIXTURE_PROBE" --dryrun >/dev/null 2>&1
|
||||
@@ -254,6 +292,132 @@ code=$?
|
||||
assert_exit_code "$code" 2 "allowed_tools not a list of strings exits 2"
|
||||
assert_contains "$output" "allowed_tools" "allowed_tools config error names the offending key"
|
||||
|
||||
BADEFFORT="$TMPDIR/bad-effort.config.json"
|
||||
cat > "$BADEFFORT" <<EOF
|
||||
{
|
||||
"parallel": 2,
|
||||
"providers": {
|
||||
"anthropic": { "profile": "$PROFILE_ANTHROPIC", "threshold_pct": 80, "five_hour_ceiling": 50, "min_idle": 5 }
|
||||
},
|
||||
"work_types": {
|
||||
"review": { "providers": ["anthropic"] },
|
||||
"draft": { "providers": ["anthropic"] },
|
||||
"research": { "providers": ["anthropic"], "effort": "ultra-mega" },
|
||||
"topic_ideas": { "providers": ["anthropic"] }
|
||||
},
|
||||
"dossiers": ["ai"],
|
||||
"review_score_threshold": 11,
|
||||
"max_unreviewed_research_per_dossier": 3,
|
||||
"max_open_topic_proposals": 6,
|
||||
"evidence_dirs": ["$TMPDIR/evidence"]
|
||||
}
|
||||
EOF
|
||||
output=$("$SCRIPT" --config "$BADEFFORT" --repo "$FIXTURE_REPO" --probe-json "$FIXTURE_PROBE" --dryrun 2>&1)
|
||||
code=$?
|
||||
assert_exit_code "$code" 2 "invalid effort level exits 2"
|
||||
assert_contains "$output" "effort" "invalid effort config error names the offending key"
|
||||
|
||||
BADMODEL="$TMPDIR/bad-model.config.json"
|
||||
cat > "$BADMODEL" <<EOF
|
||||
{
|
||||
"parallel": 2,
|
||||
"providers": {
|
||||
"anthropic": { "profile": "$PROFILE_ANTHROPIC", "threshold_pct": 80, "five_hour_ceiling": 50, "min_idle": 5 }
|
||||
},
|
||||
"work_types": {
|
||||
"review": { "providers": ["anthropic"] },
|
||||
"draft": { "providers": ["anthropic"] },
|
||||
"research": { "providers": ["anthropic"], "model": "" },
|
||||
"topic_ideas": { "providers": ["anthropic"] }
|
||||
},
|
||||
"dossiers": ["ai"],
|
||||
"review_score_threshold": 11,
|
||||
"max_unreviewed_research_per_dossier": 3,
|
||||
"max_open_topic_proposals": 6,
|
||||
"evidence_dirs": ["$TMPDIR/evidence"]
|
||||
}
|
||||
EOF
|
||||
output=$("$SCRIPT" --config "$BADMODEL" --repo "$FIXTURE_REPO" --probe-json "$FIXTURE_PROBE" --dryrun 2>&1)
|
||||
code=$?
|
||||
assert_exit_code "$code" 2 "empty model string exits 2"
|
||||
assert_contains "$output" "model" "empty model config error names the offending key"
|
||||
|
||||
echo "-- config validation: per-provider effort/model overrides (providers-list entries) --"
|
||||
BADPROVIDERNAME="$TMPDIR/bad-provider-name.config.json"
|
||||
cat > "$BADPROVIDERNAME" <<EOF
|
||||
{
|
||||
"parallel": 2,
|
||||
"providers": {
|
||||
"anthropic": { "profile": "$PROFILE_ANTHROPIC", "threshold_pct": 80, "five_hour_ceiling": 50, "min_idle": 5 }
|
||||
},
|
||||
"work_types": {
|
||||
"review": { "providers": ["anthropic"] },
|
||||
"draft": { "providers": ["anthropic"] },
|
||||
"research": { "providers": ["anthropic", {"provider": "nonexistent-provider", "effort": "high"}] },
|
||||
"topic_ideas": { "providers": ["anthropic"] }
|
||||
},
|
||||
"dossiers": ["ai"],
|
||||
"review_score_threshold": 11,
|
||||
"max_unreviewed_research_per_dossier": 3,
|
||||
"max_open_topic_proposals": 6,
|
||||
"evidence_dirs": ["$TMPDIR/evidence"]
|
||||
}
|
||||
EOF
|
||||
output=$("$SCRIPT" --config "$BADPROVIDERNAME" --repo "$FIXTURE_REPO" --probe-json "$FIXTURE_PROBE" --dryrun 2>&1)
|
||||
code=$?
|
||||
assert_exit_code "$code" 2 "providers-list entry naming an undefined provider exits 2"
|
||||
assert_contains "$output" "unknown provider" "undefined provider config error names the problem"
|
||||
|
||||
BADPROVIDERKEY="$TMPDIR/bad-provider-key.config.json"
|
||||
cat > "$BADPROVIDERKEY" <<EOF
|
||||
{
|
||||
"parallel": 2,
|
||||
"providers": {
|
||||
"anthropic": { "profile": "$PROFILE_ANTHROPIC", "threshold_pct": 80, "five_hour_ceiling": 50, "min_idle": 5 }
|
||||
},
|
||||
"work_types": {
|
||||
"review": { "providers": ["anthropic"] },
|
||||
"draft": { "providers": ["anthropic"] },
|
||||
"research": { "providers": [{"provider": "anthropic", "priority": 1}] },
|
||||
"topic_ideas": { "providers": ["anthropic"] }
|
||||
},
|
||||
"dossiers": ["ai"],
|
||||
"review_score_threshold": 11,
|
||||
"max_unreviewed_research_per_dossier": 3,
|
||||
"max_open_topic_proposals": 6,
|
||||
"evidence_dirs": ["$TMPDIR/evidence"]
|
||||
}
|
||||
EOF
|
||||
output=$("$SCRIPT" --config "$BADPROVIDERKEY" --repo "$FIXTURE_REPO" --probe-json "$FIXTURE_PROBE" --dryrun 2>&1)
|
||||
code=$?
|
||||
assert_exit_code "$code" 2 "providers-list object entry with an unknown key exits 2"
|
||||
assert_contains "$output" "unknown key" "unknown provider-entry key config error names the problem"
|
||||
|
||||
BADPROVIDEREFFORT="$TMPDIR/bad-provider-effort.config.json"
|
||||
cat > "$BADPROVIDEREFFORT" <<EOF
|
||||
{
|
||||
"parallel": 2,
|
||||
"providers": {
|
||||
"anthropic": { "profile": "$PROFILE_ANTHROPIC", "threshold_pct": 80, "five_hour_ceiling": 50, "min_idle": 5 }
|
||||
},
|
||||
"work_types": {
|
||||
"review": { "providers": ["anthropic"] },
|
||||
"draft": { "providers": ["anthropic"] },
|
||||
"research": { "providers": [{"provider": "anthropic", "effort": "super-duper"}] },
|
||||
"topic_ideas": { "providers": ["anthropic"] }
|
||||
},
|
||||
"dossiers": ["ai"],
|
||||
"review_score_threshold": 11,
|
||||
"max_unreviewed_research_per_dossier": 3,
|
||||
"max_open_topic_proposals": 6,
|
||||
"evidence_dirs": ["$TMPDIR/evidence"]
|
||||
}
|
||||
EOF
|
||||
output=$("$SCRIPT" --config "$BADPROVIDEREFFORT" --repo "$FIXTURE_REPO" --probe-json "$FIXTURE_PROBE" --dryrun 2>&1)
|
||||
code=$?
|
||||
assert_exit_code "$code" 2 "providers-list object entry with an invalid effort level exits 2"
|
||||
assert_contains "$output" "effort" "invalid provider-entry effort config error names the problem"
|
||||
|
||||
echo "-- mark subcommand round-trip --"
|
||||
MARKREPO="$TMPDIR/mark-repo"
|
||||
mkdir -p "$MARKREPO/ai"
|
||||
@@ -519,6 +683,334 @@ check("load_config: well-formed allowed_tools (list of strings) loads without er
|
||||
good_allowed_cfg["work_types"]["research"]["allowed_tools"] == ["WebSearch", "WebFetch"],
|
||||
good_allowed_cfg["work_types"]["research"])
|
||||
|
||||
# --- per-work-type effort / model: argv construction ---
|
||||
argv_effort = m.build_claude_argv("model-x", 25, [], None, "high")
|
||||
check("build_claude_argv: appends --effort when given",
|
||||
"--effort" in argv_effort and argv_effort[argv_effort.index("--effort") + 1] == "high", argv_effort)
|
||||
|
||||
argv_no_effort = m.build_claude_argv("model-x", 25, [], None, None)
|
||||
check("build_claude_argv: omits --effort when not given", "--effort" not in argv_no_effort, argv_no_effort)
|
||||
|
||||
argv_model_and_effort = m.build_claude_argv("worktype-model", 25, [], None, "xhigh")
|
||||
check("build_claude_argv: --model reflects whatever single value the caller resolved "
|
||||
"(precedence already applied upstream) -- exactly one --model flag",
|
||||
argv_model_and_effort.count("--model") == 1
|
||||
and argv_model_and_effort[argv_model_and_effort.index("--model") + 1] == "worktype-model",
|
||||
argv_model_and_effort)
|
||||
|
||||
argv_neither = m.build_claude_argv(None, 25, [], None, None)
|
||||
check("build_claude_argv: omits both --model and --effort when neither is given",
|
||||
"--model" not in argv_neither and "--effort" not in argv_neither, argv_neither)
|
||||
|
||||
# --- per-work-type effort / model: precedence resolution ---
|
||||
check("resolve_effective_model: work-type model overrides profile MODEL_ID",
|
||||
m.resolve_effective_model("profile-model", "worktype-model") == "worktype-model", "")
|
||||
check("resolve_effective_model: falls back to profile MODEL_ID when no work-type model set",
|
||||
m.resolve_effective_model("profile-model", None) == "profile-model", "")
|
||||
check("resolve_effective_model: both absent -> None",
|
||||
m.resolve_effective_model(None, None) is None, "")
|
||||
|
||||
cfg_em = {"work_types": {"research": {"providers": ["anthropic"], "effort": "medium", "model": "wt-model"},
|
||||
"draft": {"providers": ["anthropic"]}}}
|
||||
check("resolve_work_type_effort: returns configured value", m.resolve_work_type_effort(cfg_em, "research") == "medium", "")
|
||||
check("resolve_work_type_effort: returns None when absent", m.resolve_work_type_effort(cfg_em, "draft") is None, "")
|
||||
check("resolve_work_type_model: returns configured value", m.resolve_work_type_model(cfg_em, "research") == "wt-model", "")
|
||||
check("resolve_work_type_model: returns None when absent", m.resolve_work_type_model(cfg_em, "draft") is None, "")
|
||||
|
||||
# --- per-work-type effort / model: config validation ---
|
||||
bad_effort_cfg_path = tmpdir / "bad-effort-direct.config.json"
|
||||
bad_effort_cfg = json.loads((tmpdir / "idle-draft.config.json").read_text())
|
||||
bad_effort_cfg["work_types"]["research"]["effort"] = "ultra-mega"
|
||||
bad_effort_cfg_path.write_text(json.dumps(bad_effort_cfg))
|
||||
try:
|
||||
m.load_config(bad_effort_cfg_path)
|
||||
check("load_config: effort value outside CLAUDE_EFFORT_LEVELS raises ConfigError", False)
|
||||
except m.ConfigError as e:
|
||||
check("load_config: effort value outside CLAUDE_EFFORT_LEVELS raises ConfigError", "effort" in str(e), str(e))
|
||||
|
||||
bad_effort_type_path = tmpdir / "bad-effort-type.config.json"
|
||||
bad_effort_type_cfg = json.loads((tmpdir / "idle-draft.config.json").read_text())
|
||||
bad_effort_type_cfg["work_types"]["research"]["effort"] = 5
|
||||
bad_effort_type_path.write_text(json.dumps(bad_effort_type_cfg))
|
||||
try:
|
||||
m.load_config(bad_effort_type_path)
|
||||
check("load_config: non-string effort raises ConfigError", False)
|
||||
except m.ConfigError as e:
|
||||
check("load_config: non-string effort raises ConfigError", "effort" in str(e), str(e))
|
||||
|
||||
bad_model_path = tmpdir / "bad-model-direct.config.json"
|
||||
bad_model_cfg = json.loads((tmpdir / "idle-draft.config.json").read_text())
|
||||
bad_model_cfg["work_types"]["research"]["model"] = ""
|
||||
bad_model_path.write_text(json.dumps(bad_model_cfg))
|
||||
try:
|
||||
m.load_config(bad_model_path)
|
||||
check("load_config: empty-string model raises ConfigError", False)
|
||||
except m.ConfigError as e:
|
||||
check("load_config: empty-string model raises ConfigError", "model" in str(e), str(e))
|
||||
|
||||
bad_model_type_path = tmpdir / "bad-model-type.config.json"
|
||||
bad_model_type_cfg = json.loads((tmpdir / "idle-draft.config.json").read_text())
|
||||
bad_model_type_cfg["work_types"]["research"]["model"] = 42
|
||||
bad_model_type_path.write_text(json.dumps(bad_model_type_cfg))
|
||||
try:
|
||||
m.load_config(bad_model_type_path)
|
||||
check("load_config: non-string model raises ConfigError", False)
|
||||
except m.ConfigError as e:
|
||||
check("load_config: non-string model raises ConfigError", "model" in str(e), str(e))
|
||||
|
||||
good_effort_model_cfg = json.loads((tmpdir / "idle-draft.config.json").read_text())
|
||||
good_effort_model_cfg["work_types"]["research"]["effort"] = "high"
|
||||
good_effort_model_cfg["work_types"]["research"]["model"] = "claude-example-model"
|
||||
good_effort_model_path = tmpdir / "good-effort-model.config.json"
|
||||
good_effort_model_path.write_text(json.dumps(good_effort_model_cfg))
|
||||
loaded_good = m.load_config(good_effort_model_path)
|
||||
check("load_config: well-formed effort + model (in the valid set) loads without error",
|
||||
loaded_good["work_types"]["research"]["effort"] == "high"
|
||||
and loaded_good["work_types"]["research"]["model"] == "claude-example-model",
|
||||
loaded_good["work_types"]["research"])
|
||||
|
||||
for level in sorted(m.CLAUDE_EFFORT_LEVELS):
|
||||
per_level_cfg = json.loads((tmpdir / "idle-draft.config.json").read_text())
|
||||
per_level_cfg["work_types"]["research"]["effort"] = level
|
||||
per_level_path = tmpdir / f"effort-{level}.config.json"
|
||||
per_level_path.write_text(json.dumps(per_level_cfg))
|
||||
loaded = m.load_config(per_level_path)
|
||||
check(f"load_config: effort level {level!r} (member of CLAUDE_EFFORT_LEVELS) loads without error",
|
||||
loaded["work_types"]["research"]["effort"] == level, loaded["work_types"]["research"])
|
||||
|
||||
# --- per-provider effort / model overrides (within a work type) ---
|
||||
|
||||
check("provider_entry_name: plain string entry returns itself",
|
||||
m.provider_entry_name("anthropic") == "anthropic", "")
|
||||
check("provider_entry_name: object entry returns its 'provider' value",
|
||||
m.provider_entry_name({"provider": "minimax", "effort": "high"}) == "minimax", "")
|
||||
|
||||
cfg_pp = {
|
||||
"providers": {"anthropic": {}, "minimax": {}},
|
||||
"work_types": {
|
||||
"research": {
|
||||
"providers": [
|
||||
"anthropic",
|
||||
{"provider": "minimax", "model": "minimax-model", "effort": "high"},
|
||||
],
|
||||
"effort": "medium",
|
||||
"model": "worktype-model",
|
||||
},
|
||||
"draft": {"providers": ["anthropic"], "effort": "low"},
|
||||
},
|
||||
}
|
||||
|
||||
entry_a = m.find_provider_entry(cfg_pp, "research", "anthropic")
|
||||
check("find_provider_entry: returns the plain-string entry for a string-form provider",
|
||||
entry_a == "anthropic", entry_a)
|
||||
entry_m = m.find_provider_entry(cfg_pp, "research", "minimax")
|
||||
check("find_provider_entry: returns the dict entry for an object-form provider",
|
||||
isinstance(entry_m, dict) and entry_m["provider"] == "minimax", entry_m)
|
||||
entry_missing = m.find_provider_entry(cfg_pp, "research", "nonexistent")
|
||||
check("find_provider_entry: returns None when the provider isn't in this work_type's list",
|
||||
entry_missing is None, entry_missing)
|
||||
|
||||
# Precedence: per-provider entry > work-type-level > (caller supplies profile fallback separately)
|
||||
check("resolve_work_type_effort: plain-string provider entry falls through to work-type-level effort",
|
||||
m.resolve_work_type_effort(cfg_pp, "research", "anthropic") == "medium", "")
|
||||
check("resolve_work_type_model: plain-string provider entry falls through to work-type-level model",
|
||||
m.resolve_work_type_model(cfg_pp, "research", "anthropic") == "worktype-model", "")
|
||||
check("resolve_work_type_effort: object-form provider entry's own effort wins over work-type-level",
|
||||
m.resolve_work_type_effort(cfg_pp, "research", "minimax") == "high", "")
|
||||
check("resolve_work_type_model: object-form provider entry's own model wins over work-type-level",
|
||||
m.resolve_work_type_model(cfg_pp, "research", "minimax") == "minimax-model", "")
|
||||
check("resolve_work_type_effort: provider_name omitted -> work-type-level value only (back-compat)",
|
||||
m.resolve_work_type_effort(cfg_pp, "research") == "medium", "")
|
||||
check("resolve_work_type_model: provider_name omitted -> work-type-level value only (back-compat)",
|
||||
m.resolve_work_type_model(cfg_pp, "research") == "worktype-model", "")
|
||||
check("resolve_work_type_effort: work type with no per-work-type or per-provider effort -> None",
|
||||
m.resolve_work_type_effort(cfg_pp, "draft", "anthropic") == "low", "")
|
||||
|
||||
# Full chain including the ultimate profile-MODEL_ID fallback (resolve_effective_model),
|
||||
# for a provider with NO per-provider or work-type-level model at all.
|
||||
cfg_pp_no_wt_model = {
|
||||
"providers": {"anthropic": {}},
|
||||
"work_types": {"review": {"providers": ["anthropic"], "effort": "high"}},
|
||||
}
|
||||
wt_model = m.resolve_work_type_model(cfg_pp_no_wt_model, "review", "anthropic")
|
||||
check("resolve_work_type_model: no per-provider, no work-type-level -> None (falls to profile next)",
|
||||
wt_model is None, wt_model)
|
||||
check("resolve_effective_model: full chain falls back to profile MODEL_ID when neither override is set",
|
||||
m.resolve_effective_model("profile-model-id", wt_model) == "profile-model-id", "")
|
||||
|
||||
# --- per-provider effort / model overrides: dispatch_preview reflects the resolution ---
|
||||
# dispatch_preview() takes a candidate with "provider" already selected -- this exercises
|
||||
# the exact same resolution path run_task() uses, without needing a real subprocess or a
|
||||
# provider to actually be gate-eligible (credential/gate selection is select_provider()'s
|
||||
# job, tested separately; dispatch_preview only needs a chosen provider name).
|
||||
pp_repo = tmpdir / "pp-repo"
|
||||
(pp_repo / "ai").mkdir(parents=True, exist_ok=True)
|
||||
(pp_repo / "style").mkdir(parents=True, exist_ok=True)
|
||||
(pp_repo / "ai" / "01-a.overview.md").write_text("# a\n")
|
||||
(pp_repo / "ai" / "01-a.agent.md").write_text("# brief\n")
|
||||
(pp_repo / "ai" / "SOURCE-REGISTER.md").write_text("# reg\n")
|
||||
(pp_repo / "AGENTS.md").write_text("# root\n")
|
||||
(pp_repo / "ai" / "AGENTS.md").write_text("# ai\n")
|
||||
|
||||
pp_profile_anthropic = tmpdir / "pp-profiles" / "anthropic"
|
||||
pp_profile_minimax = tmpdir / "pp-profiles" / "minimax"
|
||||
pp_profile_anthropic.mkdir(parents=True, exist_ok=True)
|
||||
pp_profile_minimax.mkdir(parents=True, exist_ok=True)
|
||||
(pp_profile_anthropic / "provider.env").write_text("MODEL_ID=anthropic-profile-model\n")
|
||||
# minimax profile deliberately has no provider.env -- its --model must come entirely
|
||||
# from the per-provider config override, not any profile fallback.
|
||||
|
||||
pp_config = {
|
||||
"providers": {
|
||||
"anthropic": {"profile": str(pp_profile_anthropic), "threshold_pct": 80,
|
||||
"five_hour_ceiling": 50, "min_idle": 5},
|
||||
"minimax": {"profile": str(pp_profile_minimax), "threshold_pct": 80,
|
||||
"five_hour_ceiling": 50, "min_idle": 5},
|
||||
},
|
||||
"work_types": {
|
||||
"research": {
|
||||
"providers": [
|
||||
"anthropic",
|
||||
{"provider": "minimax", "model": "minimax-pinned-model", "effort": "xhigh"},
|
||||
],
|
||||
"effort": "medium",
|
||||
"model": "worktype-fallback-model",
|
||||
},
|
||||
},
|
||||
"dossiers": ["ai"],
|
||||
"review_score_threshold": 11,
|
||||
"max_unreviewed_research_per_dossier": 3,
|
||||
"max_open_topic_proposals": 6,
|
||||
"evidence_dirs": [],
|
||||
}
|
||||
|
||||
candidate_anthropic = {"dossier": "ai", "slug": "01-a", "work_type": "research", "provider": "anthropic"}
|
||||
preview_anthropic = m.dispatch_preview(pp_repo, pp_config, candidate_anthropic, 25)
|
||||
check("dispatch_preview: plain-string provider entry (anthropic) uses work-type-level model, not its own profile MODEL_ID",
|
||||
"worktype-fallback-model" in preview_anthropic["argv"]
|
||||
and "anthropic-profile-model" not in preview_anthropic["argv"], preview_anthropic["argv"])
|
||||
check("dispatch_preview: plain-string provider entry (anthropic) uses work-type-level effort",
|
||||
"medium" in preview_anthropic["argv"] and preview_anthropic["argv"].count("--effort") == 1,
|
||||
preview_anthropic["argv"])
|
||||
|
||||
candidate_minimax = {"dossier": "ai", "slug": "01-a", "work_type": "research", "provider": "minimax"}
|
||||
preview_minimax = m.dispatch_preview(pp_repo, pp_config, candidate_minimax, 25)
|
||||
check("dispatch_preview: object-form provider entry (minimax) uses its own pinned model over the work-type-level one",
|
||||
"minimax-pinned-model" in preview_minimax["argv"]
|
||||
and "worktype-fallback-model" not in preview_minimax["argv"], preview_minimax["argv"])
|
||||
check("dispatch_preview: object-form provider entry (minimax) uses its own pinned effort over the work-type-level one",
|
||||
"xhigh" in preview_minimax["argv"] and "medium" not in preview_minimax["argv"], preview_minimax["argv"])
|
||||
check("dispatch_preview: exactly one --model flag regardless of which provider was selected (mixed list)",
|
||||
preview_anthropic["argv"].count("--model") == 1 and preview_minimax["argv"].count("--model") == 1, "")
|
||||
|
||||
# --- per-provider effort / model overrides: config validation ---
|
||||
|
||||
def _base_pp_cfg():
|
||||
c = json.loads((tmpdir / "idle-draft.config.json").read_text())
|
||||
return c
|
||||
|
||||
bad_provider_name_str_path = tmpdir / "bad-provider-name-str.config.json"
|
||||
c = _base_pp_cfg()
|
||||
c["work_types"]["research"]["providers"] = ["anthropic", "nonexistent-provider"]
|
||||
bad_provider_name_str_path.write_text(json.dumps(c))
|
||||
try:
|
||||
m.load_config(bad_provider_name_str_path)
|
||||
check("load_config: string-form providers entry naming an unknown provider raises ConfigError", False)
|
||||
except m.ConfigError as e:
|
||||
check("load_config: string-form providers entry naming an unknown provider raises ConfigError",
|
||||
"unknown provider" in str(e) and "nonexistent-provider" in str(e), str(e))
|
||||
|
||||
bad_provider_name_obj_path = tmpdir / "bad-provider-name-obj.config.json"
|
||||
c = _base_pp_cfg()
|
||||
c["work_types"]["research"]["providers"] = ["anthropic", {"provider": "nonexistent-provider"}]
|
||||
bad_provider_name_obj_path.write_text(json.dumps(c))
|
||||
try:
|
||||
m.load_config(bad_provider_name_obj_path)
|
||||
check("load_config: object-form providers entry naming an unknown provider raises ConfigError", False)
|
||||
except m.ConfigError as e:
|
||||
check("load_config: object-form providers entry naming an unknown provider raises ConfigError",
|
||||
"unknown provider" in str(e) and "nonexistent-provider" in str(e), str(e))
|
||||
|
||||
missing_provider_key_path = tmpdir / "missing-provider-key.config.json"
|
||||
c = _base_pp_cfg()
|
||||
c["work_types"]["research"]["providers"] = ["anthropic", {"effort": "high"}]
|
||||
missing_provider_key_path.write_text(json.dumps(c))
|
||||
try:
|
||||
m.load_config(missing_provider_key_path)
|
||||
check("load_config: object-form providers entry missing 'provider' key raises ConfigError", False)
|
||||
except m.ConfigError as e:
|
||||
check("load_config: object-form providers entry missing 'provider' key raises ConfigError",
|
||||
"provider" in str(e), str(e))
|
||||
|
||||
unknown_key_path = tmpdir / "unknown-provider-entry-key.config.json"
|
||||
c = _base_pp_cfg()
|
||||
c["work_types"]["research"]["providers"] = ["anthropic", {"provider": "minimax", "bogus_key": 1}]
|
||||
unknown_key_path.write_text(json.dumps(c))
|
||||
try:
|
||||
m.load_config(unknown_key_path)
|
||||
check("load_config: object-form providers entry with an unknown key raises ConfigError", False)
|
||||
except m.ConfigError as e:
|
||||
check("load_config: object-form providers entry with an unknown key raises ConfigError",
|
||||
"unknown key" in str(e) and "bogus_key" in str(e), str(e))
|
||||
|
||||
bad_entry_effort_path = tmpdir / "bad-provider-entry-effort.config.json"
|
||||
c = _base_pp_cfg()
|
||||
c["work_types"]["research"]["providers"] = ["anthropic", {"provider": "minimax", "effort": "ultra-mega"}]
|
||||
bad_entry_effort_path.write_text(json.dumps(c))
|
||||
try:
|
||||
m.load_config(bad_entry_effort_path)
|
||||
check("load_config: object-form providers entry with invalid effort level raises ConfigError", False)
|
||||
except m.ConfigError as e:
|
||||
check("load_config: object-form providers entry with invalid effort level raises ConfigError",
|
||||
"effort" in str(e) and "minimax" in str(e), str(e))
|
||||
|
||||
bad_entry_model_path = tmpdir / "bad-provider-entry-model.config.json"
|
||||
c = _base_pp_cfg()
|
||||
c["work_types"]["research"]["providers"] = ["anthropic", {"provider": "minimax", "model": ""}]
|
||||
bad_entry_model_path.write_text(json.dumps(c))
|
||||
try:
|
||||
m.load_config(bad_entry_model_path)
|
||||
check("load_config: object-form providers entry with empty model raises ConfigError", False)
|
||||
except m.ConfigError as e:
|
||||
check("load_config: object-form providers entry with empty model raises ConfigError",
|
||||
"model" in str(e) and "minimax" in str(e), str(e))
|
||||
|
||||
bad_entry_type_path = tmpdir / "bad-provider-entry-type.config.json"
|
||||
c = _base_pp_cfg()
|
||||
c["work_types"]["research"]["providers"] = ["anthropic", 5]
|
||||
bad_entry_type_path.write_text(json.dumps(c))
|
||||
try:
|
||||
m.load_config(bad_entry_type_path)
|
||||
check("load_config: providers entry that is neither string nor object raises ConfigError", False)
|
||||
except m.ConfigError as e:
|
||||
check("load_config: providers entry that is neither string nor object raises ConfigError",
|
||||
"string or object" in str(e), str(e))
|
||||
|
||||
good_mixed_path = tmpdir / "good-mixed-providers.config.json"
|
||||
c = _base_pp_cfg()
|
||||
c["work_types"]["research"]["providers"] = ["anthropic", {"provider": "minimax", "model": "m-model", "effort": "high"}]
|
||||
good_mixed_path.write_text(json.dumps(c))
|
||||
loaded_mixed = m.load_config(good_mixed_path)
|
||||
check("load_config: well-formed mixed (string + object) providers list loads without error",
|
||||
loaded_mixed["work_types"]["research"]["providers"][0] == "anthropic"
|
||||
and loaded_mixed["work_types"]["research"]["providers"][1] == {"provider": "minimax", "model": "m-model", "effort": "high"},
|
||||
loaded_mixed["work_types"]["research"]["providers"])
|
||||
|
||||
# The shipped example config itself demonstrates the mixed form -- confirm it validates
|
||||
# and that select_provider() still walks the list in order via provider_entry_name().
|
||||
example_cfg = m.load_config(m.REPO_SELF / "data" / "idle-draft" / "config.example.json")
|
||||
example_research_providers = example_cfg["work_types"]["research"]["providers"]
|
||||
check("config.example.json: loads without error (mixed providers list included)", True, "")
|
||||
check("config.example.json: research providers mixes a plain string (anthropic) and an object (minimax)",
|
||||
example_research_providers[0] == "anthropic"
|
||||
and isinstance(example_research_providers[1], dict)
|
||||
and example_research_providers[1]["provider"] == "minimax"
|
||||
and example_research_providers[1]["effort"] == "medium",
|
||||
example_research_providers)
|
||||
check("provider_entry_name: walks config.example.json's mixed research providers list in order",
|
||||
[m.provider_entry_name(e) for e in example_research_providers] == ["anthropic", "minimax"],
|
||||
example_research_providers)
|
||||
|
||||
# --- Citation validation ---
|
||||
real_path = cred_repo / "exists.txt"
|
||||
real_path.write_text("x")
|
||||
|
||||
Reference in New Issue
Block a user