idle-draft: per-work-type and per-provider model/effort settings
--effort/--model on child argv; provider entries accept object form
{provider, model, effort}; precedence entry > work-type > profile.
Effort levels validated against the CLI's enumerated set.
Claude-Session: https://claude.ai/code/session_01YQDoWNM7XPPii28khFWoMc
This commit is contained in:
@@ -7,10 +7,11 @@
|
|||||||
"five_hour_ceiling": 50, "min_idle": 5 }
|
"five_hour_ceiling": 50, "min_idle": 5 }
|
||||||
},
|
},
|
||||||
"work_types": {
|
"work_types": {
|
||||||
"review": { "providers": ["anthropic"] },
|
"review": { "providers": ["anthropic"], "effort": "high" },
|
||||||
"draft": { "providers": ["anthropic"] },
|
"draft": { "providers": ["anthropic"], "effort": "high" },
|
||||||
"research": { "providers": ["anthropic", "minimax"], "allowed_tools": ["WebSearch", "WebFetch"] },
|
"research": { "providers": ["anthropic", { "provider": "minimax", "effort": "medium" }],
|
||||||
"topic_ideas": { "providers": ["anthropic", "minimax"], "allowed_tools": ["WebSearch", "WebFetch"] }
|
"allowed_tools": ["WebSearch", "WebFetch"], "effort": "medium" },
|
||||||
|
"topic_ideas": { "providers": ["anthropic", "minimax"], "allowed_tools": ["WebSearch", "WebFetch"], "effort": "low" }
|
||||||
},
|
},
|
||||||
"dossiers": ["ai", "ai-technical", "devops-2020", "devops-2020-technical"],
|
"dossiers": ["ai", "ai-technical", "devops-2020", "devops-2020-technical"],
|
||||||
"review_score_threshold": 11,
|
"review_score_threshold": 11,
|
||||||
|
|||||||
@@ -62,6 +62,17 @@ ALLOWED_ITEM_KEYS = {
|
|||||||
}
|
}
|
||||||
ALLOWED_ATTEMPT_WORK_TYPES = {"research", "draft", "review"}
|
ALLOWED_ATTEMPT_WORK_TYPES = {"research", "draft", "review"}
|
||||||
|
|
||||||
|
# Effort levels accepted by `claude --effort <level>`. Derived empirically on
|
||||||
|
# 2026-08-02 by running `claude --effort obviously-bogus-level -p ""` (no API
|
||||||
|
# call -- an invalid flag value is rejected before any network activity) and
|
||||||
|
# reading the CLI's own error message, which enumerated the valid set
|
||||||
|
# verbatim: "Warning: Unknown --effort value 'obviously-bogus-level' —
|
||||||
|
# ignoring it and using the default effort. Valid values: low, medium, high,
|
||||||
|
# xhigh, max." Cross-checked against `claude --help`'s `--effort <level>`
|
||||||
|
# description, which lists the same five values. If a future CLI version
|
||||||
|
# changes this set, re-run the same probe and update this constant.
|
||||||
|
CLAUDE_EFFORT_LEVELS = frozenset({"low", "medium", "high", "xhigh", "max"})
|
||||||
|
|
||||||
DEFAULT_MAX_TURNS = 25
|
DEFAULT_MAX_TURNS = 25
|
||||||
DEFAULT_TASK_TIMEOUT_SECONDS = 1800
|
DEFAULT_TASK_TIMEOUT_SECONDS = 1800
|
||||||
DEFAULT_MAX_ATTEMPTS = 2
|
DEFAULT_MAX_ATTEMPTS = 2
|
||||||
@@ -224,13 +235,77 @@ def load_config(path: Path) -> dict:
|
|||||||
for key in ("profile", "threshold_pct", "five_hour_ceiling", "min_idle"):
|
for key in ("profile", "threshold_pct", "five_hour_ceiling", "min_idle"):
|
||||||
if key not in pcfg:
|
if key not in pcfg:
|
||||||
raise ConfigError(f"provider '{pname}' missing key '{key}'")
|
raise ConfigError(f"provider '{pname}' missing key '{key}'")
|
||||||
|
known_providers = set(data["providers"].keys())
|
||||||
for wtype, wcfg in data["work_types"].items():
|
for wtype, wcfg in data["work_types"].items():
|
||||||
if "providers" not in wcfg or not isinstance(wcfg["providers"], list):
|
if "providers" not in wcfg or not isinstance(wcfg["providers"], list):
|
||||||
raise ConfigError(f"work_type '{wtype}' missing 'providers' list")
|
raise ConfigError(f"work_type '{wtype}' missing 'providers' list")
|
||||||
|
for entry in wcfg["providers"]:
|
||||||
|
# A providers-list entry is either the legacy bare provider-name
|
||||||
|
# string, or an object carrying a per-provider model/effort
|
||||||
|
# override: {"provider": "<name>", "model": "...", "effort": "..."}.
|
||||||
|
# Both forms may appear in the same list (list order still encodes
|
||||||
|
# preference for select_provider()).
|
||||||
|
if isinstance(entry, str):
|
||||||
|
pname_ref = entry
|
||||||
|
if pname_ref not in known_providers:
|
||||||
|
raise ConfigError(
|
||||||
|
f"work_type '{wtype}' providers entry names unknown provider '{pname_ref}'"
|
||||||
|
)
|
||||||
|
elif isinstance(entry, dict):
|
||||||
|
unknown_keys = set(entry.keys()) - {"provider", "model", "effort"}
|
||||||
|
if unknown_keys:
|
||||||
|
raise ConfigError(
|
||||||
|
f"work_type '{wtype}' providers entry has unknown key(s): {sorted(unknown_keys)}"
|
||||||
|
)
|
||||||
|
pname_ref = entry.get("provider")
|
||||||
|
if not isinstance(pname_ref, str) or not pname_ref:
|
||||||
|
raise ConfigError(
|
||||||
|
f"work_type '{wtype}' providers entry (object form) missing non-empty 'provider' key"
|
||||||
|
)
|
||||||
|
if pname_ref not in known_providers:
|
||||||
|
raise ConfigError(
|
||||||
|
f"work_type '{wtype}' providers entry names unknown provider '{pname_ref}'"
|
||||||
|
)
|
||||||
|
if "model" in entry:
|
||||||
|
entry_model = entry["model"]
|
||||||
|
if not isinstance(entry_model, str) or not entry_model:
|
||||||
|
raise ConfigError(
|
||||||
|
f"work_type '{wtype}' providers entry for '{pname_ref}' 'model' must be "
|
||||||
|
f"a non-empty string"
|
||||||
|
)
|
||||||
|
if "effort" in entry:
|
||||||
|
entry_effort = entry["effort"]
|
||||||
|
if not isinstance(entry_effort, str) or not entry_effort:
|
||||||
|
raise ConfigError(
|
||||||
|
f"work_type '{wtype}' providers entry for '{pname_ref}' 'effort' must be "
|
||||||
|
f"a non-empty string"
|
||||||
|
)
|
||||||
|
if entry_effort not in CLAUDE_EFFORT_LEVELS:
|
||||||
|
raise ConfigError(
|
||||||
|
f"work_type '{wtype}' providers entry for '{pname_ref}' 'effort' must be one "
|
||||||
|
f"of {sorted(CLAUDE_EFFORT_LEVELS)}, got {entry_effort!r}"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise ConfigError(
|
||||||
|
f"work_type '{wtype}' providers entry must be a string or object, "
|
||||||
|
f"got {type(entry).__name__}"
|
||||||
|
)
|
||||||
if "allowed_tools" in wcfg:
|
if "allowed_tools" in wcfg:
|
||||||
at = wcfg["allowed_tools"]
|
at = wcfg["allowed_tools"]
|
||||||
if not isinstance(at, list) or not all(isinstance(x, str) for x in at):
|
if not isinstance(at, list) or not all(isinstance(x, str) for x in at):
|
||||||
raise ConfigError(f"work_type '{wtype}' 'allowed_tools' must be a list of strings")
|
raise ConfigError(f"work_type '{wtype}' 'allowed_tools' must be a list of strings")
|
||||||
|
if "effort" in wcfg:
|
||||||
|
effort = wcfg["effort"]
|
||||||
|
if not isinstance(effort, str) or not effort:
|
||||||
|
raise ConfigError(f"work_type '{wtype}' 'effort' must be a non-empty string")
|
||||||
|
if effort not in CLAUDE_EFFORT_LEVELS:
|
||||||
|
raise ConfigError(
|
||||||
|
f"work_type '{wtype}' 'effort' must be one of {sorted(CLAUDE_EFFORT_LEVELS)}, got {effort!r}"
|
||||||
|
)
|
||||||
|
if "model" in wcfg:
|
||||||
|
model = wcfg["model"]
|
||||||
|
if not isinstance(model, str) or not model:
|
||||||
|
raise ConfigError(f"work_type '{wtype}' 'model' must be a non-empty string")
|
||||||
return data
|
return data
|
||||||
|
|
||||||
|
|
||||||
@@ -656,9 +731,19 @@ def build_ready_queue(repo: Path, config: dict, state: dict, in_flight: set) ->
|
|||||||
return candidates
|
return candidates
|
||||||
|
|
||||||
|
|
||||||
|
def provider_entry_name(entry) -> str:
|
||||||
|
"""A providers-list entry is either a plain provider-name string (the
|
||||||
|
legacy form) or an object {"provider": name, "model": ..., "effort": ...}.
|
||||||
|
This extracts just the name -- the only field select_provider() itself
|
||||||
|
needs; the model/effort overrides are resolved later, once a provider has
|
||||||
|
actually been selected, by resolve_work_type_model/_effort below."""
|
||||||
|
return entry if isinstance(entry, str) else entry["provider"]
|
||||||
|
|
||||||
|
|
||||||
def select_provider(work_type: str, config: dict, gates: dict, credential_ok: dict) -> tuple[str | None, str]:
|
def select_provider(work_type: str, config: dict, gates: dict, credential_ok: dict) -> tuple[str | None, str]:
|
||||||
providers = config["work_types"].get(work_type, {}).get("providers", [])
|
providers = config["work_types"].get(work_type, {}).get("providers", [])
|
||||||
for pname in providers:
|
for entry in providers:
|
||||||
|
pname = provider_entry_name(entry)
|
||||||
gate = gates.get(pname, {})
|
gate = gates.get(pname, {})
|
||||||
if gate.get("eligible") and credential_ok.get(pname, (False, ""))[0]:
|
if gate.get("eligible") and credential_ok.get(pname, (False, ""))[0]:
|
||||||
return pname, "eligible"
|
return pname, "eligible"
|
||||||
@@ -758,11 +843,20 @@ def validate_output(work_type: str, text: str) -> tuple[bool, str]:
|
|||||||
|
|
||||||
|
|
||||||
def build_claude_argv(
|
def build_claude_argv(
|
||||||
model_id: str | None, max_turns: int, add_dirs: list[str], allowed_tools: list[str] | None = None
|
model_id: str | None,
|
||||||
|
max_turns: int,
|
||||||
|
add_dirs: list[str],
|
||||||
|
allowed_tools: list[str] | None = None,
|
||||||
|
effort: str | None = None,
|
||||||
) -> list[str]:
|
) -> list[str]:
|
||||||
|
"""`model_id` is the already-resolved effective model (see
|
||||||
|
resolve_effective_model: work-type "model" beats the profile's MODEL_ID) --
|
||||||
|
this function emits at most one --model flag, never two."""
|
||||||
argv = ["claude", "-p", "--max-turns", str(max_turns)]
|
argv = ["claude", "-p", "--max-turns", str(max_turns)]
|
||||||
if model_id:
|
if model_id:
|
||||||
argv += ["--model", model_id]
|
argv += ["--model", model_id]
|
||||||
|
if effort:
|
||||||
|
argv += ["--effort", effort]
|
||||||
for d in add_dirs:
|
for d in add_dirs:
|
||||||
argv += ["--add-dir", d]
|
argv += ["--add-dir", d]
|
||||||
if allowed_tools:
|
if allowed_tools:
|
||||||
@@ -821,16 +915,70 @@ def resolve_allowed_tools(config: dict, work_type: str) -> list[str]:
|
|||||||
return list(config["work_types"].get(work_type, {}).get("allowed_tools") or [])
|
return list(config["work_types"].get(work_type, {}).get("allowed_tools") or [])
|
||||||
|
|
||||||
|
|
||||||
|
def find_provider_entry(config: dict, work_type: str, provider_name: str):
|
||||||
|
"""The raw providers-list entry (str or dict) for provider_name within
|
||||||
|
work_type's providers list, or None if not present. Defensive lookup --
|
||||||
|
callers only ever pass a provider name that select_provider() just chose
|
||||||
|
from this same list, so a miss shouldn't happen, but resolve_work_type_*
|
||||||
|
below treat it as "no per-provider override" rather than raising."""
|
||||||
|
for entry in config["work_types"].get(work_type, {}).get("providers", []):
|
||||||
|
if provider_entry_name(entry) == provider_name:
|
||||||
|
return entry
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_work_type_effort(config: dict, work_type: str, provider_name: str | None = None) -> str | None:
|
||||||
|
"""Effort level for (work_type, provider_name). Precedence:
|
||||||
|
per-provider entry "effort" > work-type-level "effort" > None (CLI
|
||||||
|
default). `provider_name` is optional -- omit it to get just the
|
||||||
|
work-type-level value (e.g. before a provider has been selected). Both
|
||||||
|
layers are already validated (non-empty string, member of
|
||||||
|
CLAUDE_EFFORT_LEVELS) at config-load time -- this is a plain lookup."""
|
||||||
|
wcfg = config["work_types"].get(work_type, {})
|
||||||
|
if provider_name is not None:
|
||||||
|
entry = find_provider_entry(config, work_type, provider_name)
|
||||||
|
per_provider = entry.get("effort") if isinstance(entry, dict) else None
|
||||||
|
if per_provider:
|
||||||
|
return per_provider
|
||||||
|
return wcfg.get("effort")
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_work_type_model(config: dict, work_type: str, provider_name: str | None = None) -> str | None:
|
||||||
|
"""Model override for (work_type, provider_name). Precedence:
|
||||||
|
per-provider entry "model" > work-type-level "model" > None (falls back
|
||||||
|
to the profile's MODEL_ID via resolve_effective_model). `provider_name`
|
||||||
|
is optional -- omit it to get just the work-type-level value. Both
|
||||||
|
layers are already type-checked (non-empty string) at config-load time --
|
||||||
|
this is a plain lookup."""
|
||||||
|
wcfg = config["work_types"].get(work_type, {})
|
||||||
|
if provider_name is not None:
|
||||||
|
entry = find_provider_entry(config, work_type, provider_name)
|
||||||
|
per_provider = entry.get("model") if isinstance(entry, dict) else None
|
||||||
|
if per_provider:
|
||||||
|
return per_provider
|
||||||
|
return wcfg.get("model")
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_effective_model(profile_model_id: str | None, work_type_model: str | None) -> str | None:
|
||||||
|
"""Precedence: a work-type "model" override always wins over the
|
||||||
|
profile's provider.env MODEL_ID. Pure function so the precedence rule is
|
||||||
|
independently testable without touching a profile dir."""
|
||||||
|
return work_type_model or profile_model_id
|
||||||
|
|
||||||
|
|
||||||
def dispatch_preview(repo: Path, config: dict, candidate: dict, max_turns: int) -> dict:
|
def dispatch_preview(repo: Path, config: dict, candidate: dict, max_turns: int) -> dict:
|
||||||
"""Compute everything a dryrun needs to print, without executing anything."""
|
"""Compute everything a dryrun needs to print, without executing anything."""
|
||||||
dossier = candidate["dossier"]
|
dossier = candidate["dossier"]
|
||||||
work_type = candidate["work_type"]
|
work_type = candidate["work_type"]
|
||||||
provider = candidate["provider"]
|
provider = candidate["provider"]
|
||||||
profile_dir = expand(config["providers"][provider]["profile"])
|
profile_dir = expand(config["providers"][provider]["profile"])
|
||||||
model_id = resolve_model_id(profile_dir)
|
model_id = resolve_effective_model(
|
||||||
|
resolve_model_id(profile_dir), resolve_work_type_model(config, work_type, provider)
|
||||||
|
)
|
||||||
|
effort = resolve_work_type_effort(config, work_type, provider)
|
||||||
add_dirs = resolve_add_dirs(config, work_type)
|
add_dirs = resolve_add_dirs(config, work_type)
|
||||||
allowed_tools = resolve_allowed_tools(config, work_type)
|
allowed_tools = resolve_allowed_tools(config, work_type)
|
||||||
argv = build_claude_argv(model_id, max_turns, add_dirs, allowed_tools)
|
argv = build_claude_argv(model_id, max_turns, add_dirs, allowed_tools, effort)
|
||||||
canonical = canonical_output_path(repo, candidate)
|
canonical = canonical_output_path(repo, candidate)
|
||||||
tmp_path = tmp_output_path(canonical)
|
tmp_path = tmp_output_path(canonical)
|
||||||
parsed_env = parse_provider_env(profile_dir / "provider.env")
|
parsed_env = parse_provider_env(profile_dir / "provider.env")
|
||||||
@@ -993,10 +1141,13 @@ def run_task(repo: Path, config: dict, candidate: dict, max_turns: int, task_tim
|
|||||||
prompt_text = render_prompt(work_type, mapping)
|
prompt_text = render_prompt(work_type, mapping)
|
||||||
|
|
||||||
profile_dir = expand(config["providers"][provider]["profile"])
|
profile_dir = expand(config["providers"][provider]["profile"])
|
||||||
model_id = resolve_model_id(profile_dir)
|
model_id = resolve_effective_model(
|
||||||
|
resolve_model_id(profile_dir), resolve_work_type_model(config, work_type, provider)
|
||||||
|
)
|
||||||
|
effort = resolve_work_type_effort(config, work_type, provider)
|
||||||
add_dirs = resolve_add_dirs(config, work_type)
|
add_dirs = resolve_add_dirs(config, work_type)
|
||||||
allowed_tools = resolve_allowed_tools(config, work_type)
|
allowed_tools = resolve_allowed_tools(config, work_type)
|
||||||
argv = build_claude_argv(model_id, max_turns, add_dirs, allowed_tools)
|
argv = build_claude_argv(model_id, max_turns, add_dirs, allowed_tools, effort)
|
||||||
env = build_child_env(profile_dir)
|
env = build_child_env(profile_dir)
|
||||||
|
|
||||||
timed_out = False
|
timed_out = False
|
||||||
|
|||||||
@@ -239,14 +239,21 @@ For the selected `(item_or_dossier, work_type, provider)`:
|
|||||||
invoked** (it has interactive pickers and terminal theming unsuitable for headless
|
invoked** (it has interactive pickers and terminal theming unsuitable for headless
|
||||||
cron use).
|
cron use).
|
||||||
4. Build the argv: `claude -p --max-turns <N>` (`N` = 25 unless overridden), plus
|
4. Build the argv: `claude -p --max-turns <N>` (`N` = 25 unless overridden), plus
|
||||||
`--model <id>` if `provider.env` sets `MODEL_ID`, plus `--add-dir <dir>` for each
|
`--model <id>` and `--effort <level>` resolved through the full precedence chain —
|
||||||
entry in `config["evidence_dirs"]` **only** for `research` and `topic_ideas` work
|
per-provider override on the selected provider's `providers`-list entry, then
|
||||||
types (the only ones that cite external evidence), plus `--allowedTools
|
work-type-level `model`/`effort`, then (for `model` only) `provider.env`'s
|
||||||
|
`MODEL_ID`; the flag is omitted at each layer where nothing resolves (see
|
||||||
|
"Per-work-type effort and model" and "Per-provider effort and model overrides"
|
||||||
|
below) — plus `--add-dir <dir>` for each entry in `config["evidence_dirs"]`
|
||||||
|
**only** for `research` and `topic_ideas`
|
||||||
|
work types (the only ones that cite external evidence), plus `--allowedTools
|
||||||
"<comma-joined list>"` if `config["work_types"][work_type]["allowed_tools"]` is
|
"<comma-joined list>"` if `config["work_types"][work_type]["allowed_tools"]` is
|
||||||
present and non-empty (see Config schema below). Headless `claude -p` runs cannot
|
present and non-empty (see Config schema below). Headless `claude -p` runs cannot
|
||||||
answer permission prompts, so any tool beyond the CLI's defaults must be granted
|
answer permission prompts, so any tool beyond the CLI's defaults must be granted
|
||||||
this way or the child run stalls/fails; when the key is absent or an empty list,
|
this way or the child run stalls/fails; when the `allowed_tools` key is absent or an
|
||||||
the flag is omitted entirely (unchanged pre-existing behaviour).
|
empty list, that flag is omitted entirely (unchanged pre-existing behaviour). Same for
|
||||||
|
`effort`/`model`: absent key -> no corresponding flag, argv unchanged from before
|
||||||
|
these keys existed.
|
||||||
5. Run the subprocess: `cwd=<repo>`, `env=<built env>`, prompt piped via **stdin**
|
5. Run the subprocess: `cwd=<repo>`, `env=<built env>`, prompt piped via **stdin**
|
||||||
(not as an argv element — avoids `ARG_MAX` on large rendered prompts, same lesson
|
(not as an argv element — avoids `ARG_MAX` on large rendered prompts, same lesson
|
||||||
`claude-profile` already applies to its system-prompt injection), timeout = per-task
|
`claude-profile` already applies to its system-prompt injection), timeout = per-task
|
||||||
@@ -401,6 +408,87 @@ config validation failure — exit **2** at config load, same as a missing requi
|
|||||||
before writing its Subjective stories section; the topic-ideas template requires one as
|
before writing its Subjective stories section; the topic-ideas template requires one as
|
||||||
a lighter-touch check); `draft` and `review` carry no `allowed_tools` key.
|
a lighter-touch check); `draft` and `review` carry no `allowed_tools` key.
|
||||||
|
|
||||||
|
### Per-work-type effort and model
|
||||||
|
|
||||||
|
Each entry in `work_types` may also carry two further optional keys, both consumed by
|
||||||
|
§11 step 4 when building the child's argv:
|
||||||
|
|
||||||
|
- **`effort`** (string): appends `--effort <value>` to the child argv. Validated at
|
||||||
|
config-load time against the fixed set of levels the `claude` CLI itself accepts:
|
||||||
|
`low`, `medium`, `high`, `xhigh`, `max`. This set was determined empirically on
|
||||||
|
2026-08-02 by running `claude --effort obviously-bogus-level -p ""` — an invalid
|
||||||
|
`--effort` value is rejected by the CLI's own argument parsing before any network
|
||||||
|
call is made, so the probe is safe to run offline — and reading the resulting error,
|
||||||
|
which enumerated the valid values verbatim (cross-checked against `claude --help`'s
|
||||||
|
`--effort <level>` description, which lists the same five). The set is hard-coded as
|
||||||
|
`CLAUDE_EFFORT_LEVELS` in `scripts/idle-draft`; if a future CLI version changes it,
|
||||||
|
re-run the same probe and update the constant (with a fresh date in the comment).
|
||||||
|
A value that is not a non-empty string, or not a member of this set, is a config
|
||||||
|
validation failure — exit **2**, same as a malformed `allowed_tools`.
|
||||||
|
- **`model`** (string): appends `--model <value>` to the child argv, **replacing** —
|
||||||
|
not adding to — the `--model <id>` that would otherwise come from the provider
|
||||||
|
profile's `provider.env` `MODEL_ID` (§11 step 4). Only type-checked at config-load
|
||||||
|
time (must be a non-empty string) — model identifiers are not enumerated, since new
|
||||||
|
ones ship independently of this tool and hard-coding a set would go stale. A
|
||||||
|
non-string or empty value is a config validation failure — exit **2**.
|
||||||
|
|
||||||
|
Both keys are optional and independent of each other and of `allowed_tools`. Absent on
|
||||||
|
a given work type -> the corresponding flag is omitted / the profile's model is used
|
||||||
|
unchanged, matching the tool's behaviour before these keys existed.
|
||||||
|
|
||||||
|
### Per-provider effort and model overrides (within a work type)
|
||||||
|
|
||||||
|
`model`/`effort` can be pinned even more narrowly than per-work-type: per
|
||||||
|
`(work_type, provider)` pair. Each entry in a `work_types[*].providers` list may be
|
||||||
|
**either**:
|
||||||
|
|
||||||
|
- a plain provider-name string (the original, unchanged form) — no per-provider
|
||||||
|
override, resolution falls through to the work-type-level `model`/`effort` (above);
|
||||||
|
or
|
||||||
|
- an object `{"provider": "<name>", "model": "<optional>", "effort": "<optional>"}` —
|
||||||
|
`model`/`effort` here apply only when *this* provider is the one actually selected
|
||||||
|
(§9) for a dispatch of this work type.
|
||||||
|
|
||||||
|
Both forms may be mixed freely within the same `providers` list; list order still
|
||||||
|
encodes provider preference exactly as before (§9) — the object form does not change
|
||||||
|
*which* provider is tried first, only what argv that provider gets if chosen.
|
||||||
|
`config.example.json` demonstrates this: `research`'s `providers` is
|
||||||
|
`["anthropic", {"provider": "minimax", "effort": "medium"}]` — `anthropic` stays a
|
||||||
|
plain string (falls through to `research`'s work-type-level `effort`), `minimax` pins
|
||||||
|
its own `effort` via the object form.
|
||||||
|
|
||||||
|
**Full precedence chain**, evaluated once a `(work_type, provider)` pair has been
|
||||||
|
selected (§9), for each of `model` and `effort` independently:
|
||||||
|
|
||||||
|
1. The per-provider override on the matching `providers`-list entry for that provider
|
||||||
|
(object form only; a plain-string entry has none).
|
||||||
|
2. The work-type-level `model`/`effort` (the keys directly under
|
||||||
|
`work_types[work_type]`, previous section).
|
||||||
|
3. Ultimate fallback: for `model`, the profile's `provider.env` `MODEL_ID` (or no
|
||||||
|
`--model` flag if unset); for `effort`, no `--effort` flag (the CLI's own default).
|
||||||
|
|
||||||
|
**Validation** (config-load time, exit **2** on any failure, same style as every other
|
||||||
|
config check):
|
||||||
|
|
||||||
|
- Every `providers`-list entry — string **or** object form — must name a provider that
|
||||||
|
exists as a key in the top-level `providers` map. An entry (of either form) naming an
|
||||||
|
unrecognised provider is rejected; this closes a gap that predates the object form —
|
||||||
|
a typo'd plain-string provider name was previously silently ineligible (never
|
||||||
|
selected, no error) rather than rejected at load time.
|
||||||
|
- An object-form entry must have a `"provider"` key whose value is a non-empty string.
|
||||||
|
- An object-form entry's keys are restricted to `provider`, `model`, `effort`; any other
|
||||||
|
key is rejected.
|
||||||
|
- An object-form entry's `model`, if present, must be a non-empty string (same rule as
|
||||||
|
the work-type-level `model`).
|
||||||
|
- An object-form entry's `effort`, if present, must be a non-empty string and a member
|
||||||
|
of `CLAUDE_EFFORT_LEVELS` (same rule as the work-type-level `effort`).
|
||||||
|
- A `providers`-list entry that is neither a string nor an object is rejected.
|
||||||
|
|
||||||
|
`--dryrun` prints the fully resolved argv (§11 step 4's `--model`/`--effort` included,
|
||||||
|
after the full precedence chain above), so a preview reveals exactly which provider,
|
||||||
|
model, and effort level a given work type would route to before anything is dispatched
|
||||||
|
for real.
|
||||||
|
|
||||||
## Failure classes
|
## Failure classes
|
||||||
|
|
||||||
| Class | Examples | `attempts` effect | Item outcome |
|
| Class | Examples | `attempts` effect | Item outcome |
|
||||||
@@ -438,6 +526,16 @@ a lighter-touch check); `draft` and `review` carry no `allowed_tools` key.
|
|||||||
| `--once` with no eligible work | Exits 0 immediately, no task dispatched |
|
| `--once` with no eligible work | Exits 0 immediately, no task dispatched |
|
||||||
| Config missing a required key | Exit 2 |
|
| Config missing a required key | Exit 2 |
|
||||||
| `work_types[*].allowed_tools` present but not a list of strings | Exit 2 |
|
| `work_types[*].allowed_tools` present but not a list of strings | Exit 2 |
|
||||||
|
| `work_types[*].effort` present but not one of `low`/`medium`/`high`/`xhigh`/`max` | Exit 2 |
|
||||||
|
| `work_types[*].effort` present but empty string / not a string | Exit 2 |
|
||||||
|
| `work_types[*].model` present but empty string / not a string | Exit 2 |
|
||||||
|
| `work_types[*].model` set alongside a profile whose `provider.env` also sets `MODEL_ID` | Work-type `model` wins (absent a per-provider override); argv carries exactly one `--model` flag |
|
||||||
|
| `work_types[*].providers[*]` (string or object form) names a provider not present in the top-level `providers` map | Exit 2 |
|
||||||
|
| `work_types[*].providers[*]` object form missing/empty `"provider"` key | Exit 2 |
|
||||||
|
| `work_types[*].providers[*]` object form has an unknown key (anything besides `provider`/`model`/`effort`) | Exit 2 |
|
||||||
|
| `work_types[*].providers[*]` object form `model`/`effort` invalid (same rules as the work-type-level keys) | Exit 2 |
|
||||||
|
| `work_types[*].providers[*]` entry is neither a string nor an object | Exit 2 |
|
||||||
|
| A provider's `providers`-list entry sets its own `model`/`effort`, and the work type also sets a work-type-level `model`/`effort` | Per-provider entry wins for that provider; a sibling provider in the same list with a plain-string entry still falls through to the work-type-level value |
|
||||||
| `agent-subscriptions` subprocess fails entirely | Exit 1, loud log line, no dispatch attempted |
|
| `agent-subscriptions` subprocess fails entirely | Exit 1, loud log line, no dispatch attempted |
|
||||||
|
|
||||||
## Examples
|
## Examples
|
||||||
|
|||||||
@@ -155,6 +155,44 @@ assert_contains "$output" "$TMPDIR/evidence" "dryrun argv includes configured ev
|
|||||||
assert_contains "$output" "--allowedTools" "dryrun argv for research includes --allowedTools (allowed_tools configured)"
|
assert_contains "$output" "--allowedTools" "dryrun argv for research includes --allowedTools (allowed_tools configured)"
|
||||||
assert_contains "$output" "WebSearch,WebFetch" "dryrun argv --allowedTools value is the comma-joined tool list"
|
assert_contains "$output" "WebSearch,WebFetch" "dryrun argv --allowedTools value is the comma-joined tool list"
|
||||||
|
|
||||||
|
echo "-- dryrun: work-type effort/model override profile model, appear in resolved argv --"
|
||||||
|
PROFILE_ANTHROPIC_MODEL="$TMPDIR/profiles/anthropic-with-model"
|
||||||
|
mkdir -p "$PROFILE_ANTHROPIC_MODEL"
|
||||||
|
cat > "$PROFILE_ANTHROPIC_MODEL/provider.env" <<'EOF'
|
||||||
|
MODEL_ID=profile-default-model
|
||||||
|
EOF
|
||||||
|
EFFORT_CONFIG="$TMPDIR/effort.config.json"
|
||||||
|
cat > "$EFFORT_CONFIG" <<EOF
|
||||||
|
{
|
||||||
|
"parallel": 2,
|
||||||
|
"providers": {
|
||||||
|
"anthropic": { "profile": "$PROFILE_ANTHROPIC_MODEL", "threshold_pct": 80, "five_hour_ceiling": 50, "min_idle": 5 },
|
||||||
|
"minimax": { "profile": "$PROFILE_MINIMAX", "threshold_pct": 80, "five_hour_ceiling": 50, "min_idle": 5 }
|
||||||
|
},
|
||||||
|
"work_types": {
|
||||||
|
"review": { "providers": ["anthropic"] },
|
||||||
|
"draft": { "providers": ["anthropic"] },
|
||||||
|
"research": { "providers": ["anthropic", "minimax"], "allowed_tools": ["WebSearch", "WebFetch"], "effort": "medium", "model": "worktype-override-model" },
|
||||||
|
"topic_ideas": { "providers": ["anthropic", "minimax"], "allowed_tools": ["WebSearch", "WebFetch"] }
|
||||||
|
},
|
||||||
|
"dossiers": ["ai"],
|
||||||
|
"review_score_threshold": 11,
|
||||||
|
"max_unreviewed_research_per_dossier": 3,
|
||||||
|
"max_open_topic_proposals": 6,
|
||||||
|
"evidence_dirs": ["$TMPDIR/evidence"]
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
output=$("$SCRIPT" --config "$EFFORT_CONFIG" --repo "$SOLO" --probe-json "$FIXTURE_PROBE" --dryrun --once 2>&1)
|
||||||
|
assert_contains "$output" "'--model', 'worktype-override-model'" "dryrun argv model uses work-type override, not profile MODEL_ID"
|
||||||
|
assert_not_contains "$output" "profile-default-model" "dryrun argv does not contain the profile's MODEL_ID once a work-type model overrides it"
|
||||||
|
assert_contains "$output" "'--effort', 'medium'" "dryrun argv includes --effort with the configured level"
|
||||||
|
model_flag_count=$(echo "$output" | grep -o -- "--model" | wc -l)
|
||||||
|
[[ "$model_flag_count" -eq 1 ]] && pass "resolved argv contains exactly one --model flag (no duplicate from profile)" || fail "resolved argv contains exactly one --model flag" "found $model_flag_count"
|
||||||
|
|
||||||
|
echo "-- dryrun: no effort/model configured -> flags omitted (unchanged behaviour) --"
|
||||||
|
output=$("$SCRIPT" --config "$FIXTURE_CONFIG" --repo "$SOLO" --probe-json "$FIXTURE_PROBE" --dryrun --once 2>&1)
|
||||||
|
assert_not_contains "$output" "--effort" "dryrun argv omits --effort when work_type has no effort key"
|
||||||
|
|
||||||
echo "-- dryrun: no mutation --"
|
echo "-- dryrun: no mutation --"
|
||||||
before_hash=$(cd "$FIXTURE_REPO" && git rev-parse HEAD)
|
before_hash=$(cd "$FIXTURE_REPO" && git rev-parse HEAD)
|
||||||
"$SCRIPT" --config "$FIXTURE_CONFIG" --repo "$FIXTURE_REPO" --probe-json "$FIXTURE_PROBE" --dryrun >/dev/null 2>&1
|
"$SCRIPT" --config "$FIXTURE_CONFIG" --repo "$FIXTURE_REPO" --probe-json "$FIXTURE_PROBE" --dryrun >/dev/null 2>&1
|
||||||
@@ -254,6 +292,132 @@ code=$?
|
|||||||
assert_exit_code "$code" 2 "allowed_tools not a list of strings exits 2"
|
assert_exit_code "$code" 2 "allowed_tools not a list of strings exits 2"
|
||||||
assert_contains "$output" "allowed_tools" "allowed_tools config error names the offending key"
|
assert_contains "$output" "allowed_tools" "allowed_tools config error names the offending key"
|
||||||
|
|
||||||
|
BADEFFORT="$TMPDIR/bad-effort.config.json"
|
||||||
|
cat > "$BADEFFORT" <<EOF
|
||||||
|
{
|
||||||
|
"parallel": 2,
|
||||||
|
"providers": {
|
||||||
|
"anthropic": { "profile": "$PROFILE_ANTHROPIC", "threshold_pct": 80, "five_hour_ceiling": 50, "min_idle": 5 }
|
||||||
|
},
|
||||||
|
"work_types": {
|
||||||
|
"review": { "providers": ["anthropic"] },
|
||||||
|
"draft": { "providers": ["anthropic"] },
|
||||||
|
"research": { "providers": ["anthropic"], "effort": "ultra-mega" },
|
||||||
|
"topic_ideas": { "providers": ["anthropic"] }
|
||||||
|
},
|
||||||
|
"dossiers": ["ai"],
|
||||||
|
"review_score_threshold": 11,
|
||||||
|
"max_unreviewed_research_per_dossier": 3,
|
||||||
|
"max_open_topic_proposals": 6,
|
||||||
|
"evidence_dirs": ["$TMPDIR/evidence"]
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
output=$("$SCRIPT" --config "$BADEFFORT" --repo "$FIXTURE_REPO" --probe-json "$FIXTURE_PROBE" --dryrun 2>&1)
|
||||||
|
code=$?
|
||||||
|
assert_exit_code "$code" 2 "invalid effort level exits 2"
|
||||||
|
assert_contains "$output" "effort" "invalid effort config error names the offending key"
|
||||||
|
|
||||||
|
BADMODEL="$TMPDIR/bad-model.config.json"
|
||||||
|
cat > "$BADMODEL" <<EOF
|
||||||
|
{
|
||||||
|
"parallel": 2,
|
||||||
|
"providers": {
|
||||||
|
"anthropic": { "profile": "$PROFILE_ANTHROPIC", "threshold_pct": 80, "five_hour_ceiling": 50, "min_idle": 5 }
|
||||||
|
},
|
||||||
|
"work_types": {
|
||||||
|
"review": { "providers": ["anthropic"] },
|
||||||
|
"draft": { "providers": ["anthropic"] },
|
||||||
|
"research": { "providers": ["anthropic"], "model": "" },
|
||||||
|
"topic_ideas": { "providers": ["anthropic"] }
|
||||||
|
},
|
||||||
|
"dossiers": ["ai"],
|
||||||
|
"review_score_threshold": 11,
|
||||||
|
"max_unreviewed_research_per_dossier": 3,
|
||||||
|
"max_open_topic_proposals": 6,
|
||||||
|
"evidence_dirs": ["$TMPDIR/evidence"]
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
output=$("$SCRIPT" --config "$BADMODEL" --repo "$FIXTURE_REPO" --probe-json "$FIXTURE_PROBE" --dryrun 2>&1)
|
||||||
|
code=$?
|
||||||
|
assert_exit_code "$code" 2 "empty model string exits 2"
|
||||||
|
assert_contains "$output" "model" "empty model config error names the offending key"
|
||||||
|
|
||||||
|
echo "-- config validation: per-provider effort/model overrides (providers-list entries) --"
|
||||||
|
BADPROVIDERNAME="$TMPDIR/bad-provider-name.config.json"
|
||||||
|
cat > "$BADPROVIDERNAME" <<EOF
|
||||||
|
{
|
||||||
|
"parallel": 2,
|
||||||
|
"providers": {
|
||||||
|
"anthropic": { "profile": "$PROFILE_ANTHROPIC", "threshold_pct": 80, "five_hour_ceiling": 50, "min_idle": 5 }
|
||||||
|
},
|
||||||
|
"work_types": {
|
||||||
|
"review": { "providers": ["anthropic"] },
|
||||||
|
"draft": { "providers": ["anthropic"] },
|
||||||
|
"research": { "providers": ["anthropic", {"provider": "nonexistent-provider", "effort": "high"}] },
|
||||||
|
"topic_ideas": { "providers": ["anthropic"] }
|
||||||
|
},
|
||||||
|
"dossiers": ["ai"],
|
||||||
|
"review_score_threshold": 11,
|
||||||
|
"max_unreviewed_research_per_dossier": 3,
|
||||||
|
"max_open_topic_proposals": 6,
|
||||||
|
"evidence_dirs": ["$TMPDIR/evidence"]
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
output=$("$SCRIPT" --config "$BADPROVIDERNAME" --repo "$FIXTURE_REPO" --probe-json "$FIXTURE_PROBE" --dryrun 2>&1)
|
||||||
|
code=$?
|
||||||
|
assert_exit_code "$code" 2 "providers-list entry naming an undefined provider exits 2"
|
||||||
|
assert_contains "$output" "unknown provider" "undefined provider config error names the problem"
|
||||||
|
|
||||||
|
BADPROVIDERKEY="$TMPDIR/bad-provider-key.config.json"
|
||||||
|
cat > "$BADPROVIDERKEY" <<EOF
|
||||||
|
{
|
||||||
|
"parallel": 2,
|
||||||
|
"providers": {
|
||||||
|
"anthropic": { "profile": "$PROFILE_ANTHROPIC", "threshold_pct": 80, "five_hour_ceiling": 50, "min_idle": 5 }
|
||||||
|
},
|
||||||
|
"work_types": {
|
||||||
|
"review": { "providers": ["anthropic"] },
|
||||||
|
"draft": { "providers": ["anthropic"] },
|
||||||
|
"research": { "providers": [{"provider": "anthropic", "priority": 1}] },
|
||||||
|
"topic_ideas": { "providers": ["anthropic"] }
|
||||||
|
},
|
||||||
|
"dossiers": ["ai"],
|
||||||
|
"review_score_threshold": 11,
|
||||||
|
"max_unreviewed_research_per_dossier": 3,
|
||||||
|
"max_open_topic_proposals": 6,
|
||||||
|
"evidence_dirs": ["$TMPDIR/evidence"]
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
output=$("$SCRIPT" --config "$BADPROVIDERKEY" --repo "$FIXTURE_REPO" --probe-json "$FIXTURE_PROBE" --dryrun 2>&1)
|
||||||
|
code=$?
|
||||||
|
assert_exit_code "$code" 2 "providers-list object entry with an unknown key exits 2"
|
||||||
|
assert_contains "$output" "unknown key" "unknown provider-entry key config error names the problem"
|
||||||
|
|
||||||
|
BADPROVIDEREFFORT="$TMPDIR/bad-provider-effort.config.json"
|
||||||
|
cat > "$BADPROVIDEREFFORT" <<EOF
|
||||||
|
{
|
||||||
|
"parallel": 2,
|
||||||
|
"providers": {
|
||||||
|
"anthropic": { "profile": "$PROFILE_ANTHROPIC", "threshold_pct": 80, "five_hour_ceiling": 50, "min_idle": 5 }
|
||||||
|
},
|
||||||
|
"work_types": {
|
||||||
|
"review": { "providers": ["anthropic"] },
|
||||||
|
"draft": { "providers": ["anthropic"] },
|
||||||
|
"research": { "providers": [{"provider": "anthropic", "effort": "super-duper"}] },
|
||||||
|
"topic_ideas": { "providers": ["anthropic"] }
|
||||||
|
},
|
||||||
|
"dossiers": ["ai"],
|
||||||
|
"review_score_threshold": 11,
|
||||||
|
"max_unreviewed_research_per_dossier": 3,
|
||||||
|
"max_open_topic_proposals": 6,
|
||||||
|
"evidence_dirs": ["$TMPDIR/evidence"]
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
output=$("$SCRIPT" --config "$BADPROVIDEREFFORT" --repo "$FIXTURE_REPO" --probe-json "$FIXTURE_PROBE" --dryrun 2>&1)
|
||||||
|
code=$?
|
||||||
|
assert_exit_code "$code" 2 "providers-list object entry with an invalid effort level exits 2"
|
||||||
|
assert_contains "$output" "effort" "invalid provider-entry effort config error names the problem"
|
||||||
|
|
||||||
echo "-- mark subcommand round-trip --"
|
echo "-- mark subcommand round-trip --"
|
||||||
MARKREPO="$TMPDIR/mark-repo"
|
MARKREPO="$TMPDIR/mark-repo"
|
||||||
mkdir -p "$MARKREPO/ai"
|
mkdir -p "$MARKREPO/ai"
|
||||||
@@ -519,6 +683,334 @@ check("load_config: well-formed allowed_tools (list of strings) loads without er
|
|||||||
good_allowed_cfg["work_types"]["research"]["allowed_tools"] == ["WebSearch", "WebFetch"],
|
good_allowed_cfg["work_types"]["research"]["allowed_tools"] == ["WebSearch", "WebFetch"],
|
||||||
good_allowed_cfg["work_types"]["research"])
|
good_allowed_cfg["work_types"]["research"])
|
||||||
|
|
||||||
|
# --- per-work-type effort / model: argv construction ---
|
||||||
|
argv_effort = m.build_claude_argv("model-x", 25, [], None, "high")
|
||||||
|
check("build_claude_argv: appends --effort when given",
|
||||||
|
"--effort" in argv_effort and argv_effort[argv_effort.index("--effort") + 1] == "high", argv_effort)
|
||||||
|
|
||||||
|
argv_no_effort = m.build_claude_argv("model-x", 25, [], None, None)
|
||||||
|
check("build_claude_argv: omits --effort when not given", "--effort" not in argv_no_effort, argv_no_effort)
|
||||||
|
|
||||||
|
argv_model_and_effort = m.build_claude_argv("worktype-model", 25, [], None, "xhigh")
|
||||||
|
check("build_claude_argv: --model reflects whatever single value the caller resolved "
|
||||||
|
"(precedence already applied upstream) -- exactly one --model flag",
|
||||||
|
argv_model_and_effort.count("--model") == 1
|
||||||
|
and argv_model_and_effort[argv_model_and_effort.index("--model") + 1] == "worktype-model",
|
||||||
|
argv_model_and_effort)
|
||||||
|
|
||||||
|
argv_neither = m.build_claude_argv(None, 25, [], None, None)
|
||||||
|
check("build_claude_argv: omits both --model and --effort when neither is given",
|
||||||
|
"--model" not in argv_neither and "--effort" not in argv_neither, argv_neither)
|
||||||
|
|
||||||
|
# --- per-work-type effort / model: precedence resolution ---
|
||||||
|
check("resolve_effective_model: work-type model overrides profile MODEL_ID",
|
||||||
|
m.resolve_effective_model("profile-model", "worktype-model") == "worktype-model", "")
|
||||||
|
check("resolve_effective_model: falls back to profile MODEL_ID when no work-type model set",
|
||||||
|
m.resolve_effective_model("profile-model", None) == "profile-model", "")
|
||||||
|
check("resolve_effective_model: both absent -> None",
|
||||||
|
m.resolve_effective_model(None, None) is None, "")
|
||||||
|
|
||||||
|
cfg_em = {"work_types": {"research": {"providers": ["anthropic"], "effort": "medium", "model": "wt-model"},
|
||||||
|
"draft": {"providers": ["anthropic"]}}}
|
||||||
|
check("resolve_work_type_effort: returns configured value", m.resolve_work_type_effort(cfg_em, "research") == "medium", "")
|
||||||
|
check("resolve_work_type_effort: returns None when absent", m.resolve_work_type_effort(cfg_em, "draft") is None, "")
|
||||||
|
check("resolve_work_type_model: returns configured value", m.resolve_work_type_model(cfg_em, "research") == "wt-model", "")
|
||||||
|
check("resolve_work_type_model: returns None when absent", m.resolve_work_type_model(cfg_em, "draft") is None, "")
|
||||||
|
|
||||||
|
# --- per-work-type effort / model: config validation ---
|
||||||
|
bad_effort_cfg_path = tmpdir / "bad-effort-direct.config.json"
|
||||||
|
bad_effort_cfg = json.loads((tmpdir / "idle-draft.config.json").read_text())
|
||||||
|
bad_effort_cfg["work_types"]["research"]["effort"] = "ultra-mega"
|
||||||
|
bad_effort_cfg_path.write_text(json.dumps(bad_effort_cfg))
|
||||||
|
try:
|
||||||
|
m.load_config(bad_effort_cfg_path)
|
||||||
|
check("load_config: effort value outside CLAUDE_EFFORT_LEVELS raises ConfigError", False)
|
||||||
|
except m.ConfigError as e:
|
||||||
|
check("load_config: effort value outside CLAUDE_EFFORT_LEVELS raises ConfigError", "effort" in str(e), str(e))
|
||||||
|
|
||||||
|
bad_effort_type_path = tmpdir / "bad-effort-type.config.json"
|
||||||
|
bad_effort_type_cfg = json.loads((tmpdir / "idle-draft.config.json").read_text())
|
||||||
|
bad_effort_type_cfg["work_types"]["research"]["effort"] = 5
|
||||||
|
bad_effort_type_path.write_text(json.dumps(bad_effort_type_cfg))
|
||||||
|
try:
|
||||||
|
m.load_config(bad_effort_type_path)
|
||||||
|
check("load_config: non-string effort raises ConfigError", False)
|
||||||
|
except m.ConfigError as e:
|
||||||
|
check("load_config: non-string effort raises ConfigError", "effort" in str(e), str(e))
|
||||||
|
|
||||||
|
bad_model_path = tmpdir / "bad-model-direct.config.json"
|
||||||
|
bad_model_cfg = json.loads((tmpdir / "idle-draft.config.json").read_text())
|
||||||
|
bad_model_cfg["work_types"]["research"]["model"] = ""
|
||||||
|
bad_model_path.write_text(json.dumps(bad_model_cfg))
|
||||||
|
try:
|
||||||
|
m.load_config(bad_model_path)
|
||||||
|
check("load_config: empty-string model raises ConfigError", False)
|
||||||
|
except m.ConfigError as e:
|
||||||
|
check("load_config: empty-string model raises ConfigError", "model" in str(e), str(e))
|
||||||
|
|
||||||
|
bad_model_type_path = tmpdir / "bad-model-type.config.json"
|
||||||
|
bad_model_type_cfg = json.loads((tmpdir / "idle-draft.config.json").read_text())
|
||||||
|
bad_model_type_cfg["work_types"]["research"]["model"] = 42
|
||||||
|
bad_model_type_path.write_text(json.dumps(bad_model_type_cfg))
|
||||||
|
try:
|
||||||
|
m.load_config(bad_model_type_path)
|
||||||
|
check("load_config: non-string model raises ConfigError", False)
|
||||||
|
except m.ConfigError as e:
|
||||||
|
check("load_config: non-string model raises ConfigError", "model" in str(e), str(e))
|
||||||
|
|
||||||
|
good_effort_model_cfg = json.loads((tmpdir / "idle-draft.config.json").read_text())
|
||||||
|
good_effort_model_cfg["work_types"]["research"]["effort"] = "high"
|
||||||
|
good_effort_model_cfg["work_types"]["research"]["model"] = "claude-example-model"
|
||||||
|
good_effort_model_path = tmpdir / "good-effort-model.config.json"
|
||||||
|
good_effort_model_path.write_text(json.dumps(good_effort_model_cfg))
|
||||||
|
loaded_good = m.load_config(good_effort_model_path)
|
||||||
|
check("load_config: well-formed effort + model (in the valid set) loads without error",
|
||||||
|
loaded_good["work_types"]["research"]["effort"] == "high"
|
||||||
|
and loaded_good["work_types"]["research"]["model"] == "claude-example-model",
|
||||||
|
loaded_good["work_types"]["research"])
|
||||||
|
|
||||||
|
for level in sorted(m.CLAUDE_EFFORT_LEVELS):
|
||||||
|
per_level_cfg = json.loads((tmpdir / "idle-draft.config.json").read_text())
|
||||||
|
per_level_cfg["work_types"]["research"]["effort"] = level
|
||||||
|
per_level_path = tmpdir / f"effort-{level}.config.json"
|
||||||
|
per_level_path.write_text(json.dumps(per_level_cfg))
|
||||||
|
loaded = m.load_config(per_level_path)
|
||||||
|
check(f"load_config: effort level {level!r} (member of CLAUDE_EFFORT_LEVELS) loads without error",
|
||||||
|
loaded["work_types"]["research"]["effort"] == level, loaded["work_types"]["research"])
|
||||||
|
|
||||||
|
# --- per-provider effort / model overrides (within a work type) ---
|
||||||
|
|
||||||
|
check("provider_entry_name: plain string entry returns itself",
|
||||||
|
m.provider_entry_name("anthropic") == "anthropic", "")
|
||||||
|
check("provider_entry_name: object entry returns its 'provider' value",
|
||||||
|
m.provider_entry_name({"provider": "minimax", "effort": "high"}) == "minimax", "")
|
||||||
|
|
||||||
|
cfg_pp = {
|
||||||
|
"providers": {"anthropic": {}, "minimax": {}},
|
||||||
|
"work_types": {
|
||||||
|
"research": {
|
||||||
|
"providers": [
|
||||||
|
"anthropic",
|
||||||
|
{"provider": "minimax", "model": "minimax-model", "effort": "high"},
|
||||||
|
],
|
||||||
|
"effort": "medium",
|
||||||
|
"model": "worktype-model",
|
||||||
|
},
|
||||||
|
"draft": {"providers": ["anthropic"], "effort": "low"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
entry_a = m.find_provider_entry(cfg_pp, "research", "anthropic")
|
||||||
|
check("find_provider_entry: returns the plain-string entry for a string-form provider",
|
||||||
|
entry_a == "anthropic", entry_a)
|
||||||
|
entry_m = m.find_provider_entry(cfg_pp, "research", "minimax")
|
||||||
|
check("find_provider_entry: returns the dict entry for an object-form provider",
|
||||||
|
isinstance(entry_m, dict) and entry_m["provider"] == "minimax", entry_m)
|
||||||
|
entry_missing = m.find_provider_entry(cfg_pp, "research", "nonexistent")
|
||||||
|
check("find_provider_entry: returns None when the provider isn't in this work_type's list",
|
||||||
|
entry_missing is None, entry_missing)
|
||||||
|
|
||||||
|
# Precedence: per-provider entry > work-type-level > (caller supplies profile fallback separately)
|
||||||
|
check("resolve_work_type_effort: plain-string provider entry falls through to work-type-level effort",
|
||||||
|
m.resolve_work_type_effort(cfg_pp, "research", "anthropic") == "medium", "")
|
||||||
|
check("resolve_work_type_model: plain-string provider entry falls through to work-type-level model",
|
||||||
|
m.resolve_work_type_model(cfg_pp, "research", "anthropic") == "worktype-model", "")
|
||||||
|
check("resolve_work_type_effort: object-form provider entry's own effort wins over work-type-level",
|
||||||
|
m.resolve_work_type_effort(cfg_pp, "research", "minimax") == "high", "")
|
||||||
|
check("resolve_work_type_model: object-form provider entry's own model wins over work-type-level",
|
||||||
|
m.resolve_work_type_model(cfg_pp, "research", "minimax") == "minimax-model", "")
|
||||||
|
check("resolve_work_type_effort: provider_name omitted -> work-type-level value only (back-compat)",
|
||||||
|
m.resolve_work_type_effort(cfg_pp, "research") == "medium", "")
|
||||||
|
check("resolve_work_type_model: provider_name omitted -> work-type-level value only (back-compat)",
|
||||||
|
m.resolve_work_type_model(cfg_pp, "research") == "worktype-model", "")
|
||||||
|
check("resolve_work_type_effort: work type with no per-work-type or per-provider effort -> None",
|
||||||
|
m.resolve_work_type_effort(cfg_pp, "draft", "anthropic") == "low", "")
|
||||||
|
|
||||||
|
# Full chain including the ultimate profile-MODEL_ID fallback (resolve_effective_model),
|
||||||
|
# for a provider with NO per-provider or work-type-level model at all.
|
||||||
|
cfg_pp_no_wt_model = {
|
||||||
|
"providers": {"anthropic": {}},
|
||||||
|
"work_types": {"review": {"providers": ["anthropic"], "effort": "high"}},
|
||||||
|
}
|
||||||
|
wt_model = m.resolve_work_type_model(cfg_pp_no_wt_model, "review", "anthropic")
|
||||||
|
check("resolve_work_type_model: no per-provider, no work-type-level -> None (falls to profile next)",
|
||||||
|
wt_model is None, wt_model)
|
||||||
|
check("resolve_effective_model: full chain falls back to profile MODEL_ID when neither override is set",
|
||||||
|
m.resolve_effective_model("profile-model-id", wt_model) == "profile-model-id", "")
|
||||||
|
|
||||||
|
# --- per-provider effort / model overrides: dispatch_preview reflects the resolution ---
|
||||||
|
# dispatch_preview() takes a candidate with "provider" already selected -- this exercises
|
||||||
|
# the exact same resolution path run_task() uses, without needing a real subprocess or a
|
||||||
|
# provider to actually be gate-eligible (credential/gate selection is select_provider()'s
|
||||||
|
# job, tested separately; dispatch_preview only needs a chosen provider name).
|
||||||
|
pp_repo = tmpdir / "pp-repo"
|
||||||
|
(pp_repo / "ai").mkdir(parents=True, exist_ok=True)
|
||||||
|
(pp_repo / "style").mkdir(parents=True, exist_ok=True)
|
||||||
|
(pp_repo / "ai" / "01-a.overview.md").write_text("# a\n")
|
||||||
|
(pp_repo / "ai" / "01-a.agent.md").write_text("# brief\n")
|
||||||
|
(pp_repo / "ai" / "SOURCE-REGISTER.md").write_text("# reg\n")
|
||||||
|
(pp_repo / "AGENTS.md").write_text("# root\n")
|
||||||
|
(pp_repo / "ai" / "AGENTS.md").write_text("# ai\n")
|
||||||
|
|
||||||
|
pp_profile_anthropic = tmpdir / "pp-profiles" / "anthropic"
|
||||||
|
pp_profile_minimax = tmpdir / "pp-profiles" / "minimax"
|
||||||
|
pp_profile_anthropic.mkdir(parents=True, exist_ok=True)
|
||||||
|
pp_profile_minimax.mkdir(parents=True, exist_ok=True)
|
||||||
|
(pp_profile_anthropic / "provider.env").write_text("MODEL_ID=anthropic-profile-model\n")
|
||||||
|
# minimax profile deliberately has no provider.env -- its --model must come entirely
|
||||||
|
# from the per-provider config override, not any profile fallback.
|
||||||
|
|
||||||
|
pp_config = {
|
||||||
|
"providers": {
|
||||||
|
"anthropic": {"profile": str(pp_profile_anthropic), "threshold_pct": 80,
|
||||||
|
"five_hour_ceiling": 50, "min_idle": 5},
|
||||||
|
"minimax": {"profile": str(pp_profile_minimax), "threshold_pct": 80,
|
||||||
|
"five_hour_ceiling": 50, "min_idle": 5},
|
||||||
|
},
|
||||||
|
"work_types": {
|
||||||
|
"research": {
|
||||||
|
"providers": [
|
||||||
|
"anthropic",
|
||||||
|
{"provider": "minimax", "model": "minimax-pinned-model", "effort": "xhigh"},
|
||||||
|
],
|
||||||
|
"effort": "medium",
|
||||||
|
"model": "worktype-fallback-model",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"dossiers": ["ai"],
|
||||||
|
"review_score_threshold": 11,
|
||||||
|
"max_unreviewed_research_per_dossier": 3,
|
||||||
|
"max_open_topic_proposals": 6,
|
||||||
|
"evidence_dirs": [],
|
||||||
|
}
|
||||||
|
|
||||||
|
candidate_anthropic = {"dossier": "ai", "slug": "01-a", "work_type": "research", "provider": "anthropic"}
|
||||||
|
preview_anthropic = m.dispatch_preview(pp_repo, pp_config, candidate_anthropic, 25)
|
||||||
|
check("dispatch_preview: plain-string provider entry (anthropic) uses work-type-level model, not its own profile MODEL_ID",
|
||||||
|
"worktype-fallback-model" in preview_anthropic["argv"]
|
||||||
|
and "anthropic-profile-model" not in preview_anthropic["argv"], preview_anthropic["argv"])
|
||||||
|
check("dispatch_preview: plain-string provider entry (anthropic) uses work-type-level effort",
|
||||||
|
"medium" in preview_anthropic["argv"] and preview_anthropic["argv"].count("--effort") == 1,
|
||||||
|
preview_anthropic["argv"])
|
||||||
|
|
||||||
|
candidate_minimax = {"dossier": "ai", "slug": "01-a", "work_type": "research", "provider": "minimax"}
|
||||||
|
preview_minimax = m.dispatch_preview(pp_repo, pp_config, candidate_minimax, 25)
|
||||||
|
check("dispatch_preview: object-form provider entry (minimax) uses its own pinned model over the work-type-level one",
|
||||||
|
"minimax-pinned-model" in preview_minimax["argv"]
|
||||||
|
and "worktype-fallback-model" not in preview_minimax["argv"], preview_minimax["argv"])
|
||||||
|
check("dispatch_preview: object-form provider entry (minimax) uses its own pinned effort over the work-type-level one",
|
||||||
|
"xhigh" in preview_minimax["argv"] and "medium" not in preview_minimax["argv"], preview_minimax["argv"])
|
||||||
|
check("dispatch_preview: exactly one --model flag regardless of which provider was selected (mixed list)",
|
||||||
|
preview_anthropic["argv"].count("--model") == 1 and preview_minimax["argv"].count("--model") == 1, "")
|
||||||
|
|
||||||
|
# --- per-provider effort / model overrides: config validation ---
|
||||||
|
|
||||||
|
def _base_pp_cfg():
|
||||||
|
c = json.loads((tmpdir / "idle-draft.config.json").read_text())
|
||||||
|
return c
|
||||||
|
|
||||||
|
bad_provider_name_str_path = tmpdir / "bad-provider-name-str.config.json"
|
||||||
|
c = _base_pp_cfg()
|
||||||
|
c["work_types"]["research"]["providers"] = ["anthropic", "nonexistent-provider"]
|
||||||
|
bad_provider_name_str_path.write_text(json.dumps(c))
|
||||||
|
try:
|
||||||
|
m.load_config(bad_provider_name_str_path)
|
||||||
|
check("load_config: string-form providers entry naming an unknown provider raises ConfigError", False)
|
||||||
|
except m.ConfigError as e:
|
||||||
|
check("load_config: string-form providers entry naming an unknown provider raises ConfigError",
|
||||||
|
"unknown provider" in str(e) and "nonexistent-provider" in str(e), str(e))
|
||||||
|
|
||||||
|
bad_provider_name_obj_path = tmpdir / "bad-provider-name-obj.config.json"
|
||||||
|
c = _base_pp_cfg()
|
||||||
|
c["work_types"]["research"]["providers"] = ["anthropic", {"provider": "nonexistent-provider"}]
|
||||||
|
bad_provider_name_obj_path.write_text(json.dumps(c))
|
||||||
|
try:
|
||||||
|
m.load_config(bad_provider_name_obj_path)
|
||||||
|
check("load_config: object-form providers entry naming an unknown provider raises ConfigError", False)
|
||||||
|
except m.ConfigError as e:
|
||||||
|
check("load_config: object-form providers entry naming an unknown provider raises ConfigError",
|
||||||
|
"unknown provider" in str(e) and "nonexistent-provider" in str(e), str(e))
|
||||||
|
|
||||||
|
missing_provider_key_path = tmpdir / "missing-provider-key.config.json"
|
||||||
|
c = _base_pp_cfg()
|
||||||
|
c["work_types"]["research"]["providers"] = ["anthropic", {"effort": "high"}]
|
||||||
|
missing_provider_key_path.write_text(json.dumps(c))
|
||||||
|
try:
|
||||||
|
m.load_config(missing_provider_key_path)
|
||||||
|
check("load_config: object-form providers entry missing 'provider' key raises ConfigError", False)
|
||||||
|
except m.ConfigError as e:
|
||||||
|
check("load_config: object-form providers entry missing 'provider' key raises ConfigError",
|
||||||
|
"provider" in str(e), str(e))
|
||||||
|
|
||||||
|
unknown_key_path = tmpdir / "unknown-provider-entry-key.config.json"
|
||||||
|
c = _base_pp_cfg()
|
||||||
|
c["work_types"]["research"]["providers"] = ["anthropic", {"provider": "minimax", "bogus_key": 1}]
|
||||||
|
unknown_key_path.write_text(json.dumps(c))
|
||||||
|
try:
|
||||||
|
m.load_config(unknown_key_path)
|
||||||
|
check("load_config: object-form providers entry with an unknown key raises ConfigError", False)
|
||||||
|
except m.ConfigError as e:
|
||||||
|
check("load_config: object-form providers entry with an unknown key raises ConfigError",
|
||||||
|
"unknown key" in str(e) and "bogus_key" in str(e), str(e))
|
||||||
|
|
||||||
|
bad_entry_effort_path = tmpdir / "bad-provider-entry-effort.config.json"
|
||||||
|
c = _base_pp_cfg()
|
||||||
|
c["work_types"]["research"]["providers"] = ["anthropic", {"provider": "minimax", "effort": "ultra-mega"}]
|
||||||
|
bad_entry_effort_path.write_text(json.dumps(c))
|
||||||
|
try:
|
||||||
|
m.load_config(bad_entry_effort_path)
|
||||||
|
check("load_config: object-form providers entry with invalid effort level raises ConfigError", False)
|
||||||
|
except m.ConfigError as e:
|
||||||
|
check("load_config: object-form providers entry with invalid effort level raises ConfigError",
|
||||||
|
"effort" in str(e) and "minimax" in str(e), str(e))
|
||||||
|
|
||||||
|
bad_entry_model_path = tmpdir / "bad-provider-entry-model.config.json"
|
||||||
|
c = _base_pp_cfg()
|
||||||
|
c["work_types"]["research"]["providers"] = ["anthropic", {"provider": "minimax", "model": ""}]
|
||||||
|
bad_entry_model_path.write_text(json.dumps(c))
|
||||||
|
try:
|
||||||
|
m.load_config(bad_entry_model_path)
|
||||||
|
check("load_config: object-form providers entry with empty model raises ConfigError", False)
|
||||||
|
except m.ConfigError as e:
|
||||||
|
check("load_config: object-form providers entry with empty model raises ConfigError",
|
||||||
|
"model" in str(e) and "minimax" in str(e), str(e))
|
||||||
|
|
||||||
|
bad_entry_type_path = tmpdir / "bad-provider-entry-type.config.json"
|
||||||
|
c = _base_pp_cfg()
|
||||||
|
c["work_types"]["research"]["providers"] = ["anthropic", 5]
|
||||||
|
bad_entry_type_path.write_text(json.dumps(c))
|
||||||
|
try:
|
||||||
|
m.load_config(bad_entry_type_path)
|
||||||
|
check("load_config: providers entry that is neither string nor object raises ConfigError", False)
|
||||||
|
except m.ConfigError as e:
|
||||||
|
check("load_config: providers entry that is neither string nor object raises ConfigError",
|
||||||
|
"string or object" in str(e), str(e))
|
||||||
|
|
||||||
|
good_mixed_path = tmpdir / "good-mixed-providers.config.json"
|
||||||
|
c = _base_pp_cfg()
|
||||||
|
c["work_types"]["research"]["providers"] = ["anthropic", {"provider": "minimax", "model": "m-model", "effort": "high"}]
|
||||||
|
good_mixed_path.write_text(json.dumps(c))
|
||||||
|
loaded_mixed = m.load_config(good_mixed_path)
|
||||||
|
check("load_config: well-formed mixed (string + object) providers list loads without error",
|
||||||
|
loaded_mixed["work_types"]["research"]["providers"][0] == "anthropic"
|
||||||
|
and loaded_mixed["work_types"]["research"]["providers"][1] == {"provider": "minimax", "model": "m-model", "effort": "high"},
|
||||||
|
loaded_mixed["work_types"]["research"]["providers"])
|
||||||
|
|
||||||
|
# The shipped example config itself demonstrates the mixed form -- confirm it validates
|
||||||
|
# and that select_provider() still walks the list in order via provider_entry_name().
|
||||||
|
example_cfg = m.load_config(m.REPO_SELF / "data" / "idle-draft" / "config.example.json")
|
||||||
|
example_research_providers = example_cfg["work_types"]["research"]["providers"]
|
||||||
|
check("config.example.json: loads without error (mixed providers list included)", True, "")
|
||||||
|
check("config.example.json: research providers mixes a plain string (anthropic) and an object (minimax)",
|
||||||
|
example_research_providers[0] == "anthropic"
|
||||||
|
and isinstance(example_research_providers[1], dict)
|
||||||
|
and example_research_providers[1]["provider"] == "minimax"
|
||||||
|
and example_research_providers[1]["effort"] == "medium",
|
||||||
|
example_research_providers)
|
||||||
|
check("provider_entry_name: walks config.example.json's mixed research providers list in order",
|
||||||
|
[m.provider_entry_name(e) for e in example_research_providers] == ["anthropic", "minimax"],
|
||||||
|
example_research_providers)
|
||||||
|
|
||||||
# --- Citation validation ---
|
# --- Citation validation ---
|
||||||
real_path = cred_repo / "exists.txt"
|
real_path = cred_repo / "exists.txt"
|
||||||
real_path.write_text("x")
|
real_path.write_text("x")
|
||||||
|
|||||||
Reference in New Issue
Block a user