idle-draft: per-work-type and per-provider model/effort settings
--effort/--model on child argv; provider entries accept object form
{provider, model, effort}; precedence entry > work-type > profile.
Effort levels validated against the CLI's enumerated set.
Claude-Session: https://claude.ai/code/session_01YQDoWNM7XPPii28khFWoMc
This commit is contained in:
Paul O'Reilly
@@ -155,6 +155,44 @@ assert_contains "$output" "$TMPDIR/evidence" "dryrun argv includes configured ev
|
||||
assert_contains "$output" "--allowedTools" "dryrun argv for research includes --allowedTools (allowed_tools configured)"
|
||||
assert_contains "$output" "WebSearch,WebFetch" "dryrun argv --allowedTools value is the comma-joined tool list"
|
||||
|
||||
echo "-- dryrun: work-type effort/model override profile model, appear in resolved argv --"
|
||||
PROFILE_ANTHROPIC_MODEL="$TMPDIR/profiles/anthropic-with-model"
|
||||
mkdir -p "$PROFILE_ANTHROPIC_MODEL"
|
||||
cat > "$PROFILE_ANTHROPIC_MODEL/provider.env" <<'EOF'
|
||||
MODEL_ID=profile-default-model
|
||||
EOF
|
||||
EFFORT_CONFIG="$TMPDIR/effort.config.json"
|
||||
cat > "$EFFORT_CONFIG" <<EOF
|
||||
{
|
||||
"parallel": 2,
|
||||
"providers": {
|
||||
"anthropic": { "profile": "$PROFILE_ANTHROPIC_MODEL", "threshold_pct": 80, "five_hour_ceiling": 50, "min_idle": 5 },
|
||||
"minimax": { "profile": "$PROFILE_MINIMAX", "threshold_pct": 80, "five_hour_ceiling": 50, "min_idle": 5 }
|
||||
},
|
||||
"work_types": {
|
||||
"review": { "providers": ["anthropic"] },
|
||||
"draft": { "providers": ["anthropic"] },
|
||||
"research": { "providers": ["anthropic", "minimax"], "allowed_tools": ["WebSearch", "WebFetch"], "effort": "medium", "model": "worktype-override-model" },
|
||||
"topic_ideas": { "providers": ["anthropic", "minimax"], "allowed_tools": ["WebSearch", "WebFetch"] }
|
||||
},
|
||||
"dossiers": ["ai"],
|
||||
"review_score_threshold": 11,
|
||||
"max_unreviewed_research_per_dossier": 3,
|
||||
"max_open_topic_proposals": 6,
|
||||
"evidence_dirs": ["$TMPDIR/evidence"]
|
||||
}
|
||||
EOF
|
||||
output=$("$SCRIPT" --config "$EFFORT_CONFIG" --repo "$SOLO" --probe-json "$FIXTURE_PROBE" --dryrun --once 2>&1)
|
||||
assert_contains "$output" "'--model', 'worktype-override-model'" "dryrun argv model uses work-type override, not profile MODEL_ID"
|
||||
assert_not_contains "$output" "profile-default-model" "dryrun argv does not contain the profile's MODEL_ID once a work-type model overrides it"
|
||||
assert_contains "$output" "'--effort', 'medium'" "dryrun argv includes --effort with the configured level"
|
||||
model_flag_count=$(echo "$output" | grep -o -- "--model" | wc -l)
|
||||
[[ "$model_flag_count" -eq 1 ]] && pass "resolved argv contains exactly one --model flag (no duplicate from profile)" || fail "resolved argv contains exactly one --model flag" "found $model_flag_count"
|
||||
|
||||
echo "-- dryrun: no effort/model configured -> flags omitted (unchanged behaviour) --"
|
||||
output=$("$SCRIPT" --config "$FIXTURE_CONFIG" --repo "$SOLO" --probe-json "$FIXTURE_PROBE" --dryrun --once 2>&1)
|
||||
assert_not_contains "$output" "--effort" "dryrun argv omits --effort when work_type has no effort key"
|
||||
|
||||
echo "-- dryrun: no mutation --"
|
||||
before_hash=$(cd "$FIXTURE_REPO" && git rev-parse HEAD)
|
||||
"$SCRIPT" --config "$FIXTURE_CONFIG" --repo "$FIXTURE_REPO" --probe-json "$FIXTURE_PROBE" --dryrun >/dev/null 2>&1
|
||||
@@ -254,6 +292,132 @@ code=$?
|
||||
assert_exit_code "$code" 2 "allowed_tools not a list of strings exits 2"
|
||||
assert_contains "$output" "allowed_tools" "allowed_tools config error names the offending key"
|
||||
|
||||
BADEFFORT="$TMPDIR/bad-effort.config.json"
|
||||
cat > "$BADEFFORT" <<EOF
|
||||
{
|
||||
"parallel": 2,
|
||||
"providers": {
|
||||
"anthropic": { "profile": "$PROFILE_ANTHROPIC", "threshold_pct": 80, "five_hour_ceiling": 50, "min_idle": 5 }
|
||||
},
|
||||
"work_types": {
|
||||
"review": { "providers": ["anthropic"] },
|
||||
"draft": { "providers": ["anthropic"] },
|
||||
"research": { "providers": ["anthropic"], "effort": "ultra-mega" },
|
||||
"topic_ideas": { "providers": ["anthropic"] }
|
||||
},
|
||||
"dossiers": ["ai"],
|
||||
"review_score_threshold": 11,
|
||||
"max_unreviewed_research_per_dossier": 3,
|
||||
"max_open_topic_proposals": 6,
|
||||
"evidence_dirs": ["$TMPDIR/evidence"]
|
||||
}
|
||||
EOF
|
||||
output=$("$SCRIPT" --config "$BADEFFORT" --repo "$FIXTURE_REPO" --probe-json "$FIXTURE_PROBE" --dryrun 2>&1)
|
||||
code=$?
|
||||
assert_exit_code "$code" 2 "invalid effort level exits 2"
|
||||
assert_contains "$output" "effort" "invalid effort config error names the offending key"
|
||||
|
||||
BADMODEL="$TMPDIR/bad-model.config.json"
|
||||
cat > "$BADMODEL" <<EOF
|
||||
{
|
||||
"parallel": 2,
|
||||
"providers": {
|
||||
"anthropic": { "profile": "$PROFILE_ANTHROPIC", "threshold_pct": 80, "five_hour_ceiling": 50, "min_idle": 5 }
|
||||
},
|
||||
"work_types": {
|
||||
"review": { "providers": ["anthropic"] },
|
||||
"draft": { "providers": ["anthropic"] },
|
||||
"research": { "providers": ["anthropic"], "model": "" },
|
||||
"topic_ideas": { "providers": ["anthropic"] }
|
||||
},
|
||||
"dossiers": ["ai"],
|
||||
"review_score_threshold": 11,
|
||||
"max_unreviewed_research_per_dossier": 3,
|
||||
"max_open_topic_proposals": 6,
|
||||
"evidence_dirs": ["$TMPDIR/evidence"]
|
||||
}
|
||||
EOF
|
||||
output=$("$SCRIPT" --config "$BADMODEL" --repo "$FIXTURE_REPO" --probe-json "$FIXTURE_PROBE" --dryrun 2>&1)
|
||||
code=$?
|
||||
assert_exit_code "$code" 2 "empty model string exits 2"
|
||||
assert_contains "$output" "model" "empty model config error names the offending key"
|
||||
|
||||
echo "-- config validation: per-provider effort/model overrides (providers-list entries) --"
|
||||
BADPROVIDERNAME="$TMPDIR/bad-provider-name.config.json"
|
||||
cat > "$BADPROVIDERNAME" <<EOF
|
||||
{
|
||||
"parallel": 2,
|
||||
"providers": {
|
||||
"anthropic": { "profile": "$PROFILE_ANTHROPIC", "threshold_pct": 80, "five_hour_ceiling": 50, "min_idle": 5 }
|
||||
},
|
||||
"work_types": {
|
||||
"review": { "providers": ["anthropic"] },
|
||||
"draft": { "providers": ["anthropic"] },
|
||||
"research": { "providers": ["anthropic", {"provider": "nonexistent-provider", "effort": "high"}] },
|
||||
"topic_ideas": { "providers": ["anthropic"] }
|
||||
},
|
||||
"dossiers": ["ai"],
|
||||
"review_score_threshold": 11,
|
||||
"max_unreviewed_research_per_dossier": 3,
|
||||
"max_open_topic_proposals": 6,
|
||||
"evidence_dirs": ["$TMPDIR/evidence"]
|
||||
}
|
||||
EOF
|
||||
output=$("$SCRIPT" --config "$BADPROVIDERNAME" --repo "$FIXTURE_REPO" --probe-json "$FIXTURE_PROBE" --dryrun 2>&1)
|
||||
code=$?
|
||||
assert_exit_code "$code" 2 "providers-list entry naming an undefined provider exits 2"
|
||||
assert_contains "$output" "unknown provider" "undefined provider config error names the problem"
|
||||
|
||||
BADPROVIDERKEY="$TMPDIR/bad-provider-key.config.json"
|
||||
cat > "$BADPROVIDERKEY" <<EOF
|
||||
{
|
||||
"parallel": 2,
|
||||
"providers": {
|
||||
"anthropic": { "profile": "$PROFILE_ANTHROPIC", "threshold_pct": 80, "five_hour_ceiling": 50, "min_idle": 5 }
|
||||
},
|
||||
"work_types": {
|
||||
"review": { "providers": ["anthropic"] },
|
||||
"draft": { "providers": ["anthropic"] },
|
||||
"research": { "providers": [{"provider": "anthropic", "priority": 1}] },
|
||||
"topic_ideas": { "providers": ["anthropic"] }
|
||||
},
|
||||
"dossiers": ["ai"],
|
||||
"review_score_threshold": 11,
|
||||
"max_unreviewed_research_per_dossier": 3,
|
||||
"max_open_topic_proposals": 6,
|
||||
"evidence_dirs": ["$TMPDIR/evidence"]
|
||||
}
|
||||
EOF
|
||||
output=$("$SCRIPT" --config "$BADPROVIDERKEY" --repo "$FIXTURE_REPO" --probe-json "$FIXTURE_PROBE" --dryrun 2>&1)
|
||||
code=$?
|
||||
assert_exit_code "$code" 2 "providers-list object entry with an unknown key exits 2"
|
||||
assert_contains "$output" "unknown key" "unknown provider-entry key config error names the problem"
|
||||
|
||||
BADPROVIDEREFFORT="$TMPDIR/bad-provider-effort.config.json"
|
||||
cat > "$BADPROVIDEREFFORT" <<EOF
|
||||
{
|
||||
"parallel": 2,
|
||||
"providers": {
|
||||
"anthropic": { "profile": "$PROFILE_ANTHROPIC", "threshold_pct": 80, "five_hour_ceiling": 50, "min_idle": 5 }
|
||||
},
|
||||
"work_types": {
|
||||
"review": { "providers": ["anthropic"] },
|
||||
"draft": { "providers": ["anthropic"] },
|
||||
"research": { "providers": [{"provider": "anthropic", "effort": "super-duper"}] },
|
||||
"topic_ideas": { "providers": ["anthropic"] }
|
||||
},
|
||||
"dossiers": ["ai"],
|
||||
"review_score_threshold": 11,
|
||||
"max_unreviewed_research_per_dossier": 3,
|
||||
"max_open_topic_proposals": 6,
|
||||
"evidence_dirs": ["$TMPDIR/evidence"]
|
||||
}
|
||||
EOF
|
||||
output=$("$SCRIPT" --config "$BADPROVIDEREFFORT" --repo "$FIXTURE_REPO" --probe-json "$FIXTURE_PROBE" --dryrun 2>&1)
|
||||
code=$?
|
||||
assert_exit_code "$code" 2 "providers-list object entry with an invalid effort level exits 2"
|
||||
assert_contains "$output" "effort" "invalid provider-entry effort config error names the problem"
|
||||
|
||||
echo "-- mark subcommand round-trip --"
|
||||
MARKREPO="$TMPDIR/mark-repo"
|
||||
mkdir -p "$MARKREPO/ai"
|
||||
@@ -519,6 +683,334 @@ check("load_config: well-formed allowed_tools (list of strings) loads without er
|
||||
good_allowed_cfg["work_types"]["research"]["allowed_tools"] == ["WebSearch", "WebFetch"],
|
||||
good_allowed_cfg["work_types"]["research"])
|
||||
|
||||
# --- per-work-type effort / model: argv construction ---
|
||||
argv_effort = m.build_claude_argv("model-x", 25, [], None, "high")
|
||||
check("build_claude_argv: appends --effort when given",
|
||||
"--effort" in argv_effort and argv_effort[argv_effort.index("--effort") + 1] == "high", argv_effort)
|
||||
|
||||
argv_no_effort = m.build_claude_argv("model-x", 25, [], None, None)
|
||||
check("build_claude_argv: omits --effort when not given", "--effort" not in argv_no_effort, argv_no_effort)
|
||||
|
||||
argv_model_and_effort = m.build_claude_argv("worktype-model", 25, [], None, "xhigh")
|
||||
check("build_claude_argv: --model reflects whatever single value the caller resolved "
|
||||
"(precedence already applied upstream) -- exactly one --model flag",
|
||||
argv_model_and_effort.count("--model") == 1
|
||||
and argv_model_and_effort[argv_model_and_effort.index("--model") + 1] == "worktype-model",
|
||||
argv_model_and_effort)
|
||||
|
||||
argv_neither = m.build_claude_argv(None, 25, [], None, None)
|
||||
check("build_claude_argv: omits both --model and --effort when neither is given",
|
||||
"--model" not in argv_neither and "--effort" not in argv_neither, argv_neither)
|
||||
|
||||
# --- per-work-type effort / model: precedence resolution ---
|
||||
check("resolve_effective_model: work-type model overrides profile MODEL_ID",
|
||||
m.resolve_effective_model("profile-model", "worktype-model") == "worktype-model", "")
|
||||
check("resolve_effective_model: falls back to profile MODEL_ID when no work-type model set",
|
||||
m.resolve_effective_model("profile-model", None) == "profile-model", "")
|
||||
check("resolve_effective_model: both absent -> None",
|
||||
m.resolve_effective_model(None, None) is None, "")
|
||||
|
||||
cfg_em = {"work_types": {"research": {"providers": ["anthropic"], "effort": "medium", "model": "wt-model"},
|
||||
"draft": {"providers": ["anthropic"]}}}
|
||||
check("resolve_work_type_effort: returns configured value", m.resolve_work_type_effort(cfg_em, "research") == "medium", "")
|
||||
check("resolve_work_type_effort: returns None when absent", m.resolve_work_type_effort(cfg_em, "draft") is None, "")
|
||||
check("resolve_work_type_model: returns configured value", m.resolve_work_type_model(cfg_em, "research") == "wt-model", "")
|
||||
check("resolve_work_type_model: returns None when absent", m.resolve_work_type_model(cfg_em, "draft") is None, "")
|
||||
|
||||
# --- per-work-type effort / model: config validation ---
|
||||
bad_effort_cfg_path = tmpdir / "bad-effort-direct.config.json"
|
||||
bad_effort_cfg = json.loads((tmpdir / "idle-draft.config.json").read_text())
|
||||
bad_effort_cfg["work_types"]["research"]["effort"] = "ultra-mega"
|
||||
bad_effort_cfg_path.write_text(json.dumps(bad_effort_cfg))
|
||||
try:
|
||||
m.load_config(bad_effort_cfg_path)
|
||||
check("load_config: effort value outside CLAUDE_EFFORT_LEVELS raises ConfigError", False)
|
||||
except m.ConfigError as e:
|
||||
check("load_config: effort value outside CLAUDE_EFFORT_LEVELS raises ConfigError", "effort" in str(e), str(e))
|
||||
|
||||
bad_effort_type_path = tmpdir / "bad-effort-type.config.json"
|
||||
bad_effort_type_cfg = json.loads((tmpdir / "idle-draft.config.json").read_text())
|
||||
bad_effort_type_cfg["work_types"]["research"]["effort"] = 5
|
||||
bad_effort_type_path.write_text(json.dumps(bad_effort_type_cfg))
|
||||
try:
|
||||
m.load_config(bad_effort_type_path)
|
||||
check("load_config: non-string effort raises ConfigError", False)
|
||||
except m.ConfigError as e:
|
||||
check("load_config: non-string effort raises ConfigError", "effort" in str(e), str(e))
|
||||
|
||||
bad_model_path = tmpdir / "bad-model-direct.config.json"
|
||||
bad_model_cfg = json.loads((tmpdir / "idle-draft.config.json").read_text())
|
||||
bad_model_cfg["work_types"]["research"]["model"] = ""
|
||||
bad_model_path.write_text(json.dumps(bad_model_cfg))
|
||||
try:
|
||||
m.load_config(bad_model_path)
|
||||
check("load_config: empty-string model raises ConfigError", False)
|
||||
except m.ConfigError as e:
|
||||
check("load_config: empty-string model raises ConfigError", "model" in str(e), str(e))
|
||||
|
||||
bad_model_type_path = tmpdir / "bad-model-type.config.json"
|
||||
bad_model_type_cfg = json.loads((tmpdir / "idle-draft.config.json").read_text())
|
||||
bad_model_type_cfg["work_types"]["research"]["model"] = 42
|
||||
bad_model_type_path.write_text(json.dumps(bad_model_type_cfg))
|
||||
try:
|
||||
m.load_config(bad_model_type_path)
|
||||
check("load_config: non-string model raises ConfigError", False)
|
||||
except m.ConfigError as e:
|
||||
check("load_config: non-string model raises ConfigError", "model" in str(e), str(e))
|
||||
|
||||
good_effort_model_cfg = json.loads((tmpdir / "idle-draft.config.json").read_text())
|
||||
good_effort_model_cfg["work_types"]["research"]["effort"] = "high"
|
||||
good_effort_model_cfg["work_types"]["research"]["model"] = "claude-example-model"
|
||||
good_effort_model_path = tmpdir / "good-effort-model.config.json"
|
||||
good_effort_model_path.write_text(json.dumps(good_effort_model_cfg))
|
||||
loaded_good = m.load_config(good_effort_model_path)
|
||||
check("load_config: well-formed effort + model (in the valid set) loads without error",
|
||||
loaded_good["work_types"]["research"]["effort"] == "high"
|
||||
and loaded_good["work_types"]["research"]["model"] == "claude-example-model",
|
||||
loaded_good["work_types"]["research"])
|
||||
|
||||
for level in sorted(m.CLAUDE_EFFORT_LEVELS):
|
||||
per_level_cfg = json.loads((tmpdir / "idle-draft.config.json").read_text())
|
||||
per_level_cfg["work_types"]["research"]["effort"] = level
|
||||
per_level_path = tmpdir / f"effort-{level}.config.json"
|
||||
per_level_path.write_text(json.dumps(per_level_cfg))
|
||||
loaded = m.load_config(per_level_path)
|
||||
check(f"load_config: effort level {level!r} (member of CLAUDE_EFFORT_LEVELS) loads without error",
|
||||
loaded["work_types"]["research"]["effort"] == level, loaded["work_types"]["research"])
|
||||
|
||||
# --- per-provider effort / model overrides (within a work type) ---
|
||||
|
||||
check("provider_entry_name: plain string entry returns itself",
|
||||
m.provider_entry_name("anthropic") == "anthropic", "")
|
||||
check("provider_entry_name: object entry returns its 'provider' value",
|
||||
m.provider_entry_name({"provider": "minimax", "effort": "high"}) == "minimax", "")
|
||||
|
||||
cfg_pp = {
|
||||
"providers": {"anthropic": {}, "minimax": {}},
|
||||
"work_types": {
|
||||
"research": {
|
||||
"providers": [
|
||||
"anthropic",
|
||||
{"provider": "minimax", "model": "minimax-model", "effort": "high"},
|
||||
],
|
||||
"effort": "medium",
|
||||
"model": "worktype-model",
|
||||
},
|
||||
"draft": {"providers": ["anthropic"], "effort": "low"},
|
||||
},
|
||||
}
|
||||
|
||||
entry_a = m.find_provider_entry(cfg_pp, "research", "anthropic")
|
||||
check("find_provider_entry: returns the plain-string entry for a string-form provider",
|
||||
entry_a == "anthropic", entry_a)
|
||||
entry_m = m.find_provider_entry(cfg_pp, "research", "minimax")
|
||||
check("find_provider_entry: returns the dict entry for an object-form provider",
|
||||
isinstance(entry_m, dict) and entry_m["provider"] == "minimax", entry_m)
|
||||
entry_missing = m.find_provider_entry(cfg_pp, "research", "nonexistent")
|
||||
check("find_provider_entry: returns None when the provider isn't in this work_type's list",
|
||||
entry_missing is None, entry_missing)
|
||||
|
||||
# Precedence: per-provider entry > work-type-level > (caller supplies profile fallback separately)
|
||||
check("resolve_work_type_effort: plain-string provider entry falls through to work-type-level effort",
|
||||
m.resolve_work_type_effort(cfg_pp, "research", "anthropic") == "medium", "")
|
||||
check("resolve_work_type_model: plain-string provider entry falls through to work-type-level model",
|
||||
m.resolve_work_type_model(cfg_pp, "research", "anthropic") == "worktype-model", "")
|
||||
check("resolve_work_type_effort: object-form provider entry's own effort wins over work-type-level",
|
||||
m.resolve_work_type_effort(cfg_pp, "research", "minimax") == "high", "")
|
||||
check("resolve_work_type_model: object-form provider entry's own model wins over work-type-level",
|
||||
m.resolve_work_type_model(cfg_pp, "research", "minimax") == "minimax-model", "")
|
||||
check("resolve_work_type_effort: provider_name omitted -> work-type-level value only (back-compat)",
|
||||
m.resolve_work_type_effort(cfg_pp, "research") == "medium", "")
|
||||
check("resolve_work_type_model: provider_name omitted -> work-type-level value only (back-compat)",
|
||||
m.resolve_work_type_model(cfg_pp, "research") == "worktype-model", "")
|
||||
check("resolve_work_type_effort: work type with no per-work-type or per-provider effort -> None",
|
||||
m.resolve_work_type_effort(cfg_pp, "draft", "anthropic") == "low", "")
|
||||
|
||||
# Full chain including the ultimate profile-MODEL_ID fallback (resolve_effective_model),
|
||||
# for a provider with NO per-provider or work-type-level model at all.
|
||||
cfg_pp_no_wt_model = {
|
||||
"providers": {"anthropic": {}},
|
||||
"work_types": {"review": {"providers": ["anthropic"], "effort": "high"}},
|
||||
}
|
||||
wt_model = m.resolve_work_type_model(cfg_pp_no_wt_model, "review", "anthropic")
|
||||
check("resolve_work_type_model: no per-provider, no work-type-level -> None (falls to profile next)",
|
||||
wt_model is None, wt_model)
|
||||
check("resolve_effective_model: full chain falls back to profile MODEL_ID when neither override is set",
|
||||
m.resolve_effective_model("profile-model-id", wt_model) == "profile-model-id", "")
|
||||
|
||||
# --- per-provider effort / model overrides: dispatch_preview reflects the resolution ---
|
||||
# dispatch_preview() takes a candidate with "provider" already selected -- this exercises
|
||||
# the exact same resolution path run_task() uses, without needing a real subprocess or a
|
||||
# provider to actually be gate-eligible (credential/gate selection is select_provider()'s
|
||||
# job, tested separately; dispatch_preview only needs a chosen provider name).
|
||||
pp_repo = tmpdir / "pp-repo"
|
||||
(pp_repo / "ai").mkdir(parents=True, exist_ok=True)
|
||||
(pp_repo / "style").mkdir(parents=True, exist_ok=True)
|
||||
(pp_repo / "ai" / "01-a.overview.md").write_text("# a\n")
|
||||
(pp_repo / "ai" / "01-a.agent.md").write_text("# brief\n")
|
||||
(pp_repo / "ai" / "SOURCE-REGISTER.md").write_text("# reg\n")
|
||||
(pp_repo / "AGENTS.md").write_text("# root\n")
|
||||
(pp_repo / "ai" / "AGENTS.md").write_text("# ai\n")
|
||||
|
||||
pp_profile_anthropic = tmpdir / "pp-profiles" / "anthropic"
|
||||
pp_profile_minimax = tmpdir / "pp-profiles" / "minimax"
|
||||
pp_profile_anthropic.mkdir(parents=True, exist_ok=True)
|
||||
pp_profile_minimax.mkdir(parents=True, exist_ok=True)
|
||||
(pp_profile_anthropic / "provider.env").write_text("MODEL_ID=anthropic-profile-model\n")
|
||||
# minimax profile deliberately has no provider.env -- its --model must come entirely
|
||||
# from the per-provider config override, not any profile fallback.
|
||||
|
||||
pp_config = {
|
||||
"providers": {
|
||||
"anthropic": {"profile": str(pp_profile_anthropic), "threshold_pct": 80,
|
||||
"five_hour_ceiling": 50, "min_idle": 5},
|
||||
"minimax": {"profile": str(pp_profile_minimax), "threshold_pct": 80,
|
||||
"five_hour_ceiling": 50, "min_idle": 5},
|
||||
},
|
||||
"work_types": {
|
||||
"research": {
|
||||
"providers": [
|
||||
"anthropic",
|
||||
{"provider": "minimax", "model": "minimax-pinned-model", "effort": "xhigh"},
|
||||
],
|
||||
"effort": "medium",
|
||||
"model": "worktype-fallback-model",
|
||||
},
|
||||
},
|
||||
"dossiers": ["ai"],
|
||||
"review_score_threshold": 11,
|
||||
"max_unreviewed_research_per_dossier": 3,
|
||||
"max_open_topic_proposals": 6,
|
||||
"evidence_dirs": [],
|
||||
}
|
||||
|
||||
candidate_anthropic = {"dossier": "ai", "slug": "01-a", "work_type": "research", "provider": "anthropic"}
|
||||
preview_anthropic = m.dispatch_preview(pp_repo, pp_config, candidate_anthropic, 25)
|
||||
check("dispatch_preview: plain-string provider entry (anthropic) uses work-type-level model, not its own profile MODEL_ID",
|
||||
"worktype-fallback-model" in preview_anthropic["argv"]
|
||||
and "anthropic-profile-model" not in preview_anthropic["argv"], preview_anthropic["argv"])
|
||||
check("dispatch_preview: plain-string provider entry (anthropic) uses work-type-level effort",
|
||||
"medium" in preview_anthropic["argv"] and preview_anthropic["argv"].count("--effort") == 1,
|
||||
preview_anthropic["argv"])
|
||||
|
||||
candidate_minimax = {"dossier": "ai", "slug": "01-a", "work_type": "research", "provider": "minimax"}
|
||||
preview_minimax = m.dispatch_preview(pp_repo, pp_config, candidate_minimax, 25)
|
||||
check("dispatch_preview: object-form provider entry (minimax) uses its own pinned model over the work-type-level one",
|
||||
"minimax-pinned-model" in preview_minimax["argv"]
|
||||
and "worktype-fallback-model" not in preview_minimax["argv"], preview_minimax["argv"])
|
||||
check("dispatch_preview: object-form provider entry (minimax) uses its own pinned effort over the work-type-level one",
|
||||
"xhigh" in preview_minimax["argv"] and "medium" not in preview_minimax["argv"], preview_minimax["argv"])
|
||||
check("dispatch_preview: exactly one --model flag regardless of which provider was selected (mixed list)",
|
||||
preview_anthropic["argv"].count("--model") == 1 and preview_minimax["argv"].count("--model") == 1, "")
|
||||
|
||||
# --- per-provider effort / model overrides: config validation ---
|
||||
|
||||
def _base_pp_cfg():
|
||||
c = json.loads((tmpdir / "idle-draft.config.json").read_text())
|
||||
return c
|
||||
|
||||
bad_provider_name_str_path = tmpdir / "bad-provider-name-str.config.json"
|
||||
c = _base_pp_cfg()
|
||||
c["work_types"]["research"]["providers"] = ["anthropic", "nonexistent-provider"]
|
||||
bad_provider_name_str_path.write_text(json.dumps(c))
|
||||
try:
|
||||
m.load_config(bad_provider_name_str_path)
|
||||
check("load_config: string-form providers entry naming an unknown provider raises ConfigError", False)
|
||||
except m.ConfigError as e:
|
||||
check("load_config: string-form providers entry naming an unknown provider raises ConfigError",
|
||||
"unknown provider" in str(e) and "nonexistent-provider" in str(e), str(e))
|
||||
|
||||
bad_provider_name_obj_path = tmpdir / "bad-provider-name-obj.config.json"
|
||||
c = _base_pp_cfg()
|
||||
c["work_types"]["research"]["providers"] = ["anthropic", {"provider": "nonexistent-provider"}]
|
||||
bad_provider_name_obj_path.write_text(json.dumps(c))
|
||||
try:
|
||||
m.load_config(bad_provider_name_obj_path)
|
||||
check("load_config: object-form providers entry naming an unknown provider raises ConfigError", False)
|
||||
except m.ConfigError as e:
|
||||
check("load_config: object-form providers entry naming an unknown provider raises ConfigError",
|
||||
"unknown provider" in str(e) and "nonexistent-provider" in str(e), str(e))
|
||||
|
||||
missing_provider_key_path = tmpdir / "missing-provider-key.config.json"
|
||||
c = _base_pp_cfg()
|
||||
c["work_types"]["research"]["providers"] = ["anthropic", {"effort": "high"}]
|
||||
missing_provider_key_path.write_text(json.dumps(c))
|
||||
try:
|
||||
m.load_config(missing_provider_key_path)
|
||||
check("load_config: object-form providers entry missing 'provider' key raises ConfigError", False)
|
||||
except m.ConfigError as e:
|
||||
check("load_config: object-form providers entry missing 'provider' key raises ConfigError",
|
||||
"provider" in str(e), str(e))
|
||||
|
||||
unknown_key_path = tmpdir / "unknown-provider-entry-key.config.json"
|
||||
c = _base_pp_cfg()
|
||||
c["work_types"]["research"]["providers"] = ["anthropic", {"provider": "minimax", "bogus_key": 1}]
|
||||
unknown_key_path.write_text(json.dumps(c))
|
||||
try:
|
||||
m.load_config(unknown_key_path)
|
||||
check("load_config: object-form providers entry with an unknown key raises ConfigError", False)
|
||||
except m.ConfigError as e:
|
||||
check("load_config: object-form providers entry with an unknown key raises ConfigError",
|
||||
"unknown key" in str(e) and "bogus_key" in str(e), str(e))
|
||||
|
||||
bad_entry_effort_path = tmpdir / "bad-provider-entry-effort.config.json"
|
||||
c = _base_pp_cfg()
|
||||
c["work_types"]["research"]["providers"] = ["anthropic", {"provider": "minimax", "effort": "ultra-mega"}]
|
||||
bad_entry_effort_path.write_text(json.dumps(c))
|
||||
try:
|
||||
m.load_config(bad_entry_effort_path)
|
||||
check("load_config: object-form providers entry with invalid effort level raises ConfigError", False)
|
||||
except m.ConfigError as e:
|
||||
check("load_config: object-form providers entry with invalid effort level raises ConfigError",
|
||||
"effort" in str(e) and "minimax" in str(e), str(e))
|
||||
|
||||
bad_entry_model_path = tmpdir / "bad-provider-entry-model.config.json"
|
||||
c = _base_pp_cfg()
|
||||
c["work_types"]["research"]["providers"] = ["anthropic", {"provider": "minimax", "model": ""}]
|
||||
bad_entry_model_path.write_text(json.dumps(c))
|
||||
try:
|
||||
m.load_config(bad_entry_model_path)
|
||||
check("load_config: object-form providers entry with empty model raises ConfigError", False)
|
||||
except m.ConfigError as e:
|
||||
check("load_config: object-form providers entry with empty model raises ConfigError",
|
||||
"model" in str(e) and "minimax" in str(e), str(e))
|
||||
|
||||
bad_entry_type_path = tmpdir / "bad-provider-entry-type.config.json"
|
||||
c = _base_pp_cfg()
|
||||
c["work_types"]["research"]["providers"] = ["anthropic", 5]
|
||||
bad_entry_type_path.write_text(json.dumps(c))
|
||||
try:
|
||||
m.load_config(bad_entry_type_path)
|
||||
check("load_config: providers entry that is neither string nor object raises ConfigError", False)
|
||||
except m.ConfigError as e:
|
||||
check("load_config: providers entry that is neither string nor object raises ConfigError",
|
||||
"string or object" in str(e), str(e))
|
||||
|
||||
good_mixed_path = tmpdir / "good-mixed-providers.config.json"
|
||||
c = _base_pp_cfg()
|
||||
c["work_types"]["research"]["providers"] = ["anthropic", {"provider": "minimax", "model": "m-model", "effort": "high"}]
|
||||
good_mixed_path.write_text(json.dumps(c))
|
||||
loaded_mixed = m.load_config(good_mixed_path)
|
||||
check("load_config: well-formed mixed (string + object) providers list loads without error",
|
||||
loaded_mixed["work_types"]["research"]["providers"][0] == "anthropic"
|
||||
and loaded_mixed["work_types"]["research"]["providers"][1] == {"provider": "minimax", "model": "m-model", "effort": "high"},
|
||||
loaded_mixed["work_types"]["research"]["providers"])
|
||||
|
||||
# The shipped example config itself demonstrates the mixed form -- confirm it validates
|
||||
# and that select_provider() still walks the list in order via provider_entry_name().
|
||||
example_cfg = m.load_config(m.REPO_SELF / "data" / "idle-draft" / "config.example.json")
|
||||
example_research_providers = example_cfg["work_types"]["research"]["providers"]
|
||||
check("config.example.json: loads without error (mixed providers list included)", True, "")
|
||||
check("config.example.json: research providers mixes a plain string (anthropic) and an object (minimax)",
|
||||
example_research_providers[0] == "anthropic"
|
||||
and isinstance(example_research_providers[1], dict)
|
||||
and example_research_providers[1]["provider"] == "minimax"
|
||||
and example_research_providers[1]["effort"] == "medium",
|
||||
example_research_providers)
|
||||
check("provider_entry_name: walks config.example.json's mixed research providers list in order",
|
||||
[m.provider_entry_name(e) for e in example_research_providers] == ["anthropic", "minimax"],
|
||||
example_research_providers)
|
||||
|
||||
# --- Citation validation ---
|
||||
real_path = cred_repo / "exists.txt"
|
||||
real_path.write_text("x")
|
||||
|
||||
Reference in New Issue
Block a user