test(invariants): HC-11 forge-credential layers must declare write_targets (xfail until layers are stamped)

Helper tests/invariants/forge_credentials.py detects credential carriers via
mounts[]/env/secrets_required[]/ssh_hosts[]; today gitea-admin, gitea-ssh,
gitea-ssh-accelerators, gitea-ssh-homelab are found and none declare
write_targets — the invariant xfails(strict) until the CRS-103 rollout stamps them.

Claude-Session: https://claude.ai/code/session_018LRh1S2bHJYhEgrMu3UJx8
This commit is contained in:
Paul O'Reilly
2026-09-13 13:09:24 +12:00
parent e31d472ff9
commit b7cf256c44
3 changed files with 276 additions and 0 deletions

View File

View File

@@ -0,0 +1,126 @@
"""HC-11 helper: detect harness layers that carry forge (git host) credentials.
Spec: /home/paul/dev/claude/projects/agent-runtimes/spec/harness.md HC-11
(2026-09-13 amendment, bug b1de17b4).
A layer "has forge credentials" when ANY of:
- a ``mounts[]`` entry has one of the keys
``{"secret", "secret_name", "secret_ref", "vault_path"}`` whose string
value equals a name in ``FORGE_CREDENTIAL_NAMES`` or ends with
``"/" + name`` for some such name, OR
- any ``env`` value contains such a name as a whole token.
Comments and Jinja placeholders are never matched (this module only inspects
parsed YAML string values, never raw file text).
"""
from __future__ import annotations
import re
from pathlib import Path
import yaml
# Closed set (HC-11) — maintained here per spec.
FORGE_CREDENTIAL_NAMES = {
"GITEA_SSH_KEY",
"GITEA_TOKEN",
"GITEA_DEPLOY_KEY",
"AGENT_REPO_DEPLOY_KEY",
"AGENT_REPO_TOKEN",
"SSH_KEY_AI_ENABLEMENT",
"gitea-admin",
}
_MOUNT_KEYS = {"secret", "secret_name", "secret_ref", "vault_path"}
# "whole token" — the name bounded by non-alnum/underscore characters (or
# string edges) so a substring match doesn't false-positive.
_TOKEN_RE_CACHE: dict[str, re.Pattern[str]] = {}
def _token_pattern(name: str) -> re.Pattern[str]:
pat = _TOKEN_RE_CACHE.get(name)
if pat is None:
pat = re.compile(r"(?<![A-Za-z0-9_])" + re.escape(name) + r"(?![A-Za-z0-9_])")
_TOKEN_RE_CACHE[name] = pat
return pat
def _mount_value_matches(value: object) -> bool:
if not isinstance(value, str):
return False
for name in FORGE_CREDENTIAL_NAMES:
if value == name or value.endswith("/" + name):
return True
return False
_JINJA_RE = re.compile(r"\{\{.*?\}\}")
def _env_value_matches(value: object) -> bool:
if not isinstance(value, str):
return False
# Jinja placeholders (e.g. "{{ GITEA_TOKEN }}") are never matched — they
# name a template variable, not a literal credential reference.
stripped = _JINJA_RE.sub("", value)
for name in FORGE_CREDENTIAL_NAMES:
if _token_pattern(name).search(stripped):
return True
return False
def _layer_has_forge_credentials(parsed: dict) -> bool:
mounts = parsed.get("mounts")
if isinstance(mounts, list):
for entry in mounts:
if not isinstance(entry, dict):
continue
for key in _MOUNT_KEYS:
if key in entry and _mount_value_matches(entry[key]):
return True
env = parsed.get("env")
if isinstance(env, dict):
for value in env.values():
if _env_value_matches(value):
return True
secrets_required = parsed.get("secrets_required")
if isinstance(secrets_required, list):
for entry in secrets_required:
if isinstance(entry, dict) and entry.get("name") in FORGE_CREDENTIAL_NAMES:
return True
ssh_hosts = parsed.get("ssh_hosts")
if isinstance(ssh_hosts, list):
for entry in ssh_hosts:
if isinstance(entry, dict) and entry.get("identity_secret") in FORGE_CREDENTIAL_NAMES:
return True
return False
def layers_with_forge_credentials(layers_root: Path) -> list[tuple[str, Path]]:
"""Walk every ``harnesses/contexts/*/v*/harness.yaml`` under *layers_root*.
Returns ``(layer_name, yaml_path)`` for each layer with forge credentials
per the rules above.
"""
out: list[tuple[str, Path]] = []
contexts_dir = layers_root / "contexts"
if not contexts_dir.is_dir():
return out
for yaml_path in sorted(contexts_dir.glob("*/v*/harness.yaml")):
try:
parsed = yaml.safe_load(yaml_path.read_text(encoding="utf-8"))
except (OSError, yaml.YAMLError):
continue
if not isinstance(parsed, dict):
continue
if _layer_has_forge_credentials(parsed):
name = str(parsed.get("name") or yaml_path.parent.parent.name)
out.append((name, yaml_path))
return out

View File

@@ -0,0 +1,150 @@
"""HC-11 CI invariant (bug b1de17b4, 2026-09-13 amendment): every harness layer
carrying forge (git host) credentials MUST explicitly declare
``write_targets`` (an empty list is a valid, deliberate declaration; an
absent key is not).
Spec: /home/paul/dev/claude/projects/agent-runtimes/spec/harness.md HC-11.
Runs in this repo's CI workflow (.gitea/workflows/) on every push to main.
Not committed by the authoring agent per its instructions — written here so
it can run locally against the current tree.
"""
from __future__ import annotations
from pathlib import Path
import pytest
import yaml
from tests.invariants.forge_credentials import layers_with_forge_credentials
_HARNESSES_ROOT = Path(__file__).resolve().parents[2] / "harnesses"
def test_hc11_framework_layers_with_forge_secrets_found_nonzero():
"""The invariant is non-vacuous: the helper must find at least one
forge-credential layer on the real tree (helper/closed-set regression
otherwise, not evidence the repo has no forge-credential layers).
Not xfail: this passes today. On the real tree the helper currently
finds 4 layers: gitea-admin, gitea-ssh, gitea-ssh-accelerators,
gitea-ssh-homelab (via secrets_required[].name / ssh_hosts[].identity_secret).
"""
found = layers_with_forge_credentials(_HARNESSES_ROOT)
assert found, "expected >=1 forge-credential layer on the real tree, found 0"
@pytest.mark.xfail(strict=True, reason="HC-11 not implemented")
def test_hc11_framework_layers_with_forge_secrets_declare_write_targets():
found = layers_with_forge_credentials(_HARNESSES_ROOT)
assert found, "expected >=1 forge-credential layer on the real tree, found 0"
missing = []
for name, yaml_path in found:
parsed = yaml.safe_load(yaml_path.read_text(encoding="utf-8"))
if "write_targets" not in parsed:
missing.append((name, str(yaml_path)))
# As of this session none of the 4 real layers declare write_targets
# (HC-11 has not landed in the framework repo yet), so this is expected
# to fail today — xfail(strict) so it flips to a real pass the moment
# every forge-credential layer gets a write_targets key.
assert not missing, (
"harness layers with forge credentials must declare write_targets "
f"(even an empty list): {missing}"
)
def test_hc11_helper_detects_a_mounts_secret_fixture(tmp_path):
"""Sanity check the helper against the mounts[].secret carrier shape."""
contexts = tmp_path / "contexts" / "fixture-layer" / "v1"
contexts.mkdir(parents=True)
(contexts / "harness.yaml").write_text(
"""
kind: context
name: fixture-layer
version: 1
mounts:
- secret: GITEA_TOKEN
path: /run/agent/secrets/gitea-token
"""
)
found = layers_with_forge_credentials(tmp_path)
assert [name for name, _ in found] == ["fixture-layer"]
def test_hc11_helper_detects_env_whole_token_fixture(tmp_path):
contexts = tmp_path / "contexts" / "fixture-env-layer" / "v1"
contexts.mkdir(parents=True)
(contexts / "harness.yaml").write_text(
"""
kind: context
name: fixture-env-layer
version: 1
env:
SOME_VAR: "prefix_AGENT_REPO_TOKEN_suffix"
OTHER_VAR: "AGENT_REPO_TOKEN"
"""
)
found = layers_with_forge_credentials(tmp_path)
names = [name for name, _ in found]
assert names == ["fixture-env-layer"]
def test_hc11_helper_detects_secrets_required_carrier_fixture(tmp_path):
"""secrets_required[].name is a mandatory carrier per the amended spec —
this is the shape gitea-admin/v1 actually uses on the real tree."""
contexts = tmp_path / "contexts" / "fixture-secrets-required" / "v1"
contexts.mkdir(parents=True)
(contexts / "harness.yaml").write_text(
"""
kind: context
name: fixture-secrets-required
version: 1
secrets_required:
- name: gitea-admin
account_ref: "admin.space:platform"
mount_path: /run/agent/secrets/gitea-admin
"""
)
found = layers_with_forge_credentials(tmp_path)
assert [name for name, _ in found] == ["fixture-secrets-required"]
def test_hc11_helper_detects_ssh_hosts_identity_secret_carrier_fixture(tmp_path):
"""ssh_hosts[].identity_secret is the other mandatory carrier per the
amended spec — the shape gitea-ssh/v1 (etc.) actually use."""
contexts = tmp_path / "contexts" / "fixture-ssh-hosts" / "v1"
contexts.mkdir(parents=True)
(contexts / "harness.yaml").write_text(
"""
kind: context
name: fixture-ssh-hosts
version: 1
ssh_hosts:
- alias: gitea.oreillyit.nz-ai-enablement
hostname: gitea.oreillyit.nz
identity_secret: SSH_KEY_AI_ENABLEMENT
"""
)
found = layers_with_forge_credentials(tmp_path)
assert [name for name, _ in found] == ["fixture-ssh-hosts"]
def test_hc11_helper_ignores_jinja_and_comments(tmp_path):
contexts = tmp_path / "contexts" / "clean-layer" / "v1"
contexts.mkdir(parents=True)
(contexts / "harness.yaml").write_text(
"""
# GITEA_TOKEN mentioned only in a comment — must not match.
kind: context
name: clean-layer
version: 1
env:
TEMPLATE_VAR: "{{ GITEA_TOKEN }}"
"""
)
# Jinja placeholders are never matched per HC-11 text.
found = layers_with_forge_credentials(tmp_path)
names = [name for name, _ in found]
assert names == []