test(invariants): HC-11 forge-credential layers must declare write_targets (xfail until layers are stamped)
Helper tests/invariants/forge_credentials.py detects credential carriers via mounts[]/env/secrets_required[]/ssh_hosts[]; today gitea-admin, gitea-ssh, gitea-ssh-accelerators, gitea-ssh-homelab are found and none declare write_targets — the invariant xfails(strict) until the CRS-103 rollout stamps them. Claude-Session: https://claude.ai/code/session_018LRh1S2bHJYhEgrMu3UJx8
This commit is contained in:
0
tests/invariants/__init__.py
Normal file
0
tests/invariants/__init__.py
Normal file
126
tests/invariants/forge_credentials.py
Normal file
126
tests/invariants/forge_credentials.py
Normal file
@@ -0,0 +1,126 @@
|
||||
"""HC-11 helper: detect harness layers that carry forge (git host) credentials.
|
||||
|
||||
Spec: /home/paul/dev/claude/projects/agent-runtimes/spec/harness.md HC-11
|
||||
(2026-09-13 amendment, bug b1de17b4).
|
||||
|
||||
A layer "has forge credentials" when ANY of:
|
||||
- a ``mounts[]`` entry has one of the keys
|
||||
``{"secret", "secret_name", "secret_ref", "vault_path"}`` whose string
|
||||
value equals a name in ``FORGE_CREDENTIAL_NAMES`` or ends with
|
||||
``"/" + name`` for some such name, OR
|
||||
- any ``env`` value contains such a name as a whole token.
|
||||
|
||||
Comments and Jinja placeholders are never matched (this module only inspects
|
||||
parsed YAML string values, never raw file text).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
# Closed set (HC-11) — maintained here per spec.
|
||||
FORGE_CREDENTIAL_NAMES = {
|
||||
"GITEA_SSH_KEY",
|
||||
"GITEA_TOKEN",
|
||||
"GITEA_DEPLOY_KEY",
|
||||
"AGENT_REPO_DEPLOY_KEY",
|
||||
"AGENT_REPO_TOKEN",
|
||||
"SSH_KEY_AI_ENABLEMENT",
|
||||
"gitea-admin",
|
||||
}
|
||||
|
||||
_MOUNT_KEYS = {"secret", "secret_name", "secret_ref", "vault_path"}
|
||||
|
||||
# "whole token" — the name bounded by non-alnum/underscore characters (or
|
||||
# string edges) so a substring match doesn't false-positive.
|
||||
_TOKEN_RE_CACHE: dict[str, re.Pattern[str]] = {}
|
||||
|
||||
|
||||
def _token_pattern(name: str) -> re.Pattern[str]:
|
||||
pat = _TOKEN_RE_CACHE.get(name)
|
||||
if pat is None:
|
||||
pat = re.compile(r"(?<![A-Za-z0-9_])" + re.escape(name) + r"(?![A-Za-z0-9_])")
|
||||
_TOKEN_RE_CACHE[name] = pat
|
||||
return pat
|
||||
|
||||
|
||||
def _mount_value_matches(value: object) -> bool:
|
||||
if not isinstance(value, str):
|
||||
return False
|
||||
for name in FORGE_CREDENTIAL_NAMES:
|
||||
if value == name or value.endswith("/" + name):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
_JINJA_RE = re.compile(r"\{\{.*?\}\}")
|
||||
|
||||
|
||||
def _env_value_matches(value: object) -> bool:
|
||||
if not isinstance(value, str):
|
||||
return False
|
||||
# Jinja placeholders (e.g. "{{ GITEA_TOKEN }}") are never matched — they
|
||||
# name a template variable, not a literal credential reference.
|
||||
stripped = _JINJA_RE.sub("", value)
|
||||
for name in FORGE_CREDENTIAL_NAMES:
|
||||
if _token_pattern(name).search(stripped):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _layer_has_forge_credentials(parsed: dict) -> bool:
|
||||
mounts = parsed.get("mounts")
|
||||
if isinstance(mounts, list):
|
||||
for entry in mounts:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
for key in _MOUNT_KEYS:
|
||||
if key in entry and _mount_value_matches(entry[key]):
|
||||
return True
|
||||
|
||||
env = parsed.get("env")
|
||||
if isinstance(env, dict):
|
||||
for value in env.values():
|
||||
if _env_value_matches(value):
|
||||
return True
|
||||
|
||||
secrets_required = parsed.get("secrets_required")
|
||||
if isinstance(secrets_required, list):
|
||||
for entry in secrets_required:
|
||||
if isinstance(entry, dict) and entry.get("name") in FORGE_CREDENTIAL_NAMES:
|
||||
return True
|
||||
|
||||
ssh_hosts = parsed.get("ssh_hosts")
|
||||
if isinstance(ssh_hosts, list):
|
||||
for entry in ssh_hosts:
|
||||
if isinstance(entry, dict) and entry.get("identity_secret") in FORGE_CREDENTIAL_NAMES:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def layers_with_forge_credentials(layers_root: Path) -> list[tuple[str, Path]]:
|
||||
"""Walk every ``harnesses/contexts/*/v*/harness.yaml`` under *layers_root*.
|
||||
|
||||
Returns ``(layer_name, yaml_path)`` for each layer with forge credentials
|
||||
per the rules above.
|
||||
"""
|
||||
out: list[tuple[str, Path]] = []
|
||||
contexts_dir = layers_root / "contexts"
|
||||
if not contexts_dir.is_dir():
|
||||
return out
|
||||
|
||||
for yaml_path in sorted(contexts_dir.glob("*/v*/harness.yaml")):
|
||||
try:
|
||||
parsed = yaml.safe_load(yaml_path.read_text(encoding="utf-8"))
|
||||
except (OSError, yaml.YAMLError):
|
||||
continue
|
||||
if not isinstance(parsed, dict):
|
||||
continue
|
||||
if _layer_has_forge_credentials(parsed):
|
||||
name = str(parsed.get("name") or yaml_path.parent.parent.name)
|
||||
out.append((name, yaml_path))
|
||||
|
||||
return out
|
||||
150
tests/invariants/test_forge_credentials_declare_write_targets.py
Normal file
150
tests/invariants/test_forge_credentials_declare_write_targets.py
Normal file
@@ -0,0 +1,150 @@
|
||||
"""HC-11 CI invariant (bug b1de17b4, 2026-09-13 amendment): every harness layer
|
||||
carrying forge (git host) credentials MUST explicitly declare
|
||||
``write_targets`` (an empty list is a valid, deliberate declaration; an
|
||||
absent key is not).
|
||||
|
||||
Spec: /home/paul/dev/claude/projects/agent-runtimes/spec/harness.md HC-11.
|
||||
Runs in this repo's CI workflow (.gitea/workflows/) on every push to main.
|
||||
|
||||
Not committed by the authoring agent per its instructions — written here so
|
||||
it can run locally against the current tree.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from tests.invariants.forge_credentials import layers_with_forge_credentials
|
||||
|
||||
_HARNESSES_ROOT = Path(__file__).resolve().parents[2] / "harnesses"
|
||||
|
||||
|
||||
def test_hc11_framework_layers_with_forge_secrets_found_nonzero():
|
||||
"""The invariant is non-vacuous: the helper must find at least one
|
||||
forge-credential layer on the real tree (helper/closed-set regression
|
||||
otherwise, not evidence the repo has no forge-credential layers).
|
||||
|
||||
Not xfail: this passes today. On the real tree the helper currently
|
||||
finds 4 layers: gitea-admin, gitea-ssh, gitea-ssh-accelerators,
|
||||
gitea-ssh-homelab (via secrets_required[].name / ssh_hosts[].identity_secret).
|
||||
"""
|
||||
found = layers_with_forge_credentials(_HARNESSES_ROOT)
|
||||
assert found, "expected >=1 forge-credential layer on the real tree, found 0"
|
||||
|
||||
|
||||
@pytest.mark.xfail(strict=True, reason="HC-11 not implemented")
|
||||
def test_hc11_framework_layers_with_forge_secrets_declare_write_targets():
|
||||
found = layers_with_forge_credentials(_HARNESSES_ROOT)
|
||||
assert found, "expected >=1 forge-credential layer on the real tree, found 0"
|
||||
|
||||
missing = []
|
||||
for name, yaml_path in found:
|
||||
parsed = yaml.safe_load(yaml_path.read_text(encoding="utf-8"))
|
||||
if "write_targets" not in parsed:
|
||||
missing.append((name, str(yaml_path)))
|
||||
|
||||
# As of this session none of the 4 real layers declare write_targets
|
||||
# (HC-11 has not landed in the framework repo yet), so this is expected
|
||||
# to fail today — xfail(strict) so it flips to a real pass the moment
|
||||
# every forge-credential layer gets a write_targets key.
|
||||
assert not missing, (
|
||||
"harness layers with forge credentials must declare write_targets "
|
||||
f"(even an empty list): {missing}"
|
||||
)
|
||||
|
||||
|
||||
def test_hc11_helper_detects_a_mounts_secret_fixture(tmp_path):
|
||||
"""Sanity check the helper against the mounts[].secret carrier shape."""
|
||||
contexts = tmp_path / "contexts" / "fixture-layer" / "v1"
|
||||
contexts.mkdir(parents=True)
|
||||
(contexts / "harness.yaml").write_text(
|
||||
"""
|
||||
kind: context
|
||||
name: fixture-layer
|
||||
version: 1
|
||||
mounts:
|
||||
- secret: GITEA_TOKEN
|
||||
path: /run/agent/secrets/gitea-token
|
||||
"""
|
||||
)
|
||||
found = layers_with_forge_credentials(tmp_path)
|
||||
assert [name for name, _ in found] == ["fixture-layer"]
|
||||
|
||||
|
||||
def test_hc11_helper_detects_env_whole_token_fixture(tmp_path):
|
||||
contexts = tmp_path / "contexts" / "fixture-env-layer" / "v1"
|
||||
contexts.mkdir(parents=True)
|
||||
(contexts / "harness.yaml").write_text(
|
||||
"""
|
||||
kind: context
|
||||
name: fixture-env-layer
|
||||
version: 1
|
||||
env:
|
||||
SOME_VAR: "prefix_AGENT_REPO_TOKEN_suffix"
|
||||
OTHER_VAR: "AGENT_REPO_TOKEN"
|
||||
"""
|
||||
)
|
||||
found = layers_with_forge_credentials(tmp_path)
|
||||
names = [name for name, _ in found]
|
||||
assert names == ["fixture-env-layer"]
|
||||
|
||||
|
||||
def test_hc11_helper_detects_secrets_required_carrier_fixture(tmp_path):
|
||||
"""secrets_required[].name is a mandatory carrier per the amended spec —
|
||||
this is the shape gitea-admin/v1 actually uses on the real tree."""
|
||||
contexts = tmp_path / "contexts" / "fixture-secrets-required" / "v1"
|
||||
contexts.mkdir(parents=True)
|
||||
(contexts / "harness.yaml").write_text(
|
||||
"""
|
||||
kind: context
|
||||
name: fixture-secrets-required
|
||||
version: 1
|
||||
secrets_required:
|
||||
- name: gitea-admin
|
||||
account_ref: "admin.space:platform"
|
||||
mount_path: /run/agent/secrets/gitea-admin
|
||||
"""
|
||||
)
|
||||
found = layers_with_forge_credentials(tmp_path)
|
||||
assert [name for name, _ in found] == ["fixture-secrets-required"]
|
||||
|
||||
|
||||
def test_hc11_helper_detects_ssh_hosts_identity_secret_carrier_fixture(tmp_path):
|
||||
"""ssh_hosts[].identity_secret is the other mandatory carrier per the
|
||||
amended spec — the shape gitea-ssh/v1 (etc.) actually use."""
|
||||
contexts = tmp_path / "contexts" / "fixture-ssh-hosts" / "v1"
|
||||
contexts.mkdir(parents=True)
|
||||
(contexts / "harness.yaml").write_text(
|
||||
"""
|
||||
kind: context
|
||||
name: fixture-ssh-hosts
|
||||
version: 1
|
||||
ssh_hosts:
|
||||
- alias: gitea.oreillyit.nz-ai-enablement
|
||||
hostname: gitea.oreillyit.nz
|
||||
identity_secret: SSH_KEY_AI_ENABLEMENT
|
||||
"""
|
||||
)
|
||||
found = layers_with_forge_credentials(tmp_path)
|
||||
assert [name for name, _ in found] == ["fixture-ssh-hosts"]
|
||||
|
||||
|
||||
def test_hc11_helper_ignores_jinja_and_comments(tmp_path):
|
||||
contexts = tmp_path / "contexts" / "clean-layer" / "v1"
|
||||
contexts.mkdir(parents=True)
|
||||
(contexts / "harness.yaml").write_text(
|
||||
"""
|
||||
# GITEA_TOKEN mentioned only in a comment — must not match.
|
||||
kind: context
|
||||
name: clean-layer
|
||||
version: 1
|
||||
env:
|
||||
TEMPLATE_VAR: "{{ GITEA_TOKEN }}"
|
||||
"""
|
||||
)
|
||||
# Jinja placeholders are never matched per HC-11 text.
|
||||
found = layers_with_forge_credentials(tmp_path)
|
||||
names = [name for name, _ in found]
|
||||
assert names == []
|
||||
Reference in New Issue
Block a user