f5968cfab5
Brings the framework CRS repo up to date with all content that was living in agent-runtimes (local-dev fallback) but hadn't been promoted. New composites: feature-delivery-loop, integration-direct, scaffolding-repo, sonnet-impl-narrow, sonnet-manager, test-writing-repo New contexts: integration/v1, scaffolding/v1, sonnet-manager/v1, z-ai/v1, airouter/v1/bin (anthropic-compat-wrapper.sh), cp-harness/v1/init.sh New task-templates: sonnet-integrator.yaml, workflow/* (17 typed workflow task templates for the Epic 1 pipeline) Updated: agent-repo/v1/finalize.sh — adds AR-38/F97 empty-deliverable audit (SKIP_BRANCH_PUSH support, boilerplate-path filtering, ci_metadata.json flag) Also adds MEMORY.md index and memory/ topic files for the framework repo.
429 lines
16 KiB
Bash
429 lines
16 KiB
Bash
#!/bin/bash
|
|
# Agent repo finalize script — auto-commit and push changes
|
|
# AR-19, AR-20, AR-8, AR-32, AR-38, F66, F97, BUG-5, BUG-20
|
|
set -euo pipefail
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# AR-38 / F97: Post-commit empty-deliverable audit configuration
|
|
# Paths matching this regex are treated as boilerplate / metadata only —
|
|
# a commit containing ONLY paths matching this pattern is rejected.
|
|
# Overridable so harnesses with a different notion of "trivial" can adjust
|
|
# without forking this script.
|
|
# ---------------------------------------------------------------------------
|
|
DEFAULT_BOILERPLATE_PATHS_REGEX='^(\.gitignore|AGENTS\.md|ci_metadata\.json|memory/log/.*|\.agent-output/.*)$'
|
|
BOILERPLATE_PATHS_REGEX="${AGENT_BOILERPLATE_PATHS_REGEX:-$DEFAULT_BOILERPLATE_PATHS_REGEX}"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# F66: Content-based secret scanning
|
|
# Scans staged diff for credential patterns before committing.
|
|
# Returns 0 if clean, 1 if secrets detected.
|
|
# ---------------------------------------------------------------------------
|
|
scan_for_secrets() {
|
|
local diff found=0
|
|
diff=$(cat)
|
|
|
|
# Extract only added lines (skip diff headers +++)
|
|
local added
|
|
added=$(printf '%s\n' "$diff" | grep '^+' | grep -v '^+++' || true)
|
|
|
|
if [[ -z "$added" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
# Pattern 1: AWS access key — AKIA followed by exactly 16 uppercase letters/digits
|
|
if printf '%s\n' "$added" | grep -qE 'AKIA[0-9A-Z]{16}'; then
|
|
echo "AWS access key pattern detected (AKIA...)"
|
|
found=1
|
|
fi
|
|
|
|
# Pattern 2: PEM private key header (RSA, EC, OPENSSH, etc.)
|
|
if printf '%s\n' "$added" | grep -qE -- '-----BEGIN .* PRIVATE KEY-----'; then
|
|
echo "PEM private key header detected"
|
|
found=1
|
|
fi
|
|
|
|
# Pattern 3: High-entropy credential assignment (40+ char value near key/token/secret keywords)
|
|
if printf '%s\n' "$added" | grep -qiE \
|
|
'(secret[._]?key|api[._]?key|access[._]?key|auth[._]?token|\bsecret\b|\btoken\b|\bpassword\b|\bpasswd\b)[[:space:]]*[=:][[:space:]]*['"'"'""]?[A-Za-z0-9+/=_-]{40,}'; then
|
|
echo "High-entropy credential assignment detected"
|
|
found=1
|
|
fi
|
|
|
|
return $found
|
|
}
|
|
|
|
# Allow sourcing for unit tests without running main
|
|
if [[ "${BASH_SOURCE[0]}" != "${0}" ]]; then
|
|
return 0 2>/dev/null || true
|
|
fi
|
|
|
|
echo "=== agent-repo/v1 finalize.sh ==="
|
|
set -x # BUG-20: trace all commands for diagnostic visibility
|
|
|
|
WORKING_DIR="${AGENT_WORKING_DIR:-/workspace/project}"
|
|
AGENT_OUTPUT_DIR="${AGENT_OUTPUT_DIR:-/workspace/.agent-output}"
|
|
METADATA_FILE="$AGENT_OUTPUT_DIR/ci_metadata.json"
|
|
AGENT_EXIT="${AGENT_EXIT_CODE:-0}"
|
|
TASK_ID="${AGENT_TASK_ID:-unknown}"
|
|
BRANCH="${AGENT_BRANCH:-}"
|
|
REPO_URL="${AGENT_REPO_URL:-}"
|
|
# BUG-20: Push retry
|
|
PUSH_RETRIES="${AGENT_PUSH_RETRIES:-1}"
|
|
# When true, skip the branch push entirely (e.g. direct-push integrators that
|
|
# push to origin main themselves rather than using branch persistence).
|
|
SKIP_BRANCH_PUSH="${AGENT_SKIP_BRANCH_PUSH:-false}"
|
|
|
|
mkdir -p "$AGENT_OUTPUT_DIR"
|
|
|
|
# AR-19: Mark working dir as safe (finalize may run as different user than agent)
|
|
git config --global --add safe.directory "$WORKING_DIR"
|
|
|
|
# Change to working directory
|
|
cd "$WORKING_DIR" || {
|
|
echo "ERROR: Failed to change to working directory: $WORKING_DIR" >&2
|
|
exit 1
|
|
}
|
|
|
|
# Verify this is a git repository (defensive: catches misconfigured payloads
|
|
# where agent_repo field is missing but agent-repo harness is in the composite)
|
|
if [ ! -d ".git" ]; then
|
|
echo "ERROR: $WORKING_DIR is not a git repository." >&2
|
|
echo "Hint: Ensure the task payload includes 'agent_repo: {repo_url: ..., branch: ...}'" >&2
|
|
echo " so init.sh clones the repo before the agent runs." >&2
|
|
exit 1
|
|
fi
|
|
|
|
# AR-32: Restore .gitignore from harness template to prevent removing entries
|
|
HARNESS_GITIGNORE="/opt/harness/contexts/agent-repo/v1/.gitignore_template"
|
|
if [ -f "$HARNESS_GITIGNORE" ]; then
|
|
cat "$HARNESS_GITIGNORE" >> .gitignore
|
|
fi
|
|
|
|
# AR-37/F73-10: Exclude protected test folders from commit.
|
|
# PROTECTED_TEST_FOLDERS is set by entrypoint.py when test_pass_required is active.
|
|
if [ -n "${PROTECTED_TEST_FOLDERS:-}" ]; then
|
|
IFS=',' read -ra _PROTECTED_FOLDERS <<< "$PROTECTED_TEST_FOLDERS"
|
|
for _folder in "${_PROTECTED_FOLDERS[@]}"; do
|
|
_folder=$(echo "$_folder" | xargs) # trim whitespace
|
|
if [ -n "$_folder" ]; then
|
|
echo "F73: Excluding test folder from commit: $_folder"
|
|
git reset HEAD -- "$_folder" 2>/dev/null || true
|
|
git checkout HEAD -- "$_folder" 2>/dev/null || true
|
|
git clean -fd -- "$_folder" 2>/dev/null || true
|
|
fi
|
|
done
|
|
# Also exclude test infrastructure files
|
|
for _infra in conftest.py pyproject.toml pytest.ini setup.cfg tox.ini; do
|
|
git reset HEAD -- "$_infra" 2>/dev/null || true
|
|
git checkout HEAD -- "$_infra" 2>/dev/null || true
|
|
git clean -fd -- "$_infra" 2>/dev/null || true
|
|
done
|
|
fi
|
|
|
|
# AR-8: Check if there are any changes to commit
|
|
echo "Running: git add -A"
|
|
git add -A
|
|
|
|
# BUG-5 correction: detect wrong-path writes to .agent-output/
|
|
# These files are gitignored and will not be committed — move them to working dir
|
|
if [ -d "$WORKING_DIR/.agent-output" ]; then
|
|
# Find files in .agent-output/ that are NOT metadata/sentinel files
|
|
CORRECTION_FILES=$(ls "$WORKING_DIR/.agent-output/" 2>/dev/null | grep -v -E '^(ci_metadata|finalize-error|correction-prompt|pre-test-result)' || true)
|
|
if [ -n "$CORRECTION_FILES" ]; then
|
|
echo "WARNING: Files found in .agent-output/ — moving to working directory for commit"
|
|
echo "Offending files:"
|
|
echo "$CORRECTION_FILES"
|
|
for _f in $CORRECTION_FILES; do
|
|
if [ -f "$WORKING_DIR/.agent-output/$_f" ]; then
|
|
cp "$WORKING_DIR/.agent-output/$_f" "$WORKING_DIR/$_f"
|
|
echo " Moved: $_f -> $WORKING_DIR/$_f"
|
|
fi
|
|
done
|
|
# Write correction notice that will trigger re-invoke in entrypoint
|
|
cat > "$AGENT_OUTPUT_DIR/.correction-prompt.txt" << 'CORRECTION'
|
|
NOTE: You wrote output to /workspace/.agent-output/ instead of /workspace/project/.
|
|
Those files are gitignored and will NOT be committed automatically.
|
|
|
|
The following files have been moved to /workspace/project/ and WILL be committed:
|
|
CORRECTION
|
|
echo "$CORRECTION_FILES" >> "$AGENT_OUTPUT_DIR/.correction-prompt.txt"
|
|
cat >> "$AGENT_OUTPUT_DIR/.correction-prompt.txt" << 'CORRECTION'
|
|
|
|
ALWAYS write deliverable output directly to /workspace/project/<target-path>.
|
|
Never use /workspace/.agent-output/ when /workspace/project/ exists.
|
|
CORRECTION
|
|
echo "Correction prompt written to $AGENT_OUTPUT_DIR/.correction-prompt.txt"
|
|
fi
|
|
fi
|
|
|
|
# Check again after potential moves
|
|
if [ -z "$(git status --porcelain)" ]; then
|
|
echo "No changes to commit — skipping push (AR-8)"
|
|
# AR-38 / F97: AR-8 no-commit path is semantically equivalent to
|
|
# "empty deliverable" — the agent produced no work product at all.
|
|
# When the audit is enabled, surface this as empty_deliverable=true so
|
|
# operators triaging via ci_metadata.json can distinguish "agent did
|
|
# nothing" from "audit disabled by operator config" (the only path
|
|
# that produces an absent key).
|
|
AR8_EMPTY_DELIVERABLE_ENABLED="true"
|
|
if [ "${AGENT_EMPTY_DELIVERABLE_CHECK:-true}" = "false" ]; then
|
|
AR8_EMPTY_DELIVERABLE_ENABLED="false"
|
|
fi
|
|
python3 -c "
|
|
import json, os
|
|
out = '$METADATA_FILE'
|
|
meta = {}
|
|
if os.path.isfile(out):
|
|
try:
|
|
with open(out) as f:
|
|
meta = json.load(f)
|
|
except Exception:
|
|
meta = {}
|
|
meta['agent_branch'] = '$BRANCH'
|
|
meta['agent_repo_url'] = '$REPO_URL'
|
|
meta['agent_branch_pushed'] = False
|
|
if '$AR8_EMPTY_DELIVERABLE_ENABLED' == 'true':
|
|
meta['empty_deliverable'] = True
|
|
with open(out, 'w') as f:
|
|
json.dump(meta, f)
|
|
print('Wrote ci_metadata.json (no-op: no changes)')
|
|
"
|
|
exit 0
|
|
fi
|
|
|
|
# F66: Scan staged content for secrets before committing
|
|
SCAN_FINDINGS=""
|
|
SCAN_EXIT=0
|
|
set +e
|
|
SCAN_FINDINGS=$(git diff --cached | scan_for_secrets)
|
|
SCAN_EXIT=$?
|
|
set -e
|
|
|
|
if [ "$SCAN_EXIT" -ne 0 ]; then
|
|
echo "SECRET SCAN BLOCKED COMMIT — credentials detected in staged content"
|
|
echo "$SCAN_FINDINGS"
|
|
python3 -c "
|
|
import json, os
|
|
out = '$METADATA_FILE'
|
|
meta = {}
|
|
if os.path.isfile(out):
|
|
try:
|
|
with open(out) as f:
|
|
meta = json.load(f)
|
|
except Exception:
|
|
meta = {}
|
|
meta['agent_branch'] = '$BRANCH'
|
|
meta['agent_repo_url'] = '$REPO_URL'
|
|
meta['agent_branch_pushed'] = False
|
|
meta['secret_scan_blocked'] = True
|
|
meta['scan_findings'] = '''$SCAN_FINDINGS'''
|
|
with open(out, 'w') as f:
|
|
json.dump(meta, f)
|
|
print('Wrote ci_metadata.json (blocked by secret scan)')
|
|
"
|
|
exit 0
|
|
fi
|
|
|
|
# V3: Diff size check (warning only, does not block push)
|
|
DIFF_LINES=$(git diff --cached --stat | tail -1 | grep -oP '\d+(?= insertion)' || echo "0")
|
|
DIFF_KB=$(echo "$DIFF_LINES" | awk '{printf "%.1f", $1/1000}')
|
|
MAX_DIFF_KB="${AGENT_MAX_DIFF_KB:-500}"
|
|
if awk "BEGIN {exit !($DIFF_KB > $MAX_DIFF_KB)}"; then
|
|
echo "WARNING: Large diff detected: ${DIFF_KB}KB (threshold: ${MAX_DIFF_KB}KB)"
|
|
fi
|
|
|
|
# AR-6, AR-18: Build commit message based on agent outcome
|
|
if [ "$AGENT_EXIT" = "0" ]; then
|
|
OUTCOME="succeeded"
|
|
else
|
|
OUTCOME="failed (exit $AGENT_EXIT)"
|
|
fi
|
|
|
|
# Get first line of prompt (safe, via tempfile)
|
|
PROMPT_FIRST_LINE="${AGENT_PROMPT:-}"
|
|
if [ -n "$PROMPT_FIRST_LINE" ]; then
|
|
PROMPT_FIRST_LINE=$(echo "$PROMPT_FIRST_LINE" | head -1 | cut -c1-100)
|
|
fi
|
|
|
|
# Write commit message to tempfile (AR-19: safe handling)
|
|
COMMIT_MSG_FILE=$(mktemp)
|
|
trap "rm -f $COMMIT_MSG_FILE" EXIT
|
|
|
|
if [ -n "$PROMPT_FIRST_LINE" ]; then
|
|
echo "Agent task $TASK_ID ($OUTCOME): $PROMPT_FIRST_LINE" > "$COMMIT_MSG_FILE"
|
|
else
|
|
echo "Agent task $TASK_ID ($OUTCOME)" > "$COMMIT_MSG_FILE"
|
|
fi
|
|
|
|
# Commit
|
|
echo "Running: git commit"
|
|
git commit -F "$COMMIT_MSG_FILE"
|
|
|
|
# Get commit SHA
|
|
COMMIT_SHA=$(git rev-parse HEAD)
|
|
echo "Committed as: $COMMIT_SHA"
|
|
|
|
# AR-38 / F97: Post-commit empty-deliverable audit ------------------------
|
|
# Inspect the committed diff. If every changed path is boilerplate
|
|
# (.gitignore, AGENTS.md, ci_metadata.json, memory/log/*, .agent-output/*),
|
|
# flag the commit as empty_deliverable so the operator can distinguish a
|
|
# real successful run from a no-op that happened to advance HEAD.
|
|
#
|
|
# Sentinel values for EMPTY_DELIVERABLE:
|
|
# "true" — audit ran, every committed path was boilerplate
|
|
# "false" — audit ran, at least one path was substantive
|
|
# "skipped" — audit disabled by AGENT_EMPTY_DELIVERABLE_CHECK=false
|
|
#
|
|
# Wrapped in `set +eo pipefail` because a piped grep that finds nothing
|
|
# returns 1, which would otherwise abort the script (see gotchas-agent-repo
|
|
# "AR-21 set-e/pipefail aborts finalize on diff-verify pipeline").
|
|
EMPTY_DELIVERABLE="false"
|
|
if [ "${AGENT_EMPTY_DELIVERABLE_CHECK:-true}" = "false" ]; then
|
|
echo "AR-38: empty-deliverable audit disabled by AGENT_EMPTY_DELIVERABLE_CHECK=false"
|
|
EMPTY_DELIVERABLE="skipped"
|
|
else
|
|
set +eo pipefail
|
|
COMMITTED_PATHS=$(git diff-tree --no-commit-id --name-only -r "$COMMIT_SHA")
|
|
SUBSTANTIVE_PATHS=$(printf '%s\n' "$COMMITTED_PATHS" \
|
|
| grep -v '^[[:space:]]*$' \
|
|
| grep -vxE "$BOILERPLATE_PATHS_REGEX")
|
|
set -eo pipefail
|
|
if [ -z "$SUBSTANTIVE_PATHS" ]; then
|
|
echo "ERROR: AR-38 empty-deliverable audit FAILED for commit $COMMIT_SHA" >&2
|
|
echo "ERROR: every committed path matched the boilerplate regex:" >&2
|
|
echo "ERROR: regex: $BOILERPLATE_PATHS_REGEX" >&2
|
|
echo "ERROR: committed paths:" >&2
|
|
printf '%s\n' "$COMMITTED_PATHS" | sed 's/^/ERROR: /' >&2
|
|
EMPTY_DELIVERABLE="true"
|
|
else
|
|
echo "AR-38: audit passed — substantive path(s) found in commit:"
|
|
printf '%s\n' "$SUBSTANTIVE_PATHS" | sed 's/^/ /'
|
|
fi
|
|
fi
|
|
|
|
# BUG-20: Output validation — check expected output file exists and is non-empty
|
|
# AGENT_EXPECTED_OUTPUT: absolute path to required output file (e.g., /workspace/project/spec/f94-auth.md)
|
|
# AGENT_MIN_OUTPUT_BYTES: minimum size in bytes (default 0 = any non-empty)
|
|
OUTPUT_VALIDATED=true
|
|
OUTPUT_MISSING=""
|
|
if [ -n "${AGENT_EXPECTED_OUTPUT:-}" ]; then
|
|
echo "Validating expected output: $AGENT_EXPECTED_OUTPUT"
|
|
if [ ! -f "$AGENT_EXPECTED_OUTPUT" ]; then
|
|
echo "ERROR: Expected output file missing: $AGENT_EXPECTED_OUTPUT"
|
|
OUTPUT_VALIDATED=false
|
|
OUTPUT_MISSING="$AGENT_EXPECTED_OUTPUT"
|
|
else
|
|
SIZE=$(stat -c%s "$AGENT_EXPECTED_OUTPUT" 2>/dev/null || stat -f%z "$AGENT_EXPECTED_OUTPUT" 2>/dev/null || echo "0")
|
|
MIN_SIZE="${AGENT_MIN_OUTPUT_BYTES:-0}"
|
|
if [ "$SIZE" -lt "$MIN_SIZE" ]; then
|
|
echo "WARNING: Output file $AGENT_EXPECTED_OUTPUT is ${SIZE} bytes (minimum: ${MIN_SIZE})"
|
|
else
|
|
echo "Output validated: $AGENT_EXPECTED_OUTPUT (${SIZE} bytes)"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# AGENT_SKIP_BRANCH_PUSH: direct-push agents (e.g. integrators) push to origin
|
|
# main themselves and don't want a task branch created as a side-effect.
|
|
if [ "$SKIP_BRANCH_PUSH" = "true" ]; then
|
|
echo "AGENT_SKIP_BRANCH_PUSH=true — skipping task branch push"
|
|
python3 -c "
|
|
import json, os
|
|
out = '$METADATA_FILE'
|
|
meta = {}
|
|
if os.path.isfile(out):
|
|
try:
|
|
with open(out) as f:
|
|
meta = json.load(f)
|
|
except Exception:
|
|
meta = {}
|
|
meta['agent_branch'] = '$BRANCH'
|
|
meta['agent_repo_url'] = '$REPO_URL'
|
|
meta['agent_branch_pushed'] = False
|
|
meta['direct_push_mode'] = True
|
|
with open(out, 'w') as f:
|
|
json.dump(meta, f)
|
|
print('Wrote ci_metadata.json (direct_push_mode: branch push skipped)')
|
|
"
|
|
echo "=== agent-repo/v1 finalize.sh complete (direct_push_mode) ==="
|
|
exit 0
|
|
fi
|
|
|
|
# AR-19: Push with retry — attempt up to PUSH_RETRIES+1 times (default 2: initial + 1 retry)
|
|
echo "Pushing branch $BRANCH to $REPO_URL..."
|
|
PUSHED=false
|
|
PUSH_EXIT=0
|
|
for attempt in $(seq 1 $((PUSH_RETRIES + 1))); do
|
|
if [ "$attempt" -gt 1 ]; then
|
|
echo "Push attempt $attempt — sleeping 5s before retry"
|
|
sleep 5
|
|
echo "Retry $attempt: git push $REPO_URL HEAD:refs/heads/$BRANCH --force"
|
|
fi
|
|
if timeout 120 git push "$REPO_URL" "HEAD:refs/heads/$BRANCH" --force 2>&1; then
|
|
echo "Push succeeded (attempt $attempt)"
|
|
PUSHED=true
|
|
PUSH_EXIT=0
|
|
break
|
|
else
|
|
PUSH_EXIT=$?
|
|
echo "Push attempt $attempt failed with exit code $PUSH_EXIT"
|
|
if [ "$attempt" -lt $((PUSH_RETRIES + 1)) ]; then
|
|
echo "Will retry..."
|
|
fi
|
|
fi
|
|
done
|
|
|
|
# Log committed files for diagnosis
|
|
echo "Files in commit:"
|
|
git diff-tree --no-commit-id --name-only -r "$COMMIT_SHA" | while read f; do echo " $f"; done
|
|
|
|
if [ "$PUSHED" = "false" ]; then
|
|
echo "ERROR: Push failed after $((PUSH_RETRIES + 1)) attempt(s) — branch $BRANCH was committed but not pushed" >&2
|
|
fi
|
|
|
|
# Write ci_metadata.json
|
|
python3 -c "
|
|
import json, os
|
|
out = '$METADATA_FILE'
|
|
meta = {}
|
|
if os.path.isfile(out):
|
|
try:
|
|
with open(out) as f:
|
|
meta = json.load(f)
|
|
except Exception:
|
|
meta = {}
|
|
meta['agent_branch'] = '$BRANCH'
|
|
meta['agent_sha'] = '$COMMIT_SHA'
|
|
meta['agent_repo_url'] = '$REPO_URL'
|
|
meta['agent_branch_pushed'] = $( [ '$PUSHED' = 'true' ] && echo 'True' || echo 'False' )
|
|
meta['diff_kb'] = float('$DIFF_KB') if '$DIFF_KB' else 0.0
|
|
meta['output_validated'] = $( [ '$OUTPUT_VALIDATED' = 'true' ] && echo 'True' || echo 'False' )
|
|
if '$OUTPUT_MISSING':
|
|
meta['output_missing'] = '$OUTPUT_MISSING'
|
|
# AR-38 / F97: record empty-deliverable audit outcome.
|
|
# Only emit the key when the audit ran. Skipped state omits the key so that
|
|
# downstream consumers can tell apart ran-and-passed from did-not-run.
|
|
ed = '$EMPTY_DELIVERABLE'
|
|
if ed == 'true':
|
|
meta['empty_deliverable'] = True
|
|
elif ed == 'false':
|
|
meta['empty_deliverable'] = False
|
|
with open(out, 'w') as f:
|
|
json.dump(meta, f)
|
|
print('Wrote ci_metadata.json')
|
|
"
|
|
|
|
# AR-38 / F97: exit non-zero (dedicated code 2) when audit flagged the commit.
|
|
# Take precedence over push failure (code 1) — empty deliverable is the more
|
|
# actionable signal for the operator. Exit only after push attempt above so
|
|
# the commit is still pushed for forensics.
|
|
if [ "$EMPTY_DELIVERABLE" = "true" ]; then
|
|
echo "ERROR: exiting 2 — AR-38 empty-deliverable audit failed" >&2
|
|
exit 2
|
|
fi
|
|
|
|
if [ "$PUSHED" = "false" ]; then
|
|
exit 1
|
|
fi
|
|
|
|
echo "=== agent-repo/v1 finalize.sh complete ===" |