BUG-5/BUG-20 fixes: pre_test.sh, output validation, push retry, workflow output tags
- pre_test.sh: harness-level test folder revert script, replaces inline
_revert_test_folders() in entrypoint. Writes {"reverted": N} atomically.
- finalize.sh: set -x debug tracing; push retry (AGENT_PUSH_RETRIES, default 1
retry after 5s); output validation (AGENT_EXPECTED_OUTPUT env var); wrong-path
detection moves .agent-output/ files to working dir and writes
.correction-prompt.txt for entrypoint re-invoke
- harness.yaml: add scripts.pre_test
- spec-planning.yaml: all 17 nodes tagged with output:{path,min_bytes}
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Paul O'Reilly
@@ -1,6 +1,6 @@
|
||||
#!/bin/bash
|
||||
# Agent repo finalize script — auto-commit and push changes
|
||||
# AR-19, AR-20, AR-8, AR-32, F66
|
||||
# AR-19, AR-20, AR-8, AR-32, F66, BUG-5, BUG-20
|
||||
set -euo pipefail
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -48,6 +48,7 @@ if [[ "${BASH_SOURCE[0]}" != "${0}" ]]; then
|
||||
fi
|
||||
|
||||
echo "=== agent-repo/v1 finalize.sh ==="
|
||||
set -x # BUG-20: trace all commands for diagnostic visibility
|
||||
|
||||
WORKING_DIR="${AGENT_WORKING_DIR:-/workspace/project}"
|
||||
AGENT_OUTPUT_DIR="${AGENT_OUTPUT_DIR:-/workspace/.agent-output}"
|
||||
@@ -56,6 +57,8 @@ AGENT_EXIT="${AGENT_EXIT_CODE:-0}"
|
||||
TASK_ID="${AGENT_TASK_ID:-unknown}"
|
||||
BRANCH="${AGENT_BRANCH:-}"
|
||||
REPO_URL="${AGENT_REPO_URL:-}"
|
||||
# BUG-20: Push retry
|
||||
PUSH_RETRIES="${AGENT_PUSH_RETRIES:-1}"
|
||||
|
||||
mkdir -p "$AGENT_OUTPUT_DIR"
|
||||
|
||||
@@ -83,13 +86,8 @@ if [ -f "$HARNESS_GITIGNORE" ]; then
|
||||
cat "$HARNESS_GITIGNORE" >> .gitignore
|
||||
fi
|
||||
|
||||
# AR-8: Check if there are any changes to commit
|
||||
git add -A
|
||||
|
||||
# AR-37/F73-10: Exclude protected test folders from commit.
|
||||
# PROTECTED_TEST_FOLDERS is set by entrypoint.py when test_pass_required is active.
|
||||
# This is defense-in-depth: prevents test file changes reaching the agent branch even
|
||||
# if the entrypoint revert had a window between the last test run and finalize.
|
||||
if [ -n "${PROTECTED_TEST_FOLDERS:-}" ]; then
|
||||
IFS=',' read -ra _PROTECTED_FOLDERS <<< "$PROTECTED_TEST_FOLDERS"
|
||||
for _folder in "${_PROTECTED_FOLDERS[@]}"; do
|
||||
@@ -109,9 +107,45 @@ if [ -n "${PROTECTED_TEST_FOLDERS:-}" ]; then
|
||||
done
|
||||
fi
|
||||
|
||||
# AR-8: Check if there are any changes to commit
|
||||
echo "Running: git add -A"
|
||||
git add -A
|
||||
|
||||
# BUG-5 correction: detect wrong-path writes to .agent-output/
|
||||
# These files are gitignored and will not be committed — move them to working dir
|
||||
if [ -d "$WORKING_DIR/.agent-output" ]; then
|
||||
# Find files in .agent-output/ that are NOT metadata/sentinel files
|
||||
CORRECTION_FILES=$(ls "$WORKING_DIR/.agent-output/" 2>/dev/null | grep -v -E '^(ci_metadata|finalize-error|correction-prompt|pre-test-result)' || true)
|
||||
if [ -n "$CORRECTION_FILES" ]; then
|
||||
echo "WARNING: Files found in .agent-output/ — moving to working directory for commit"
|
||||
echo "Offending files:"
|
||||
echo "$CORRECTION_FILES"
|
||||
for _f in $CORRECTION_FILES; do
|
||||
if [ -f "$WORKING_DIR/.agent-output/$_f" ]; then
|
||||
cp "$WORKING_DIR/.agent-output/$_f" "$WORKING_DIR/$_f"
|
||||
echo " Moved: $_f -> $WORKING_DIR/$_f"
|
||||
fi
|
||||
done
|
||||
# Write correction notice that will trigger re-invoke in entrypoint
|
||||
cat > "$AGENT_OUTPUT_DIR/.correction-prompt.txt" << 'CORRECTION'
|
||||
NOTE: You wrote output to /workspace/.agent-output/ instead of /workspace/project/.
|
||||
Those files are gitignored and will NOT be committed automatically.
|
||||
|
||||
The following files have been moved to /workspace/project/ and WILL be committed:
|
||||
CORRECTION
|
||||
echo "$CORRECTION_FILES" >> "$AGENT_OUTPUT_DIR/.correction-prompt.txt"
|
||||
cat >> "$AGENT_OUTPUT_DIR/.correction-prompt.txt" << 'CORRECTION'
|
||||
|
||||
ALWAYS write deliverable output directly to /workspace/project/<target-path>.
|
||||
Never use /workspace/.agent-output/ when /workspace/project/ exists.
|
||||
CORRECTION
|
||||
echo "Correction prompt written to $AGENT_OUTPUT_DIR/.correction-prompt.txt"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check again after potential moves
|
||||
if [ -z "$(git status --porcelain)" ]; then
|
||||
echo "No changes to commit — skipping push (AR-8)"
|
||||
# Write metadata with agent_branch_pushed=false
|
||||
python3 -c "
|
||||
import json, os
|
||||
out = '$METADATA_FILE'
|
||||
@@ -143,7 +177,6 @@ set -e
|
||||
if [ "$SCAN_EXIT" -ne 0 ]; then
|
||||
echo "SECRET SCAN BLOCKED COMMIT — credentials detected in staged content"
|
||||
echo "$SCAN_FINDINGS"
|
||||
# Write metadata indicating scan blocked the commit
|
||||
python3 -c "
|
||||
import json, os
|
||||
out = '$METADATA_FILE'
|
||||
@@ -198,20 +231,65 @@ else
|
||||
fi
|
||||
|
||||
# Commit
|
||||
echo "Running: git commit"
|
||||
git commit -F "$COMMIT_MSG_FILE"
|
||||
|
||||
# Get commit SHA
|
||||
COMMIT_SHA=$(git rev-parse HEAD)
|
||||
echo "Committed as: $COMMIT_SHA"
|
||||
|
||||
# AR-19: Always push — even if tests failed, partial work is better than lost work.
|
||||
# The task exit code and ci_metadata.json track whether tests passed.
|
||||
# BUG-20: Output validation — check expected output file exists and is non-empty
|
||||
# AGENT_EXPECTED_OUTPUT: absolute path to required output file (e.g., /workspace/project/spec/f94-auth.md)
|
||||
# AGENT_MIN_OUTPUT_BYTES: minimum size in bytes (default 0 = any non-empty)
|
||||
OUTPUT_VALIDATED=true
|
||||
OUTPUT_MISSING=""
|
||||
if [ -n "${AGENT_EXPECTED_OUTPUT:-}" ]; then
|
||||
echo "Validating expected output: $AGENT_EXPECTED_OUTPUT"
|
||||
if [ ! -f "$AGENT_EXPECTED_OUTPUT" ]; then
|
||||
echo "ERROR: Expected output file missing: $AGENT_EXPECTED_OUTPUT"
|
||||
OUTPUT_VALIDATED=false
|
||||
OUTPUT_MISSING="$AGENT_EXPECTED_OUTPUT"
|
||||
else
|
||||
SIZE=$(stat -c%s "$AGENT_EXPECTED_OUTPUT" 2>/dev/null || stat -f%z "$AGENT_EXPECTED_OUTPUT" 2>/dev/null || echo "0")
|
||||
MIN_SIZE="${AGENT_MIN_OUTPUT_BYTES:-0}"
|
||||
if [ "$SIZE" -lt "$MIN_SIZE" ]; then
|
||||
echo "WARNING: Output file $AGENT_EXPECTED_OUTPUT is ${SIZE} bytes (minimum: ${MIN_SIZE})"
|
||||
else
|
||||
echo "Output validated: $AGENT_EXPECTED_OUTPUT (${SIZE} bytes)"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# AR-19: Push with retry — attempt up to PUSH_RETRIES+1 times (default 2: initial + 1 retry)
|
||||
echo "Pushing branch $BRANCH to $REPO_URL..."
|
||||
if timeout 120 git push "$REPO_URL" "HEAD:refs/heads/$BRANCH" --force; then
|
||||
echo "Push succeeded"
|
||||
PUSHED=true
|
||||
else
|
||||
echo "ERROR: Push failed" >&2
|
||||
PUSHED=false
|
||||
PUSHED=false
|
||||
PUSH_EXIT=0
|
||||
for attempt in $(seq 1 $((PUSH_RETRIES + 1))); do
|
||||
if [ "$attempt" -gt 1 ]; then
|
||||
echo "Push attempt $attempt — sleeping 5s before retry"
|
||||
sleep 5
|
||||
echo "Retry $attempt: git push $REPO_URL HEAD:refs/heads/$BRANCH --force"
|
||||
fi
|
||||
if timeout 120 git push "$REPO_URL" "HEAD:refs/heads/$BRANCH" --force 2>&1; then
|
||||
echo "Push succeeded (attempt $attempt)"
|
||||
PUSHED=true
|
||||
PUSH_EXIT=0
|
||||
break
|
||||
else
|
||||
PUSH_EXIT=$?
|
||||
echo "Push attempt $attempt failed with exit code $PUSH_EXIT"
|
||||
if [ "$attempt" -lt $((PUSH_RETRIES + 1)) ]; then
|
||||
echo "Will retry..."
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# Log committed files for diagnosis
|
||||
echo "Files in commit:"
|
||||
git diff-tree --no-commit-id --name-only -r "$COMMIT_SHA" | while read f; do echo " $f"; done
|
||||
|
||||
if [ "$PUSHED" = "false" ]; then
|
||||
echo "ERROR: Push failed after $((PUSH_RETRIES + 1)) attempt(s) — branch $BRANCH was committed but not pushed" >&2
|
||||
fi
|
||||
|
||||
# Write ci_metadata.json
|
||||
@@ -230,14 +308,16 @@ meta['agent_sha'] = '$COMMIT_SHA'
|
||||
meta['agent_repo_url'] = '$REPO_URL'
|
||||
meta['agent_branch_pushed'] = $( [ '$PUSHED' = 'true' ] && echo 'True' || echo 'False' )
|
||||
meta['diff_kb'] = float('$DIFF_KB') if '$DIFF_KB' else 0.0
|
||||
meta['output_validated'] = $( [ '$OUTPUT_VALIDATED' = 'true' ] && echo 'True' || echo 'False' )
|
||||
if '$OUTPUT_MISSING':
|
||||
meta['output_missing'] = '$OUTPUT_MISSING'
|
||||
with open(out, 'w') as f:
|
||||
json.dump(meta, f)
|
||||
print('Wrote ci_metadata.json')
|
||||
"
|
||||
|
||||
if [ "$PUSHED" = "false" ]; then
|
||||
echo "ERROR: Push failed — branch $BRANCH was committed but not pushed" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=== agent-repo/v1 finalize.sh complete ==="
|
||||
echo "=== agent-repo/v1 finalize.sh complete ==="
|
||||
@@ -8,6 +8,7 @@ provides: [project-repo]
|
||||
scripts:
|
||||
init: "./init.sh"
|
||||
finalize: "./finalize.sh"
|
||||
pre_test: "./pre_test.sh"
|
||||
|
||||
env:
|
||||
AGENT_WORKING_DIR: "/workspace/project"
|
||||
|
||||
64
harnesses/contexts/agent-repo/v1/pre_test.sh
Normal file
64
harnesses/contexts/agent-repo/v1/pre_test.sh
Normal file
@@ -0,0 +1,64 @@
|
||||
#!/bin/bash
|
||||
# pre_test.sh — Revert agent modifications to test folders before each test run
|
||||
# F73 / pre_test spec
|
||||
set -euo pipefail
|
||||
|
||||
echo "=== agent-repo/v1 pre_test.sh ==="
|
||||
|
||||
WORKING_DIR="${AGENT_WORKING_DIR:-/workspace/project}"
|
||||
AGENT_OUTPUT_DIR="${AGENT_OUTPUT_DIR:-/workspace/.agent-output}"
|
||||
PROTECTED_FOLDERS="${PROTECTED_TEST_FOLDERS:-}"
|
||||
|
||||
mkdir -p "$AGENT_OUTPUT_DIR"
|
||||
|
||||
if [ -z "$PROTECTED_FOLDERS" ]; then
|
||||
echo "No PROTECTED_TEST_FOLDERS set — nothing to revert"
|
||||
echo '{"reverted": 0}' > "$AGENT_OUTPUT_DIR/.pre-test-result.json"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
total_reverted=0
|
||||
|
||||
IFS=',' read -ra FOLDERS <<< "$PROTECTED_FOLDERS"
|
||||
for folder in "${FOLDERS[@]}"; do
|
||||
folder=$(echo "$folder" | xargs) # trim whitespace
|
||||
[ -z "$folder" ] && continue
|
||||
|
||||
echo "Reverting: $folder"
|
||||
|
||||
# Revert tracked modifications
|
||||
git -C "$WORKING_DIR" reset HEAD -- "$folder" 2>/dev/null || true
|
||||
git -C "$WORKING_DIR" checkout HEAD -- "$folder" 2>/dev/null || true
|
||||
|
||||
# Remove untracked files
|
||||
git -C "$WORKING_DIR" clean -fd -- "$folder" 2>/dev/null || true
|
||||
|
||||
# Count what was reverted
|
||||
modified=$(git -C "$WORKING_DIR" diff --name-only HEAD -- "$folder" 2>/dev/null | wc -l)
|
||||
untracked=$(git -C "$WORKING_DIR" ls-files --others --exclude-standard -- "$folder" 2>/dev/null | wc -l)
|
||||
count=$((modified + untracked))
|
||||
if [ "$count" -gt 0 ]; then
|
||||
echo " Reverted $count item(s) from $folder"
|
||||
total_reverted=$((total_reverted + count))
|
||||
fi
|
||||
done
|
||||
|
||||
# Also revert test infrastructure files
|
||||
for infra in conftest.py pyproject.toml pytest.ini setup.cfg tox.ini; do
|
||||
if git -C "$WORKING_DIR" diff --name-only HEAD -- "$infra" 2>/dev/null | grep -q .; then
|
||||
git -C "$WORKING_DIR" checkout HEAD -- "$infra" 2>/dev/null || true
|
||||
echo " Reverted infrastructure file: $infra"
|
||||
total_reverted=$((total_reverted + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Total reverted: $total_reverted"
|
||||
|
||||
# Atomic write via mktemp + mv (required contract)
|
||||
RESULT_FILE=$(mktemp)
|
||||
echo "{\"reverted\": $total_reverted}" > "$RESULT_FILE"
|
||||
mv "$RESULT_FILE" "$AGENT_OUTPUT_DIR/.pre-test-result.json"
|
||||
chmod 0644 "$AGENT_OUTPUT_DIR/.pre-test-result.json"
|
||||
|
||||
echo "=== pre_test.sh complete ==="
|
||||
exit 0
|
||||
Reference in New Issue
Block a user