Add validate-skill: SKILL.md linter for Claude Code skills
Checks for common issues that silently break skills — non-ASCII frontmatter, $VAR in paths, uncovered binaries in allowed-tools, etc. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Paul O'Reilly
381
tests/test-validate-skill.sh
Executable file
381
tests/test-validate-skill.sh
Executable file
@@ -0,0 +1,381 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
SCRIPT="$SCRIPT_DIR/../scripts/validate-skill"
|
||||
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
RESET='\033[0m'
|
||||
|
||||
PASS=0
|
||||
FAIL=0
|
||||
|
||||
assert_exit() {
|
||||
local desc="$1" expected="$2"
|
||||
shift 2
|
||||
local actual
|
||||
set +e
|
||||
"$@" >/dev/null 2>&1
|
||||
actual=$?
|
||||
set -e
|
||||
if [[ "$actual" -eq "$expected" ]]; then
|
||||
echo -e "${GREEN}PASS${RESET}: $desc"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
echo -e "${RED}FAIL${RESET}: $desc (expected exit $expected, got $actual)"
|
||||
FAIL=$((FAIL + 1))
|
||||
fi
|
||||
}
|
||||
|
||||
assert_contains() {
|
||||
local desc="$1" pattern="$2"
|
||||
shift 2
|
||||
local output
|
||||
set +e
|
||||
output=$("$@" 2>&1)
|
||||
set -e
|
||||
if echo "$output" | grep -qF "$pattern"; then
|
||||
echo -e "${GREEN}PASS${RESET}: $desc"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
echo -e "${RED}FAIL${RESET}: $desc (output missing: '$pattern')"
|
||||
echo " Got: $output"
|
||||
FAIL=$((FAIL + 1))
|
||||
fi
|
||||
}
|
||||
|
||||
assert_not_contains() {
|
||||
local desc="$1" pattern="$2"
|
||||
shift 2
|
||||
local output
|
||||
set +e
|
||||
output=$("$@" 2>&1)
|
||||
set -e
|
||||
if echo "$output" | grep -qF "$pattern"; then
|
||||
echo -e "${RED}FAIL${RESET}: $desc (output should NOT contain: '$pattern')"
|
||||
echo " Got: $output"
|
||||
FAIL=$((FAIL + 1))
|
||||
else
|
||||
echo -e "${GREEN}PASS${RESET}: $desc"
|
||||
PASS=$((PASS + 1))
|
||||
fi
|
||||
}
|
||||
|
||||
# Setup temp directory with test fixtures
|
||||
TMPDIR="$(mktemp -d)"
|
||||
trap 'rm -rf "$TMPDIR"' EXIT
|
||||
|
||||
# --- Fixture: valid skill ---
|
||||
mkdir -p "$TMPDIR/valid"
|
||||
cat > "$TMPDIR/valid/SKILL.md" <<'EOF'
|
||||
---
|
||||
name: test-skill
|
||||
description: A test skill for validation
|
||||
allowed-tools: Read, Bash(cat *), Bash(ls *), Bash(date *)
|
||||
---
|
||||
|
||||
# Test Skill
|
||||
|
||||
## Context
|
||||
!`cat README.md 2>/dev/null || echo "no readme"`
|
||||
!`ls -1 scripts/ 2>/dev/null || echo "no scripts"`
|
||||
!`date +%Y-%m-%d`
|
||||
EOF
|
||||
|
||||
# --- Fixture: missing frontmatter ---
|
||||
mkdir -p "$TMPDIR/no-frontmatter"
|
||||
cat > "$TMPDIR/no-frontmatter/SKILL.md" <<'EOF'
|
||||
# No frontmatter here
|
||||
Just some content.
|
||||
EOF
|
||||
|
||||
# --- Fixture: missing closing delimiter ---
|
||||
mkdir -p "$TMPDIR/no-close"
|
||||
cat > "$TMPDIR/no-close/SKILL.md" <<'EOF'
|
||||
---
|
||||
name: broken
|
||||
description: Missing closing delimiter
|
||||
EOF
|
||||
|
||||
# --- Fixture: bad name ---
|
||||
mkdir -p "$TMPDIR/bad-name"
|
||||
cat > "$TMPDIR/bad-name/SKILL.md" <<'EOF'
|
||||
---
|
||||
name: My_Skill!
|
||||
description: A skill with a bad name
|
||||
---
|
||||
|
||||
# Bad Name Skill
|
||||
EOF
|
||||
|
||||
# --- Fixture: missing name ---
|
||||
mkdir -p "$TMPDIR/no-name"
|
||||
cat > "$TMPDIR/no-name/SKILL.md" <<'EOF'
|
||||
---
|
||||
description: A skill without a name
|
||||
---
|
||||
|
||||
# No Name Skill
|
||||
EOF
|
||||
|
||||
# --- Fixture: missing description ---
|
||||
mkdir -p "$TMPDIR/no-desc"
|
||||
cat > "$TMPDIR/no-desc/SKILL.md" <<'EOF'
|
||||
---
|
||||
name: no-desc
|
||||
---
|
||||
|
||||
# No Description
|
||||
EOF
|
||||
|
||||
# --- Fixture: ${VAR} in bang-command ---
|
||||
mkdir -p "$TMPDIR/curly-var"
|
||||
cat > "$TMPDIR/curly-var/SKILL.md" <<'EOF'
|
||||
---
|
||||
name: curly-var
|
||||
description: Uses curly brace variable
|
||||
allowed-tools: Read, Bash(cat *)
|
||||
---
|
||||
|
||||
# Curly Var
|
||||
!`cat ${HOME}/some/path`
|
||||
EOF
|
||||
|
||||
# --- Fixture: $() in bang-command ---
|
||||
mkdir -p "$TMPDIR/subshell"
|
||||
cat > "$TMPDIR/subshell/SKILL.md" <<'EOF'
|
||||
---
|
||||
name: subshell
|
||||
description: Uses command substitution
|
||||
allowed-tools: Read, Bash(git *)
|
||||
---
|
||||
|
||||
# Subshell
|
||||
!`git log --since="$(date -d '7 days ago')"`
|
||||
EOF
|
||||
|
||||
# --- Fixture: ~/ in bang-command ---
|
||||
mkdir -p "$TMPDIR/tilde"
|
||||
cat > "$TMPDIR/tilde/SKILL.md" <<'EOF'
|
||||
---
|
||||
name: tilde
|
||||
description: Uses tilde path
|
||||
allowed-tools: Read, Bash(cat *)
|
||||
---
|
||||
|
||||
# Tilde
|
||||
!`cat ~/dev/claude/settings.yaml`
|
||||
EOF
|
||||
|
||||
# --- Fixture: ../ in bang-command ---
|
||||
mkdir -p "$TMPDIR/dotdot"
|
||||
cat > "$TMPDIR/dotdot/SKILL.md" <<'EOF'
|
||||
---
|
||||
name: dotdot
|
||||
description: Uses relative path
|
||||
allowed-tools: Read, Bash(cat *)
|
||||
---
|
||||
|
||||
# Dotdot
|
||||
!`cat ../claude-foundations/settings.yaml`
|
||||
EOF
|
||||
|
||||
# --- Fixture: uncovered binary ---
|
||||
mkdir -p "$TMPDIR/uncovered"
|
||||
cat > "$TMPDIR/uncovered/SKILL.md" <<'EOF'
|
||||
---
|
||||
name: uncovered
|
||||
description: Uses a binary not in allowed-tools
|
||||
allowed-tools: Read, Bash(cat *)
|
||||
---
|
||||
|
||||
# Uncovered
|
||||
!`cat README.md`
|
||||
!`head -5 README.md`
|
||||
EOF
|
||||
|
||||
# --- Fixture: /home/ hardcoded path (warning only) ---
|
||||
mkdir -p "$TMPDIR/home-path"
|
||||
cat > "$TMPDIR/home-path/SKILL.md" <<'EOF'
|
||||
---
|
||||
name: home-path
|
||||
description: Uses hardcoded home path
|
||||
allowed-tools: Read, Bash(cat *)
|
||||
---
|
||||
|
||||
# Home Path
|
||||
!`cat /home/paul/dev/claude/settings.yaml`
|
||||
EOF
|
||||
|
||||
# --- Fixture: broad Bash(git *) (warning only) ---
|
||||
mkdir -p "$TMPDIR/broad-git"
|
||||
cat > "$TMPDIR/broad-git/SKILL.md" <<'EOF'
|
||||
---
|
||||
name: broad-git
|
||||
description: Uses overly broad git pattern
|
||||
allowed-tools: Read, Bash(git *)
|
||||
---
|
||||
|
||||
# Broad Git
|
||||
!`git log --oneline -5`
|
||||
EOF
|
||||
|
||||
# --- Fixture: no bang-commands, no allowed-tools (valid) ---
|
||||
mkdir -p "$TMPDIR/no-bangs"
|
||||
cat > "$TMPDIR/no-bangs/SKILL.md" <<'EOF'
|
||||
---
|
||||
name: no-bangs
|
||||
description: A simple instruction-only skill
|
||||
---
|
||||
|
||||
# No Bang Commands
|
||||
|
||||
Just instructions, no dynamic context.
|
||||
EOF
|
||||
|
||||
# --- Fixture: $HOME in path (should be error) ---
|
||||
mkdir -p "$TMPDIR/dollar-home"
|
||||
cat > "$TMPDIR/dollar-home/SKILL.md" <<'EOF'
|
||||
---
|
||||
name: dollar-home
|
||||
description: Uses dollar home in path
|
||||
allowed-tools: Read, Bash(cat *)
|
||||
---
|
||||
|
||||
# Dollar Home
|
||||
!`cat $HOME/dev/claude/settings.yaml 2>/dev/null || echo "not found"`
|
||||
EOF
|
||||
|
||||
# --- Fixture: $VAR not in path context (should be fine) ---
|
||||
mkdir -p "$TMPDIR/dollar-nopath"
|
||||
cat > "$TMPDIR/dollar-nopath/SKILL.md" <<'EOF'
|
||||
---
|
||||
name: dollar-nopath
|
||||
description: Uses dollar var not as a path
|
||||
allowed-tools: Read, Bash(date *), Bash(echo *)
|
||||
---
|
||||
|
||||
# Dollar No Path
|
||||
!`date +%Y-%m-%d`
|
||||
!`echo $USER said hello`
|
||||
EOF
|
||||
|
||||
# --- Fixture: multi-line description with > ---
|
||||
mkdir -p "$TMPDIR/multiline-desc"
|
||||
cat > "$TMPDIR/multiline-desc/SKILL.md" <<'EOF'
|
||||
---
|
||||
name: multiline-desc
|
||||
description: >
|
||||
A skill with a multi-line description
|
||||
that spans multiple lines
|
||||
---
|
||||
|
||||
# Multi-line Description
|
||||
EOF
|
||||
|
||||
# --- Fixture: non-ASCII in frontmatter (em dash) ---
|
||||
mkdir -p "$TMPDIR/non-ascii"
|
||||
printf -- '---\nname: non-ascii\ndescription: >\n A skill that takes no action \xe2\x80\x94 information only\n---\n\n# Non-ASCII\n' > "$TMPDIR/non-ascii/SKILL.md"
|
||||
|
||||
# --- Fixture: bang-commands but no allowed-tools ---
|
||||
mkdir -p "$TMPDIR/bangs-no-tools"
|
||||
cat > "$TMPDIR/bangs-no-tools/SKILL.md" <<'EOF'
|
||||
---
|
||||
name: bangs-no-tools
|
||||
description: Has bang-commands but no allowed-tools
|
||||
---
|
||||
|
||||
# Bangs No Tools
|
||||
!`cat README.md`
|
||||
EOF
|
||||
|
||||
# ============================
|
||||
# Tests
|
||||
# ============================
|
||||
|
||||
echo "=== validate-skill tests ==="
|
||||
echo
|
||||
|
||||
echo "-- basic functionality --"
|
||||
assert_exit "--help exits 0" 0 "$SCRIPT" --help
|
||||
assert_contains "--help shows usage" "Usage:" "$SCRIPT" --help
|
||||
assert_exit "valid skill exits 0" 0 "$SCRIPT" "$TMPDIR/valid/SKILL.md"
|
||||
assert_contains "valid skill shows no issues" "No issues found" "$SCRIPT" "$TMPDIR/valid/SKILL.md"
|
||||
echo
|
||||
|
||||
echo "-- dryrun --"
|
||||
assert_exit "--dryrun always exits 0" 0 "$SCRIPT" --dryrun "$TMPDIR"
|
||||
assert_contains "--dryrun lists files" "[dryrun]" "$SCRIPT" --dryrun "$TMPDIR"
|
||||
echo
|
||||
|
||||
echo "-- frontmatter errors --"
|
||||
assert_exit "missing frontmatter exits 1" 1 "$SCRIPT" "$TMPDIR/no-frontmatter/SKILL.md"
|
||||
assert_contains "reports missing ---" "does not start with ---" "$SCRIPT" "$TMPDIR/no-frontmatter/SKILL.md"
|
||||
assert_exit "missing closing --- exits 1" 1 "$SCRIPT" "$TMPDIR/no-close/SKILL.md"
|
||||
assert_contains "reports missing closing ---" "Missing closing ---" "$SCRIPT" "$TMPDIR/no-close/SKILL.md"
|
||||
assert_exit "bad name exits 1" 1 "$SCRIPT" "$TMPDIR/bad-name/SKILL.md"
|
||||
assert_contains "reports invalid name" "invalid characters" "$SCRIPT" "$TMPDIR/bad-name/SKILL.md"
|
||||
assert_exit "missing name exits 1" 1 "$SCRIPT" "$TMPDIR/no-name/SKILL.md"
|
||||
assert_contains "reports missing name" "Missing 'name:'" "$SCRIPT" "$TMPDIR/no-name/SKILL.md"
|
||||
assert_exit "missing description exits 1" 1 "$SCRIPT" "$TMPDIR/no-desc/SKILL.md"
|
||||
assert_contains "reports missing description" "Missing 'description:'" "$SCRIPT" "$TMPDIR/no-desc/SKILL.md"
|
||||
echo
|
||||
|
||||
echo "-- bang-command errors --"
|
||||
assert_exit '${VAR} detected exits 1' 1 "$SCRIPT" "$TMPDIR/curly-var/SKILL.md"
|
||||
assert_contains '${VAR} message shown' '${VAR} syntax' "$SCRIPT" "$TMPDIR/curly-var/SKILL.md"
|
||||
assert_exit '$() detected exits 1' 1 "$SCRIPT" "$TMPDIR/subshell/SKILL.md"
|
||||
assert_contains '$() message shown' '$() command substitution' "$SCRIPT" "$TMPDIR/subshell/SKILL.md"
|
||||
assert_exit '../ detected exits 1' 1 "$SCRIPT" "$TMPDIR/dotdot/SKILL.md"
|
||||
assert_contains '../ message shown' '../ relative path' "$SCRIPT" "$TMPDIR/dotdot/SKILL.md"
|
||||
echo
|
||||
|
||||
echo "-- allowed-tools coverage --"
|
||||
assert_exit "uncovered binary exits 1" 1 "$SCRIPT" "$TMPDIR/uncovered/SKILL.md"
|
||||
assert_contains "reports uncovered binary" "not covered" "$SCRIPT" "$TMPDIR/uncovered/SKILL.md"
|
||||
assert_contains "identifies the binary" "'head'" "$SCRIPT" "$TMPDIR/uncovered/SKILL.md"
|
||||
echo
|
||||
|
||||
echo "-- path errors --"
|
||||
assert_exit '/home/ path exits 1' 1 "$SCRIPT" "$TMPDIR/home-path/SKILL.md"
|
||||
assert_contains '/home/ suggests ~/' "use ~/" "$SCRIPT" "$TMPDIR/home-path/SKILL.md"
|
||||
assert_exit '$HOME in path exits 1' 1 "$SCRIPT" "$TMPDIR/dollar-home/SKILL.md"
|
||||
assert_contains '$HOME in path message' '$VAR in path' "$SCRIPT" "$TMPDIR/dollar-home/SKILL.md"
|
||||
echo
|
||||
|
||||
echo "-- warnings (should not cause exit 1) --"
|
||||
assert_exit "~/ path is warning only" 0 "$SCRIPT" "$TMPDIR/tilde/SKILL.md"
|
||||
assert_contains "~/ warning shown" "WARN" "$SCRIPT" "$TMPDIR/tilde/SKILL.md"
|
||||
assert_exit "broad git pattern is warning only" 0 "$SCRIPT" "$TMPDIR/broad-git/SKILL.md"
|
||||
assert_contains "broad git warning shown" "overly broad" "$SCRIPT" "$TMPDIR/broad-git/SKILL.md"
|
||||
assert_exit "bangs without allowed-tools is warning only" 0 "$SCRIPT" "$TMPDIR/bangs-no-tools/SKILL.md"
|
||||
assert_contains "bangs without allowed-tools warning shown" "WARN" "$SCRIPT" "$TMPDIR/bangs-no-tools/SKILL.md"
|
||||
echo
|
||||
|
||||
echo "-- non-ASCII in frontmatter --"
|
||||
assert_exit "non-ASCII in frontmatter exits 1" 1 "$SCRIPT" "$TMPDIR/non-ascii/SKILL.md"
|
||||
assert_contains "reports non-ASCII" "non-ASCII" "$SCRIPT" "$TMPDIR/non-ascii/SKILL.md"
|
||||
echo
|
||||
|
||||
echo "-- edge cases --"
|
||||
assert_exit "no bang-commands is valid" 0 "$SCRIPT" "$TMPDIR/no-bangs/SKILL.md"
|
||||
assert_exit '$VAR not in path context is valid' 0 "$SCRIPT" "$TMPDIR/dollar-nopath/SKILL.md"
|
||||
assert_not_contains '$VAR not in path has no errors' "ERROR" "$SCRIPT" "$TMPDIR/dollar-nopath/SKILL.md"
|
||||
assert_exit "multi-line description is valid" 0 "$SCRIPT" "$TMPDIR/multiline-desc/SKILL.md"
|
||||
|
||||
# No files found
|
||||
EMPTY="$(mktemp -d)"
|
||||
assert_contains "no files message" "No SKILL.md files found" "$SCRIPT" "$EMPTY"
|
||||
assert_exit "no files exits 0" 0 "$SCRIPT" "$EMPTY"
|
||||
rmdir "$EMPTY"
|
||||
echo
|
||||
|
||||
echo "-- directory scan --"
|
||||
assert_exit "directory scan finds errors" 1 "$SCRIPT" "$TMPDIR"
|
||||
assert_contains "summary shows file count" "files checked" "$SCRIPT" "$TMPDIR"
|
||||
echo
|
||||
|
||||
echo "---"
|
||||
echo -e "Results: ${GREEN}$PASS passed${RESET}, ${RED}$FAIL failed${RESET}"
|
||||
[[ $FAIL -eq 0 ]]
|
||||
Reference in New Issue
Block a user