Add validate-skill: SKILL.md linter for Claude Code skills
Checks for common issues that silently break skills — non-ASCII frontmatter, $VAR in paths, uncovered binaries in allowed-tools, etc. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Paul O'Reilly
284
scripts/validate-skill
Executable file
284
scripts/validate-skill
Executable file
@@ -0,0 +1,284 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# Colours
|
||||||
|
RED='\033[0;31m'
|
||||||
|
GREEN='\033[0;32m'
|
||||||
|
YELLOW='\033[0;33m'
|
||||||
|
BOLD='\033[1m'
|
||||||
|
RESET='\033[0m'
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<'USAGE'
|
||||||
|
Usage: validate-skill [OPTIONS] <PATH>
|
||||||
|
|
||||||
|
Validate Claude Code SKILL.md files against known restrictions.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
PATH A SKILL.md file or directory to scan recursively (default: .)
|
||||||
|
|
||||||
|
Options:
|
||||||
|
-n, --dryrun List files that would be checked without validating
|
||||||
|
-h, --help Show this help message
|
||||||
|
|
||||||
|
Checks for:
|
||||||
|
- Frontmatter: missing delimiters, name/description fields, invalid name format
|
||||||
|
- Bang-commands: ${VAR} syntax, $() substitution, ~/ and ../ paths
|
||||||
|
- allowed-tools: uncovered command binaries, overly broad patterns
|
||||||
|
USAGE
|
||||||
|
}
|
||||||
|
|
||||||
|
DRYRUN=false
|
||||||
|
TARGET="."
|
||||||
|
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
|
case "$1" in
|
||||||
|
-n|--dryrun) DRYRUN=true; shift ;;
|
||||||
|
-h|--help) usage; exit 0 ;;
|
||||||
|
-*) echo "Unknown option: $1" >&2; usage >&2; exit 2 ;;
|
||||||
|
*) TARGET="$1"; shift ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
# Discovery
|
||||||
|
files=()
|
||||||
|
if [[ -f "$TARGET" ]]; then
|
||||||
|
files+=("$(cd "$(dirname "$TARGET")" && pwd)/$(basename "$TARGET")")
|
||||||
|
elif [[ -d "$TARGET" ]]; then
|
||||||
|
TARGET="$(cd "$TARGET" && pwd)"
|
||||||
|
while IFS= read -r f; do
|
||||||
|
files+=("$f")
|
||||||
|
done < <(find "$TARGET" -name "SKILL.md" -type f 2>/dev/null | sort)
|
||||||
|
else
|
||||||
|
echo "Error: $TARGET is not a file or directory" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ ${#files[@]} -eq 0 ]]; then
|
||||||
|
echo "No SKILL.md files found in $TARGET"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Dryrun
|
||||||
|
if [[ "$DRYRUN" == true ]]; then
|
||||||
|
echo "[dryrun] Would validate ${#files[@]} files:"
|
||||||
|
for f in "${files[@]}"; do
|
||||||
|
if [[ -d "$TARGET" ]]; then
|
||||||
|
echo " ${f#"$TARGET"/}"
|
||||||
|
else
|
||||||
|
echo " $f"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Validation
|
||||||
|
total_errors=0
|
||||||
|
total_warnings=0
|
||||||
|
files_with_errors=0
|
||||||
|
|
||||||
|
for file in "${files[@]}"; do
|
||||||
|
file_errors=0
|
||||||
|
file_warnings=0
|
||||||
|
issues=()
|
||||||
|
|
||||||
|
# Display path
|
||||||
|
if [[ -d "$TARGET" ]]; then
|
||||||
|
display="${f#"$TARGET"/}"
|
||||||
|
else
|
||||||
|
display="$file"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- Frontmatter checks ---
|
||||||
|
|
||||||
|
first_line=$(head -1 "$file")
|
||||||
|
if [[ "$first_line" != "---" ]]; then
|
||||||
|
issues+=("$(printf " ${RED}ERROR${RESET} Line 1: File does not start with --- frontmatter delimiter")")
|
||||||
|
file_errors=$((file_errors + 1))
|
||||||
|
# Can't do further frontmatter checks without delimiters
|
||||||
|
else
|
||||||
|
# Find closing ---
|
||||||
|
second_delim=$(awk 'NR>1 && /^---[[:space:]]*$/ {print NR; exit}' "$file")
|
||||||
|
if [[ -z "$second_delim" ]]; then
|
||||||
|
issues+=("$(printf " ${RED}ERROR${RESET} Frontmatter: Missing closing --- delimiter")")
|
||||||
|
file_errors=$((file_errors + 1))
|
||||||
|
else
|
||||||
|
# Extract frontmatter
|
||||||
|
frontmatter=$(sed -n "2,$((second_delim - 1))p" "$file")
|
||||||
|
|
||||||
|
# Check name field
|
||||||
|
name_value=$(echo "$frontmatter" | grep '^name:' | sed 's/^name:[[:space:]]*//' | head -1 || true)
|
||||||
|
if [[ -z "$name_value" ]]; then
|
||||||
|
issues+=("$(printf " ${RED}ERROR${RESET} Frontmatter: Missing 'name:' field")")
|
||||||
|
file_errors=$((file_errors + 1))
|
||||||
|
else
|
||||||
|
if [[ ! "$name_value" =~ ^[a-z0-9-]+$ ]]; then
|
||||||
|
issues+=("$(printf " ${RED}ERROR${RESET} Frontmatter: name '${name_value}' contains invalid characters (must be lowercase+numbers+hyphens)")")
|
||||||
|
file_errors=$((file_errors + 1))
|
||||||
|
fi
|
||||||
|
if [[ ${#name_value} -gt 64 ]]; then
|
||||||
|
issues+=("$(printf " ${RED}ERROR${RESET} Frontmatter: name exceeds 64 characters")")
|
||||||
|
file_errors=$((file_errors + 1))
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check for non-ASCII characters in frontmatter (can silently break YAML parsing)
|
||||||
|
non_ascii_lines=$(echo "$frontmatter" | grep -nP '[^\x00-\x7F]' || true)
|
||||||
|
if [[ -n "$non_ascii_lines" ]]; then
|
||||||
|
while IFS= read -r nal; do
|
||||||
|
fm_ln="${nal%%:*}"
|
||||||
|
# Offset by 1 for the opening --- line
|
||||||
|
actual_ln=$((fm_ln + 1))
|
||||||
|
issues+=("$(printf " ${RED}ERROR${RESET} Line ${actual_ln}: Frontmatter contains non-ASCII characters (em dashes, smart quotes, etc.) — can silently prevent skill loading")")
|
||||||
|
file_errors=$((file_errors + 1))
|
||||||
|
done <<< "$non_ascii_lines"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check description field
|
||||||
|
desc_line=$(echo "$frontmatter" | grep '^description:' || true)
|
||||||
|
if [[ -z "$desc_line" ]]; then
|
||||||
|
issues+=("$(printf " ${RED}ERROR${RESET} Frontmatter: Missing 'description:' field")")
|
||||||
|
file_errors=$((file_errors + 1))
|
||||||
|
else
|
||||||
|
desc_value=$(echo "$desc_line" | sed 's/^description:[[:space:]]*//')
|
||||||
|
# Multi-line > or | is non-empty; empty string after colon is not
|
||||||
|
if [[ -z "$desc_value" ]]; then
|
||||||
|
issues+=("$(printf " ${RED}ERROR${RESET} Frontmatter: 'description:' field is empty")")
|
||||||
|
file_errors=$((file_errors + 1))
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Extract allowed-tools
|
||||||
|
allowed_tools_line=$(echo "$frontmatter" | grep '^allowed-tools:' || true)
|
||||||
|
allowed_tools_value=""
|
||||||
|
bash_patterns=()
|
||||||
|
if [[ -n "$allowed_tools_line" ]]; then
|
||||||
|
allowed_tools_value=$(echo "$allowed_tools_line" | sed 's/^allowed-tools:[[:space:]]*//')
|
||||||
|
# Parse Bash(...) patterns
|
||||||
|
while IFS= read -r pattern; do
|
||||||
|
pattern=$(echo "$pattern" | xargs) # trim whitespace
|
||||||
|
if [[ "$pattern" == Bash\(*\) ]]; then
|
||||||
|
bash_patterns+=("$pattern")
|
||||||
|
fi
|
||||||
|
done < <(echo "$allowed_tools_value" | tr ',' '\n')
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check for overly broad Bash(git *) pattern
|
||||||
|
for pattern in "${bash_patterns[@]+"${bash_patterns[@]}"}"; do
|
||||||
|
if [[ "$pattern" == "Bash(git *)" ]]; then
|
||||||
|
issues+=("$(printf " ${YELLOW}WARN${RESET} Frontmatter: 'Bash(git *)' is overly broad — use specific subcommands like 'Bash(git log *)', 'Bash(git diff *)'")")
|
||||||
|
file_warnings=$((file_warnings + 1))
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- Bang-command checks ---
|
||||||
|
|
||||||
|
bang_commands=()
|
||||||
|
while IFS= read -r line; do
|
||||||
|
[[ -n "$line" ]] && bang_commands+=("$line")
|
||||||
|
done < <(grep -n '^!`' "$file" 2>/dev/null || true)
|
||||||
|
|
||||||
|
# Warn if bang-commands exist but no allowed-tools
|
||||||
|
if [[ ${#bang_commands[@]} -gt 0 && -z "$allowed_tools_value" && "$first_line" == "---" && -n "${second_delim:-}" ]]; then
|
||||||
|
issues+=("$(printf " ${YELLOW}WARN${RESET} Bang-commands found but no 'allowed-tools' declared — commands won't be pre-authorised")")
|
||||||
|
file_warnings=$((file_warnings + 1))
|
||||||
|
fi
|
||||||
|
|
||||||
|
for entry in "${bang_commands[@]+"${bang_commands[@]}"}"; do
|
||||||
|
ln="${entry%%:*}"
|
||||||
|
line_content="${entry#*:}"
|
||||||
|
# Extract command between backticks
|
||||||
|
cmd=$(echo "$line_content" | sed 's/^!`//' | sed 's/`$//')
|
||||||
|
|
||||||
|
# Check for ${VAR} syntax (e.g., ${HOME}, ${VAR:-default})
|
||||||
|
if echo "$cmd" | grep -qE '\$\{[^}]+\}'; then
|
||||||
|
issues+=("$(printf " ${RED}ERROR${RESET} Line ${ln}: Bang-command uses \${VAR} syntax (rejected by permission checker)")")
|
||||||
|
file_errors=$((file_errors + 1))
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check for $VAR shell expansion in paths (e.g., $HOME/dev/...)
|
||||||
|
# Match $WORD followed by / (path context) but not $() or ${} which are caught above
|
||||||
|
if echo "$cmd" | grep -qE '\$[A-Za-z_][A-Za-z0-9_]*/'; then
|
||||||
|
issues+=("$(printf " ${RED}ERROR${RESET} Line ${ln}: Bang-command uses \$VAR in path (permission checker rejects shell expansion in paths — move to skill instructions instead)")")
|
||||||
|
file_errors=$((file_errors + 1))
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check for $() command substitution
|
||||||
|
if echo "$cmd" | grep -qE '\$\('; then
|
||||||
|
issues+=("$(printf " ${RED}ERROR${RESET} Line ${ln}: Bang-command uses \$() command substitution (rejected by permission checker)")")
|
||||||
|
file_errors=$((file_errors + 1))
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check for /home/ hardcoded paths (prefer ~/)
|
||||||
|
if echo "$cmd" | grep -qF '/home/'; then
|
||||||
|
issues+=("$(printf " ${RED}ERROR${RESET} Line ${ln}: Bang-command uses hardcoded /home/ path (use ~/ for portability)")")
|
||||||
|
file_errors=$((file_errors + 1))
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check for ~/ paths (more portable but may still be rejected by some permission modes)
|
||||||
|
if echo "$cmd" | grep -qF '~/'; then
|
||||||
|
issues+=("$(printf " ${YELLOW}WARN${RESET} Line ${ln}: Bang-command uses ~/ path (may be rejected by sandbox — consider moving to skill instructions)")")
|
||||||
|
file_warnings=$((file_warnings + 1))
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check for ../ relative paths
|
||||||
|
if echo "$cmd" | grep -qF '../'; then
|
||||||
|
issues+=("$(printf " ${RED}ERROR${RESET} Line ${ln}: Bang-command uses ../ relative path (fragile, breaks when CWD changes)")")
|
||||||
|
file_errors=$((file_errors + 1))
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check allowed-tools coverage
|
||||||
|
if [[ -n "$allowed_tools_value" && ${#bash_patterns[@]} -ge 0 ]]; then
|
||||||
|
binary=$(echo "$cmd" | awk '{print $1}')
|
||||||
|
matched=false
|
||||||
|
for pattern in "${bash_patterns[@]+"${bash_patterns[@]}"}"; do
|
||||||
|
# Extract binary from Bash(binary ...) pattern
|
||||||
|
pattern_binary=$(echo "$pattern" | sed 's/^Bash(\([^ )]*\).*/\1/')
|
||||||
|
if [[ "$binary" == "$pattern_binary" ]]; then
|
||||||
|
matched=true
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
if [[ "$matched" == false ]]; then
|
||||||
|
issues+=("$(printf " ${RED}ERROR${RESET} Line ${ln}: Command binary '${binary}' not covered by any Bash() pattern in allowed-tools")")
|
||||||
|
file_errors=$((file_errors + 1))
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
# --- Print results for this file ---
|
||||||
|
|
||||||
|
if [[ -d "${TARGET:-}" ]]; then
|
||||||
|
rel="${file#"$TARGET"/}"
|
||||||
|
else
|
||||||
|
rel="$file"
|
||||||
|
fi
|
||||||
|
echo -e "── ${BOLD}${rel}${RESET} ──"
|
||||||
|
|
||||||
|
if [[ ${#issues[@]} -eq 0 ]]; then
|
||||||
|
echo -e " ${GREEN}No issues found.${RESET}"
|
||||||
|
else
|
||||||
|
for issue in "${issues[@]}"; do
|
||||||
|
echo -e "$issue"
|
||||||
|
done
|
||||||
|
echo -e " ${file_errors} errors, ${file_warnings} warnings"
|
||||||
|
fi
|
||||||
|
echo
|
||||||
|
|
||||||
|
total_errors=$((total_errors + file_errors))
|
||||||
|
total_warnings=$((total_warnings + file_warnings))
|
||||||
|
if [[ $file_errors -gt 0 ]]; then
|
||||||
|
files_with_errors=$((files_with_errors + 1))
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
# Summary
|
||||||
|
echo -e "Summary: ${#files[@]} files checked, ${total_errors} errors, ${total_warnings} warnings"
|
||||||
|
|
||||||
|
if [[ $total_errors -gt 0 ]]; then
|
||||||
|
exit 1
|
||||||
|
else
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
127
specs/validate-skill.spec.md
Normal file
127
specs/validate-skill.spec.md
Normal file
@@ -0,0 +1,127 @@
|
|||||||
|
# validate-skill
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Validate Claude Code SKILL.md files against known restrictions that cause permission checker failures, sandbox errors, and other breakages.
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
```
|
||||||
|
validate-skill [OPTIONS] <PATH>
|
||||||
|
```
|
||||||
|
|
||||||
|
### Arguments
|
||||||
|
|
||||||
|
| Argument | Default | Description |
|
||||||
|
|----------|---------|-------------|
|
||||||
|
| `PATH` | `.` (current directory) | A SKILL.md file or a directory to scan recursively |
|
||||||
|
|
||||||
|
### Flags
|
||||||
|
|
||||||
|
| Flag | Short | Description |
|
||||||
|
|------|-------|-------------|
|
||||||
|
| `--dryrun` | `-n` | List SKILL.md files that would be checked without validating |
|
||||||
|
| `--help` | `-h` | Show usage information |
|
||||||
|
|
||||||
|
## Behaviour
|
||||||
|
|
||||||
|
1. **Discovery phase:** If `PATH` is a file, validate that one file. If `PATH` is a directory, find all files named `SKILL.md` recursively.
|
||||||
|
|
||||||
|
2. **Validation phase:** For each SKILL.md file, run these checks:
|
||||||
|
|
||||||
|
**Frontmatter checks:**
|
||||||
|
a. File starts with `---` on line 1
|
||||||
|
b. A closing `---` delimiter exists on a subsequent line
|
||||||
|
c. `name:` field exists, contains only lowercase letters, numbers, and hyphens, max 64 characters
|
||||||
|
d. `description:` field exists and is non-empty (multi-line `>` syntax counts as non-empty)
|
||||||
|
|
||||||
|
e. Non-ASCII characters in frontmatter (em dashes, smart quotes, curly apostrophes, etc.) — these can silently break YAML parsing and prevent the skill from loading
|
||||||
|
|
||||||
|
**Bang-command checks** (lines matching `!` followed by a backtick):
|
||||||
|
e. No `${VAR}` syntax — must use `$VAR` instead
|
||||||
|
f. No `$()` command substitution
|
||||||
|
g. No `~/` paths (sandbox-fragile)
|
||||||
|
h. No `../` relative paths (CWD-fragile)
|
||||||
|
i. Warning (not error) for hardcoded `/home/` paths
|
||||||
|
|
||||||
|
**allowed-tools coverage checks:**
|
||||||
|
j. For each bang-command, the command binary (first word) must match a `Bash(binary *)` or `Bash(binary)` pattern in `allowed-tools`
|
||||||
|
k. Warning for overly broad patterns like `Bash(git *)`
|
||||||
|
l. Warning if bang-commands exist but no `allowed-tools` is declared
|
||||||
|
|
||||||
|
3. **Report phase:** For each file:
|
||||||
|
- Print the file path
|
||||||
|
- Print each issue with severity (`ERROR`/`WARN`), line number, and description
|
||||||
|
- Print per-file summary: `N errors, M warnings`
|
||||||
|
- At the end, print total summary across all files
|
||||||
|
|
||||||
|
4. **Exit code:**
|
||||||
|
- `0` — no errors (warnings are OK)
|
||||||
|
- `1` — at least one error found
|
||||||
|
- `2` — usage error (bad arguments)
|
||||||
|
|
||||||
|
## Dryrun Behaviour
|
||||||
|
|
||||||
|
When `--dryrun` is passed:
|
||||||
|
- Perform the discovery phase only
|
||||||
|
- Print each SKILL.md path, one per line
|
||||||
|
- Prefix output with `[dryrun] Would validate N files:`
|
||||||
|
- Exit code is always `0`
|
||||||
|
|
||||||
|
## Edge Cases
|
||||||
|
|
||||||
|
| Scenario | Handling |
|
||||||
|
|----------|----------|
|
||||||
|
| No SKILL.md files found | Print "No SKILL.md files found in <path>" and exit 0 |
|
||||||
|
| File is not named SKILL.md (when given directly) | Validate it anyway (user explicitly chose it) |
|
||||||
|
| No frontmatter at all | Report missing `---` delimiter error, skip field checks |
|
||||||
|
| No bang-commands and no allowed-tools | Valid (like the linter skill) |
|
||||||
|
| Bang-commands exist but no allowed-tools declared | Warning: commands won't be pre-authorised |
|
||||||
|
| `$HOME` (without braces) in bang-command | No error — `$HOME` works fine, only `${HOME}` is rejected |
|
||||||
|
| Multi-line `description: >` | Treat as non-empty |
|
||||||
|
| Commands with `\|\| echo "fallback"` | Binary is the first word only |
|
||||||
|
| Bang-commands inside markdown code blocks | Still checked (they execute regardless of markdown context) |
|
||||||
|
|
||||||
|
## Examples
|
||||||
|
|
||||||
|
### Clean file
|
||||||
|
```
|
||||||
|
── skills/log/SKILL.md ──
|
||||||
|
No issues found.
|
||||||
|
|
||||||
|
Summary: 1 file checked, 0 errors, 0 warnings
|
||||||
|
```
|
||||||
|
|
||||||
|
### File with issues
|
||||||
|
```
|
||||||
|
── skills/distill/SKILL.md ──
|
||||||
|
ERROR Line 18: Bang-command uses ${VAR} syntax (use $VAR instead)
|
||||||
|
ERROR Line 22: Bang-command uses $() command substitution
|
||||||
|
WARN Line 8: 'Bash(git *)' is overly broad — use specific subcommands
|
||||||
|
2 errors, 1 warning
|
||||||
|
|
||||||
|
Summary: 1 file checked, 2 errors, 1 warning
|
||||||
|
```
|
||||||
|
|
||||||
|
### Directory scan
|
||||||
|
```
|
||||||
|
── skills/log/SKILL.md ──
|
||||||
|
No issues found.
|
||||||
|
|
||||||
|
── skills/reflect/SKILL.md ──
|
||||||
|
No issues found.
|
||||||
|
|
||||||
|
── skills/distill/SKILL.md ──
|
||||||
|
ERROR Line 18: Bang-command uses ${VAR} syntax (use $VAR instead)
|
||||||
|
1 error, 0 warnings
|
||||||
|
|
||||||
|
Summary: 3 files checked, 1 error, 0 warnings
|
||||||
|
```
|
||||||
|
|
||||||
|
### Dryrun
|
||||||
|
```
|
||||||
|
[dryrun] Would validate 3 files:
|
||||||
|
skills/log/SKILL.md
|
||||||
|
skills/reflect/SKILL.md
|
||||||
|
skills/distill/SKILL.md
|
||||||
|
```
|
||||||
381
tests/test-validate-skill.sh
Executable file
381
tests/test-validate-skill.sh
Executable file
@@ -0,0 +1,381 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||||
|
SCRIPT="$SCRIPT_DIR/../scripts/validate-skill"
|
||||||
|
|
||||||
|
RED='\033[0;31m'
|
||||||
|
GREEN='\033[0;32m'
|
||||||
|
RESET='\033[0m'
|
||||||
|
|
||||||
|
PASS=0
|
||||||
|
FAIL=0
|
||||||
|
|
||||||
|
assert_exit() {
|
||||||
|
local desc="$1" expected="$2"
|
||||||
|
shift 2
|
||||||
|
local actual
|
||||||
|
set +e
|
||||||
|
"$@" >/dev/null 2>&1
|
||||||
|
actual=$?
|
||||||
|
set -e
|
||||||
|
if [[ "$actual" -eq "$expected" ]]; then
|
||||||
|
echo -e "${GREEN}PASS${RESET}: $desc"
|
||||||
|
PASS=$((PASS + 1))
|
||||||
|
else
|
||||||
|
echo -e "${RED}FAIL${RESET}: $desc (expected exit $expected, got $actual)"
|
||||||
|
FAIL=$((FAIL + 1))
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_contains() {
|
||||||
|
local desc="$1" pattern="$2"
|
||||||
|
shift 2
|
||||||
|
local output
|
||||||
|
set +e
|
||||||
|
output=$("$@" 2>&1)
|
||||||
|
set -e
|
||||||
|
if echo "$output" | grep -qF "$pattern"; then
|
||||||
|
echo -e "${GREEN}PASS${RESET}: $desc"
|
||||||
|
PASS=$((PASS + 1))
|
||||||
|
else
|
||||||
|
echo -e "${RED}FAIL${RESET}: $desc (output missing: '$pattern')"
|
||||||
|
echo " Got: $output"
|
||||||
|
FAIL=$((FAIL + 1))
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_not_contains() {
|
||||||
|
local desc="$1" pattern="$2"
|
||||||
|
shift 2
|
||||||
|
local output
|
||||||
|
set +e
|
||||||
|
output=$("$@" 2>&1)
|
||||||
|
set -e
|
||||||
|
if echo "$output" | grep -qF "$pattern"; then
|
||||||
|
echo -e "${RED}FAIL${RESET}: $desc (output should NOT contain: '$pattern')"
|
||||||
|
echo " Got: $output"
|
||||||
|
FAIL=$((FAIL + 1))
|
||||||
|
else
|
||||||
|
echo -e "${GREEN}PASS${RESET}: $desc"
|
||||||
|
PASS=$((PASS + 1))
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# Setup temp directory with test fixtures
|
||||||
|
TMPDIR="$(mktemp -d)"
|
||||||
|
trap 'rm -rf "$TMPDIR"' EXIT
|
||||||
|
|
||||||
|
# --- Fixture: valid skill ---
|
||||||
|
mkdir -p "$TMPDIR/valid"
|
||||||
|
cat > "$TMPDIR/valid/SKILL.md" <<'EOF'
|
||||||
|
---
|
||||||
|
name: test-skill
|
||||||
|
description: A test skill for validation
|
||||||
|
allowed-tools: Read, Bash(cat *), Bash(ls *), Bash(date *)
|
||||||
|
---
|
||||||
|
|
||||||
|
# Test Skill
|
||||||
|
|
||||||
|
## Context
|
||||||
|
!`cat README.md 2>/dev/null || echo "no readme"`
|
||||||
|
!`ls -1 scripts/ 2>/dev/null || echo "no scripts"`
|
||||||
|
!`date +%Y-%m-%d`
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# --- Fixture: missing frontmatter ---
|
||||||
|
mkdir -p "$TMPDIR/no-frontmatter"
|
||||||
|
cat > "$TMPDIR/no-frontmatter/SKILL.md" <<'EOF'
|
||||||
|
# No frontmatter here
|
||||||
|
Just some content.
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# --- Fixture: missing closing delimiter ---
|
||||||
|
mkdir -p "$TMPDIR/no-close"
|
||||||
|
cat > "$TMPDIR/no-close/SKILL.md" <<'EOF'
|
||||||
|
---
|
||||||
|
name: broken
|
||||||
|
description: Missing closing delimiter
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# --- Fixture: bad name ---
|
||||||
|
mkdir -p "$TMPDIR/bad-name"
|
||||||
|
cat > "$TMPDIR/bad-name/SKILL.md" <<'EOF'
|
||||||
|
---
|
||||||
|
name: My_Skill!
|
||||||
|
description: A skill with a bad name
|
||||||
|
---
|
||||||
|
|
||||||
|
# Bad Name Skill
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# --- Fixture: missing name ---
|
||||||
|
mkdir -p "$TMPDIR/no-name"
|
||||||
|
cat > "$TMPDIR/no-name/SKILL.md" <<'EOF'
|
||||||
|
---
|
||||||
|
description: A skill without a name
|
||||||
|
---
|
||||||
|
|
||||||
|
# No Name Skill
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# --- Fixture: missing description ---
|
||||||
|
mkdir -p "$TMPDIR/no-desc"
|
||||||
|
cat > "$TMPDIR/no-desc/SKILL.md" <<'EOF'
|
||||||
|
---
|
||||||
|
name: no-desc
|
||||||
|
---
|
||||||
|
|
||||||
|
# No Description
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# --- Fixture: ${VAR} in bang-command ---
|
||||||
|
mkdir -p "$TMPDIR/curly-var"
|
||||||
|
cat > "$TMPDIR/curly-var/SKILL.md" <<'EOF'
|
||||||
|
---
|
||||||
|
name: curly-var
|
||||||
|
description: Uses curly brace variable
|
||||||
|
allowed-tools: Read, Bash(cat *)
|
||||||
|
---
|
||||||
|
|
||||||
|
# Curly Var
|
||||||
|
!`cat ${HOME}/some/path`
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# --- Fixture: $() in bang-command ---
|
||||||
|
mkdir -p "$TMPDIR/subshell"
|
||||||
|
cat > "$TMPDIR/subshell/SKILL.md" <<'EOF'
|
||||||
|
---
|
||||||
|
name: subshell
|
||||||
|
description: Uses command substitution
|
||||||
|
allowed-tools: Read, Bash(git *)
|
||||||
|
---
|
||||||
|
|
||||||
|
# Subshell
|
||||||
|
!`git log --since="$(date -d '7 days ago')"`
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# --- Fixture: ~/ in bang-command ---
|
||||||
|
mkdir -p "$TMPDIR/tilde"
|
||||||
|
cat > "$TMPDIR/tilde/SKILL.md" <<'EOF'
|
||||||
|
---
|
||||||
|
name: tilde
|
||||||
|
description: Uses tilde path
|
||||||
|
allowed-tools: Read, Bash(cat *)
|
||||||
|
---
|
||||||
|
|
||||||
|
# Tilde
|
||||||
|
!`cat ~/dev/claude/settings.yaml`
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# --- Fixture: ../ in bang-command ---
|
||||||
|
mkdir -p "$TMPDIR/dotdot"
|
||||||
|
cat > "$TMPDIR/dotdot/SKILL.md" <<'EOF'
|
||||||
|
---
|
||||||
|
name: dotdot
|
||||||
|
description: Uses relative path
|
||||||
|
allowed-tools: Read, Bash(cat *)
|
||||||
|
---
|
||||||
|
|
||||||
|
# Dotdot
|
||||||
|
!`cat ../claude-foundations/settings.yaml`
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# --- Fixture: uncovered binary ---
|
||||||
|
mkdir -p "$TMPDIR/uncovered"
|
||||||
|
cat > "$TMPDIR/uncovered/SKILL.md" <<'EOF'
|
||||||
|
---
|
||||||
|
name: uncovered
|
||||||
|
description: Uses a binary not in allowed-tools
|
||||||
|
allowed-tools: Read, Bash(cat *)
|
||||||
|
---
|
||||||
|
|
||||||
|
# Uncovered
|
||||||
|
!`cat README.md`
|
||||||
|
!`head -5 README.md`
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# --- Fixture: /home/ hardcoded path (warning only) ---
|
||||||
|
mkdir -p "$TMPDIR/home-path"
|
||||||
|
cat > "$TMPDIR/home-path/SKILL.md" <<'EOF'
|
||||||
|
---
|
||||||
|
name: home-path
|
||||||
|
description: Uses hardcoded home path
|
||||||
|
allowed-tools: Read, Bash(cat *)
|
||||||
|
---
|
||||||
|
|
||||||
|
# Home Path
|
||||||
|
!`cat /home/paul/dev/claude/settings.yaml`
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# --- Fixture: broad Bash(git *) (warning only) ---
|
||||||
|
mkdir -p "$TMPDIR/broad-git"
|
||||||
|
cat > "$TMPDIR/broad-git/SKILL.md" <<'EOF'
|
||||||
|
---
|
||||||
|
name: broad-git
|
||||||
|
description: Uses overly broad git pattern
|
||||||
|
allowed-tools: Read, Bash(git *)
|
||||||
|
---
|
||||||
|
|
||||||
|
# Broad Git
|
||||||
|
!`git log --oneline -5`
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# --- Fixture: no bang-commands, no allowed-tools (valid) ---
|
||||||
|
mkdir -p "$TMPDIR/no-bangs"
|
||||||
|
cat > "$TMPDIR/no-bangs/SKILL.md" <<'EOF'
|
||||||
|
---
|
||||||
|
name: no-bangs
|
||||||
|
description: A simple instruction-only skill
|
||||||
|
---
|
||||||
|
|
||||||
|
# No Bang Commands
|
||||||
|
|
||||||
|
Just instructions, no dynamic context.
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# --- Fixture: $HOME in path (should be error) ---
|
||||||
|
mkdir -p "$TMPDIR/dollar-home"
|
||||||
|
cat > "$TMPDIR/dollar-home/SKILL.md" <<'EOF'
|
||||||
|
---
|
||||||
|
name: dollar-home
|
||||||
|
description: Uses dollar home in path
|
||||||
|
allowed-tools: Read, Bash(cat *)
|
||||||
|
---
|
||||||
|
|
||||||
|
# Dollar Home
|
||||||
|
!`cat $HOME/dev/claude/settings.yaml 2>/dev/null || echo "not found"`
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# --- Fixture: $VAR not in path context (should be fine) ---
|
||||||
|
mkdir -p "$TMPDIR/dollar-nopath"
|
||||||
|
cat > "$TMPDIR/dollar-nopath/SKILL.md" <<'EOF'
|
||||||
|
---
|
||||||
|
name: dollar-nopath
|
||||||
|
description: Uses dollar var not as a path
|
||||||
|
allowed-tools: Read, Bash(date *), Bash(echo *)
|
||||||
|
---
|
||||||
|
|
||||||
|
# Dollar No Path
|
||||||
|
!`date +%Y-%m-%d`
|
||||||
|
!`echo $USER said hello`
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# --- Fixture: multi-line description with > ---
|
||||||
|
mkdir -p "$TMPDIR/multiline-desc"
|
||||||
|
cat > "$TMPDIR/multiline-desc/SKILL.md" <<'EOF'
|
||||||
|
---
|
||||||
|
name: multiline-desc
|
||||||
|
description: >
|
||||||
|
A skill with a multi-line description
|
||||||
|
that spans multiple lines
|
||||||
|
---
|
||||||
|
|
||||||
|
# Multi-line Description
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# --- Fixture: non-ASCII in frontmatter (em dash) ---
|
||||||
|
mkdir -p "$TMPDIR/non-ascii"
|
||||||
|
printf -- '---\nname: non-ascii\ndescription: >\n A skill that takes no action \xe2\x80\x94 information only\n---\n\n# Non-ASCII\n' > "$TMPDIR/non-ascii/SKILL.md"
|
||||||
|
|
||||||
|
# --- Fixture: bang-commands but no allowed-tools ---
|
||||||
|
mkdir -p "$TMPDIR/bangs-no-tools"
|
||||||
|
cat > "$TMPDIR/bangs-no-tools/SKILL.md" <<'EOF'
|
||||||
|
---
|
||||||
|
name: bangs-no-tools
|
||||||
|
description: Has bang-commands but no allowed-tools
|
||||||
|
---
|
||||||
|
|
||||||
|
# Bangs No Tools
|
||||||
|
!`cat README.md`
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# ============================
|
||||||
|
# Tests
|
||||||
|
# ============================
|
||||||
|
|
||||||
|
echo "=== validate-skill tests ==="
|
||||||
|
echo
|
||||||
|
|
||||||
|
echo "-- basic functionality --"
|
||||||
|
assert_exit "--help exits 0" 0 "$SCRIPT" --help
|
||||||
|
assert_contains "--help shows usage" "Usage:" "$SCRIPT" --help
|
||||||
|
assert_exit "valid skill exits 0" 0 "$SCRIPT" "$TMPDIR/valid/SKILL.md"
|
||||||
|
assert_contains "valid skill shows no issues" "No issues found" "$SCRIPT" "$TMPDIR/valid/SKILL.md"
|
||||||
|
echo
|
||||||
|
|
||||||
|
echo "-- dryrun --"
|
||||||
|
assert_exit "--dryrun always exits 0" 0 "$SCRIPT" --dryrun "$TMPDIR"
|
||||||
|
assert_contains "--dryrun lists files" "[dryrun]" "$SCRIPT" --dryrun "$TMPDIR"
|
||||||
|
echo
|
||||||
|
|
||||||
|
echo "-- frontmatter errors --"
|
||||||
|
assert_exit "missing frontmatter exits 1" 1 "$SCRIPT" "$TMPDIR/no-frontmatter/SKILL.md"
|
||||||
|
assert_contains "reports missing ---" "does not start with ---" "$SCRIPT" "$TMPDIR/no-frontmatter/SKILL.md"
|
||||||
|
assert_exit "missing closing --- exits 1" 1 "$SCRIPT" "$TMPDIR/no-close/SKILL.md"
|
||||||
|
assert_contains "reports missing closing ---" "Missing closing ---" "$SCRIPT" "$TMPDIR/no-close/SKILL.md"
|
||||||
|
assert_exit "bad name exits 1" 1 "$SCRIPT" "$TMPDIR/bad-name/SKILL.md"
|
||||||
|
assert_contains "reports invalid name" "invalid characters" "$SCRIPT" "$TMPDIR/bad-name/SKILL.md"
|
||||||
|
assert_exit "missing name exits 1" 1 "$SCRIPT" "$TMPDIR/no-name/SKILL.md"
|
||||||
|
assert_contains "reports missing name" "Missing 'name:'" "$SCRIPT" "$TMPDIR/no-name/SKILL.md"
|
||||||
|
assert_exit "missing description exits 1" 1 "$SCRIPT" "$TMPDIR/no-desc/SKILL.md"
|
||||||
|
assert_contains "reports missing description" "Missing 'description:'" "$SCRIPT" "$TMPDIR/no-desc/SKILL.md"
|
||||||
|
echo
|
||||||
|
|
||||||
|
echo "-- bang-command errors --"
|
||||||
|
assert_exit '${VAR} detected exits 1' 1 "$SCRIPT" "$TMPDIR/curly-var/SKILL.md"
|
||||||
|
assert_contains '${VAR} message shown' '${VAR} syntax' "$SCRIPT" "$TMPDIR/curly-var/SKILL.md"
|
||||||
|
assert_exit '$() detected exits 1' 1 "$SCRIPT" "$TMPDIR/subshell/SKILL.md"
|
||||||
|
assert_contains '$() message shown' '$() command substitution' "$SCRIPT" "$TMPDIR/subshell/SKILL.md"
|
||||||
|
assert_exit '../ detected exits 1' 1 "$SCRIPT" "$TMPDIR/dotdot/SKILL.md"
|
||||||
|
assert_contains '../ message shown' '../ relative path' "$SCRIPT" "$TMPDIR/dotdot/SKILL.md"
|
||||||
|
echo
|
||||||
|
|
||||||
|
echo "-- allowed-tools coverage --"
|
||||||
|
assert_exit "uncovered binary exits 1" 1 "$SCRIPT" "$TMPDIR/uncovered/SKILL.md"
|
||||||
|
assert_contains "reports uncovered binary" "not covered" "$SCRIPT" "$TMPDIR/uncovered/SKILL.md"
|
||||||
|
assert_contains "identifies the binary" "'head'" "$SCRIPT" "$TMPDIR/uncovered/SKILL.md"
|
||||||
|
echo
|
||||||
|
|
||||||
|
echo "-- path errors --"
|
||||||
|
assert_exit '/home/ path exits 1' 1 "$SCRIPT" "$TMPDIR/home-path/SKILL.md"
|
||||||
|
assert_contains '/home/ suggests ~/' "use ~/" "$SCRIPT" "$TMPDIR/home-path/SKILL.md"
|
||||||
|
assert_exit '$HOME in path exits 1' 1 "$SCRIPT" "$TMPDIR/dollar-home/SKILL.md"
|
||||||
|
assert_contains '$HOME in path message' '$VAR in path' "$SCRIPT" "$TMPDIR/dollar-home/SKILL.md"
|
||||||
|
echo
|
||||||
|
|
||||||
|
echo "-- warnings (should not cause exit 1) --"
|
||||||
|
assert_exit "~/ path is warning only" 0 "$SCRIPT" "$TMPDIR/tilde/SKILL.md"
|
||||||
|
assert_contains "~/ warning shown" "WARN" "$SCRIPT" "$TMPDIR/tilde/SKILL.md"
|
||||||
|
assert_exit "broad git pattern is warning only" 0 "$SCRIPT" "$TMPDIR/broad-git/SKILL.md"
|
||||||
|
assert_contains "broad git warning shown" "overly broad" "$SCRIPT" "$TMPDIR/broad-git/SKILL.md"
|
||||||
|
assert_exit "bangs without allowed-tools is warning only" 0 "$SCRIPT" "$TMPDIR/bangs-no-tools/SKILL.md"
|
||||||
|
assert_contains "bangs without allowed-tools warning shown" "WARN" "$SCRIPT" "$TMPDIR/bangs-no-tools/SKILL.md"
|
||||||
|
echo
|
||||||
|
|
||||||
|
echo "-- non-ASCII in frontmatter --"
|
||||||
|
assert_exit "non-ASCII in frontmatter exits 1" 1 "$SCRIPT" "$TMPDIR/non-ascii/SKILL.md"
|
||||||
|
assert_contains "reports non-ASCII" "non-ASCII" "$SCRIPT" "$TMPDIR/non-ascii/SKILL.md"
|
||||||
|
echo
|
||||||
|
|
||||||
|
echo "-- edge cases --"
|
||||||
|
assert_exit "no bang-commands is valid" 0 "$SCRIPT" "$TMPDIR/no-bangs/SKILL.md"
|
||||||
|
assert_exit '$VAR not in path context is valid' 0 "$SCRIPT" "$TMPDIR/dollar-nopath/SKILL.md"
|
||||||
|
assert_not_contains '$VAR not in path has no errors' "ERROR" "$SCRIPT" "$TMPDIR/dollar-nopath/SKILL.md"
|
||||||
|
assert_exit "multi-line description is valid" 0 "$SCRIPT" "$TMPDIR/multiline-desc/SKILL.md"
|
||||||
|
|
||||||
|
# No files found
|
||||||
|
EMPTY="$(mktemp -d)"
|
||||||
|
assert_contains "no files message" "No SKILL.md files found" "$SCRIPT" "$EMPTY"
|
||||||
|
assert_exit "no files exits 0" 0 "$SCRIPT" "$EMPTY"
|
||||||
|
rmdir "$EMPTY"
|
||||||
|
echo
|
||||||
|
|
||||||
|
echo "-- directory scan --"
|
||||||
|
assert_exit "directory scan finds errors" 1 "$SCRIPT" "$TMPDIR"
|
||||||
|
assert_contains "summary shows file count" "files checked" "$SCRIPT" "$TMPDIR"
|
||||||
|
echo
|
||||||
|
|
||||||
|
echo "---"
|
||||||
|
echo -e "Results: ${GREEN}$PASS passed${RESET}, ${RED}$FAIL failed${RESET}"
|
||||||
|
[[ $FAIL -eq 0 ]]
|
||||||
Reference in New Issue
Block a user