ci(security): harden pull request validation (#332)
This commit is contained in:
Darren Hinde
GitHub
parent
ef3836efd6
commit
0c99ddc6d1
5 changed files with 564 additions and 386 deletions
88
.github/workflows/pr-checks.yml
vendored
88
.github/workflows/pr-checks.yml
vendored
|
|
@ -5,6 +5,9 @@ on:
|
||||||
branches: [main, dev]
|
branches: [main, dev]
|
||||||
types: [opened, edited, synchronize, reopened]
|
types: [opened, edited, synchronize, reopened]
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
pr-title-check:
|
pr-title-check:
|
||||||
name: Validate PR Title
|
name: Validate PR Title
|
||||||
|
|
@ -101,15 +104,9 @@ jobs:
|
||||||
Edit your PR title to match the format above.
|
Edit your PR title to match the format above.
|
||||||
`;
|
`;
|
||||||
|
|
||||||
|
core.info(message);
|
||||||
|
await core.summary.addRaw(message).write();
|
||||||
core.setFailed(message);
|
core.setFailed(message);
|
||||||
|
|
||||||
// Also post as a comment
|
|
||||||
await github.rest.issues.createComment({
|
|
||||||
owner: context.repo.owner,
|
|
||||||
repo: context.repo.repo,
|
|
||||||
issue_number: context.payload.pull_request.number,
|
|
||||||
body: message
|
|
||||||
});
|
|
||||||
} else {
|
} else {
|
||||||
const [type] = matchedType;
|
const [type] = matchedType;
|
||||||
let versionBump = 'patch (0.3.0 → 0.3.1)';
|
let versionBump = 'patch (0.3.0 → 0.3.1)';
|
||||||
|
|
@ -160,52 +157,29 @@ jobs:
|
||||||
needs: pr-title-check
|
needs: pr-title-check
|
||||||
if: needs.pr-title-check.outputs.title-valid == 'true'
|
if: needs.pr-title-check.outputs.title-valid == 'true'
|
||||||
outputs:
|
outputs:
|
||||||
has-evals: ${{ steps.filter.outputs.evals }}
|
has-evals: ${{ steps.filter.outputs.has-evals }}
|
||||||
has-docs: ${{ steps.filter.outputs.docs }}
|
has-docs: ${{ steps.filter.outputs.has-docs }}
|
||||||
has-workflows: ${{ steps.filter.outputs.workflows }}
|
has-workflows: ${{ steps.filter.outputs.has-workflows }}
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
|
persist-credentials: false
|
||||||
|
|
||||||
|
- name: Setup Bun
|
||||||
|
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
|
||||||
|
with:
|
||||||
|
bun-version: 1.3.14
|
||||||
|
|
||||||
- name: Check changed files
|
- name: Check changed files
|
||||||
id: filter
|
id: filter
|
||||||
run: |
|
run: |
|
||||||
# Get list of changed files
|
echo "Changed files:" >&2
|
||||||
git fetch origin ${{ github.base_ref }}
|
git diff --name-only -z "${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }}" |
|
||||||
CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD)
|
tee >(while IFS= read -r -d '' path; do printf ' %q\n' "$path" >&2; done) |
|
||||||
|
bun run scripts/validation/detect-pr-changes.ts
|
||||||
echo "Changed files:"
|
|
||||||
echo "$CHANGED_FILES"
|
|
||||||
|
|
||||||
# Check for evals changes
|
|
||||||
if echo "$CHANGED_FILES" | grep -q "^evals/"; then
|
|
||||||
echo "has-evals=true" >> $GITHUB_OUTPUT
|
|
||||||
echo "✅ Evals changes detected"
|
|
||||||
else
|
|
||||||
echo "has-evals=false" >> $GITHUB_OUTPUT
|
|
||||||
echo "ℹ️ No evals changes"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Check for docs changes
|
|
||||||
if echo "$CHANGED_FILES" | grep -q "^docs/"; then
|
|
||||||
echo "has-docs=true" >> $GITHUB_OUTPUT
|
|
||||||
echo "✅ Docs changes detected"
|
|
||||||
else
|
|
||||||
echo "has-docs=false" >> $GITHUB_OUTPUT
|
|
||||||
echo "ℹ️ No docs changes"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Check for workflow changes
|
|
||||||
if echo "$CHANGED_FILES" | grep -q "^.github/workflows/"; then
|
|
||||||
echo "has-workflows=true" >> $GITHUB_OUTPUT
|
|
||||||
echo "✅ Workflow changes detected"
|
|
||||||
else
|
|
||||||
echo "has-workflows=false" >> $GITHUB_OUTPUT
|
|
||||||
echo "ℹ️ No workflow changes"
|
|
||||||
fi
|
|
||||||
|
|
||||||
build-check:
|
build-check:
|
||||||
name: Build & Validate
|
name: Build & Validate
|
||||||
|
|
@ -219,6 +193,8 @@ jobs:
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
persist-credentials: false
|
||||||
|
|
||||||
- name: Setup Node.js
|
- name: Setup Node.js
|
||||||
uses: actions/setup-node@v4
|
uses: actions/setup-node@v4
|
||||||
|
|
@ -295,24 +271,41 @@ jobs:
|
||||||
if [ "${{ needs.check-changes.outputs.has-workflows }}" == "true" ]; then
|
if [ "${{ needs.check-changes.outputs.has-workflows }}" == "true" ]; then
|
||||||
echo " - ⚙️ Workflow changes detected" >> $GITHUB_STEP_SUMMARY
|
echo " - ⚙️ Workflow changes detected" >> $GITHUB_STEP_SUMMARY
|
||||||
fi
|
fi
|
||||||
else
|
elif [ "${{ needs.check-changes.result }}" == "failure" ]; then
|
||||||
|
echo "❌ **Changed Files:** Detection failed - check logs" >> $GITHUB_STEP_SUMMARY
|
||||||
|
elif [ "${{ needs.check-changes.result }}" == "cancelled" ]; then
|
||||||
|
echo "🚫 **Changed Files:** Detection cancelled" >> $GITHUB_STEP_SUMMARY
|
||||||
|
elif [ "${{ needs.check-changes.result }}" == "skipped" ]; then
|
||||||
echo "⏭️ **Changed Files:** Skipped (title validation failed)" >> $GITHUB_STEP_SUMMARY
|
echo "⏭️ **Changed Files:** Skipped (title validation failed)" >> $GITHUB_STEP_SUMMARY
|
||||||
|
else
|
||||||
|
echo "❓ **Changed Files:** Unknown result" >> $GITHUB_STEP_SUMMARY
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Build Check
|
# Build Check
|
||||||
if [ "${{ needs.build-check.result }}" == "success" ]; then
|
if [ "${{ needs.build-check.result }}" == "success" ]; then
|
||||||
echo "✅ **Build & Validate:** Passed" >> $GITHUB_STEP_SUMMARY
|
echo "✅ **Build & Validate:** Passed" >> $GITHUB_STEP_SUMMARY
|
||||||
elif [ "${{ needs.build-check.result }}" == "skipped" ]; then
|
elif [ "${{ needs.build-check.result }}" == "skipped" ]; then
|
||||||
echo "⏭️ **Build & Validate:** Skipped (no evals changes)" >> $GITHUB_STEP_SUMMARY
|
if [ "${{ needs.check-changes.result }}" != "success" ]; then
|
||||||
|
echo "⏭️ **Build & Validate:** Skipped (change detection did not succeed)" >> $GITHUB_STEP_SUMMARY
|
||||||
|
elif [ "${{ needs.check-changes.outputs.has-evals }}" != "true" ]; then
|
||||||
|
echo "⏭️ **Build & Validate:** Skipped (no evals changes)" >> $GITHUB_STEP_SUMMARY
|
||||||
|
else
|
||||||
|
echo "⏭️ **Build & Validate:** Skipped (prerequisite not met)" >> $GITHUB_STEP_SUMMARY
|
||||||
|
fi
|
||||||
elif [ "${{ needs.build-check.result }}" == "failure" ]; then
|
elif [ "${{ needs.build-check.result }}" == "failure" ]; then
|
||||||
echo "❌ **Build & Validate:** Failed - check logs" >> $GITHUB_STEP_SUMMARY
|
echo "❌ **Build & Validate:** Failed - check logs" >> $GITHUB_STEP_SUMMARY
|
||||||
|
elif [ "${{ needs.build-check.result }}" == "cancelled" ]; then
|
||||||
|
echo "🚫 **Build & Validate:** Cancelled" >> $GITHUB_STEP_SUMMARY
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "" >> $GITHUB_STEP_SUMMARY
|
echo "" >> $GITHUB_STEP_SUMMARY
|
||||||
|
|
||||||
# Overall status
|
# Overall status
|
||||||
if [ "${{ needs.pr-title-check.result }}" == "success" ] && \
|
if [ "${{ needs.pr-title-check.result }}" == "success" ] && \
|
||||||
([ "${{ needs.build-check.result }}" == "success" ] || [ "${{ needs.build-check.result }}" == "skipped" ]); then
|
[ "${{ needs.check-changes.result }}" == "success" ] && \
|
||||||
|
{ [ "${{ needs.build-check.result }}" == "success" ] || \
|
||||||
|
{ [ "${{ needs.build-check.result }}" == "skipped" ] && \
|
||||||
|
[ "${{ needs.check-changes.outputs.has-evals }}" != "true" ]; }; }; then
|
||||||
echo "### ✅ All Required Checks Passed!" >> $GITHUB_STEP_SUMMARY
|
echo "### ✅ All Required Checks Passed!" >> $GITHUB_STEP_SUMMARY
|
||||||
echo "" >> $GITHUB_STEP_SUMMARY
|
echo "" >> $GITHUB_STEP_SUMMARY
|
||||||
echo "This PR is ready for review." >> $GITHUB_STEP_SUMMARY
|
echo "This PR is ready for review." >> $GITHUB_STEP_SUMMARY
|
||||||
|
|
@ -320,4 +313,5 @@ jobs:
|
||||||
echo "### ❌ Some Checks Failed" >> $GITHUB_STEP_SUMMARY
|
echo "### ❌ Some Checks Failed" >> $GITHUB_STEP_SUMMARY
|
||||||
echo "" >> $GITHUB_STEP_SUMMARY
|
echo "" >> $GITHUB_STEP_SUMMARY
|
||||||
echo "Please fix the failing checks before merging." >> $GITHUB_STEP_SUMMARY
|
echo "Please fix the failing checks before merging." >> $GITHUB_STEP_SUMMARY
|
||||||
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
|
||||||
386
.github/workflows/validate-registry.yml
vendored
386
.github/workflows/validate-registry.yml
vendored
|
|
@ -1,379 +1,87 @@
|
||||||
name: Validate Registry on PR
|
name: Validate Registry on PR
|
||||||
|
|
||||||
# This workflow validates the registry.json and prompt library structure on all PRs.
|
# Contributor code must run only with a read-only token and no secrets.
|
||||||
#
|
# Never change this workflow back to pull_request_target while it checks out
|
||||||
# For bot-created PRs (like automated version bumps), the workflow won't trigger automatically
|
# and executes pull-request code.
|
||||||
# due to GitHub's security restrictions. In those cases, you can manually trigger this workflow:
|
|
||||||
#
|
|
||||||
# Option 1 - Run Validation:
|
|
||||||
# 1. Go to Actions > Validate Registry on PR > Run workflow
|
|
||||||
# 2. Enter the PR number (e.g., 106)
|
|
||||||
# 3. Leave "skip_validation" unchecked
|
|
||||||
# 4. Click "Run workflow"
|
|
||||||
#
|
|
||||||
# Option 2 - Admin Bypass (for trusted bot PRs):
|
|
||||||
# 1. Go to Actions > Validate Registry on PR > Run workflow
|
|
||||||
# 2. Enter the PR number (e.g., 106)
|
|
||||||
# 3. Check "skip_validation" checkbox
|
|
||||||
# 4. Click "Run workflow"
|
|
||||||
# 5. The check will pass immediately without running validation
|
|
||||||
|
|
||||||
on:
|
on:
|
||||||
# Use pull_request_target to allow running on bot-created PRs
|
pull_request:
|
||||||
# This also allows the workflow to write to the PR branch
|
|
||||||
pull_request_target:
|
|
||||||
branches:
|
branches:
|
||||||
- main
|
- main
|
||||||
- dev
|
- dev
|
||||||
# Removed paths filter - this check is required by repository ruleset
|
|
||||||
# so it must run on ALL PRs to prevent blocking merges
|
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
inputs:
|
|
||||||
pr_number:
|
|
||||||
description: 'PR number to validate (for manual runs on bot-created PRs)'
|
|
||||||
required: false
|
|
||||||
type: number
|
|
||||||
skip_validation:
|
|
||||||
description: 'Skip validation checks (maintainer override)'
|
|
||||||
required: false
|
|
||||||
type: boolean
|
|
||||||
default: false
|
|
||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
contents: write
|
contents: read
|
||||||
pull-requests: write
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
validate-and-update:
|
validate-and-update:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 10
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Admin bypass check
|
- name: Checkout code without persisted credentials
|
||||||
if: github.event_name == 'workflow_dispatch' && github.event.inputs.skip_validation == 'true'
|
|
||||||
run: |
|
|
||||||
echo "## ✅ Validation Bypassed (Admin Override)" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "Validation checks skipped by maintainer." >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "PR: #${{ github.event.inputs.pr_number }}" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "The workflow will complete successfully without running validation steps." >> $GITHUB_STEP_SUMMARY
|
|
||||||
|
|
||||||
- name: Checkout repository (for manual runs)
|
|
||||||
if: github.event_name == 'workflow_dispatch' && github.event.inputs.skip_validation != 'true'
|
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
token: ${{ secrets.GITHUB_TOKEN }}
|
persist-credentials: false
|
||||||
|
|
||||||
- name: Get PR details (for manual runs)
|
- name: Install system dependencies
|
||||||
if: github.event_name == 'workflow_dispatch' && github.event.inputs.pr_number != '' && github.event.inputs.skip_validation != 'true'
|
|
||||||
id: get_pr
|
|
||||||
run: |
|
|
||||||
PR_DATA=$(gh pr view ${{ github.event.inputs.pr_number }} --json headRefName,headRepository,headRepositoryOwner)
|
|
||||||
echo "head_ref=$(echo $PR_DATA | jq -r '.headRefName')" >> $GITHUB_OUTPUT
|
|
||||||
echo "head_repo=$(echo $PR_DATA | jq -r '.headRepositoryOwner.login + "/" + .headRepository.name')" >> $GITHUB_OUTPUT
|
|
||||||
env:
|
|
||||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
|
|
||||||
- name: Checkout PR branch
|
|
||||||
if: github.event.inputs.skip_validation != 'true'
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
# For manual runs: use PR details from get_pr step
|
|
||||||
# For PR events: use event data
|
|
||||||
repository: ${{ github.event_name == 'workflow_dispatch' && steps.get_pr.outputs.head_repo || github.event.pull_request.head.repo.full_name }}
|
|
||||||
ref: ${{ github.event_name == 'workflow_dispatch' && steps.get_pr.outputs.head_ref || github.event.pull_request.head.ref }}
|
|
||||||
fetch-depth: 0
|
|
||||||
token: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
|
|
||||||
- name: Detect fork PR
|
|
||||||
if: github.event.inputs.skip_validation != 'true'
|
|
||||||
id: fork_check
|
|
||||||
run: |
|
|
||||||
# For manual runs, use the fetched PR data
|
|
||||||
if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
|
|
||||||
HEAD_REPO="${{ steps.get_pr.outputs.head_repo }}"
|
|
||||||
else
|
|
||||||
HEAD_REPO="${{ github.event.pull_request.head.repo.full_name }}"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ "$HEAD_REPO" != "${{ github.repository }}" ]; then
|
|
||||||
echo "is_fork=true" >> $GITHUB_OUTPUT
|
|
||||||
echo "🔀 Fork PR detected from: $HEAD_REPO"
|
|
||||||
else
|
|
||||||
echo "is_fork=false" >> $GITHUB_OUTPUT
|
|
||||||
echo "📝 Internal PR detected"
|
|
||||||
fi
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
if: github.event.inputs.skip_validation != 'true'
|
|
||||||
run: |
|
run: |
|
||||||
sudo apt-get update
|
sudo apt-get update
|
||||||
sudo apt-get install -y jq
|
sudo apt-get install -y jq
|
||||||
|
|
||||||
- name: Install Bun
|
- name: Install Bun
|
||||||
if: github.event.inputs.skip_validation != 'true'
|
|
||||||
uses: oven-sh/setup-bun@v2
|
uses: oven-sh/setup-bun@v2
|
||||||
with:
|
with:
|
||||||
bun-version: latest
|
bun-version: latest
|
||||||
|
|
||||||
- name: Install dependencies
|
- name: Install project dependencies
|
||||||
if: github.event.inputs.skip_validation != 'true'
|
run: bun install --frozen-lockfile
|
||||||
run: |
|
|
||||||
# Install root dependencies (glob package needed for validation script)
|
- name: Make validation scripts executable
|
||||||
bun install --frozen-lockfile
|
|
||||||
|
|
||||||
- name: Make scripts executable
|
|
||||||
if: github.event.inputs.skip_validation != 'true'
|
|
||||||
run: |
|
run: |
|
||||||
chmod +x scripts/registry/validate-registry.sh
|
chmod +x scripts/registry/validate-registry.sh
|
||||||
chmod +x scripts/registry/auto-detect-components.sh
|
chmod +x scripts/registry/auto-detect-components.sh
|
||||||
chmod +x scripts/registry/register-component.sh
|
chmod +x scripts/registry/register-component.sh
|
||||||
chmod +x scripts/prompts/validate-pr.sh
|
chmod +x scripts/prompts/validate-pr.sh
|
||||||
|
|
||||||
- name: Auto-detect new components
|
- name: Report registry drift
|
||||||
if: github.event.inputs.skip_validation != 'true'
|
id: registry_drift
|
||||||
id: auto_detect
|
|
||||||
run: |
|
run: |
|
||||||
echo "## 🔍 Auto-Detection Results" >> $GITHUB_STEP_SUMMARY
|
./scripts/registry/auto-detect-components.sh --dry-run > /tmp/detect-output.txt 2>&1
|
||||||
echo "" >> $GITHUB_STEP_SUMMARY
|
cat /tmp/detect-output.txt
|
||||||
|
sed $'s/\033\\[[0-9;]*m//g' /tmp/detect-output.txt > /tmp/detect-output-plain.txt
|
||||||
# Run auto-detect in dry-run mode first to see what would be added
|
|
||||||
if ./scripts/registry/auto-detect-components.sh --dry-run > /tmp/detect-output.txt 2>&1; then
|
if grep -Eq 'New Components:[[:space:]]*[1-9][0-9]*([[:space:]]|$)' /tmp/detect-output-plain.txt; then
|
||||||
cat /tmp/detect-output.txt >> $GITHUB_STEP_SUMMARY
|
echo "drift_detected=true" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "## Registry drift detected" >> "$GITHUB_STEP_SUMMARY"
|
||||||
# Check if new components were found
|
echo "" >> "$GITHUB_STEP_SUMMARY"
|
||||||
if grep -q "Found.*new component" /tmp/detect-output.txt; then
|
echo "The repository already contains components not represented in registry.json." >> "$GITHUB_STEP_SUMMARY"
|
||||||
echo "new_components=true" >> $GITHUB_OUTPUT
|
echo "This is reported but does not block security validation until the existing drift is reconciled." >> "$GITHUB_STEP_SUMMARY"
|
||||||
echo "" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "⚠️ New components detected - will auto-add to registry" >> $GITHUB_STEP_SUMMARY
|
|
||||||
else
|
|
||||||
echo "new_components=false" >> $GITHUB_OUTPUT
|
|
||||||
echo "✅ No new components found" >> $GITHUB_STEP_SUMMARY
|
|
||||||
fi
|
|
||||||
else
|
else
|
||||||
echo "new_components=false" >> $GITHUB_OUTPUT
|
echo "drift_detected=false" >> "$GITHUB_OUTPUT"
|
||||||
echo "❌ Auto-detection failed" >> $GITHUB_STEP_SUMMARY
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
- name: Add new components to registry
|
|
||||||
if: steps.auto_detect.outputs.new_components == 'true' && github.event.inputs.skip_validation != 'true'
|
|
||||||
run: |
|
|
||||||
echo "## 📝 Adding New Components" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "" >> $GITHUB_STEP_SUMMARY
|
|
||||||
|
|
||||||
./scripts/registry/auto-detect-components.sh --auto-add | tee -a $GITHUB_STEP_SUMMARY
|
|
||||||
|
|
||||||
- name: Validate prompt library structure
|
- name: Validate prompt library structure
|
||||||
if: github.event.inputs.skip_validation != 'true'
|
run: ./scripts/prompts/validate-pr.sh
|
||||||
id: validate_prompts
|
|
||||||
run: |
|
|
||||||
echo "## 🔍 Prompt Library Validation" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "" >> $GITHUB_STEP_SUMMARY
|
|
||||||
|
|
||||||
if ./scripts/prompts/validate-pr.sh > /tmp/prompt-validation.txt 2>&1; then
|
|
||||||
echo "prompt_validation=passed" >> $GITHUB_OUTPUT
|
|
||||||
echo "✅ Prompt library structure is valid!" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
|
|
||||||
cat /tmp/prompt-validation.txt >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
|
|
||||||
else
|
|
||||||
echo "prompt_validation=failed" >> $GITHUB_OUTPUT
|
|
||||||
echo "❌ Prompt library validation failed!" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
|
|
||||||
cat /tmp/prompt-validation.txt >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "**Architecture:**" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "- Agent files (.opencode/agent/**/*.md) = Canonical defaults" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "- Prompt variants (.opencode/prompts/<agent>/<model>.md) = Model-specific" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "- default.md files should NOT exist" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "- Agents organized in category subdirectories (core/, development/, content/, etc.)" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "See [CONTRIBUTING.md](docs/contributing/CONTRIBUTING.md) for details" >> $GITHUB_STEP_SUMMARY
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
- name: Validate markdown context links
|
- name: Validate markdown context links
|
||||||
if: github.event.inputs.skip_validation != 'true'
|
run: npm run validate:context-links
|
||||||
id: validate_context_links
|
|
||||||
run: |
|
|
||||||
echo "## 🔗 Context Link Validation" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "" >> $GITHUB_STEP_SUMMARY
|
|
||||||
|
|
||||||
if bun run scripts/validation/validate-markdown-links.ts > /tmp/context-link-validation.txt 2>&1; then
|
|
||||||
echo "context_links=passed" >> $GITHUB_OUTPUT
|
|
||||||
echo "✅ Context markdown links are valid!" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
|
|
||||||
cat /tmp/context-link-validation.txt >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
|
|
||||||
else
|
|
||||||
echo "context_links=failed" >> $GITHUB_OUTPUT
|
|
||||||
echo "❌ Context markdown link validation failed!" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
|
|
||||||
cat /tmp/context-link-validation.txt >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
- name: Validate registry
|
- name: Validate registry
|
||||||
if: github.event.inputs.skip_validation != 'true'
|
run: npm run validate:registry
|
||||||
id: validate
|
|
||||||
run: |
|
|
||||||
echo "## ✅ Registry Validation" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "" >> $GITHUB_STEP_SUMMARY
|
|
||||||
|
|
||||||
# Use TypeScript validator (fast and reliable)
|
|
||||||
# Run validation and capture output (show in logs AND save to file)
|
|
||||||
if bun run scripts/registry/validate-registry.ts 2>&1 | tee /tmp/validation-output.txt; then
|
|
||||||
echo "validation=passed" >> $GITHUB_OUTPUT
|
|
||||||
echo "✅ All registry paths are valid!" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
|
|
||||||
cat /tmp/validation-output.txt >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
|
|
||||||
else
|
|
||||||
echo "validation=failed" >> $GITHUB_OUTPUT
|
|
||||||
echo "❌ Registry validation failed!" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
|
|
||||||
cat /tmp/validation-output.txt >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "**Check the logs above for detailed error output**" >> $GITHUB_STEP_SUMMARY
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
- name: Commit registry updates (Internal PRs only)
|
|
||||||
if: |
|
|
||||||
github.event.inputs.skip_validation != 'true' &&
|
|
||||||
steps.fork_check.outputs.is_fork == 'false' &&
|
|
||||||
steps.auto_detect.outputs.new_components == 'true' &&
|
|
||||||
steps.validate_prompts.outputs.prompt_validation == 'passed' &&
|
|
||||||
steps.validate.outputs.validation == 'passed'
|
|
||||||
run: |
|
|
||||||
git config --local user.email "github-actions[bot]@users.noreply.github.com"
|
|
||||||
git config --local user.name "github-actions[bot]"
|
|
||||||
|
|
||||||
if ! git diff --quiet registry.json; then
|
|
||||||
git add registry.json
|
|
||||||
git commit -m "chore: auto-update registry with new components [skip ci]"
|
|
||||||
|
|
||||||
# For manual runs, use the fetched branch name
|
|
||||||
BRANCH_NAME="${{ github.event_name == 'workflow_dispatch' && steps.get_pr.outputs.head_ref || github.event.pull_request.head.ref }}"
|
|
||||||
git push origin "$BRANCH_NAME"
|
|
||||||
|
|
||||||
echo "## 🚀 Registry Updated" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "Registry has been automatically updated with new components." >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "Changes have been pushed to this PR branch." >> $GITHUB_STEP_SUMMARY
|
|
||||||
else
|
|
||||||
echo "## ℹ️ No Changes to Commit" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "Registry is already up to date." >> $GITHUB_STEP_SUMMARY
|
|
||||||
fi
|
|
||||||
|
|
||||||
- name: Fork PR notice
|
|
||||||
if: |
|
|
||||||
github.event.inputs.skip_validation != 'true' &&
|
|
||||||
steps.fork_check.outputs.is_fork == 'true' &&
|
|
||||||
steps.auto_detect.outputs.new_components == 'true' &&
|
|
||||||
steps.validate_prompts.outputs.prompt_validation == 'passed' &&
|
|
||||||
steps.validate.outputs.validation == 'passed'
|
|
||||||
uses: actions/github-script@v7
|
|
||||||
with:
|
|
||||||
script: |
|
|
||||||
github.rest.issues.createComment({
|
|
||||||
issue_number: context.issue.number,
|
|
||||||
owner: context.repo.owner,
|
|
||||||
repo: context.repo.repo,
|
|
||||||
body: `## 📝 Registry Update Needed
|
|
||||||
|
|
||||||
Hi @${{ github.event.pull_request.user.login }}! 👋
|
|
||||||
|
|
||||||
New components were detected in your PR. Since this is a fork PR, I can't auto-commit the registry updates for security reasons.
|
|
||||||
|
|
||||||
**Please run these commands locally:**
|
|
||||||
\`\`\`bash
|
|
||||||
./scripts/registry/auto-detect-components.sh --auto-add
|
|
||||||
git add registry.json
|
|
||||||
git commit -m "chore: update registry"
|
|
||||||
git push
|
|
||||||
\`\`\`
|
|
||||||
|
|
||||||
Once you push the updated registry, the checks will pass! ✅`
|
|
||||||
});
|
|
||||||
|
|
||||||
- name: Fork PR summary
|
|
||||||
if: steps.fork_check.outputs.is_fork == 'true' && github.event.inputs.skip_validation != 'true'
|
|
||||||
run: |
|
|
||||||
echo "## 🔀 Fork PR Detected" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "This is an external contribution - thank you! 🎉" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "" >> $GITHUB_STEP_SUMMARY
|
|
||||||
|
|
||||||
if [ "${{ steps.auto_detect.outputs.new_components }}" == "true" ]; then
|
|
||||||
echo "⚠️ **Action Required:** New components detected" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "A comment has been posted with instructions to update the registry." >> $GITHUB_STEP_SUMMARY
|
|
||||||
else
|
|
||||||
echo "✅ No registry updates needed" >> $GITHUB_STEP_SUMMARY
|
|
||||||
fi
|
|
||||||
|
|
||||||
- name: Post validation summary
|
|
||||||
if: always()
|
|
||||||
run: |
|
|
||||||
echo "" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "---" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "" >> $GITHUB_STEP_SUMMARY
|
|
||||||
|
|
||||||
PROMPT_VALID="${{ steps.validate_prompts.outputs.prompt_validation }}"
|
|
||||||
CONTEXT_LINKS_VALID="${{ steps.validate_context_links.outputs.context_links }}"
|
|
||||||
REGISTRY_VALID="${{ steps.validate.outputs.validation }}"
|
|
||||||
|
|
||||||
if [ "$PROMPT_VALID" = "passed" ] && [ "$CONTEXT_LINKS_VALID" = "passed" ] && [ "$REGISTRY_VALID" = "passed" ]; then
|
- name: Validation summary
|
||||||
echo "### ✅ All Validations Passed" >> $GITHUB_STEP_SUMMARY
|
if: success()
|
||||||
echo "" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "- ✅ Prompt library structure is valid" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "- ✅ Context markdown links are valid" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "- ✅ Registry paths are valid" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "This PR is ready for review!" >> $GITHUB_STEP_SUMMARY
|
|
||||||
else
|
|
||||||
echo "### ❌ Validation Failed" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "" >> $GITHUB_STEP_SUMMARY
|
|
||||||
|
|
||||||
if [ "$PROMPT_VALID" != "passed" ]; then
|
|
||||||
echo "- ❌ Prompt library validation failed" >> $GITHUB_STEP_SUMMARY
|
|
||||||
else
|
|
||||||
echo "- ✅ Prompt library validation passed" >> $GITHUB_STEP_SUMMARY
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ "$CONTEXT_LINKS_VALID" != "passed" ]; then
|
|
||||||
echo "- ❌ Context markdown link validation failed" >> $GITHUB_STEP_SUMMARY
|
|
||||||
else
|
|
||||||
echo "- ✅ Context markdown link validation passed" >> $GITHUB_STEP_SUMMARY
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ "$REGISTRY_VALID" != "passed" ]; then
|
|
||||||
echo "- ❌ Registry validation failed" >> $GITHUB_STEP_SUMMARY
|
|
||||||
else
|
|
||||||
echo "- ✅ Registry validation passed" >> $GITHUB_STEP_SUMMARY
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "" >> $GITHUB_STEP_SUMMARY
|
|
||||||
echo "Please fix the issues above before merging." >> $GITHUB_STEP_SUMMARY
|
|
||||||
fi
|
|
||||||
|
|
||||||
- name: Fail if validation failed
|
|
||||||
if: |
|
|
||||||
(steps.validate_prompts.outputs.prompt_validation == 'failed' || steps.validate_context_links.outputs.context_links == 'failed' || steps.validate.outputs.validation == 'failed') &&
|
|
||||||
github.event.inputs.skip_validation != 'true'
|
|
||||||
run: |
|
run: |
|
||||||
echo "❌ Validation failed - blocking PR merge"
|
echo "## All registry validations passed" >> "$GITHUB_STEP_SUMMARY"
|
||||||
echo "Maintainer can override by running workflow manually with 'skip_validation' enabled"
|
echo "" >> "$GITHUB_STEP_SUMMARY"
|
||||||
exit 1
|
echo "- Prompt library structure is valid" >> "$GITHUB_STEP_SUMMARY"
|
||||||
|
echo "- Context markdown links are valid" >> "$GITHUB_STEP_SUMMARY"
|
||||||
|
echo "- Registry paths and dependencies are valid" >> "$GITHUB_STEP_SUMMARY"
|
||||||
|
if [ "${{ steps.registry_drift.outputs.drift_detected }}" = "true" ]; then
|
||||||
|
echo "- Existing registry drift was reported separately" >> "$GITHUB_STEP_SUMMARY"
|
||||||
|
else
|
||||||
|
echo "- No unregistered components were detected" >> "$GITHUB_STEP_SUMMARY"
|
||||||
|
fi
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,192 @@
|
||||||
|
# CI Threat Model and Baseline
|
||||||
|
|
||||||
|
**Date:** 2026-07-14
|
||||||
|
**Scope:** GitHub Actions, repository security settings, and required checks
|
||||||
|
**Status:** Baseline only — no security fix is implemented by this document
|
||||||
|
|
||||||
|
## Security Objective
|
||||||
|
|
||||||
|
Pull requests must be able to test contributor code without giving that code repository write access, secrets, identity tokens, or a path into later privileged jobs. Trusted automation must use narrowly scoped permissions and must not silently bypass review.
|
||||||
|
|
||||||
|
## Threat Actors
|
||||||
|
|
||||||
|
- A malicious external contributor opening a fork pull request.
|
||||||
|
- A compromised contributor, maintainer, bot, action publisher, or dependency.
|
||||||
|
- A well-intentioned contributor whose workflow or script has unsafe side effects.
|
||||||
|
- An attacker posting crafted issue or PR comments to trigger automation.
|
||||||
|
- A compromised upstream action referenced by a mutable tag or branch.
|
||||||
|
|
||||||
|
## Protected Assets
|
||||||
|
|
||||||
|
- `GITHUB_TOKEN` write permissions and repository contents.
|
||||||
|
- Pull requests, issues, releases, tags, branches, and rulesets.
|
||||||
|
- `ANTHROPIC_API_KEY`, `OPENCODE_API_KEY`, and any repository secrets.
|
||||||
|
- OIDC identity available through `id-token: write`.
|
||||||
|
- Published npm packages, release artifacts, registry data, and generated documentation.
|
||||||
|
- Maintainer trust and the integrity of `main`.
|
||||||
|
|
||||||
|
## Trust Boundaries
|
||||||
|
|
||||||
|
1. **Fork pull-request code is untrusted.** Repository scripts, package manifests, lockfiles, actions, generated files, and artifacts from the fork are attacker-controlled.
|
||||||
|
2. **Default-branch code is trusted only after review.** A merge can still introduce compromised automation.
|
||||||
|
3. **Third-party actions are external code.** Mutable tags and branches can change after review.
|
||||||
|
4. **Artifacts and caches cross job boundaries.** Data from an untrusted job must never become executable input to a privileged job.
|
||||||
|
5. **Comments and workflow inputs are untrusted strings.** Authorization checks do not make their content safe for shell interpolation.
|
||||||
|
6. **GitHub settings are part of the security boundary.** Safe workflow files are insufficient if default token permissions and rulesets remain weak.
|
||||||
|
|
||||||
|
## Critical Untrusted Execution Path
|
||||||
|
|
||||||
|
`.github/workflows/validate-registry.yml` currently creates a direct privileged path:
|
||||||
|
|
||||||
|
1. It runs on `pull_request_target` (`lines 21–29`).
|
||||||
|
2. It grants `contents: write` and `pull-requests: write` (`lines 42–44`).
|
||||||
|
3. It checks out the pull request head repository and branch (`lines 78–87`).
|
||||||
|
4. It runs `bun install` using the contributor-controlled lockfile/package graph (`lines 120–124`).
|
||||||
|
5. It executes contributor-controlled shell and TypeScript files (`lines 126–248`).
|
||||||
|
6. The same job retains its write-capable token while running this code.
|
||||||
|
|
||||||
|
**Impact:** A malicious fork can attempt arbitrary code execution with a repository write token. Fork-specific logic later in the job does not remove the token or undo earlier execution.
|
||||||
|
|
||||||
|
**Required remediation direction:** Move untrusted validation to a `pull_request` workflow with `contents: read` and no secrets. If a `pull_request_target` workflow remains, it must be metadata-only and must never check out or execute the contributor head.
|
||||||
|
|
||||||
|
## Workflow Inventory
|
||||||
|
|
||||||
|
| Workflow | Trigger | Current privilege or secret | Baseline concern |
|
||||||
|
|---|---|---|---|
|
||||||
|
| `validate-registry.yml` | `pull_request_target`, manual | Contents and PR write | **Critical:** executes fork code with write token |
|
||||||
|
| `pr-checks.yml` | Pull request | Implicit defaults | Output names do not match values written; relevant builds are skipped |
|
||||||
|
| `installer-checks.yml` | Pull request, push, manual | Implicit defaults | Mutable `action-shellcheck@master`; final status ignores compatibility/profile failures |
|
||||||
|
| `validate-test-suites.yml` | Pull request, push, manual | Implicit defaults; PR comment attempt | No explicit permissions; artifact contents originate from PR code |
|
||||||
|
| `update-registry.yml` | Push to `main`, manual | Contents write | Direct push to `main`; validation failure only warns; generated change bypasses PR review |
|
||||||
|
| `sync-docs.yml` | Push to `main`, manual | Contents, PR, and issues write | Broad workflow-level permissions; creates branches/issues and deletes branches on failure |
|
||||||
|
| `post-merge-pr.yml` | Push to `main`, manual | Contents and PR write | Broad workflow-level permissions; creates version branches and PRs |
|
||||||
|
| `create-release.yml` | Push to `main`, manual | Contents write | Tag/release authority; actions are tag-pinned rather than SHA-pinned |
|
||||||
|
| `opencode.yml` | Issue comment | OIDC, contents, PR, issues write; Anthropic key | `sst/opencode/github@latest` is mutable and receives powerful credentials |
|
||||||
|
| `evals/run-evaluations.yml` | Schedule, push to `main`, manual | OpenCode API key | User-supplied workflow inputs reach a shell command without a strict allowlist |
|
||||||
|
|
||||||
|
## Confirmed Configuration Baseline
|
||||||
|
|
||||||
|
Read-only GitHub API inspection on 2026-07-14 reported:
|
||||||
|
|
||||||
|
- Default workflow token permission: **write**.
|
||||||
|
- Actions may approve pull-request reviews: **enabled**.
|
||||||
|
- Allowed actions: **all**.
|
||||||
|
- Full-SHA pinning requirement: **disabled**.
|
||||||
|
- Fork workflow approval policy: **first-time contributors only**.
|
||||||
|
- Private vulnerability reporting: **disabled**.
|
||||||
|
- CodeQL default setup: **not configured**.
|
||||||
|
- Dependabot security updates: **disabled**.
|
||||||
|
- Secret scanning: **enabled**.
|
||||||
|
- Secret scanning push protection: **enabled**.
|
||||||
|
- Automatic branch deletion after merge: **disabled**.
|
||||||
|
- No classic branch protection rule; an active repository ruleset protects `main`.
|
||||||
|
|
||||||
|
### Active `main` ruleset
|
||||||
|
|
||||||
|
- Pull requests are required.
|
||||||
|
- Required approving reviews: **0**.
|
||||||
|
- Code-owner review: **not required**.
|
||||||
|
- Latest-push approval: **not required**.
|
||||||
|
- Conversation resolution: **not required**.
|
||||||
|
- Strict up-to-date status checks: **disabled**.
|
||||||
|
- Only required check: `validate-and-update`.
|
||||||
|
- No bypass actors are configured.
|
||||||
|
|
||||||
|
The only required check is the workflow containing the critical `pull_request_target` vulnerability.
|
||||||
|
|
||||||
|
## PR Check Correctness
|
||||||
|
|
||||||
|
`.github/workflows/pr-checks.yml` declares:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
has-evals: ${{ steps.filter.outputs.evals }}
|
||||||
|
has-docs: ${{ steps.filter.outputs.docs }}
|
||||||
|
has-workflows: ${{ steps.filter.outputs.workflows }}
|
||||||
|
```
|
||||||
|
|
||||||
|
The `filter` step actually writes `has-evals`, `has-docs`, and `has-workflows`. Therefore dependent jobs can see empty outputs and skip required validation. The workflow builds and validates eval suites but does not run Vitest.
|
||||||
|
|
||||||
|
## Action Pinning Baseline
|
||||||
|
|
||||||
|
Current workflows use mutable references including:
|
||||||
|
|
||||||
|
- `sst/opencode/github@latest`
|
||||||
|
- `ludeeus/action-shellcheck@master`
|
||||||
|
- Major-version tags such as `actions/checkout@v4`, `actions/setup-node@v4`, `actions/github-script@v7`, `actions/upload-artifact@v4`, and `oven-sh/setup-bun@v2`
|
||||||
|
|
||||||
|
Current repository settings neither restrict allowed actions nor require full commit SHA pinning.
|
||||||
|
|
||||||
|
## Additional High-Risk Findings
|
||||||
|
|
||||||
|
### Registry writes after validation failure
|
||||||
|
|
||||||
|
`update-registry.yml` validates generated registry state but intentionally does not fail on an invalid registry. It can then commit and push directly to `main` using `[skip ci]`.
|
||||||
|
|
||||||
|
### Installer summary can report false success
|
||||||
|
|
||||||
|
`installer-checks.yml` displays compatibility and profile results, but its final `FAILED` calculation checks only shellcheck, syntax, non-interactive, and end-to-end jobs.
|
||||||
|
|
||||||
|
### Comment-triggered privileged AI action
|
||||||
|
|
||||||
|
`opencode.yml` limits triggers to repository owners/members, but gives a mutable third-party action an API key, OIDC, and write access to contents, PRs, and issues. A compromised action version would inherit all of them.
|
||||||
|
|
||||||
|
### Missing governance files
|
||||||
|
|
||||||
|
The repository currently has no:
|
||||||
|
|
||||||
|
- Root `SECURITY.md`.
|
||||||
|
- `.github/CODEOWNERS`.
|
||||||
|
- `.github/dependabot.yml`.
|
||||||
|
|
||||||
|
## Target Security Invariants
|
||||||
|
|
||||||
|
The implementation tasks following this baseline must establish:
|
||||||
|
|
||||||
|
1. Untrusted PR code runs only with read permissions and no secrets.
|
||||||
|
2. Privileged workflows never execute or source untrusted code or artifacts.
|
||||||
|
3. Workflow and job permissions are explicitly declared and minimized.
|
||||||
|
4. External actions are pinned to reviewed full commit SHAs.
|
||||||
|
5. Registry and documentation generators create reviewed PRs instead of pushing unvalidated changes directly.
|
||||||
|
6. Required checks exercise the behavior affected by a PR and cannot report success after a required job fails.
|
||||||
|
7. Security-sensitive workflow changes require code-owner review.
|
||||||
|
8. Vulnerability reporters have a private channel and documented response expectations.
|
||||||
|
9. Dependency and code-scanning results participate in merge protection where available.
|
||||||
|
|
||||||
|
## Reproducible Static Inspection
|
||||||
|
|
||||||
|
These commands are read-only:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Locate privileged triggers, permissions, secrets, and mutable actions
|
||||||
|
rg -n 'pull_request_target|workflow_run|issue_comment|permissions:|contents: write|pull-requests: write|id-token: write|secrets\.|uses:.*@(main|master|latest|v[0-9]+)' .github/workflows
|
||||||
|
|
||||||
|
# Inspect PR output wiring
|
||||||
|
rg -n 'has-evals|outputs\.evals|has-docs|outputs\.docs|has-workflows|outputs\.workflows' .github/workflows/pr-checks.yml
|
||||||
|
|
||||||
|
# Validate YAML syntax with Ruby's safe parser
|
||||||
|
ruby -e 'require "yaml"; Dir[".github/workflows/*.{yml,yaml}"].each { |f| YAML.safe_load(File.read(f), aliases: true); puts f }'
|
||||||
|
|
||||||
|
# Inspect repository-side security settings
|
||||||
|
gh api repos/darrenhinde/OpenAgentsControl/actions/permissions/workflow
|
||||||
|
gh api repos/darrenhinde/OpenAgentsControl/actions/permissions
|
||||||
|
gh api repos/darrenhinde/OpenAgentsControl/actions/permissions/fork-pr-contributor-approval
|
||||||
|
gh api repos/darrenhinde/OpenAgentsControl/rulesets
|
||||||
|
gh api repos/darrenhinde/OpenAgentsControl/private-vulnerability-reporting
|
||||||
|
gh api repos/darrenhinde/OpenAgentsControl/code-scanning/default-setup
|
||||||
|
```
|
||||||
|
|
||||||
|
## Validation for This Baseline
|
||||||
|
|
||||||
|
- Confirm every active workflow is represented in the inventory.
|
||||||
|
- Confirm cited line ranges against current files.
|
||||||
|
- Run Markdown internal-link validation.
|
||||||
|
- Run `git diff --check`.
|
||||||
|
- Do not alter workflows or GitHub settings in this task.
|
||||||
|
|
||||||
|
## References
|
||||||
|
|
||||||
|
- [Secure use of `pull_request_target`](https://docs.github.com/en/actions/reference/security/securely-using-pull_request_target)
|
||||||
|
- [Security hardening for GitHub Actions](https://docs.github.com/en/actions/security-for-github-actions/security-guides/security-hardening-for-github-actions)
|
||||||
|
- [GitHub Actions permission syntax](https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#permissions)
|
||||||
|
- [Adding a repository security policy](https://docs.github.com/en/code-security/getting-started/adding-a-security-policy-to-your-repository)
|
||||||
|
- [Configuring private vulnerability reporting](https://docs.github.com/en/code-security/security-advisories/working-with-repository-security-advisories/configuring-private-vulnerability-reporting-for-a-repository)
|
||||||
234
scripts/validation/detect-pr-changes.test.ts
Normal file
234
scripts/validation/detect-pr-changes.test.ts
Normal file
|
|
@ -0,0 +1,234 @@
|
||||||
|
import { describe, expect, test } from 'bun:test'
|
||||||
|
import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'
|
||||||
|
import { tmpdir } from 'node:os'
|
||||||
|
import { join } from 'node:path'
|
||||||
|
|
||||||
|
import {
|
||||||
|
classifyChangedPaths,
|
||||||
|
formatGitHubOutput,
|
||||||
|
parseChangedPaths,
|
||||||
|
} from './detect-pr-changes'
|
||||||
|
|
||||||
|
const NO_CHANGES = {
|
||||||
|
'has-evals': false,
|
||||||
|
'has-docs': false,
|
||||||
|
'has-workflows': false,
|
||||||
|
}
|
||||||
|
|
||||||
|
const SCRIPT_PATH = join(import.meta.dir, 'detect-pr-changes.ts')
|
||||||
|
const WORKFLOW_PATH = join(import.meta.dir, '..', '..', '.github', 'workflows', 'pr-checks.yml')
|
||||||
|
|
||||||
|
type CliResult = {
|
||||||
|
exitCode: number
|
||||||
|
stderr: string
|
||||||
|
stdout: string
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runCli(input: string, outputPath?: string): Promise<CliResult> {
|
||||||
|
const env = { ...process.env }
|
||||||
|
if (outputPath === undefined) delete env.GITHUB_OUTPUT
|
||||||
|
else env.GITHUB_OUTPUT = outputPath
|
||||||
|
|
||||||
|
const subprocess = Bun.spawn([process.execPath, 'run', SCRIPT_PATH], {
|
||||||
|
env,
|
||||||
|
stderr: 'pipe',
|
||||||
|
stdin: new Blob([input]),
|
||||||
|
stdout: 'pipe',
|
||||||
|
})
|
||||||
|
|
||||||
|
const [exitCode, stderr, stdout] = await Promise.all([
|
||||||
|
subprocess.exited,
|
||||||
|
new Response(subprocess.stderr).text(),
|
||||||
|
new Response(subprocess.stdout).text(),
|
||||||
|
])
|
||||||
|
|
||||||
|
return { exitCode, stderr, stdout }
|
||||||
|
}
|
||||||
|
|
||||||
|
async function withTempDir<T>(run: (directory: string) => Promise<T>): Promise<T> {
|
||||||
|
const directory = await mkdtemp(join(tmpdir(), 'detect-pr-changes-'))
|
||||||
|
try {
|
||||||
|
return await run(directory)
|
||||||
|
} finally {
|
||||||
|
await rm(directory, { force: true, recursive: true })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('classifyChangedPaths', () => {
|
||||||
|
test('detects eval changes', () => {
|
||||||
|
expect(classifyChangedPaths(['evals/framework/src/index.ts'])).toEqual({
|
||||||
|
...NO_CHANGES,
|
||||||
|
'has-evals': true,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
test('detects docs changes', () => {
|
||||||
|
expect(classifyChangedPaths(['docs/maintenance/guide.md'])).toEqual({
|
||||||
|
...NO_CHANGES,
|
||||||
|
'has-docs': true,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
test('detects workflow changes', () => {
|
||||||
|
expect(classifyChangedPaths(['.github/workflows/pr-checks.yml'])).toEqual({
|
||||||
|
...NO_CHANGES,
|
||||||
|
'has-workflows': true,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
test('detects mixed changes', () => {
|
||||||
|
expect(
|
||||||
|
classifyChangedPaths([
|
||||||
|
'evals/framework/package.json',
|
||||||
|
'docs/README.md',
|
||||||
|
'.github/workflows/release.yml',
|
||||||
|
'src/index.ts',
|
||||||
|
]),
|
||||||
|
).toEqual({
|
||||||
|
'has-evals': true,
|
||||||
|
'has-docs': true,
|
||||||
|
'has-workflows': true,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
test('returns false for every category when paths do not match', () => {
|
||||||
|
expect(classifyChangedPaths(['README.md', 'scripts/check.ts'])).toEqual(NO_CHANGES)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('returns false for every category when input is empty', () => {
|
||||||
|
expect(classifyChangedPaths([])).toEqual(NO_CHANGES)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('rejects near-prefix paths', () => {
|
||||||
|
expect(
|
||||||
|
classifyChangedPaths([
|
||||||
|
'evals-old/test.ts',
|
||||||
|
'docs.md',
|
||||||
|
'.github/workflows-old/check.yml',
|
||||||
|
'nested/evals/test.ts',
|
||||||
|
]),
|
||||||
|
).toEqual(NO_CHANGES)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('parseChangedPaths', () => {
|
||||||
|
test('preserves path identity and removes empty NUL records', () => {
|
||||||
|
expect(parseChangedPaths(' evals/test.ts \0\0docs/雪\n$HOME; file.md\0')).toEqual([
|
||||||
|
' evals/test.ts ',
|
||||||
|
'docs/雪\n$HOME; file.md',
|
||||||
|
])
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('formatGitHubOutput', () => {
|
||||||
|
test('formats newline-delimited GitHub outputs', () => {
|
||||||
|
expect(
|
||||||
|
formatGitHubOutput({
|
||||||
|
'has-evals': true,
|
||||||
|
'has-docs': false,
|
||||||
|
'has-workflows': true,
|
||||||
|
}),
|
||||||
|
).toBe('has-evals=true\nhas-docs=false\nhas-workflows=true\n')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('CLI', () => {
|
||||||
|
test('appends outputs to GITHUB_OUTPUT', async () => {
|
||||||
|
await withTempDir(async (directory) => {
|
||||||
|
const outputPath = join(directory, 'github-output')
|
||||||
|
await writeFile(outputPath, 'existing=value\n')
|
||||||
|
|
||||||
|
const result = await runCli('evals/test.ts\0', outputPath)
|
||||||
|
|
||||||
|
expect(result).toEqual({ exitCode: 0, stderr: '', stdout: '' })
|
||||||
|
expect(await readFile(outputPath, 'utf8')).toBe(
|
||||||
|
'existing=value\nhas-evals=true\nhas-docs=false\nhas-workflows=false\n',
|
||||||
|
)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
test('writes false outputs for empty stdin', async () => {
|
||||||
|
await withTempDir(async (directory) => {
|
||||||
|
const outputPath = join(directory, 'github-output')
|
||||||
|
|
||||||
|
const result = await runCli('', outputPath)
|
||||||
|
|
||||||
|
expect(result.exitCode).toBe(0)
|
||||||
|
expect(await readFile(outputPath, 'utf8')).toBe(
|
||||||
|
'has-evals=false\nhas-docs=false\nhas-workflows=false\n',
|
||||||
|
)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
test('fails clearly when GITHUB_OUTPUT is missing', async () => {
|
||||||
|
const result = await runCli('evals/test.ts\0')
|
||||||
|
|
||||||
|
expect(result.exitCode).not.toBe(0)
|
||||||
|
expect(result.stderr).toContain('GITHUB_OUTPUT is required but was not set')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('writes an error to stderr and exits nonzero for an unwritable target', async () => {
|
||||||
|
await withTempDir(async (directory) => {
|
||||||
|
const outputPath = join(directory, 'output-directory')
|
||||||
|
await mkdir(outputPath)
|
||||||
|
|
||||||
|
const result = await runCli('docs/guide.md\0', outputPath)
|
||||||
|
|
||||||
|
expect(result.exitCode).not.toBe(0)
|
||||||
|
expect(result.stderr).toContain('Unable to append change detection outputs to GITHUB_OUTPUT')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
test('handles NUL-delimited Unicode and metacharacter filenames without changing identity', async () => {
|
||||||
|
await withTempDir(async (directory) => {
|
||||||
|
const outputPath = join(directory, 'github-output')
|
||||||
|
const paths = [
|
||||||
|
'evals/ leading and trailing .ts ',
|
||||||
|
'docs/雪\n$HOME;$(touch never).md',
|
||||||
|
'.github/workflows/[check]& weird.yml',
|
||||||
|
]
|
||||||
|
|
||||||
|
const result = await runCli(`${paths.join('\0')}\0`, outputPath)
|
||||||
|
|
||||||
|
expect(result.exitCode).toBe(0)
|
||||||
|
expect(await readFile(outputPath, 'utf8')).toBe(
|
||||||
|
'has-evals=true\nhas-docs=true\nhas-workflows=true\n',
|
||||||
|
)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
test('does not trim leading whitespace into a matching path', async () => {
|
||||||
|
await withTempDir(async (directory) => {
|
||||||
|
const outputPath = join(directory, 'github-output')
|
||||||
|
|
||||||
|
const result = await runCli(' evals/not-under-evals.ts\0', outputPath)
|
||||||
|
|
||||||
|
expect(result.exitCode).toBe(0)
|
||||||
|
expect(await readFile(outputPath, 'utf8')).toContain('has-evals=false\n')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('PR checks workflow contract', () => {
|
||||||
|
test('uses the NUL-delimited detector and exact output names', async () => {
|
||||||
|
const workflow = await readFile(WORKFLOW_PATH, 'utf8')
|
||||||
|
|
||||||
|
expect(workflow).toContain('git diff --name-only -z')
|
||||||
|
expect(workflow).toMatch(/git diff --name-only -z[^\n]*\|[\s\S]*bun run scripts\/validation\/detect-pr-changes\.ts/)
|
||||||
|
expect(workflow).toContain('has-evals: ${{ steps.filter.outputs.has-evals }}')
|
||||||
|
expect(workflow).toContain('has-docs: ${{ steps.filter.outputs.has-docs }}')
|
||||||
|
expect(workflow).toContain('has-workflows: ${{ steps.filter.outputs.has-workflows }}')
|
||||||
|
expect(workflow).not.toMatch(/steps\.filter\.outputs\.(evals|docs|workflows)(?:\s|})/)
|
||||||
|
expect(workflow).toContain('oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6')
|
||||||
|
expect(workflow).toContain('bun-version: 1.3.14')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('requires successful change detection before reporting overall success', async () => {
|
||||||
|
const workflow = await readFile(WORKFLOW_PATH, 'utf8')
|
||||||
|
const overallStatus = workflow.slice(workflow.indexOf('# Overall status'))
|
||||||
|
|
||||||
|
expect(overallStatus).toContain('needs.check-changes.result }}" == "success"')
|
||||||
|
expect(overallStatus).toContain('needs.check-changes.outputs.has-evals }}" != "true"')
|
||||||
|
expect(overallStatus).toContain('exit 1')
|
||||||
|
})
|
||||||
|
})
|
||||||
50
scripts/validation/detect-pr-changes.ts
Normal file
50
scripts/validation/detect-pr-changes.ts
Normal file
|
|
@ -0,0 +1,50 @@
|
||||||
|
#!/usr/bin/env bun
|
||||||
|
|
||||||
|
import { appendFile } from 'node:fs/promises'
|
||||||
|
|
||||||
|
export type ChangeFlags = {
|
||||||
|
'has-evals': boolean
|
||||||
|
'has-docs': boolean
|
||||||
|
'has-workflows': boolean
|
||||||
|
}
|
||||||
|
|
||||||
|
export function parseChangedPaths(input: string): string[] {
|
||||||
|
return input.split('\0').filter((path) => path.length > 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
export function classifyChangedPaths(paths: readonly string[]): ChangeFlags {
|
||||||
|
return {
|
||||||
|
'has-evals': paths.some((path) => path.startsWith('evals/')),
|
||||||
|
'has-docs': paths.some((path) => path.startsWith('docs/')),
|
||||||
|
'has-workflows': paths.some((path) => path.startsWith('.github/workflows/')),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function formatGitHubOutput(flags: ChangeFlags): string {
|
||||||
|
return [
|
||||||
|
`has-evals=${flags['has-evals']}`,
|
||||||
|
`has-docs=${flags['has-docs']}`,
|
||||||
|
`has-workflows=${flags['has-workflows']}`,
|
||||||
|
].join('\n') + '\n'
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main(): Promise<void> {
|
||||||
|
const outputPath = process.env.GITHUB_OUTPUT
|
||||||
|
if (!outputPath) throw new Error('GITHUB_OUTPUT is required but was not set')
|
||||||
|
|
||||||
|
const paths = parseChangedPaths(await Bun.stdin.text())
|
||||||
|
const output = formatGitHubOutput(classifyChangedPaths(paths))
|
||||||
|
|
||||||
|
await appendFile(outputPath, output, 'utf8').catch((error: unknown) => {
|
||||||
|
const message = error instanceof Error ? error.message : String(error)
|
||||||
|
throw new Error(`Unable to append change detection outputs to GITHUB_OUTPUT (${outputPath}): ${message}`)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
if (import.meta.main) {
|
||||||
|
await main().catch((error: unknown) => {
|
||||||
|
const message = error instanceof Error ? error.message : String(error)
|
||||||
|
console.error(`detect-pr-changes: ${message}`)
|
||||||
|
process.exitCode = 1
|
||||||
|
})
|
||||||
|
}
|
||||||
Loading…
Reference in a new issue