diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index e2bc3a5..52018b6 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -5,6 +5,9 @@ on: branches: [main, dev] types: [opened, edited, synchronize, reopened] +permissions: + contents: read + jobs: pr-title-check: name: Validate PR Title @@ -101,15 +104,9 @@ jobs: Edit your PR title to match the format above. `; + core.info(message); + await core.summary.addRaw(message).write(); core.setFailed(message); - - // Also post as a comment - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.payload.pull_request.number, - body: message - }); } else { const [type] = matchedType; let versionBump = 'patch (0.3.0 → 0.3.1)'; @@ -160,52 +157,29 @@ jobs: needs: pr-title-check if: needs.pr-title-check.outputs.title-valid == 'true' outputs: - has-evals: ${{ steps.filter.outputs.evals }} - has-docs: ${{ steps.filter.outputs.docs }} - has-workflows: ${{ steps.filter.outputs.workflows }} + has-evals: ${{ steps.filter.outputs.has-evals }} + has-docs: ${{ steps.filter.outputs.has-docs }} + has-workflows: ${{ steps.filter.outputs.has-workflows }} steps: - name: Checkout code uses: actions/checkout@v4 with: fetch-depth: 0 + persist-credentials: false + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.3.14 - name: Check changed files id: filter run: | - # Get list of changed files - git fetch origin ${{ github.base_ref }} - CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD) - - echo "Changed files:" - echo "$CHANGED_FILES" - - # Check for evals changes - if echo "$CHANGED_FILES" | grep -q "^evals/"; then - echo "has-evals=true" >> $GITHUB_OUTPUT - echo "✅ Evals changes detected" - else - echo "has-evals=false" >> $GITHUB_OUTPUT - echo "ℹ️ No evals changes" - fi - - # Check for docs changes - if echo "$CHANGED_FILES" | grep -q "^docs/"; then - echo "has-docs=true" >> $GITHUB_OUTPUT - echo "✅ Docs changes detected" - else - echo "has-docs=false" >> $GITHUB_OUTPUT - echo "ℹ️ No docs changes" - fi - - # Check for workflow changes - if echo "$CHANGED_FILES" | grep -q "^.github/workflows/"; then - echo "has-workflows=true" >> $GITHUB_OUTPUT - echo "✅ Workflow changes detected" - else - echo "has-workflows=false" >> $GITHUB_OUTPUT - echo "ℹ️ No workflow changes" - fi + echo "Changed files:" >&2 + git diff --name-only -z "${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }}" | + tee >(while IFS= read -r -d '' path; do printf ' %q\n' "$path" >&2; done) | + bun run scripts/validation/detect-pr-changes.ts build-check: name: Build & Validate @@ -219,6 +193,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@v4 @@ -295,24 +271,41 @@ jobs: if [ "${{ needs.check-changes.outputs.has-workflows }}" == "true" ]; then echo " - ⚙️ Workflow changes detected" >> $GITHUB_STEP_SUMMARY fi - else + elif [ "${{ needs.check-changes.result }}" == "failure" ]; then + echo "❌ **Changed Files:** Detection failed - check logs" >> $GITHUB_STEP_SUMMARY + elif [ "${{ needs.check-changes.result }}" == "cancelled" ]; then + echo "🚫 **Changed Files:** Detection cancelled" >> $GITHUB_STEP_SUMMARY + elif [ "${{ needs.check-changes.result }}" == "skipped" ]; then echo "⏭️ **Changed Files:** Skipped (title validation failed)" >> $GITHUB_STEP_SUMMARY + else + echo "❓ **Changed Files:** Unknown result" >> $GITHUB_STEP_SUMMARY fi # Build Check if [ "${{ needs.build-check.result }}" == "success" ]; then echo "✅ **Build & Validate:** Passed" >> $GITHUB_STEP_SUMMARY elif [ "${{ needs.build-check.result }}" == "skipped" ]; then - echo "⏭️ **Build & Validate:** Skipped (no evals changes)" >> $GITHUB_STEP_SUMMARY + if [ "${{ needs.check-changes.result }}" != "success" ]; then + echo "⏭️ **Build & Validate:** Skipped (change detection did not succeed)" >> $GITHUB_STEP_SUMMARY + elif [ "${{ needs.check-changes.outputs.has-evals }}" != "true" ]; then + echo "⏭️ **Build & Validate:** Skipped (no evals changes)" >> $GITHUB_STEP_SUMMARY + else + echo "⏭️ **Build & Validate:** Skipped (prerequisite not met)" >> $GITHUB_STEP_SUMMARY + fi elif [ "${{ needs.build-check.result }}" == "failure" ]; then echo "❌ **Build & Validate:** Failed - check logs" >> $GITHUB_STEP_SUMMARY + elif [ "${{ needs.build-check.result }}" == "cancelled" ]; then + echo "🚫 **Build & Validate:** Cancelled" >> $GITHUB_STEP_SUMMARY fi echo "" >> $GITHUB_STEP_SUMMARY # Overall status if [ "${{ needs.pr-title-check.result }}" == "success" ] && \ - ([ "${{ needs.build-check.result }}" == "success" ] || [ "${{ needs.build-check.result }}" == "skipped" ]); then + [ "${{ needs.check-changes.result }}" == "success" ] && \ + { [ "${{ needs.build-check.result }}" == "success" ] || \ + { [ "${{ needs.build-check.result }}" == "skipped" ] && \ + [ "${{ needs.check-changes.outputs.has-evals }}" != "true" ]; }; }; then echo "### ✅ All Required Checks Passed!" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "This PR is ready for review." >> $GITHUB_STEP_SUMMARY @@ -320,4 +313,5 @@ jobs: echo "### ❌ Some Checks Failed" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "Please fix the failing checks before merging." >> $GITHUB_STEP_SUMMARY + exit 1 fi diff --git a/.github/workflows/validate-registry.yml b/.github/workflows/validate-registry.yml index b6df352..df78cc0 100644 --- a/.github/workflows/validate-registry.yml +++ b/.github/workflows/validate-registry.yml @@ -1,379 +1,87 @@ name: Validate Registry on PR -# This workflow validates the registry.json and prompt library structure on all PRs. -# -# For bot-created PRs (like automated version bumps), the workflow won't trigger automatically -# due to GitHub's security restrictions. In those cases, you can manually trigger this workflow: -# -# Option 1 - Run Validation: -# 1. Go to Actions > Validate Registry on PR > Run workflow -# 2. Enter the PR number (e.g., 106) -# 3. Leave "skip_validation" unchecked -# 4. Click "Run workflow" -# -# Option 2 - Admin Bypass (for trusted bot PRs): -# 1. Go to Actions > Validate Registry on PR > Run workflow -# 2. Enter the PR number (e.g., 106) -# 3. Check "skip_validation" checkbox -# 4. Click "Run workflow" -# 5. The check will pass immediately without running validation +# Contributor code must run only with a read-only token and no secrets. +# Never change this workflow back to pull_request_target while it checks out +# and executes pull-request code. on: - # Use pull_request_target to allow running on bot-created PRs - # This also allows the workflow to write to the PR branch - pull_request_target: + pull_request: branches: - main - dev - # Removed paths filter - this check is required by repository ruleset - # so it must run on ALL PRs to prevent blocking merges workflow_dispatch: - inputs: - pr_number: - description: 'PR number to validate (for manual runs on bot-created PRs)' - required: false - type: number - skip_validation: - description: 'Skip validation checks (maintainer override)' - required: false - type: boolean - default: false permissions: - contents: write - pull-requests: write + contents: read jobs: validate-and-update: runs-on: ubuntu-latest - + timeout-minutes: 10 + steps: - - name: Admin bypass check - if: github.event_name == 'workflow_dispatch' && github.event.inputs.skip_validation == 'true' - run: | - echo "## ✅ Validation Bypassed (Admin Override)" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "Validation checks skipped by maintainer." >> $GITHUB_STEP_SUMMARY - echo "PR: #${{ github.event.inputs.pr_number }}" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "The workflow will complete successfully without running validation steps." >> $GITHUB_STEP_SUMMARY - - - name: Checkout repository (for manual runs) - if: github.event_name == 'workflow_dispatch' && github.event.inputs.skip_validation != 'true' + - name: Checkout code without persisted credentials uses: actions/checkout@v4 with: fetch-depth: 0 - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Get PR details (for manual runs) - if: github.event_name == 'workflow_dispatch' && github.event.inputs.pr_number != '' && github.event.inputs.skip_validation != 'true' - id: get_pr - run: | - PR_DATA=$(gh pr view ${{ github.event.inputs.pr_number }} --json headRefName,headRepository,headRepositoryOwner) - echo "head_ref=$(echo $PR_DATA | jq -r '.headRefName')" >> $GITHUB_OUTPUT - echo "head_repo=$(echo $PR_DATA | jq -r '.headRepositoryOwner.login + "/" + .headRepository.name')" >> $GITHUB_OUTPUT - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Checkout PR branch - if: github.event.inputs.skip_validation != 'true' - uses: actions/checkout@v4 - with: - # For manual runs: use PR details from get_pr step - # For PR events: use event data - repository: ${{ github.event_name == 'workflow_dispatch' && steps.get_pr.outputs.head_repo || github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event_name == 'workflow_dispatch' && steps.get_pr.outputs.head_ref || github.event.pull_request.head.ref }} - fetch-depth: 0 - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Detect fork PR - if: github.event.inputs.skip_validation != 'true' - id: fork_check - run: | - # For manual runs, use the fetched PR data - if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - HEAD_REPO="${{ steps.get_pr.outputs.head_repo }}" - else - HEAD_REPO="${{ github.event.pull_request.head.repo.full_name }}" - fi - - if [ "$HEAD_REPO" != "${{ github.repository }}" ]; then - echo "is_fork=true" >> $GITHUB_OUTPUT - echo "🔀 Fork PR detected from: $HEAD_REPO" - else - echo "is_fork=false" >> $GITHUB_OUTPUT - echo "📝 Internal PR detected" - fi - - - name: Install dependencies - if: github.event.inputs.skip_validation != 'true' + persist-credentials: false + + - name: Install system dependencies run: | sudo apt-get update sudo apt-get install -y jq - + - name: Install Bun - if: github.event.inputs.skip_validation != 'true' uses: oven-sh/setup-bun@v2 with: bun-version: latest - - - name: Install dependencies - if: github.event.inputs.skip_validation != 'true' - run: | - # Install root dependencies (glob package needed for validation script) - bun install --frozen-lockfile - - - name: Make scripts executable - if: github.event.inputs.skip_validation != 'true' + + - name: Install project dependencies + run: bun install --frozen-lockfile + + - name: Make validation scripts executable run: | chmod +x scripts/registry/validate-registry.sh chmod +x scripts/registry/auto-detect-components.sh chmod +x scripts/registry/register-component.sh chmod +x scripts/prompts/validate-pr.sh - - - name: Auto-detect new components - if: github.event.inputs.skip_validation != 'true' - id: auto_detect + + - name: Report registry drift + id: registry_drift run: | - echo "## 🔍 Auto-Detection Results" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - - # Run auto-detect in dry-run mode first to see what would be added - if ./scripts/registry/auto-detect-components.sh --dry-run > /tmp/detect-output.txt 2>&1; then - cat /tmp/detect-output.txt >> $GITHUB_STEP_SUMMARY - - # Check if new components were found - if grep -q "Found.*new component" /tmp/detect-output.txt; then - echo "new_components=true" >> $GITHUB_OUTPUT - echo "" >> $GITHUB_STEP_SUMMARY - echo "⚠️ New components detected - will auto-add to registry" >> $GITHUB_STEP_SUMMARY - else - echo "new_components=false" >> $GITHUB_OUTPUT - echo "✅ No new components found" >> $GITHUB_STEP_SUMMARY - fi + ./scripts/registry/auto-detect-components.sh --dry-run > /tmp/detect-output.txt 2>&1 + cat /tmp/detect-output.txt + sed $'s/\033\\[[0-9;]*m//g' /tmp/detect-output.txt > /tmp/detect-output-plain.txt + + if grep -Eq 'New Components:[[:space:]]*[1-9][0-9]*([[:space:]]|$)' /tmp/detect-output-plain.txt; then + echo "drift_detected=true" >> "$GITHUB_OUTPUT" + echo "## Registry drift detected" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "The repository already contains components not represented in registry.json." >> "$GITHUB_STEP_SUMMARY" + echo "This is reported but does not block security validation until the existing drift is reconciled." >> "$GITHUB_STEP_SUMMARY" else - echo "new_components=false" >> $GITHUB_OUTPUT - echo "❌ Auto-detection failed" >> $GITHUB_STEP_SUMMARY + echo "drift_detected=false" >> "$GITHUB_OUTPUT" fi - - - name: Add new components to registry - if: steps.auto_detect.outputs.new_components == 'true' && github.event.inputs.skip_validation != 'true' - run: | - echo "## 📝 Adding New Components" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - - ./scripts/registry/auto-detect-components.sh --auto-add | tee -a $GITHUB_STEP_SUMMARY - + - name: Validate prompt library structure - if: github.event.inputs.skip_validation != 'true' - id: validate_prompts - run: | - echo "## 🔍 Prompt Library Validation" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - - if ./scripts/prompts/validate-pr.sh > /tmp/prompt-validation.txt 2>&1; then - echo "prompt_validation=passed" >> $GITHUB_OUTPUT - echo "✅ Prompt library structure is valid!" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "\`\`\`" >> $GITHUB_STEP_SUMMARY - cat /tmp/prompt-validation.txt >> $GITHUB_STEP_SUMMARY - echo "\`\`\`" >> $GITHUB_STEP_SUMMARY - else - echo "prompt_validation=failed" >> $GITHUB_OUTPUT - echo "❌ Prompt library validation failed!" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "\`\`\`" >> $GITHUB_STEP_SUMMARY - cat /tmp/prompt-validation.txt >> $GITHUB_STEP_SUMMARY - echo "\`\`\`" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "**Architecture:**" >> $GITHUB_STEP_SUMMARY - echo "- Agent files (.opencode/agent/**/*.md) = Canonical defaults" >> $GITHUB_STEP_SUMMARY - echo "- Prompt variants (.opencode/prompts//.md) = Model-specific" >> $GITHUB_STEP_SUMMARY - echo "- default.md files should NOT exist" >> $GITHUB_STEP_SUMMARY - echo "- Agents organized in category subdirectories (core/, development/, content/, etc.)" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "See [CONTRIBUTING.md](docs/contributing/CONTRIBUTING.md) for details" >> $GITHUB_STEP_SUMMARY - exit 1 - fi + run: ./scripts/prompts/validate-pr.sh - name: Validate markdown context links - if: github.event.inputs.skip_validation != 'true' - id: validate_context_links - run: | - echo "## 🔗 Context Link Validation" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY + run: npm run validate:context-links - if bun run scripts/validation/validate-markdown-links.ts > /tmp/context-link-validation.txt 2>&1; then - echo "context_links=passed" >> $GITHUB_OUTPUT - echo "✅ Context markdown links are valid!" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "\`\`\`" >> $GITHUB_STEP_SUMMARY - cat /tmp/context-link-validation.txt >> $GITHUB_STEP_SUMMARY - echo "\`\`\`" >> $GITHUB_STEP_SUMMARY - else - echo "context_links=failed" >> $GITHUB_OUTPUT - echo "❌ Context markdown link validation failed!" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "\`\`\`" >> $GITHUB_STEP_SUMMARY - cat /tmp/context-link-validation.txt >> $GITHUB_STEP_SUMMARY - echo "\`\`\`" >> $GITHUB_STEP_SUMMARY - exit 1 - fi - - name: Validate registry - if: github.event.inputs.skip_validation != 'true' - id: validate - run: | - echo "## ✅ Registry Validation" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - - # Use TypeScript validator (fast and reliable) - # Run validation and capture output (show in logs AND save to file) - if bun run scripts/registry/validate-registry.ts 2>&1 | tee /tmp/validation-output.txt; then - echo "validation=passed" >> $GITHUB_OUTPUT - echo "✅ All registry paths are valid!" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "\`\`\`" >> $GITHUB_STEP_SUMMARY - cat /tmp/validation-output.txt >> $GITHUB_STEP_SUMMARY - echo "\`\`\`" >> $GITHUB_STEP_SUMMARY - else - echo "validation=failed" >> $GITHUB_OUTPUT - echo "❌ Registry validation failed!" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "\`\`\`" >> $GITHUB_STEP_SUMMARY - cat /tmp/validation-output.txt >> $GITHUB_STEP_SUMMARY - echo "\`\`\`" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "**Check the logs above for detailed error output**" >> $GITHUB_STEP_SUMMARY - exit 1 - fi - - - name: Commit registry updates (Internal PRs only) - if: | - github.event.inputs.skip_validation != 'true' && - steps.fork_check.outputs.is_fork == 'false' && - steps.auto_detect.outputs.new_components == 'true' && - steps.validate_prompts.outputs.prompt_validation == 'passed' && - steps.validate.outputs.validation == 'passed' - run: | - git config --local user.email "github-actions[bot]@users.noreply.github.com" - git config --local user.name "github-actions[bot]" - - if ! git diff --quiet registry.json; then - git add registry.json - git commit -m "chore: auto-update registry with new components [skip ci]" - - # For manual runs, use the fetched branch name - BRANCH_NAME="${{ github.event_name == 'workflow_dispatch' && steps.get_pr.outputs.head_ref || github.event.pull_request.head.ref }}" - git push origin "$BRANCH_NAME" - - echo "## 🚀 Registry Updated" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "Registry has been automatically updated with new components." >> $GITHUB_STEP_SUMMARY - echo "Changes have been pushed to this PR branch." >> $GITHUB_STEP_SUMMARY - else - echo "## ℹ️ No Changes to Commit" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "Registry is already up to date." >> $GITHUB_STEP_SUMMARY - fi - - - name: Fork PR notice - if: | - github.event.inputs.skip_validation != 'true' && - steps.fork_check.outputs.is_fork == 'true' && - steps.auto_detect.outputs.new_components == 'true' && - steps.validate_prompts.outputs.prompt_validation == 'passed' && - steps.validate.outputs.validation == 'passed' - uses: actions/github-script@v7 - with: - script: | - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: `## 📝 Registry Update Needed - - Hi @${{ github.event.pull_request.user.login }}! 👋 - - New components were detected in your PR. Since this is a fork PR, I can't auto-commit the registry updates for security reasons. - - **Please run these commands locally:** - \`\`\`bash - ./scripts/registry/auto-detect-components.sh --auto-add - git add registry.json - git commit -m "chore: update registry" - git push - \`\`\` - - Once you push the updated registry, the checks will pass! ✅` - }); - - - name: Fork PR summary - if: steps.fork_check.outputs.is_fork == 'true' && github.event.inputs.skip_validation != 'true' - run: | - echo "## 🔀 Fork PR Detected" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "This is an external contribution - thank you! 🎉" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - - if [ "${{ steps.auto_detect.outputs.new_components }}" == "true" ]; then - echo "⚠️ **Action Required:** New components detected" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "A comment has been posted with instructions to update the registry." >> $GITHUB_STEP_SUMMARY - else - echo "✅ No registry updates needed" >> $GITHUB_STEP_SUMMARY - fi - - - name: Post validation summary - if: always() - run: | - echo "" >> $GITHUB_STEP_SUMMARY - echo "---" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - - PROMPT_VALID="${{ steps.validate_prompts.outputs.prompt_validation }}" - CONTEXT_LINKS_VALID="${{ steps.validate_context_links.outputs.context_links }}" - REGISTRY_VALID="${{ steps.validate.outputs.validation }}" + run: npm run validate:registry - if [ "$PROMPT_VALID" = "passed" ] && [ "$CONTEXT_LINKS_VALID" = "passed" ] && [ "$REGISTRY_VALID" = "passed" ]; then - echo "### ✅ All Validations Passed" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "- ✅ Prompt library structure is valid" >> $GITHUB_STEP_SUMMARY - echo "- ✅ Context markdown links are valid" >> $GITHUB_STEP_SUMMARY - echo "- ✅ Registry paths are valid" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "This PR is ready for review!" >> $GITHUB_STEP_SUMMARY - else - echo "### ❌ Validation Failed" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - - if [ "$PROMPT_VALID" != "passed" ]; then - echo "- ❌ Prompt library validation failed" >> $GITHUB_STEP_SUMMARY - else - echo "- ✅ Prompt library validation passed" >> $GITHUB_STEP_SUMMARY - fi - - if [ "$CONTEXT_LINKS_VALID" != "passed" ]; then - echo "- ❌ Context markdown link validation failed" >> $GITHUB_STEP_SUMMARY - else - echo "- ✅ Context markdown link validation passed" >> $GITHUB_STEP_SUMMARY - fi - - if [ "$REGISTRY_VALID" != "passed" ]; then - echo "- ❌ Registry validation failed" >> $GITHUB_STEP_SUMMARY - else - echo "- ✅ Registry validation passed" >> $GITHUB_STEP_SUMMARY - fi - - echo "" >> $GITHUB_STEP_SUMMARY - echo "Please fix the issues above before merging." >> $GITHUB_STEP_SUMMARY - fi - - - name: Fail if validation failed - if: | - (steps.validate_prompts.outputs.prompt_validation == 'failed' || steps.validate_context_links.outputs.context_links == 'failed' || steps.validate.outputs.validation == 'failed') && - github.event.inputs.skip_validation != 'true' + - name: Validation summary + if: success() run: | - echo "❌ Validation failed - blocking PR merge" - echo "Maintainer can override by running workflow manually with 'skip_validation' enabled" - exit 1 + echo "## All registry validations passed" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "- Prompt library structure is valid" >> "$GITHUB_STEP_SUMMARY" + echo "- Context markdown links are valid" >> "$GITHUB_STEP_SUMMARY" + echo "- Registry paths and dependencies are valid" >> "$GITHUB_STEP_SUMMARY" + if [ "${{ steps.registry_drift.outputs.drift_detected }}" = "true" ]; then + echo "- Existing registry drift was reported separately" >> "$GITHUB_STEP_SUMMARY" + else + echo "- No unregistered components were detected" >> "$GITHUB_STEP_SUMMARY" + fi diff --git a/docs/maintenance/repository-recovery/ci-threat-model-and-baseline.md b/docs/maintenance/repository-recovery/ci-threat-model-and-baseline.md new file mode 100644 index 0000000..8a9f34d --- /dev/null +++ b/docs/maintenance/repository-recovery/ci-threat-model-and-baseline.md @@ -0,0 +1,192 @@ +# CI Threat Model and Baseline + +**Date:** 2026-07-14 +**Scope:** GitHub Actions, repository security settings, and required checks +**Status:** Baseline only — no security fix is implemented by this document + +## Security Objective + +Pull requests must be able to test contributor code without giving that code repository write access, secrets, identity tokens, or a path into later privileged jobs. Trusted automation must use narrowly scoped permissions and must not silently bypass review. + +## Threat Actors + +- A malicious external contributor opening a fork pull request. +- A compromised contributor, maintainer, bot, action publisher, or dependency. +- A well-intentioned contributor whose workflow or script has unsafe side effects. +- An attacker posting crafted issue or PR comments to trigger automation. +- A compromised upstream action referenced by a mutable tag or branch. + +## Protected Assets + +- `GITHUB_TOKEN` write permissions and repository contents. +- Pull requests, issues, releases, tags, branches, and rulesets. +- `ANTHROPIC_API_KEY`, `OPENCODE_API_KEY`, and any repository secrets. +- OIDC identity available through `id-token: write`. +- Published npm packages, release artifacts, registry data, and generated documentation. +- Maintainer trust and the integrity of `main`. + +## Trust Boundaries + +1. **Fork pull-request code is untrusted.** Repository scripts, package manifests, lockfiles, actions, generated files, and artifacts from the fork are attacker-controlled. +2. **Default-branch code is trusted only after review.** A merge can still introduce compromised automation. +3. **Third-party actions are external code.** Mutable tags and branches can change after review. +4. **Artifacts and caches cross job boundaries.** Data from an untrusted job must never become executable input to a privileged job. +5. **Comments and workflow inputs are untrusted strings.** Authorization checks do not make their content safe for shell interpolation. +6. **GitHub settings are part of the security boundary.** Safe workflow files are insufficient if default token permissions and rulesets remain weak. + +## Critical Untrusted Execution Path + +`.github/workflows/validate-registry.yml` currently creates a direct privileged path: + +1. It runs on `pull_request_target` (`lines 21–29`). +2. It grants `contents: write` and `pull-requests: write` (`lines 42–44`). +3. It checks out the pull request head repository and branch (`lines 78–87`). +4. It runs `bun install` using the contributor-controlled lockfile/package graph (`lines 120–124`). +5. It executes contributor-controlled shell and TypeScript files (`lines 126–248`). +6. The same job retains its write-capable token while running this code. + +**Impact:** A malicious fork can attempt arbitrary code execution with a repository write token. Fork-specific logic later in the job does not remove the token or undo earlier execution. + +**Required remediation direction:** Move untrusted validation to a `pull_request` workflow with `contents: read` and no secrets. If a `pull_request_target` workflow remains, it must be metadata-only and must never check out or execute the contributor head. + +## Workflow Inventory + +| Workflow | Trigger | Current privilege or secret | Baseline concern | +|---|---|---|---| +| `validate-registry.yml` | `pull_request_target`, manual | Contents and PR write | **Critical:** executes fork code with write token | +| `pr-checks.yml` | Pull request | Implicit defaults | Output names do not match values written; relevant builds are skipped | +| `installer-checks.yml` | Pull request, push, manual | Implicit defaults | Mutable `action-shellcheck@master`; final status ignores compatibility/profile failures | +| `validate-test-suites.yml` | Pull request, push, manual | Implicit defaults; PR comment attempt | No explicit permissions; artifact contents originate from PR code | +| `update-registry.yml` | Push to `main`, manual | Contents write | Direct push to `main`; validation failure only warns; generated change bypasses PR review | +| `sync-docs.yml` | Push to `main`, manual | Contents, PR, and issues write | Broad workflow-level permissions; creates branches/issues and deletes branches on failure | +| `post-merge-pr.yml` | Push to `main`, manual | Contents and PR write | Broad workflow-level permissions; creates version branches and PRs | +| `create-release.yml` | Push to `main`, manual | Contents write | Tag/release authority; actions are tag-pinned rather than SHA-pinned | +| `opencode.yml` | Issue comment | OIDC, contents, PR, issues write; Anthropic key | `sst/opencode/github@latest` is mutable and receives powerful credentials | +| `evals/run-evaluations.yml` | Schedule, push to `main`, manual | OpenCode API key | User-supplied workflow inputs reach a shell command without a strict allowlist | + +## Confirmed Configuration Baseline + +Read-only GitHub API inspection on 2026-07-14 reported: + +- Default workflow token permission: **write**. +- Actions may approve pull-request reviews: **enabled**. +- Allowed actions: **all**. +- Full-SHA pinning requirement: **disabled**. +- Fork workflow approval policy: **first-time contributors only**. +- Private vulnerability reporting: **disabled**. +- CodeQL default setup: **not configured**. +- Dependabot security updates: **disabled**. +- Secret scanning: **enabled**. +- Secret scanning push protection: **enabled**. +- Automatic branch deletion after merge: **disabled**. +- No classic branch protection rule; an active repository ruleset protects `main`. + +### Active `main` ruleset + +- Pull requests are required. +- Required approving reviews: **0**. +- Code-owner review: **not required**. +- Latest-push approval: **not required**. +- Conversation resolution: **not required**. +- Strict up-to-date status checks: **disabled**. +- Only required check: `validate-and-update`. +- No bypass actors are configured. + +The only required check is the workflow containing the critical `pull_request_target` vulnerability. + +## PR Check Correctness + +`.github/workflows/pr-checks.yml` declares: + +```yaml +has-evals: ${{ steps.filter.outputs.evals }} +has-docs: ${{ steps.filter.outputs.docs }} +has-workflows: ${{ steps.filter.outputs.workflows }} +``` + +The `filter` step actually writes `has-evals`, `has-docs`, and `has-workflows`. Therefore dependent jobs can see empty outputs and skip required validation. The workflow builds and validates eval suites but does not run Vitest. + +## Action Pinning Baseline + +Current workflows use mutable references including: + +- `sst/opencode/github@latest` +- `ludeeus/action-shellcheck@master` +- Major-version tags such as `actions/checkout@v4`, `actions/setup-node@v4`, `actions/github-script@v7`, `actions/upload-artifact@v4`, and `oven-sh/setup-bun@v2` + +Current repository settings neither restrict allowed actions nor require full commit SHA pinning. + +## Additional High-Risk Findings + +### Registry writes after validation failure + +`update-registry.yml` validates generated registry state but intentionally does not fail on an invalid registry. It can then commit and push directly to `main` using `[skip ci]`. + +### Installer summary can report false success + +`installer-checks.yml` displays compatibility and profile results, but its final `FAILED` calculation checks only shellcheck, syntax, non-interactive, and end-to-end jobs. + +### Comment-triggered privileged AI action + +`opencode.yml` limits triggers to repository owners/members, but gives a mutable third-party action an API key, OIDC, and write access to contents, PRs, and issues. A compromised action version would inherit all of them. + +### Missing governance files + +The repository currently has no: + +- Root `SECURITY.md`. +- `.github/CODEOWNERS`. +- `.github/dependabot.yml`. + +## Target Security Invariants + +The implementation tasks following this baseline must establish: + +1. Untrusted PR code runs only with read permissions and no secrets. +2. Privileged workflows never execute or source untrusted code or artifacts. +3. Workflow and job permissions are explicitly declared and minimized. +4. External actions are pinned to reviewed full commit SHAs. +5. Registry and documentation generators create reviewed PRs instead of pushing unvalidated changes directly. +6. Required checks exercise the behavior affected by a PR and cannot report success after a required job fails. +7. Security-sensitive workflow changes require code-owner review. +8. Vulnerability reporters have a private channel and documented response expectations. +9. Dependency and code-scanning results participate in merge protection where available. + +## Reproducible Static Inspection + +These commands are read-only: + +```bash +# Locate privileged triggers, permissions, secrets, and mutable actions +rg -n 'pull_request_target|workflow_run|issue_comment|permissions:|contents: write|pull-requests: write|id-token: write|secrets\.|uses:.*@(main|master|latest|v[0-9]+)' .github/workflows + +# Inspect PR output wiring +rg -n 'has-evals|outputs\.evals|has-docs|outputs\.docs|has-workflows|outputs\.workflows' .github/workflows/pr-checks.yml + +# Validate YAML syntax with Ruby's safe parser +ruby -e 'require "yaml"; Dir[".github/workflows/*.{yml,yaml}"].each { |f| YAML.safe_load(File.read(f), aliases: true); puts f }' + +# Inspect repository-side security settings +gh api repos/darrenhinde/OpenAgentsControl/actions/permissions/workflow +gh api repos/darrenhinde/OpenAgentsControl/actions/permissions +gh api repos/darrenhinde/OpenAgentsControl/actions/permissions/fork-pr-contributor-approval +gh api repos/darrenhinde/OpenAgentsControl/rulesets +gh api repos/darrenhinde/OpenAgentsControl/private-vulnerability-reporting +gh api repos/darrenhinde/OpenAgentsControl/code-scanning/default-setup +``` + +## Validation for This Baseline + +- Confirm every active workflow is represented in the inventory. +- Confirm cited line ranges against current files. +- Run Markdown internal-link validation. +- Run `git diff --check`. +- Do not alter workflows or GitHub settings in this task. + +## References + +- [Secure use of `pull_request_target`](https://docs.github.com/en/actions/reference/security/securely-using-pull_request_target) +- [Security hardening for GitHub Actions](https://docs.github.com/en/actions/security-for-github-actions/security-guides/security-hardening-for-github-actions) +- [GitHub Actions permission syntax](https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#permissions) +- [Adding a repository security policy](https://docs.github.com/en/code-security/getting-started/adding-a-security-policy-to-your-repository) +- [Configuring private vulnerability reporting](https://docs.github.com/en/code-security/security-advisories/working-with-repository-security-advisories/configuring-private-vulnerability-reporting-for-a-repository) diff --git a/scripts/validation/detect-pr-changes.test.ts b/scripts/validation/detect-pr-changes.test.ts new file mode 100644 index 0000000..668f6f6 --- /dev/null +++ b/scripts/validation/detect-pr-changes.test.ts @@ -0,0 +1,234 @@ +import { describe, expect, test } from 'bun:test' +import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { + classifyChangedPaths, + formatGitHubOutput, + parseChangedPaths, +} from './detect-pr-changes' + +const NO_CHANGES = { + 'has-evals': false, + 'has-docs': false, + 'has-workflows': false, +} + +const SCRIPT_PATH = join(import.meta.dir, 'detect-pr-changes.ts') +const WORKFLOW_PATH = join(import.meta.dir, '..', '..', '.github', 'workflows', 'pr-checks.yml') + +type CliResult = { + exitCode: number + stderr: string + stdout: string +} + +async function runCli(input: string, outputPath?: string): Promise { + const env = { ...process.env } + if (outputPath === undefined) delete env.GITHUB_OUTPUT + else env.GITHUB_OUTPUT = outputPath + + const subprocess = Bun.spawn([process.execPath, 'run', SCRIPT_PATH], { + env, + stderr: 'pipe', + stdin: new Blob([input]), + stdout: 'pipe', + }) + + const [exitCode, stderr, stdout] = await Promise.all([ + subprocess.exited, + new Response(subprocess.stderr).text(), + new Response(subprocess.stdout).text(), + ]) + + return { exitCode, stderr, stdout } +} + +async function withTempDir(run: (directory: string) => Promise): Promise { + const directory = await mkdtemp(join(tmpdir(), 'detect-pr-changes-')) + try { + return await run(directory) + } finally { + await rm(directory, { force: true, recursive: true }) + } +} + +describe('classifyChangedPaths', () => { + test('detects eval changes', () => { + expect(classifyChangedPaths(['evals/framework/src/index.ts'])).toEqual({ + ...NO_CHANGES, + 'has-evals': true, + }) + }) + + test('detects docs changes', () => { + expect(classifyChangedPaths(['docs/maintenance/guide.md'])).toEqual({ + ...NO_CHANGES, + 'has-docs': true, + }) + }) + + test('detects workflow changes', () => { + expect(classifyChangedPaths(['.github/workflows/pr-checks.yml'])).toEqual({ + ...NO_CHANGES, + 'has-workflows': true, + }) + }) + + test('detects mixed changes', () => { + expect( + classifyChangedPaths([ + 'evals/framework/package.json', + 'docs/README.md', + '.github/workflows/release.yml', + 'src/index.ts', + ]), + ).toEqual({ + 'has-evals': true, + 'has-docs': true, + 'has-workflows': true, + }) + }) + + test('returns false for every category when paths do not match', () => { + expect(classifyChangedPaths(['README.md', 'scripts/check.ts'])).toEqual(NO_CHANGES) + }) + + test('returns false for every category when input is empty', () => { + expect(classifyChangedPaths([])).toEqual(NO_CHANGES) + }) + + test('rejects near-prefix paths', () => { + expect( + classifyChangedPaths([ + 'evals-old/test.ts', + 'docs.md', + '.github/workflows-old/check.yml', + 'nested/evals/test.ts', + ]), + ).toEqual(NO_CHANGES) + }) +}) + +describe('parseChangedPaths', () => { + test('preserves path identity and removes empty NUL records', () => { + expect(parseChangedPaths(' evals/test.ts \0\0docs/雪\n$HOME; file.md\0')).toEqual([ + ' evals/test.ts ', + 'docs/雪\n$HOME; file.md', + ]) + }) +}) + +describe('formatGitHubOutput', () => { + test('formats newline-delimited GitHub outputs', () => { + expect( + formatGitHubOutput({ + 'has-evals': true, + 'has-docs': false, + 'has-workflows': true, + }), + ).toBe('has-evals=true\nhas-docs=false\nhas-workflows=true\n') + }) +}) + +describe('CLI', () => { + test('appends outputs to GITHUB_OUTPUT', async () => { + await withTempDir(async (directory) => { + const outputPath = join(directory, 'github-output') + await writeFile(outputPath, 'existing=value\n') + + const result = await runCli('evals/test.ts\0', outputPath) + + expect(result).toEqual({ exitCode: 0, stderr: '', stdout: '' }) + expect(await readFile(outputPath, 'utf8')).toBe( + 'existing=value\nhas-evals=true\nhas-docs=false\nhas-workflows=false\n', + ) + }) + }) + + test('writes false outputs for empty stdin', async () => { + await withTempDir(async (directory) => { + const outputPath = join(directory, 'github-output') + + const result = await runCli('', outputPath) + + expect(result.exitCode).toBe(0) + expect(await readFile(outputPath, 'utf8')).toBe( + 'has-evals=false\nhas-docs=false\nhas-workflows=false\n', + ) + }) + }) + + test('fails clearly when GITHUB_OUTPUT is missing', async () => { + const result = await runCli('evals/test.ts\0') + + expect(result.exitCode).not.toBe(0) + expect(result.stderr).toContain('GITHUB_OUTPUT is required but was not set') + }) + + test('writes an error to stderr and exits nonzero for an unwritable target', async () => { + await withTempDir(async (directory) => { + const outputPath = join(directory, 'output-directory') + await mkdir(outputPath) + + const result = await runCli('docs/guide.md\0', outputPath) + + expect(result.exitCode).not.toBe(0) + expect(result.stderr).toContain('Unable to append change detection outputs to GITHUB_OUTPUT') + }) + }) + + test('handles NUL-delimited Unicode and metacharacter filenames without changing identity', async () => { + await withTempDir(async (directory) => { + const outputPath = join(directory, 'github-output') + const paths = [ + 'evals/ leading and trailing .ts ', + 'docs/雪\n$HOME;$(touch never).md', + '.github/workflows/[check]& weird.yml', + ] + + const result = await runCli(`${paths.join('\0')}\0`, outputPath) + + expect(result.exitCode).toBe(0) + expect(await readFile(outputPath, 'utf8')).toBe( + 'has-evals=true\nhas-docs=true\nhas-workflows=true\n', + ) + }) + }) + + test('does not trim leading whitespace into a matching path', async () => { + await withTempDir(async (directory) => { + const outputPath = join(directory, 'github-output') + + const result = await runCli(' evals/not-under-evals.ts\0', outputPath) + + expect(result.exitCode).toBe(0) + expect(await readFile(outputPath, 'utf8')).toContain('has-evals=false\n') + }) + }) +}) + +describe('PR checks workflow contract', () => { + test('uses the NUL-delimited detector and exact output names', async () => { + const workflow = await readFile(WORKFLOW_PATH, 'utf8') + + expect(workflow).toContain('git diff --name-only -z') + expect(workflow).toMatch(/git diff --name-only -z[^\n]*\|[\s\S]*bun run scripts\/validation\/detect-pr-changes\.ts/) + expect(workflow).toContain('has-evals: ${{ steps.filter.outputs.has-evals }}') + expect(workflow).toContain('has-docs: ${{ steps.filter.outputs.has-docs }}') + expect(workflow).toContain('has-workflows: ${{ steps.filter.outputs.has-workflows }}') + expect(workflow).not.toMatch(/steps\.filter\.outputs\.(evals|docs|workflows)(?:\s|})/) + expect(workflow).toContain('oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6') + expect(workflow).toContain('bun-version: 1.3.14') + }) + + test('requires successful change detection before reporting overall success', async () => { + const workflow = await readFile(WORKFLOW_PATH, 'utf8') + const overallStatus = workflow.slice(workflow.indexOf('# Overall status')) + + expect(overallStatus).toContain('needs.check-changes.result }}" == "success"') + expect(overallStatus).toContain('needs.check-changes.outputs.has-evals }}" != "true"') + expect(overallStatus).toContain('exit 1') + }) +}) diff --git a/scripts/validation/detect-pr-changes.ts b/scripts/validation/detect-pr-changes.ts new file mode 100644 index 0000000..f7349d1 --- /dev/null +++ b/scripts/validation/detect-pr-changes.ts @@ -0,0 +1,50 @@ +#!/usr/bin/env bun + +import { appendFile } from 'node:fs/promises' + +export type ChangeFlags = { + 'has-evals': boolean + 'has-docs': boolean + 'has-workflows': boolean +} + +export function parseChangedPaths(input: string): string[] { + return input.split('\0').filter((path) => path.length > 0) +} + +export function classifyChangedPaths(paths: readonly string[]): ChangeFlags { + return { + 'has-evals': paths.some((path) => path.startsWith('evals/')), + 'has-docs': paths.some((path) => path.startsWith('docs/')), + 'has-workflows': paths.some((path) => path.startsWith('.github/workflows/')), + } +} + +export function formatGitHubOutput(flags: ChangeFlags): string { + return [ + `has-evals=${flags['has-evals']}`, + `has-docs=${flags['has-docs']}`, + `has-workflows=${flags['has-workflows']}`, + ].join('\n') + '\n' +} + +async function main(): Promise { + const outputPath = process.env.GITHUB_OUTPUT + if (!outputPath) throw new Error('GITHUB_OUTPUT is required but was not set') + + const paths = parseChangedPaths(await Bun.stdin.text()) + const output = formatGitHubOutput(classifyChangedPaths(paths)) + + await appendFile(outputPath, output, 'utf8').catch((error: unknown) => { + const message = error instanceof Error ? error.message : String(error) + throw new Error(`Unable to append change detection outputs to GITHUB_OUTPUT (${outputPath}): ${message}`) + }) +} + +if (import.meta.main) { + await main().catch((error: unknown) => { + const message = error instanceof Error ? error.message : String(error) + console.error(`detect-pr-changes: ${message}`) + process.exitCode = 1 + }) +}