OpenWriterControle/.github/workflows/pr-checks.yml

name: PR Checks

on:
  pull_request:
    branches: [main, dev]
    types: [opened, edited, synchronize, reopened]

permissions:
  contents: read

jobs:
  pr-title-check:
    name: Validate PR Title
    runs-on: ubuntu-latest
    outputs:
      title-valid: ${{ steps.validate.outputs.valid }}
    
    steps:
      - name: Check PR title format
        id: validate
        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
        with:
          script: |
            const prTitle = context.payload.pull_request.title;
            
            // Conventional commit patterns
            const patterns = {
              feat: /^feat(\(.+\))?!?:\s.+/,
              fix: /^fix(\(.+\))?!?:\s.+/,
              docs: /^docs(\(.+\))?:\s.+/,
              style: /^style(\(.+\))?:\s.+/,
              refactor: /^refactor(\(.+\))?:\s.+/,
              perf: /^perf(\(.+\))?:\s.+/,
              test: /^test(\(.+\))?:\s.+/,
              chore: /^chore(\(.+\))?:\s.+/,
              ci: /^ci(\(.+\))?:\s.+/,
              build: /^build(\(.+\))?:\s.+/,
              revert: /^revert(\(.+\))?:\s.+/,
              alpha: /^\[alpha\]\s.+/,
              beta: /^\[beta\]\s.+/,
              rc: /^\[rc\]\s.+/
            };
            
            // Check if title matches any pattern
            const matchedType = Object.entries(patterns).find(([type, pattern]) => 
              pattern.test(prTitle)
            );
            
            if (!matchedType) {
              const validExamples = [
                '✅ feat(evals): add new evaluator',
                '✅ fix(agents): correct delegation logic',
                '✅ docs(readme): update installation guide',
                '✅ test(evals): add execution-balance tests',
                '✅ chore(deps): update dependencies',
                '✅ feat!: breaking API change',
                '✅ [alpha] experimental feature'
              ];
              
              const message = `
              ## ❌ Invalid PR Title Format
              
              **Current title:** \`${prTitle}\`
              
              ### Required Format
              PR titles must follow [Conventional Commits](https://www.conventionalcommits.org/) format:
              
              \`\`\`
              <type>(<scope>): <description>
              \`\`\`
              
              ### Valid Types
              - **feat** - New feature (triggers minor version bump: 0.3.0 → 0.4.0)
              - **fix** - Bug fix (triggers patch version bump: 0.3.0 → 0.3.1)
              - **docs** - Documentation changes (triggers patch bump)
              - **test** - Test additions/changes (triggers patch bump)
              - **refactor** - Code refactoring (triggers patch bump)
              - **chore** - Maintenance tasks (triggers patch bump)
              - **ci** - CI/CD changes (triggers patch bump)
              - **perf** - Performance improvements (triggers patch bump)
              - **style** - Code style changes (triggers patch bump)
              - **build** - Build system changes (triggers patch bump)
              - **revert** - Revert previous commit (triggers patch bump)
              
              ### Breaking Changes
              - **feat!:** or **fix!:** - Breaking change (triggers major version bump: 0.3.0 → 1.0.0)
              - **BREAKING CHANGE:** in description
              
              ### Pre-release Tags
              - **[alpha]** - Alpha release (0.3.0 → 0.3.1-alpha.1)
              - **[beta]** - Beta release (0.3.0 → 0.3.1-beta.1)
              - **[rc]** - Release candidate (0.3.0 → 0.3.1-rc.1)
              
              ### Valid Examples
              ${validExamples.map(ex => `- ${ex}`).join('\n')}
              
              ### Why This Matters
              - ✅ Enables automatic semantic versioning
              - ✅ Generates meaningful changelogs
              - ✅ Makes commit history searchable
              - ✅ Clarifies the impact of changes
              
              ### How to Fix
              Edit your PR title to match the format above.
              `;
              
              core.info(message);
              await core.summary.addRaw(message).write();
              core.setFailed(message);
            } else {
              const [type] = matchedType;
              let versionBump = 'patch (0.3.0 → 0.3.1)';
              
              if (type === 'feat' && prTitle.includes('!')) {
                versionBump = 'major (0.3.0 → 1.0.0) - BREAKING CHANGE';
              } else if (type === 'feat') {
                versionBump = 'minor (0.3.0 → 0.4.0)';
              } else if (type === 'fix' && prTitle.includes('!')) {
                versionBump = 'major (0.3.0 → 1.0.0) - BREAKING CHANGE';
              } else if (type === 'alpha') {
                versionBump = 'alpha (0.3.0 → 0.3.1-alpha.1)';
              } else if (type === 'beta') {
                versionBump = 'beta (0.3.0 → 0.3.1-beta.1)';
              } else if (type === 'rc') {
                versionBump = 'rc (0.3.0 → 0.3.1-rc.1)';
              }
              
              const message = `
              ## ✅ PR Title Valid
              
              **Title:** \`${prTitle}\`
              **Type:** \`${type}\`
              **Version bump:** ${versionBump}
              
              When this PR is merged using **"Squash and Merge"**, the version will be automatically bumped.
              `;
              
              core.info(message);
              
              // Set output for summary
              core.summary
                .addHeading('✅ PR Title Validation Passed', 2)
                .addRaw(`**Title:** \`${prTitle}\``)
                .addBreak()
                .addRaw(`**Type:** \`${type}\``)
                .addBreak()
                .addRaw(`**Version bump:** ${versionBump}`)
                .write();
              
              // Set output for dependent jobs
              core.setOutput('valid', 'true');
            }            

  check-changes:
    name: Detect Changed Files
    runs-on: ubuntu-latest
    needs: pr-title-check
    if: needs.pr-title-check.outputs.title-valid == 'true'
    outputs:
      has-evals: ${{ steps.filter.outputs.has-evals }}
      has-docs: ${{ steps.filter.outputs.has-docs }}
      has-workflows: ${{ steps.filter.outputs.has-workflows }}
    
    steps:
      - name: Checkout code
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          fetch-depth: 0
          persist-credentials: false

      - name: Setup Bun
        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
        with:
          bun-version: 1.3.14
      
      - name: Check changed files
        id: filter
        run: |
          echo "Changed files:" >&2
          git diff --name-only -z "${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }}" |
            tee >(while IFS= read -r -d '' path; do printf '  %q\n' "$path" >&2; done) |
            bun run scripts/validation/detect-pr-changes.ts          

  build-check:
    name: Build & Validate
    runs-on: ubuntu-latest
    timeout-minutes: 5
    needs: [pr-title-check, check-changes]
    if: |
      needs.pr-title-check.outputs.title-valid == 'true' &&
      needs.check-changes.outputs.has-evals == 'true'      
    
    steps:
      - name: Checkout code
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          persist-credentials: false
      
      - name: Setup Node.js
        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
        with:
          node-version: '20'
          cache: 'npm'
          cache-dependency-path: 'evals/framework/package-lock.json'
      
      - name: Install dependencies
        working-directory: evals/framework
        run: npm ci
      
      - name: Build framework
        working-directory: evals/framework
        run: npm run build
      
      - name: Validate test suites
        working-directory: evals/framework
        run: npm run validate:suites:all

      - name: Run deterministic tests
        working-directory: evals/framework
        run: npm run test:ci

      - name: Summary
        if: success()
        run: |
          echo "## ✅ Build Check Passed" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "- ✅ TypeScript compilation successful" >> $GITHUB_STEP_SUMMARY
          echo "- ✅ Test suite validation passed" >> $GITHUB_STEP_SUMMARY
          echo "- ✅ Deterministic tests passed (\`npm run test:ci\`)" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "**Note:** \`test:ci\` runs the offline, deterministic Vitest allowlist only." >> $GITHUB_STEP_SUMMARY
          echo "Model- or network-dependent agent tests (\`eval:sdk\`, integration suites) are not run on PRs." >> $GITHUB_STEP_SUMMARY          
      
      - name: Failure summary
        if: failure()
        run: |
          echo "## ❌ Build Check Failed" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "Please check the logs above for details." >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "**Common fixes:**" >> $GITHUB_STEP_SUMMARY
          echo "- TypeScript errors: Fix type issues in \`evals/framework/src/\`" >> $GITHUB_STEP_SUMMARY
          echo "- YAML validation: Check test files in \`evals/agents/*/tests/\`" >> $GITHUB_STEP_SUMMARY          

  summary:
    name: PR Checks Summary
    runs-on: ubuntu-latest
    needs: [pr-title-check, check-changes, build-check]
    if: always()
    
    steps:
      - name: Generate summary
        run: |
          echo "## 📊 PR Checks Summary" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          
          # PR Title Check
          if [ "${{ needs.pr-title-check.result }}" == "success" ]; then
            echo "✅ **PR Title:** Valid conventional commit format" >> $GITHUB_STEP_SUMMARY
          else
            echo "❌ **PR Title:** Invalid format - please fix" >> $GITHUB_STEP_SUMMARY
          fi
          
          # Changed Files Detection
          if [ "${{ needs.check-changes.result }}" == "success" ]; then
            echo "✅ **Changed Files:** Detected successfully" >> $GITHUB_STEP_SUMMARY
            
            if [ "${{ needs.check-changes.outputs.has-evals }}" == "true" ]; then
              echo "  - 📦 Evals changes detected" >> $GITHUB_STEP_SUMMARY
            fi
            
            if [ "${{ needs.check-changes.outputs.has-docs }}" == "true" ]; then
              echo "  - 📚 Docs changes detected" >> $GITHUB_STEP_SUMMARY
            fi
            
            if [ "${{ needs.check-changes.outputs.has-workflows }}" == "true" ]; then
              echo "  - ⚙️ Workflow changes detected" >> $GITHUB_STEP_SUMMARY
            fi
          elif [ "${{ needs.check-changes.result }}" == "failure" ]; then
            echo "❌ **Changed Files:** Detection failed - check logs" >> $GITHUB_STEP_SUMMARY
          elif [ "${{ needs.check-changes.result }}" == "cancelled" ]; then
            echo "🚫 **Changed Files:** Detection cancelled" >> $GITHUB_STEP_SUMMARY
          elif [ "${{ needs.check-changes.result }}" == "skipped" ]; then
            echo "⏭️ **Changed Files:** Skipped (title validation failed)" >> $GITHUB_STEP_SUMMARY
          else
            echo "❓ **Changed Files:** Unknown result" >> $GITHUB_STEP_SUMMARY
          fi
          
          # Build Check
          if [ "${{ needs.build-check.result }}" == "success" ]; then
            echo "✅ **Build & Validate:** Passed" >> $GITHUB_STEP_SUMMARY
          elif [ "${{ needs.build-check.result }}" == "skipped" ]; then
            if [ "${{ needs.check-changes.result }}" != "success" ]; then
              echo "⏭️ **Build & Validate:** Skipped (change detection did not succeed)" >> $GITHUB_STEP_SUMMARY
            elif [ "${{ needs.check-changes.outputs.has-evals }}" != "true" ]; then
              echo "⏭️ **Build & Validate:** Skipped (no evals changes)" >> $GITHUB_STEP_SUMMARY
            else
              echo "⏭️ **Build & Validate:** Skipped (prerequisite not met)" >> $GITHUB_STEP_SUMMARY
            fi
          elif [ "${{ needs.build-check.result }}" == "failure" ]; then
            echo "❌ **Build & Validate:** Failed - check logs" >> $GITHUB_STEP_SUMMARY
          elif [ "${{ needs.build-check.result }}" == "cancelled" ]; then
            echo "🚫 **Build & Validate:** Cancelled" >> $GITHUB_STEP_SUMMARY
          fi
          
          echo "" >> $GITHUB_STEP_SUMMARY
          
          # Overall status
          if [ "${{ needs.pr-title-check.result }}" == "success" ] && \
             [ "${{ needs.check-changes.result }}" == "success" ] && \
             { [ "${{ needs.build-check.result }}" == "success" ] || \
               { [ "${{ needs.build-check.result }}" == "skipped" ] && \
                 [ "${{ needs.check-changes.outputs.has-evals }}" != "true" ]; }; }; then
            echo "### ✅ All Required Checks Passed!" >> $GITHUB_STEP_SUMMARY
            echo "" >> $GITHUB_STEP_SUMMARY
            echo "This PR is ready for review." >> $GITHUB_STEP_SUMMARY
          else
            echo "### ❌ Some Checks Failed" >> $GITHUB_STEP_SUMMARY
            echo "" >> $GITHUB_STEP_SUMMARY
            echo "Please fix the failing checks before merging." >> $GITHUB_STEP_SUMMARY
            exit 1
          fi