fix(evals): repair deterministic eval baseline (behavioral regressions + tsc build) (#334)

* fix(evals): repair six eval-framework behavioral regressions

Restore the deterministic eval baseline by fixing false-negatives and
identity/discovery bugs across the framework:

- error-handling: correct severity-boundary fixtures so mixed-severity
  cases actually cross the >1000 warning threshold (deterministic
  error/warning/info), plus explicit 1000-vs-1001 boundary tests.
- context-loading-evaluator: emit `no-context-loaded` and
  `context-loaded-after-execution` as errors so missing/late context
  fails (was a passing warning); update focused test accordingly.
- tool-usage-evaluator: enforce dedicated tools — bash `cat`/`ls`/`ls -la`
  now fail as `bash-antipattern-*` errors (ls de-allowlisted); npm/git/
  piped commands still pass.
- test-case-loader: canonicalize legacy agent ids via normalizeAgentId
  (`openagent` -> `core/openagent`), preserving canonical/unknown ids.
- suite-validator: accept category-based agent ids, resolve test paths
  under the canonical dir, add requested-vs-declared conflict detection;
  new suite-validator.test.ts.
- validate-suites-cli: recursive suite discovery (nested <cat>/<agent>/
  config), deterministic order, fail-closed on zero suites; new
  validate-suites-cli.test.ts.

Deterministic Vitest allowlist green; validate:suites:all non-vacuous
and exits 0. No model/network/eval:sdk execution.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* fix(evals): clear pre-existing TypeScript build errors (tsc 39 -> 0)

`npm run build` (tsc) failed on 39 pre-existing errors in files outside
the behavioral-repair scope, blocking the baseline gate. Root causes:

- glob: drop obsolete @types/glob ^8 so glob v13's own types (globSync)
  resolve (fixes run-sdk-tests.ts).
- task-type-detector: introduce a local DetectedTaskType union for this
  (dead/unused) detector; shared TaskType union left unchanged.
- behavior-evaluator: the expectedResponse check filtered a nonexistent
  `message` event type and was silently dead — route through
  base-evaluator.getAssistantMessages (assistant_message/text). Adds
  three deterministic tests exercising the now-live check.
- integration tests: remove unused `expectedOutcome` fixture metadata
  (not on TestCase, never asserted) and add the required `category`
  field the excess-property error had masked.

tsc now exits 0. Vitest allowlist 112/112. Lint remains unconfigured in
evals/framework (no ESLint config has ever existed) and is deferred to a
separate follow-up; it is not part of this change.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* fix(evals): sync bun.lock after @types/glob removal

The previous commit removed @types/glob via npm, updating package-lock.json
but leaving the primary bun.lock stale, which broke CI's
`bun install --frozen-lockfile`. Regenerate bun.lock so both lockfiles
agree (drops @types/glob and its transitive @types/minimatch only).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* ci(evals): add evals/framework/package-lock.json for build-check

The pr-checks `build-check` job caches on and runs `npm ci` in
evals/framework, but that workspace had no local lockfile, so the job
failed at Setup Node.js ("Some specified paths were not resolved") and
would fail `npm ci`. Add a standalone, in-sync package-lock.json
(lockfileVersion 3, no @types/glob) so the job's cache path resolves and
`npm ci` succeeds. Validated locally with `npm ci --dry-run`.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>

bun.lock

@@ -18,7 +18,6 @@
         "zod": "^3.25.76",
       },
       "devDependencies": {
-        "@types/glob": "^8.1.0",
         "@types/node": "^20.10.0",
         "@typescript-eslint/eslint-plugin": "^6.13.0",
         "@typescript-eslint/parser": "^6.13.0",
@@ -234,14 +233,10 @@
 
     "@types/estree": ["@types/estree@1.0.8", "", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="],
 
-    "@types/glob": ["@types/glob@8.1.0", "", { "dependencies": { "@types/minimatch": "^5.1.2", "@types/node": "*" } }, ""],
-
     "@types/js-yaml": ["@types/js-yaml@4.0.9", "", {}, "sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg=="],
 
     "@types/json-schema": ["@types/json-schema@7.0.15", "", {}, ""],
 
-    "@types/minimatch": ["@types/minimatch@5.1.2", "", {}, ""],
-
     "@types/node": ["@types/node@20.19.25", "", { "dependencies": { "undici-types": "~6.21.0" } }, ""],
 
     "@types/semver": ["@types/semver@7.7.1", "", {}, ""],

evals/framework/package.json

@@ -38,7 +38,6 @@
     "zod": "^3.25.76"
   },
   "devDependencies": {
-    "@types/glob": "^8.1.0",
     "@types/node": "^20.10.0",
     "@typescript-eslint/eslint-plugin": "^6.13.0",
     "@typescript-eslint/parser": "^6.13.0",

evals/framework/src/__tests__/error-handling.test.ts

@@ -419,6 +419,30 @@ describe('Severity Levels', () => {
       expect(warningViolation!.type).toBe('code-length');
     });
 
+    it('does not report a warning when code length is exactly 1000', () => {
+      // Arrange
+      const code = 'x'.repeat(1000);
+
+      // Act
+      const result = validateCode(code);
+
+      // Assert
+      expect(code).toHaveLength(1000);
+      expect(result.violations.some(v => v.severity === 'warning')).toBe(false);
+    });
+
+    it('reports a warning when code length is exactly 1001', () => {
+      // Arrange
+      const code = 'x'.repeat(1001);
+
+      // Act
+      const result = validateCode(code);
+
+      // Assert
+      expect(code).toHaveLength(1001);
+      expect(result.violations.some(v => v.severity === 'warning')).toBe(true);
+    });
+
     it('passes with info-level violations', () => {
       // Arrange
       const code = 'const x = 10;'; // No strict mode
@@ -435,23 +459,21 @@ describe('Severity Levels', () => {
 
     it('reports multiple violations with different severities', () => {
       // Arrange
-      const code = 'eval("bad");'.repeat(100); // error + warning + info
+      const code = 'eval("bad");' + 'x'.repeat(989);
 
       // Act
       const result = validateCode(code);
 
       // Assert
+      expect(code).toHaveLength(1001);
       expect(result.passed).toBe(false); // Error causes failure
-      expect(result.violations.length).toBeGreaterThanOrEqual(2);
-      
       const severities = result.violations.map(v => v.severity);
-      expect(severities).toContain('error');
+      expect(severities).toEqual(['error', 'warning', 'info']);
-      expect(severities).toContain('warning');
     });
 
     it('distinguishes between severity levels correctly', () => {
       // Arrange
-      const code = 'eval("x");'.repeat(100);
+      const code = 'eval("x");'.repeat(101); // 1010 chars: crosses the >1000 warning boundary
 
       // Act
       const result = validateCode(code);

evals/framework/src/__tests__/eval-pipeline-integration.test.ts

@@ -70,13 +70,10 @@ describe.skipIf(skipIntegration)('Eval Pipeline Integration', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'Say "Hello Integration Test" and nothing else.',
         timeout: 15000,
+        category: 'developer',
         approvalStrategy: {
           type: 'auto-approve',
         },
-        expectedOutcome: {
-          type: 'text-response',
-          contains: ['Hello Integration Test'],
-        },
       };
 
       const result = await runner.runTest(testCase);
@@ -115,13 +112,10 @@ describe.skipIf(skipIntegration)('Eval Pipeline Integration', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'List files in the current directory using the List tool.',
         timeout: 20000,
+        category: 'developer',
         approvalStrategy: {
           type: 'auto-approve',
         },
-        expectedOutcome: {
-          type: 'tool-execution',
-          tools: ['list'],
-        },
       };
 
       const result = await runner.runTest(testCase);
@@ -159,12 +153,10 @@ describe.skipIf(skipIntegration)('Eval Pipeline Integration', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'Create a file named test.txt with content "test".',
         timeout: 20000,
+        category: 'developer',
         approvalStrategy: {
           type: 'auto-deny', // Deny all approvals
         },
-        expectedOutcome: {
-          type: 'approval-denied',
-        },
       };
 
       const result = await runner.runTest(testCase);
@@ -192,8 +184,8 @@ describe.skipIf(skipIntegration)('Eval Pipeline Integration', () => {
           model: 'opencode/grok-code-fast',
           prompt: 'Say "Test 1".',
           timeout: 15000,
+          category: 'developer',
           approvalStrategy: { type: 'auto-approve' },
-          expectedOutcome: { type: 'text-response', contains: ['Test 1'] },
         },
         {
           id: 'integration-multi-2',
@@ -203,8 +195,8 @@ describe.skipIf(skipIntegration)('Eval Pipeline Integration', () => {
           model: 'opencode/grok-code-fast',
           prompt: 'Say "Test 2".',
           timeout: 15000,
+          category: 'developer',
           approvalStrategy: { type: 'auto-approve' },
-          expectedOutcome: { type: 'text-response', contains: ['Test 2'] },
         },
       ];
 
@@ -236,8 +228,8 @@ describe.skipIf(skipIntegration)('Eval Pipeline Integration', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'List files in current directory.',
         timeout: 20000,
+        category: 'developer',
         approvalStrategy: { type: 'auto-approve' },
-        expectedOutcome: { type: 'tool-execution', tools: ['list'] },
       };
 
       const result = await runner.runTest(testCase);
@@ -271,8 +263,8 @@ describe.skipIf(skipIntegration)('Eval Pipeline Integration', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'Use cat command to read a file.', // Should trigger tool-usage violation
         timeout: 20000,
+        category: 'developer',
         approvalStrategy: { type: 'auto-approve' },
-        expectedOutcome: { type: 'tool-execution' },
       };
 
       const result = await runner.runTest(testCase);
@@ -312,8 +304,8 @@ describe.skipIf(skipIntegration)('Eval Pipeline Integration', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'List files and then say "Done".',
         timeout: 20000,
+        category: 'developer',
         approvalStrategy: { type: 'auto-approve' },
-        expectedOutcome: { type: 'text-response', contains: ['Done'] },
       };
 
       const result = await runner.runTest(testCase);
@@ -347,8 +339,8 @@ describe.skipIf(skipIntegration)('Eval Pipeline Integration', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'Say "No tools needed" and nothing else.',
         timeout: 15000,
+        category: 'developer',
         approvalStrategy: { type: 'auto-approve' },
-        expectedOutcome: { type: 'text-response', contains: ['No tools needed'] },
       };
 
       const result = await runner.runTest(testCase);
@@ -380,8 +372,8 @@ describe.skipIf(skipIntegration)('Eval Pipeline Integration', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'Perform a very long task that takes forever.',
         timeout: 5000, // Very short timeout
+        category: 'developer',
         approvalStrategy: { type: 'auto-approve' },
-        expectedOutcome: { type: 'text-response' },
       };
 
       const result = await runner.runTest(testCase);
@@ -421,11 +413,8 @@ describe.skipIf(skipIntegration)('Eval Pipeline Integration', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'Say "Validation Test".',
         timeout: 15000,
+        category: 'developer',
         approvalStrategy: { type: 'auto-approve' },
-        expectedOutcome: {
-          type: 'text-response',
-          contains: ['Validation Test'],
-        },
       };
 
       const result = await runner.runTest(testCase);
@@ -472,8 +461,8 @@ describe.skipIf(skipIntegration)('Eval Pipeline Integration', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'Say "Score Test".',
         timeout: 15000,
+        category: 'developer',
         approvalStrategy: { type: 'auto-approve' },
-        expectedOutcome: { type: 'text-response', contains: ['Score Test'] },
       };
 
       const result = await runner.runTest(testCase);
@@ -505,8 +494,8 @@ describe.skipIf(skipIntegration)('Eval Pipeline Integration', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'Say "Report Test".',
         timeout: 15000,
+        category: 'developer',
         approvalStrategy: { type: 'auto-approve' },
-        expectedOutcome: { type: 'text-response', contains: ['Report Test'] },
       };
 
       const result = await runner.runTest(testCase);
@@ -549,8 +538,8 @@ describe.skipIf(skipIntegration)('Eval Pipeline Integration', () => {
           model: 'opencode/grok-code-fast',
           prompt: 'Say "Batch 1".',
           timeout: 15000,
+          category: 'developer',
           approvalStrategy: { type: 'auto-approve' },
-          expectedOutcome: { type: 'text-response', contains: ['Batch 1'] },
         },
         {
           id: 'integration-batch-2',
@@ -560,8 +549,8 @@ describe.skipIf(skipIntegration)('Eval Pipeline Integration', () => {
           model: 'opencode/grok-code-fast',
           prompt: 'Say "Batch 2".',
           timeout: 15000,
+          category: 'developer',
           approvalStrategy: { type: 'auto-approve' },
-          expectedOutcome: { type: 'text-response', contains: ['Batch 2'] },
         },
       ];
 

evals/framework/src/__tests__/llm-integration.test.ts

@@ -78,12 +78,10 @@ describe.skipIf(skipLLMTests)('LLM Integration Tests', () => {
           },
         ],
         timeout: 30000,
+        category: 'developer',
         approvalStrategy: {
           type: 'auto-approve',
         },
-        expectedOutcome: {
-          type: 'text-response',
-        },
       };
 
       const result = await runner.runTest(testCase);
@@ -122,13 +120,10 @@ describe.skipIf(skipLLMTests)('LLM Integration Tests', () => {
           },
         ],
         timeout: 30000,
+        category: 'developer',
         approvalStrategy: {
           type: 'auto-approve',
         },
-        expectedOutcome: {
-          type: 'text-response',
-          contains: ['blue'],
-        },
       };
 
       const result = await runner.runTest(testCase);
@@ -150,13 +145,10 @@ describe.skipIf(skipLLMTests)('LLM Integration Tests', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'Read the package.json file',
         timeout: 30000,
+        category: 'developer',
         approvalStrategy: {
           type: 'auto-approve',
         },
-        expectedOutcome: {
-          type: 'tool-execution',
-          tools: ['read'],
-        },
         behavior: {
           requiresApproval: true,
         },
@@ -186,12 +178,10 @@ describe.skipIf(skipLLMTests)('LLM Integration Tests', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'Create a new file called test.txt with content "Hello World"',
         timeout: 30000,
+        category: 'developer',
         approvalStrategy: {
           type: 'auto-deny',
         },
-        expectedOutcome: {
-          type: 'approval-denied',
-        },
       };
 
       const result = await runner.runTest(testCase);
@@ -219,13 +209,10 @@ describe.skipIf(skipLLMTests)('LLM Integration Tests', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'Say "Performance test complete"',
         timeout: 15000,
+        category: 'developer',
         approvalStrategy: {
           type: 'auto-approve',
         },
-        expectedOutcome: {
-          type: 'text-response',
-          contains: ['Performance test complete'],
-        },
       };
 
       const startTime = Date.now();
@@ -253,13 +240,10 @@ describe.skipIf(skipLLMTests)('LLM Integration Tests', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'Read a file that does not exist: /nonexistent/file.txt',
         timeout: 30000,
+        category: 'developer',
         approvalStrategy: {
           type: 'auto-approve',
         },
-        expectedOutcome: {
-          type: 'tool-execution',
-          tools: ['read'],
-        },
       };
 
       const result = await runner.runTest(testCase);
@@ -286,13 +270,10 @@ describe.skipIf(skipLLMTests)('LLM Integration Tests', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'Read the package.json file and list all files in the current directory',
         timeout: 30000,
+        category: 'developer',
         approvalStrategy: {
           type: 'auto-approve',
         },
-        expectedOutcome: {
-          type: 'tool-execution',
-          tools: ['read', 'glob'],
-        },
         behavior: {
           mustUseDedicatedTools: true,
           mustUseAnyOf: [
@@ -333,13 +314,10 @@ describe.skipIf(skipLLMTests)('LLM Integration Tests', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'Create a new TypeScript file called math.ts with a function called add that adds two numbers',
         timeout: 45000,
+        category: 'developer',
         approvalStrategy: {
           type: 'auto-approve',
         },
-        expectedOutcome: {
-          type: 'tool-execution',
-          tools: ['write'],
-        },
         behavior: {
           requiresContext: true,
           expectedContextFiles: [
@@ -397,12 +375,10 @@ describe.skipIf(skipLLMTests)('LLM Integration Tests', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'Tell me about the package.json file',
         timeout: 30000,
+        category: 'developer',
         approvalStrategy: {
           type: 'auto-approve',
         },
-        expectedOutcome: {
-          type: 'text-response',
-        },
         behavior: {
           mustNotUseTools: ['bash'], // Should not use bash for this task
           mayUseTools: ['read', 'glob'], // Can use these if needed
@@ -445,13 +421,10 @@ describe.skipIf(skipLLMTests)('LLM Integration Tests', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'Read the package.json file using the Read tool',
         timeout: 30000,
+        category: 'developer',
         approvalStrategy: {
           type: 'auto-approve',
         },
-        expectedOutcome: {
-          type: 'tool-execution',
-          tools: ['read'],
-        },
       };
 
       const result = await runner.runTest(testCase);

evals/framework/src/evaluators/__tests__/context-loading-evaluator.test.ts

@@ -98,11 +98,11 @@ describe('ContextLoadingEvaluator', () => {
 
       const result = await evaluator.evaluate(timeline, mockSessionInfo);
 
-      // Context loading violation is a warning, not error, so passed is still true
+      // No context loaded before execution is an error, so passed is false
-      // But contextLoadedBeforeExecution should be false
+      // And contextLoadedBeforeExecution should be false
       expect(result.metadata?.contextLoadedBeforeExecution).toBe(false);
       expect(result.violations.length).toBeGreaterThan(0);
-      expect(result.violations[0].severity).toBe('warning');
+      expect(result.violations[0].severity).toBe('error');
     });
   });
 

evals/framework/src/evaluators/__tests__/evaluator-reliability.test.ts

@@ -212,6 +212,53 @@ describe('Evaluator Reliability - False Negatives', () => {
       expect(result.violations.length).toBeGreaterThan(0);
       expect(result.violations[0].type).toBe('insufficient-tool-calls');
     });
+
+    it('MUST FAIL: Assistant response missing expected content', async () => {
+      const evaluator = new BehaviorEvaluator({
+        expectedResponse: {
+          contains: ['Task completed successfully'],
+        },
+      });
+
+      const timeline: TimelineEvent[] = [
+        createUserMessage('Do the task', 1000),
+        createAssistantMessage('I ran into an error and stopped.', 2000),
+      ];
+
+      const result = await evaluator.evaluate(timeline, mockSessionInfo);
+
+      // Expected string is absent from the assistant message -> violation.
+      expect(result.passed).toBe(false);
+      const missingContent = result.violations.find(
+        v => v.type === 'missing-expected-content'
+      );
+      expect(missingContent).toBeDefined();
+      expect(missingContent?.evidence?.missingStrings).toContain(
+        'Task completed successfully'
+      );
+    });
+
+    it('MUST FAIL: Assistant response contains forbidden content', async () => {
+      const evaluator = new BehaviorEvaluator({
+        expectedResponse: {
+          notContains: ['rm -rf /'],
+        },
+      });
+
+      const timeline: TimelineEvent[] = [
+        createUserMessage('Clean up', 1000),
+        createAssistantMessage('Sure, running rm -rf / now.', 2000),
+      ];
+
+      const result = await evaluator.evaluate(timeline, mockSessionInfo);
+
+      expect(result.passed).toBe(false);
+      const forbidden = result.violations.find(
+        v => v.type === 'forbidden-content-found'
+      );
+      expect(forbidden).toBeDefined();
+      expect(forbidden?.evidence?.foundForbiddenStrings).toContain('rm -rf /');
+    });
   });
 
   describe('StopOnFailureEvaluator', () => {
@@ -397,6 +444,33 @@ describe('Evaluator Reliability - False Positives', () => {
       expect(result.passed).toBe(true);
       expect(result.violations.length).toBe(0);
     });
+
+    it('MUST PASS: Assistant response satisfies expectedResponse', async () => {
+      const evaluator = new BehaviorEvaluator({
+        expectedResponse: {
+          contains: ['Task completed successfully'],
+          notContains: ['rm -rf /'],
+        },
+      });
+
+      const timeline: TimelineEvent[] = [
+        createUserMessage('Do the task', 1000),
+        createAssistantMessage('Task completed successfully. All good.', 2000),
+      ];
+
+      const result = await evaluator.evaluate(timeline, mockSessionInfo);
+
+      // Assistant message satisfies both contains and notContains -> no
+      // expectedResponse violations.
+      expect(
+        result.violations.some(
+          v =>
+            v.type === 'missing-expected-content' ||
+            v.type === 'forbidden-content-found'
+        )
+      ).toBe(false);
+      expect(result.passed).toBe(true);
+    });
   });
 
   describe('ToolUsageEvaluator', () => {

evals/framework/src/evaluators/__tests__/tool-usage-enhanced.test.ts

@@ -41,7 +41,7 @@ describe('ToolUsageEvaluator - Enhanced Features', () => {
       const result = await evaluator.evaluate(timeline, mockSessionInfo);
       
       expect(result.violations.length).toBeGreaterThan(0);
-      expect(result.violations[0].type).toBe('suboptimal-tool-usage');
+      expect(result.violations[0].type).toBe('bash-antipattern-edit');
       expect(result.violations[0].evidence.suggestedTool).toBe('edit');
     });
 
@@ -179,7 +179,7 @@ describe('ToolUsageEvaluator - Enhanced Features', () => {
   });
 
   describe('Severity Levels', () => {
-    it('should use warning severity for cat usage', async () => {
+    it('should use error severity for cat usage', async () => {
       const timeline: TimelineEvent[] = [
         {
           timestamp: 1000,
@@ -193,10 +193,12 @@ describe('ToolUsageEvaluator - Enhanced Features', () => {
 
       const result = await evaluator.evaluate(timeline, mockSessionInfo);
 
-      expect(result.violations[0].severity).toBe('warning');
+      expect(result.violations[0].severity).toBe('error');
+      expect(result.violations[0].type).toBe('bash-antipattern-read');
+      expect(result.passed).toBe(false);
     });
 
-    it('should use info severity for ls usage', async () => {
+    it('should use error severity for ls usage', async () => {
       const timeline: TimelineEvent[] = [
         {
           timestamp: 1000,
@@ -210,7 +212,9 @@ describe('ToolUsageEvaluator - Enhanced Features', () => {
 
       const result = await evaluator.evaluate(timeline, mockSessionInfo);
 
-      expect(result.violations[0].severity).toBe('info');
+      expect(result.violations[0].severity).toBe('error');
+      expect(result.violations[0].type).toBe('bash-antipattern-list');
+      expect(result.passed).toBe(false);
     });
   });
 
@@ -267,7 +271,7 @@ describe('ToolUsageEvaluator - Enhanced Features', () => {
       expect(result.passed).toBe(true);
     });
 
-    it('should allow ls -la for detailed info', async () => {
+    it('should flag ls -la as a dedicated-listing-tool violation', async () => {
       const timeline: TimelineEvent[] = [
         {
           timestamp: 1000,
@@ -281,7 +285,9 @@ describe('ToolUsageEvaluator - Enhanced Features', () => {
 
       const result = await evaluator.evaluate(timeline, mockSessionInfo);
 
-      expect(result.passed).toBe(true);
+      expect(result.passed).toBe(false);
+      expect(result.violations.length).toBeGreaterThan(0);
+      expect(result.violations[0].type).toBe('bash-antipattern-list');
     });
 
     it('should allow echo to stdout (no redirection)', async () => {

evals/framework/src/evaluators/behavior-evaluator.ts

@@ -464,9 +464,7 @@ export class BehaviorEvaluator extends BaseEvaluator {
 
     // Check 8: expectedResponse (validate response content)
     if (this.behavior.expectedResponse) {
-      const assistantMessages = timeline.filter(
+      const assistantMessages = this.getAssistantMessages(timeline);
-        e => e.type === 'message' && e.data?.role === 'assistant' && e.data?.text
-      );
       
       // Combine all assistant messages into one text for validation
       const fullResponse = assistantMessages

evals/framework/src/evaluators/context-loading-evaluator.ts

@@ -369,7 +369,7 @@ export class ContextLoadingEvaluator extends BaseEvaluator {
       violations.push(
         this.createViolation(
           'no-context-loaded',
-          'warning',
+          'error',
           'Task execution started without loading any context files',
           firstExecution.timestamp,
           {
@@ -402,7 +402,7 @@ export class ContextLoadingEvaluator extends BaseEvaluator {
       violations.push(
         this.createViolation(
           'context-loaded-after-execution',
-          'warning',
+          'error',
           'Some executions happened before context was loaded',
           firstExecution.timestamp,
           {

evals/framework/src/evaluators/task-type-detector.ts

@@ -7,10 +7,33 @@
 
 import type { TimelineEvent, TaskType } from '../types/index.js';
 
+/**
+ * Task type taxonomy specific to the detector.
+ *
+ * This is intentionally distinct from (and richer than) the shared `TaskType`
+ * union in `../types/index.js`. The detector distinguishes file-operation
+ * intents (create/modify/delete) and conversational/read-only sessions that the
+ * shared union does not model. Every member below corresponds to a literal
+ * returned by `detectTaskType` and a key used in the applicability matrix.
+ */
+export type DetectedTaskType =
+  | 'read-only'
+  | 'create-new-file'
+  | 'modify-existing-file'
+  | 'delete-file'
+  | 'conversational'
+  | 'delegation'
+  | 'bash-only'
+  | 'code'
+  | 'docs'
+  | 'tests'
+  | 'review'
+  | 'unknown';
+
 /**
  * Detect task type from user message and timeline events
  */
-export function detectTaskType(userMessage: string | any, timeline: TimelineEvent[]): TaskType {
+export function detectTaskType(userMessage: string | any, timeline: TimelineEvent[]): DetectedTaskType {
   // Extract text from userMessage (could be string or object)
   const messageText = typeof userMessage === 'string' 
     ? userMessage 
@@ -140,7 +163,7 @@ export function getEvaluatorApplicability(
   evaluatorName: string,
   taskType: TaskType
 ): { applicable: boolean; reason?: string } {
-  const matrix: Record<string, Partial<Record<TaskType, { applicable: boolean; reason?: string }>>> = {
+  const matrix: Record<string, Partial<Record<DetectedTaskType, { applicable: boolean; reason?: string }>>> = {
     'approval-gate': {
       'create-new-file': { applicable: true },
       'modify-existing-file': { applicable: true },

evals/framework/src/evaluators/tool-usage-evaluator.ts

@@ -37,7 +37,7 @@ export class ToolUsageEvaluator extends BaseEvaluator {
       pattern: /\bcat\s+([^\s|>]+)(?!\s*[|>])/i,
       tool: 'read',
       message: 'Use Read tool instead of cat for reading files',
-      severity: 'warning' as const,
+      severity: 'error' as const,
       example: 'read(filePath: "path/to/file")'
     },
     { 
@@ -55,10 +55,10 @@ export class ToolUsageEvaluator extends BaseEvaluator {
       example: 'read(filePath: "file", offset: -10)'
     },
     {
-      pattern: /\bls\s+(?!-[al]+\s)([^\s]*)/i, 
+      pattern: /\bls(\s+|$)/i,
       tool: 'list',
-      message: 'Use List tool instead of ls (unless ls -la for detailed info)',
+      message: 'Use List tool (or Glob for patterns) instead of ls',
-      severity: 'info' as const,
+      severity: 'error' as const,
       example: 'list(path: "directory")'
     },
     { 
@@ -116,7 +116,6 @@ export class ToolUsageEvaluator extends BaseEvaluator {
     /^\s*mv\s+/i,            // moving files
     /^\s*cp\s+/i,            // copying files
     /^\s*chmod\s+/i,         // permissions
-    /^\s*ls\s+-[la]+/i,      // ls -la for detailed directory info
     /^\s*cd\s+/i,            // navigation
     /^\s*pwd\s*/i,           // current directory
     /^\s*which\s+/i,         // command location
@@ -195,7 +194,7 @@ export class ToolUsageEvaluator extends BaseEvaluator {
         // Add violation with appropriate severity
         violations.push(
           this.createViolation(
-            'suboptimal-tool-usage',
+            `bash-antipattern-${antiPattern.tool}`,
             antiPattern.severity,
             antiPattern.message,
             bashCall.timestamp,

evals/framework/src/sdk/__tests__/suite-validator.test.ts

@@ -0,0 +1,217 @@
+/**
+ * Tests for SuiteValidator category-based identity and path resolution.
+ *
+ * Uses the REAL evals/agents directory as agentsDir so that the canonical
+ * `core/openagent` core suite and its nested test paths are exercised
+ * end-to-end. Conflict / missing-test / duplicate-id cases use small temp
+ * fixture files whose paths are passed directly to the validator.
+ *
+ * NOTE: Validation is filesystem-only. No network, model, or agent execution.
+ */
+
+import { describe, it, expect, beforeAll, afterAll } from 'vitest';
+import { SuiteValidator } from '../suite-validator.js';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+import { mkdtempSync, writeFileSync, rmSync } from 'fs';
+import { tmpdir } from 'os';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+// Real agents directory (matches the layout used by the validate:suites CLI)
+const agentsDir = join(__dirname, '../../../../agents');
+const coreSuitePath = join(
+  agentsDir,
+  'core/openagent/config/core-suite.json'
+);
+
+/**
+ * Minimal but schema-valid suite object used for temp fixtures.
+ */
+function makeSuite(overrides: Record<string, unknown> = {}): Record<string, unknown> {
+  return {
+    name: 'Fixture Suite',
+    description: 'Temporary fixture suite for validator tests',
+    version: '1.0.0',
+    agent: 'core/openagent',
+    totalTests: 1,
+    estimatedRuntime: '1-2 minutes',
+    tests: [
+      {
+        id: 1,
+        name: 'Approval Gate',
+        path: '01-critical-rules/approval-gate/05-approval-before-execution-positive.yaml',
+        category: 'critical-rules',
+        priority: 'critical'
+      }
+    ],
+    ...overrides
+  };
+}
+
+describe('SuiteValidator - category-based identities', () => {
+  let tmpDir: string;
+
+  beforeAll(() => {
+    tmpDir = mkdtempSync(join(tmpdir(), 'suite-validator-'));
+  });
+
+  afterAll(() => {
+    rmSync(tmpDir, { recursive: true, force: true });
+  });
+
+  function writeFixture(name: string, data: unknown): string {
+    const filePath = join(tmpDir, name);
+    writeFileSync(filePath, JSON.stringify(data, null, 2), 'utf8');
+    return filePath;
+  }
+
+  describe('canonical id resolution (criterion 1)', () => {
+    it('accepts canonical core/openagent suite and resolves its test paths', () => {
+      // Arrange
+      const validator = new SuiteValidator(agentsDir);
+
+      // Act
+      const result = validator.validateSuiteFile('core/openagent', coreSuitePath);
+
+      // Assert
+      expect(result.valid).toBe(true);
+      expect(result.errors).toHaveLength(0);
+      expect(result.suite?.agent).toBe('core/openagent');
+      // Required tests were located under evals/agents/core/openagent/tests
+      expect(result.missingTests).toHaveLength(0);
+      expect(result.suite?.tests.length).toBeGreaterThan(0);
+    });
+  });
+
+  describe('legacy id resolution (criterion 2)', () => {
+    it('accepts legacy "openagent" request and resolves to the canonical dir', () => {
+      // Arrange
+      const validator = new SuiteValidator(agentsDir);
+
+      // Act: request with the legacy flat id against the canonical suite file
+      const result = validator.validateSuiteFile('openagent', coreSuitePath);
+
+      // Assert: legacy and canonical forms of the SAME agent must match
+      expect(result.valid).toBe(true);
+      expect(result.errors).toHaveLength(0);
+      // Paths still resolved under core/openagent/tests (nothing missing)
+      expect(result.missingTests).toHaveLength(0);
+    });
+  });
+
+  describe('conflict detection (criterion 3)', () => {
+    it('fails with an "agent" field error when requested agent differs from declared', () => {
+      // Arrange: suite declares a DIFFERENT agent than the one requested
+      const fixturePath = writeFixture(
+        'conflict.json',
+        makeSuite({ agent: 'core/opencoder' })
+      );
+      const validator = new SuiteValidator(agentsDir);
+
+      // Act
+      const result = validator.validateSuiteFile('core/openagent', fixturePath);
+
+      // Assert
+      expect(result.valid).toBe(false);
+      const agentError = result.errors.find(e => e.field === 'agent');
+      expect(agentError).toBeDefined();
+      expect(agentError?.message).toContain('does not match requested agent');
+    });
+
+    it('does NOT flag a conflict for legacy vs canonical forms of the same agent', () => {
+      // Arrange: suite declares canonical, request uses legacy flat id
+      const fixturePath = writeFixture(
+        'no-conflict.json',
+        makeSuite({ agent: 'core/openagent' })
+      );
+      const validator = new SuiteValidator(agentsDir);
+
+      // Act
+      const result = validator.validateSuiteFile('openagent', fixturePath);
+
+      // Assert: no agent-field conflict error
+      const agentError = result.errors.find(e => e.field === 'agent');
+      expect(agentError).toBeUndefined();
+      expect(result.valid).toBe(true);
+    });
+  });
+
+  describe('required test presence and unique ids (criterion 4)', () => {
+    it('fails when a required test file is missing', () => {
+      // Arrange: required test with a path that does not exist
+      const fixturePath = writeFixture(
+        'missing-test.json',
+        makeSuite({
+          tests: [
+            {
+              id: 1,
+              name: 'Nonexistent Test',
+              path: 'does-not-exist/nope.yaml',
+              category: 'critical-rules',
+              priority: 'critical',
+              required: true
+            }
+          ]
+        })
+      );
+      const validator = new SuiteValidator(agentsDir);
+
+      // Act
+      const result = validator.validateSuiteFile('core/openagent', fixturePath);
+
+      // Assert
+      expect(result.valid).toBe(false);
+      expect(result.missingTests).toContain('does-not-exist/nope.yaml');
+      expect(
+        result.errors.some(e => e.message.includes('Required test file not found'))
+      ).toBe(true);
+    });
+
+    it('fails when duplicate test IDs are present', () => {
+      // Arrange: two tests sharing id 1 (both point at a real file so the
+      // failure is attributable to the duplicate id, not a missing path)
+      const realPath =
+        '01-critical-rules/approval-gate/05-approval-before-execution-positive.yaml';
+      const fixturePath = writeFixture(
+        'dup-ids.json',
+        makeSuite({
+          totalTests: 2,
+          tests: [
+            { id: 1, name: 'A', path: realPath, category: 'critical-rules', priority: 'critical' },
+            { id: 1, name: 'B', path: realPath, category: 'critical-rules', priority: 'high' }
+          ]
+        })
+      );
+      const validator = new SuiteValidator(agentsDir);
+
+      // Act
+      const result = validator.validateSuiteFile('core/openagent', fixturePath);
+
+      // Assert
+      expect(result.valid).toBe(false);
+      expect(
+        result.errors.some(e => e.message.includes('Duplicate test IDs'))
+      ).toBe(true);
+    });
+  });
+
+  describe('schema acceptance matrix', () => {
+    it('accepts all supported core agent id forms and rejects unknown flat ids', () => {
+      // Arrange
+      const validator = new SuiteValidator(agentsDir);
+
+      // Act + Assert: supported forms validate at the data/schema level
+      for (const agent of ['openagent', 'opencoder', 'core/openagent', 'core/opencoder']) {
+        const res = validator.validateSuiteData(makeSuite({ agent }));
+        expect(res.valid, `expected ${agent} to be schema-valid`).toBe(true);
+      }
+
+      // Unknown flat id (no slash, not normalizable) is rejected by the schema
+      const bad = validator.validateSuiteData(makeSuite({ agent: 'totally-unknown-agent' }));
+      expect(bad.valid).toBe(false);
+      expect(bad.errors.some(e => e.field === 'agent')).toBe(true);
+    });
+  });
+});

evals/framework/src/sdk/__tests__/validate-suites-cli.test.ts

@@ -0,0 +1,124 @@
+/**
+ * Tests for the validate-suites CLI helpers.
+ *
+ * Exercises the pure, exported discovery and exit-code functions using temp
+ * directory fixtures (mkdtempSync). No CLI/process.exit is triggered because
+ * the module uses an ESM entrypoint guard, so importing it is side-effect free.
+ *
+ * NOTE: Filesystem-only. No network, model, paid API, or agent execution.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { mkdtempSync, mkdirSync, rmSync } from 'fs';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+import { tmpdir } from 'os';
+
+import {
+  discoverAgents,
+  computeExitCode
+} from '../validate-suites-cli.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+// Real agents directory (matches the layout used by the validate:suites CLI).
+const realAgentsDir = join(__dirname, '../../../../agents');
+
+describe('validate-suites-cli - discoverAgents', () => {
+  let tmpRoot: string;
+
+  beforeEach(() => {
+    tmpRoot = mkdtempSync(join(tmpdir(), 'discover-agents-'));
+  });
+
+  afterEach(() => {
+    rmSync(tmpRoot, { recursive: true, force: true });
+  });
+
+  it('recursively finds category-based agent config dirs and ignores non-agent siblings', () => {
+    // Arrange: nested <category>/<agent>/config layout.
+    mkdirSync(join(tmpRoot, 'core', 'openagent', 'config'), { recursive: true });
+    mkdirSync(join(tmpRoot, 'core', 'opencoder', 'config'), { recursive: true });
+    mkdirSync(join(tmpRoot, 'development', 'frontend-specialist', 'config'), { recursive: true });
+    // Sibling directory WITHOUT a config subdir -> must be ignored.
+    mkdirSync(join(tmpRoot, 'content', 'copywriter', 'docs'), { recursive: true });
+    // Noise directories that must be skipped.
+    mkdirSync(join(tmpRoot, '.hidden', 'config'), { recursive: true });
+    mkdirSync(join(tmpRoot, 'node_modules', 'pkg', 'config'), { recursive: true });
+
+    // Act
+    const agents = discoverAgents(tmpRoot);
+
+    // Assert: category-based ids, sorted, deterministic; no non-agent dirs.
+    expect(agents).toEqual([
+      'core/opencoder',
+      'core/openagent',
+      'development/frontend-specialist'
+    ].sort());
+    expect(agents).not.toContain('content/copywriter');
+    expect(agents).not.toContain('.hidden');
+    expect(agents.some(a => a.includes('node_modules'))).toBe(false);
+  });
+
+  it('returns a sorted, deterministic order', () => {
+    // Arrange
+    mkdirSync(join(tmpRoot, 'zeta', 'zzz', 'config'), { recursive: true });
+    mkdirSync(join(tmpRoot, 'alpha', 'aaa', 'config'), { recursive: true });
+    mkdirSync(join(tmpRoot, 'core', 'mmm', 'config'), { recursive: true });
+
+    // Act
+    const agents = discoverAgents(tmpRoot);
+
+    // Assert
+    expect(agents).toEqual([...agents].sort());
+    expect(agents).toEqual(['alpha/aaa', 'core/mmm', 'zeta/zzz']);
+  });
+
+  it('returns an empty array for an empty tree', () => {
+    // Arrange: tmpRoot exists but has no agent config dirs.
+    // Act
+    const agents = discoverAgents(tmpRoot);
+
+    // Assert
+    expect(agents).toEqual([]);
+  });
+
+  it('returns an empty array for a nonexistent directory', () => {
+    // Act
+    const agents = discoverAgents(join(tmpRoot, 'does-not-exist'));
+
+    // Assert
+    expect(agents).toEqual([]);
+  });
+
+  it('discovers core/openagent in the real agents directory', () => {
+    // Act
+    const agents = discoverAgents(realAgentsDir);
+
+    // Assert
+    expect(agents).toContain('core/openagent');
+  });
+});
+
+describe('validate-suites-cli - computeExitCode (fail-closed)', () => {
+  it('returns nonzero when zero suites were discovered', () => {
+    // Assert
+    expect(computeExitCode({ totalSuites: 0, invalidSuites: 0 })).not.toBe(0);
+  });
+
+  it('returns nonzero when any suite is invalid', () => {
+    // Assert
+    expect(computeExitCode({ totalSuites: 5, invalidSuites: 1 })).not.toBe(0);
+  });
+
+  it('returns zero only when total > 0 and every suite is valid', () => {
+    // Assert
+    expect(computeExitCode({ totalSuites: 3, invalidSuites: 0 })).toBe(0);
+  });
+
+  it('treats zero suites as a failure even with no invalid suites', () => {
+    // Assert: fail-closed — empty discovery must not be reported as success.
+    expect(computeExitCode({ totalSuites: 0, invalidSuites: 0 })).toBe(1);
+  });
+});

evals/framework/src/sdk/suite-validator.ts

@@ -7,6 +7,7 @@
 import { z } from 'zod';
 import { readFileSync, existsSync } from 'fs';
 import { join } from 'path';
+import { normalizeAgentId } from '../config.js';
 
 /**
  * Zod schema for test definition
@@ -39,7 +40,14 @@ const TestSuiteSchema = z.object({
   name: z.string().min(1),
   description: z.string().min(1),
   version: z.string().regex(/^\d+\.\d+\.\d+$/, 'Version must be semver (e.g., 1.0.0)'),
-  agent: z.enum(['openagent', 'opencoder']),
+  // Accept both legacy flat ids (e.g. "openagent") and canonical category-based
+  // ids (e.g. "core/openagent"). A value is valid when it is already
+  // category-based (contains "/") or maps to a known canonical id via
+  // normalizeAgentId. Arbitrary/unknown flat ids are rejected.
+  agent: z.string().min(1).refine(
+    (val) => val.includes('/') || normalizeAgentId(val) !== val,
+    { message: 'Agent must be a known agent id (e.g., "openagent") or a category-based path (e.g., "core/openagent")' }
+  ),
   totalTests: z.number().int().positive(),
   estimatedRuntime: z.string().regex(/^\d+-\d+ (minutes|seconds|hours)$/),
   coverage: z.record(z.boolean()).optional(),
@@ -145,8 +153,20 @@ export class SuiteValidator {
     const suite = parseResult.data;
     result.suite = suite;
 
-    // Validate test paths exist
+    // Conflict detection: the requested agent must resolve to the same
-    const testsDir = join(this.agentsDir, agent, 'tests');
+    // canonical id as the agent declared in the suite. Legacy vs canonical
+    // forms of the SAME agent (e.g. "openagent" vs "core/openagent") match.
+    if (suite.agent && normalizeAgentId(agent) !== normalizeAgentId(suite.agent)) {
+      result.valid = false;
+      result.errors.push({
+        field: 'agent',
+        message: `Suite agent "${suite.agent}" (resolves to "${normalizeAgentId(suite.agent)}") does not match requested agent "${agent}" (resolves to "${normalizeAgentId(agent)}")`
+      });
+      return result;
+    }
+
+    // Validate test paths exist (resolve legacy ids to canonical dirs)
+    const testsDir = join(this.agentsDir, normalizeAgentId(agent), 'tests');
     
     if (!existsSync(testsDir)) {
       result.valid = false;
@@ -217,7 +237,18 @@ export class SuiteValidator {
       suite
     };
 
-    const testsDir = join(this.agentsDir, agent, 'tests');
+    // Conflict detection: requested agent must resolve to the same canonical
+    // id as the declared suite agent (legacy vs canonical forms still match).
+    if (suite.agent && normalizeAgentId(agent) !== normalizeAgentId(suite.agent)) {
+      result.valid = false;
+      result.errors.push({
+        field: 'agent',
+        message: `Suite agent "${suite.agent}" (resolves to "${normalizeAgentId(suite.agent)}") does not match requested agent "${agent}" (resolves to "${normalizeAgentId(agent)}")`
+      });
+      return result;
+    }
+
+    const testsDir = join(this.agentsDir, normalizeAgentId(agent), 'tests');
     
     if (!existsSync(testsDir)) {
       result.valid = false;
@@ -317,9 +348,9 @@ export class SuiteValidator {
     const content = readFileSync(suitePath, 'utf8');
     const suite = JSON.parse(content) as TestSuite;
     
-    // Ensure agent field is set
+    // Ensure agent field is set (canonicalize the requested id)
     if (!suite.agent) {
-      suite.agent = agent as 'openagent' | 'opencoder';
+      suite.agent = normalizeAgentId(agent);
     }
     
     return suite;
@@ -329,7 +360,7 @@ export class SuiteValidator {
    * Get absolute paths for all tests in a suite
    */
   getTestPaths(agent: string, suite: TestSuite): string[] {
-    const testsDir = join(this.agentsDir, agent, 'tests');
+    const testsDir = join(this.agentsDir, normalizeAgentId(agent), 'tests');
     const paths: string[] = [];
     
     for (const test of suite.tests) {

evals/framework/src/sdk/test-case-loader.ts

@@ -1,6 +1,7 @@
 import { readFile } from 'fs/promises';
 import { parse as parseYaml } from 'yaml';
 import { TestCaseSchema, TestSuiteSchema, type TestCase, type TestSuite } from './test-case-schema.js';
+import { normalizeAgentId } from '../config.js';
 
 /**
  * Load a single test case from a YAML file
@@ -12,6 +13,12 @@ export async function loadTestCase(filePath: string): Promise<TestCase> {
   try {
     const testCase = TestCaseSchema.parse(data);
 
+    // Canonicalize legacy agent identifiers to category-based IDs (e.g. "openagent" → "core/openagent").
+    // Unknown identifiers are preserved unchanged by normalizeAgentId.
+    if (testCase.agent) {
+      testCase.agent = normalizeAgentId(testCase.agent);
+    }
+
     // Warn about deprecated schema
     if (testCase.expected && !testCase.behavior && !testCase.expectedViolations) {
       console.warn(`⚠️  Test ${testCase.id} uses deprecated "expected" schema.`);

evals/framework/src/sdk/validate-suites-cli.ts

@@ -4,12 +4,12 @@
  *
  * Usage:
  *   npm run validate:suites
- *   npm run validate:suites -- openagent
+ *   npm run validate:suites -- core/openagent
  *   npm run validate:suites -- --all
  */
 
 import { SuiteValidator } from './suite-validator.js';
-import { readFileSync, existsSync, readdirSync } from 'fs';
+import { existsSync, readdirSync } from 'fs';
 import { join, dirname } from 'path';
 import { fileURLToPath } from 'url';
 
@@ -25,7 +25,7 @@ const colors = {
   blue: '\x1b[34m'
 };
 
-interface ValidationStats {
+export interface ValidationStats {
   totalSuites: number;
   validSuites: number;
   invalidSuites: number;
@@ -33,6 +33,98 @@ interface ValidationStats {
   totalWarnings: number;
 }
 
+/**
+ * Recursively discover agent ids under a nested agents directory.
+ *
+ * An "agent" is any directory that CONTAINS a `config` subdirectory. Agents
+ * live at a category-based path such as `core/openagent`, so the returned ids
+ * are the relative path from `agentsDir` to each such directory. Discovery:
+ *   - recurses through category directories (which have no direct `config`),
+ *   - stops descending once a `config` subdir is found (agents don't nest),
+ *   - skips hidden dirs and `node_modules`,
+ *   - returns a SORTED array for deterministic ordering.
+ */
+export function discoverAgents(agentsDir: string): string[] {
+  const results: string[] = [];
+
+  function walk(dir: string, relPrefix: string): void {
+    let entries;
+    try {
+      entries = readdirSync(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+
+    for (const entry of entries) {
+      if (!entry.isDirectory()) continue;
+
+      const name = entry.name;
+      if (name.startsWith('.') || name === 'node_modules') continue;
+
+      const rel = relPrefix ? `${relPrefix}/${name}` : name;
+      const full = join(dir, name);
+
+      if (existsSync(join(full, 'config'))) {
+        // This directory is an agent; record it and do not descend further.
+        results.push(rel);
+      } else {
+        walk(full, rel);
+      }
+    }
+  }
+
+  walk(agentsDir, '');
+  return results.sort();
+}
+
+/**
+ * Collect suite JSON files for a single agent.
+ *
+ * Looks in `<config>/suites/*.json` (new location) and `<config>/*.json`
+ * (legacy location, excluding the suite schema). Returns absolute paths.
+ */
+export function collectSuiteFiles(agentsDir: string, agentId: string): string[] {
+  const agentConfigDir = join(agentsDir, agentId, 'config');
+  const suiteFiles: string[] = [];
+
+  if (!existsSync(agentConfigDir)) {
+    return suiteFiles;
+  }
+
+  // New location: suites directory
+  const suitesDir = join(agentConfigDir, 'suites');
+  if (existsSync(suitesDir)) {
+    readdirSync(suitesDir)
+      .filter(f => f.endsWith('.json'))
+      .forEach(f => suiteFiles.push(join(suitesDir, f)));
+  }
+
+  // Legacy location: JSON files directly in config directory
+  readdirSync(agentConfigDir)
+    .filter(f => f.endsWith('.json') && f !== 'suite-schema.json')
+    .forEach(f => {
+      const filePath = join(agentConfigDir, f);
+      if (!suiteFiles.includes(filePath)) {
+        suiteFiles.push(filePath);
+      }
+    });
+
+  return suiteFiles;
+}
+
+/**
+ * Fail-closed exit-code rule.
+ *
+ * Returns nonzero when NO suites were discovered (`totalSuites === 0`) OR when
+ * any discovered suite is invalid (`invalidSuites > 0`). Returns 0 only when at
+ * least one suite was discovered and every one of them validated.
+ */
+export function computeExitCode(stats: Pick<ValidationStats, 'totalSuites' | 'invalidSuites'>): number {
+  if (stats.totalSuites === 0) return 1;
+  if (stats.invalidSuites > 0) return 1;
+  return 0;
+}
+
 function validateSuite(agent: string, suitePath: string, agentsDir: string): boolean {
   const suiteName = suitePath.split('/').pop()?.replace('.json', '') || 'unknown';
 
@@ -92,12 +184,13 @@ function main() {
   };
 
   const agentsToValidate = validateAll
-    ? readdirSync(agentsDir).filter(f => {
+    ? discoverAgents(agentsDir)
-        const agentPath = join(agentsDir, f);
-        return existsSync(join(agentPath, 'config'));
-      })
     : [agent!];
 
+  if (validateAll && agentsToValidate.length === 0) {
+    console.log(`${colors.yellow}⚠️  No agents with a config directory found under: ${agentsDir}${colors.reset}\n`);
+  }
+
   for (const agentName of agentsToValidate) {
     const agentConfigDir = join(agentsDir, agentName, 'config');
 
@@ -106,27 +199,7 @@ function main() {
       continue;
     }
 
-    // Check for suites directory
+    const suiteFiles = collectSuiteFiles(agentsDir, agentName);
-    const suitesDir = join(agentConfigDir, 'suites');
-    const suiteFiles: string[] = [];
-    
-    if (existsSync(suitesDir)) {
-      const files = readdirSync(suitesDir);
-      files.filter(f => f.endsWith('.json')).forEach(f => {
-        suiteFiles.push(join(suitesDir, f));
-      });
-    }
-    
-    // Check for suite files in config directory (legacy location)
-    const configFiles = readdirSync(agentConfigDir);
-    configFiles
-      .filter(f => f.endsWith('.json') && f !== 'suite-schema.json')
-      .forEach(f => {
-        const filePath = join(agentConfigDir, f);
-        if (!suiteFiles.includes(filePath)) {
-          suiteFiles.push(filePath);
-        }
-      });
 
     if (suiteFiles.length === 0) {
       console.log(`${colors.yellow}⚠️  No test suites found for agent: ${agentName}${colors.reset}\n`);
@@ -159,13 +232,24 @@ function main() {
 
   console.log();
 
-  if (stats.invalidSuites > 0) {
+  const exitCode = computeExitCode(stats);
+
+  if (exitCode !== 0) {
+    if (stats.totalSuites === 0) {
+      console.log(`${colors.red}❌ Validation failed: no test suites were discovered${colors.reset}`);
+    } else {
       console.log(`${colors.red}❌ Validation failed${colors.reset}`);
-    process.exit(1);
+    }
+    process.exit(exitCode);
   } else {
     console.log(`${colors.green}✅ All suites valid${colors.reset}`);
-    process.exit(0);
+    process.exit(exitCode);
   }
 }
 
+// ESM entrypoint guard: only run the CLI when this module is executed directly
+// (e.g. via `tsx src/sdk/validate-suites-cli.ts`). Importing it in tests must
+// NOT trigger main()/process.exit().
+if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
   main();
+}