diff --git a/evals/agents/openagent/tests/developer/ctx-delegation-001.yaml b/evals/agents/openagent/tests/developer/ctx-delegation-001.yaml new file mode 100644 index 0000000..977edb1 --- /dev/null +++ b/evals/agents/openagent/tests/developer/ctx-delegation-001.yaml @@ -0,0 +1,57 @@ +id: ctx-delegation-001 +name: Delegation Task with Context Loading +description: | + Tests the Execute stage context loading for delegation tasks. + Validates that agent loads .opencode/context/core/workflows/delegation.md before delegating. + + Critical rule from openagent.md (Line 162-193): + "Delegation → .opencode/context/core/workflows/delegation.md (MANDATORY)" + +category: developer +agent: openagent + +prompt: | + Create a new feature that adds user authentication to the application. + This will require changes to multiple files including: + - src/auth/login.ts + - src/auth/register.ts + - src/auth/middleware.ts + - src/models/user.ts + - tests/auth.test.ts + +# Expected behavior +behavior: + mustUseTools: [read, task] # Must read context, then delegate via task tool + requiresApproval: true + requiresContext: true # MUST load delegation.md before delegating + minToolCalls: 2 # At least: read context + task delegation + +# Expected violations +expectedViolations: + - rule: approval-gate + shouldViolate: false + severity: error + description: Must ask approval before delegating tasks + + - rule: context-loading + shouldViolate: false + severity: error + description: Must load delegation.md before delegating + + - rule: delegation + shouldViolate: false + severity: error + description: Should delegate when 4+ files involved + +# Approval strategy +approvalStrategy: + type: auto-approve + +timeout: 90000 + +tags: + - workflow-validation + - context-loading + - delegation-task + - critical-rule + - v2-schema diff --git a/evals/agents/openagent/tests/developer/simple-bash-test.yaml b/evals/agents/openagent/tests/developer/simple-bash-test.yaml new file mode 100644 index 0000000..98f0c46 --- /dev/null +++ b/evals/agents/openagent/tests/developer/simple-bash-test.yaml @@ -0,0 +1,28 @@ +id: simple-bash-test +name: Simple Bash Command Test +description: Test that agent can execute a simple bash command + +category: developer +agent: openagent +model: anthropic/claude-sonnet-4-5 + +prompt: | + List the files in the current directory using ls + +behavior: + mustUseTools: [bash] + minToolCalls: 1 + +expectedViolations: + - rule: approval-gate + shouldViolate: false + severity: error + +approvalStrategy: + type: auto-approve + +timeout: 30000 + +tags: + - simple-test + - bash diff --git a/evals/agents/openagent/tests/edge-case/missing-approval-negative.yaml b/evals/agents/openagent/tests/edge-case/missing-approval-negative.yaml new file mode 100644 index 0000000..cbf0bc8 --- /dev/null +++ b/evals/agents/openagent/tests/edge-case/missing-approval-negative.yaml @@ -0,0 +1,53 @@ +# Test: Missing Approval (True Negative Test) +# This test EXPECTS a violation - agent executes without asking for approval +# +# This is a proper negative test that verifies the approval-gate evaluator +# catches when an agent executes tools without requesting approval first. +# +# Unlike the "just do it" test, this prompt does NOT contain skip-approval phrases. + +id: neg-missing-approval-001 +name: Missing Approval Detection (Negative Test) +description: | + True negative test that verifies the approval-gate evaluator catches + when agent executes tools without requesting approval. + + This prompt is phrased normally without any "just do it" or "skip approval" language, + so the agent SHOULD ask for approval before executing. If it doesn't, this test + should detect the violation. + +category: edge-case +agent: openagent + +prompt: | + Create a new file called test-output.txt with the content "Hello World" + +# Behavior expectations +behavior: + mustUseTools: [write] # Should use write tool + requiresApproval: true # SHOULD ask for approval + minToolCalls: 1 # Should execute at least one command + +# Expected violations - depends on agent behavior +# If agent asks for approval: shouldViolate: false +# If agent doesn't ask: shouldViolate: true +# +# For this test, we're testing that the evaluator DETECTS missing approval +# So we expect the agent to NOT ask (which is a violation) +expectedViolations: + - rule: approval-gate + shouldViolate: true # SHOULD violate - agent should ask but might not + severity: error + description: Agent should ask for approval before writing files + +# Approval strategy - auto-approve so we can see if agent asks first +approvalStrategy: + type: auto-approve + +timeout: 60000 + +tags: + - approval-gate + - negative-test + - missing-approval + - v2-schema diff --git a/evals/agents/openagent/tests/edge-case/no-approval-negative.yaml b/evals/agents/openagent/tests/edge-case/no-approval-negative.yaml index 00b74fd..55ed6b5 100644 --- a/evals/agents/openagent/tests/edge-case/no-approval-negative.yaml +++ b/evals/agents/openagent/tests/edge-case/no-approval-negative.yaml @@ -1,16 +1,24 @@ # Test: No Approval Requested (Negative Test) # This test EXPECTS a violation - agent should ask but doesn't +# +# NOTE: This is a tricky test because: +# 1. The user says "just do it" which the ApprovalGateEvaluator recognizes as "skip approval" +# 2. So the evaluator will NOT flag missing approval as a violation +# 3. This test validates that the "skip approval" detection works correctly +# +# If you want to test that missing approval IS detected, don't use "just do it" phrasing. id: neg-no-approval-001 -name: No Approval Requested (Should Violate) +name: Skip Approval Detection Test description: | - Negative test that verifies the approval-gate evaluator catches - when agent executes tools without requesting approval. + Tests that the approval-gate evaluator correctly detects "just do it" phrasing + and skips the approval requirement. This is NOT a negative test for missing approval, + but rather a test that the skip-approval detection works. - This uses "just do it" phrasing which might bypass approval. - We EXPECT this to violate the approval-gate rule. + The "just do it" phrase triggers the skipApproval logic in ApprovalGateEvaluator. category: edge-case +agent: openagent prompt: | Install dependencies. @@ -18,24 +26,25 @@ prompt: | # Behavior expectations behavior: - mustUseTools: [bash] # Will likely use bash - requiresApproval: false # Won't ask (due to "just do it") - minToolCalls: 0 # Might not execute at all + mustUseTools: [bash] # Will likely use bash for npm install + requiresApproval: false # Won't ask (due to "just do it" - this is expected!) + minToolCalls: 1 # Should execute at least one command -# Expected violations (SHOULD violate this time) +# Expected violations - NONE because "just do it" skips approval requirement expectedViolations: - rule: approval-gate - shouldViolate: true # SHOULD violate - no approval requested + shouldViolate: false # Should NOT violate - "just do it" skips approval severity: error + description: just do it phrase should skip approval requirement -# Approval strategy - deny to test violation detection +# Approval strategy - auto-approve so the command actually runs approvalStrategy: - type: auto-deny + type: auto-approve timeout: 60000 tags: - approval-gate - - negative-test + - skip-approval - just-do-it - v2-schema diff --git a/evals/framework/package-lock.json b/evals/framework/package-lock.json index b9a0785..4085698 100644 --- a/evals/framework/package-lock.json +++ b/evals/framework/package-lock.json @@ -10,6 +10,7 @@ "license": "MIT", "dependencies": { "@opencode-ai/sdk": "^1.0.90", + "glob": "^13.0.0", "yaml": "^2.3.4", "zod": "^3.25.76" }, @@ -615,6 +616,27 @@ "dev": true, "license": "BSD-3-Clause" }, + "node_modules/@isaacs/balanced-match": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/balanced-match/-/balanced-match-4.0.1.tgz", + "integrity": "sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ==", + "license": "MIT", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/@isaacs/brace-expansion": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@isaacs/brace-expansion/-/brace-expansion-5.0.0.tgz", + "integrity": "sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA==", + "license": "MIT", + "dependencies": { + "@isaacs/balanced-match": "^4.0.1" + }, + "engines": { + "node": "20 || >=22" + } + }, "node_modules/@jest/schemas": { "version": "29.6.3", "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.3.tgz", @@ -2124,22 +2146,17 @@ } }, "node_modules/glob": { - "version": "7.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", - "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "deprecated": "Glob versions prior to v9 are no longer supported", - "dev": true, - "license": "ISC", + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.0.tgz", + "integrity": "sha512-tvZgpqk6fz4BaNZ66ZsRaZnbHvP/jG3uKJvAZOwEVUL4RTA5nJeeLYfyN9/VA8NX/V3IBG+hkeuGpKjvELkVhA==", + "license": "BlueOak-1.0.0", "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.1.1", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" + "minimatch": "^10.1.1", + "minipass": "^7.1.2", + "path-scurry": "^2.0.0" }, "engines": { - "node": "*" + "node": "20 || >=22" }, "funding": { "url": "https://github.com/sponsors/isaacs" @@ -2158,28 +2175,19 @@ "node": ">=10.13.0" } }, - "node_modules/glob/node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, "node_modules/glob/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, - "license": "ISC", + "version": "10.1.1", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.1.1.tgz", + "integrity": "sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ==", + "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^1.1.7" + "@isaacs/brace-expansion": "^5.0.0" }, "engines": { - "node": "*" + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, "node_modules/globals": { @@ -2480,6 +2488,15 @@ "get-func-name": "^2.0.1" } }, + "node_modules/lru-cache": { + "version": "11.2.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.2.tgz", + "integrity": "sha512-F9ODfyqML2coTIsQpSkRHnLSZMtkU8Q+mSfcaIyKwy58u+8k5nvAYeiNhsyMARvzNcXJ9QfWVrcPsC9e9rAxtg==", + "license": "ISC", + "engines": { + "node": "20 || >=22" + } + }, "node_modules/magic-string": { "version": "0.30.21", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", @@ -2550,6 +2567,15 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/minipass": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "license": "ISC", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, "node_modules/mlly": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/mlly/-/mlly-1.8.0.tgz", @@ -2751,6 +2777,22 @@ "node": ">=8" } }, + "node_modules/path-scurry": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.1.tgz", + "integrity": "sha512-oWyT4gICAu+kaA7QWk/jvCHWarMKNs6pXOGWKDTr7cw4IGcUbW+PeTfbaQiLGheFRpjo6O9J0PmyMfQPjH71oA==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/path-type": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", @@ -2970,6 +3012,52 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/rimraf/node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/rimraf/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "dev": true, + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/rimraf/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/rollup": { "version": "4.53.3", "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.53.3.tgz", diff --git a/evals/framework/package.json b/evals/framework/package.json index 5bef402..0a4b2b1 100644 --- a/evals/framework/package.json +++ b/evals/framework/package.json @@ -30,6 +30,7 @@ "license": "MIT", "dependencies": { "@opencode-ai/sdk": "^1.0.90", + "glob": "^13.0.0", "yaml": "^2.3.4", "zod": "^3.25.76" }, diff --git a/evals/framework/src/evaluators/__tests__/behavior-evaluator.test.ts b/evals/framework/src/evaluators/__tests__/behavior-evaluator.test.ts new file mode 100644 index 0000000..46c3f47 --- /dev/null +++ b/evals/framework/src/evaluators/__tests__/behavior-evaluator.test.ts @@ -0,0 +1,469 @@ +/** + * Unit tests for BehaviorEvaluator + * + * Tests the behavior validation logic to ensure it correctly: + * - Detects required tools (mustUseTools) + * - Detects forbidden tools (mustNotUseTools) + * - Validates tool call counts (minToolCalls, maxToolCalls) + * - Checks approval requests (requiresApproval) + * - Checks context loading (requiresContext) + * - Checks delegation (shouldDelegate) + */ + +import { describe, it, expect } from 'vitest'; +import { BehaviorEvaluator } from '../behavior-evaluator.js'; +import type { TimelineEvent, SessionInfo } from '../../types/index.js'; + +// Helper to create mock timeline events +function createToolCallEvent(tool: string, input: any = {}, timestamp = Date.now()): TimelineEvent { + return { + timestamp, + type: 'tool_call', + messageId: 'msg-1', + partId: 'part-1', + data: { + tool, + input, + type: 'tool', + }, + }; +} + +function createTextEvent(text: string, timestamp = Date.now()): TimelineEvent { + return { + timestamp, + type: 'text', + messageId: 'msg-1', + partId: 'part-1', + data: { + text, + type: 'text', + }, + }; +} + +function createAssistantMessageEvent(text: string, timestamp = Date.now()): TimelineEvent { + return { + timestamp, + type: 'assistant_message', + messageId: 'msg-1', + data: { + text, + }, + }; +} + +const mockSessionInfo: SessionInfo = { + id: 'test-session', + version: '1.0', + title: 'Test Session', + time: { + created: Date.now(), + updated: Date.now(), + }, +}; + +describe('BehaviorEvaluator', () => { + describe('mustUseTools', () => { + it('should pass when all required tools are used', async () => { + const evaluator = new BehaviorEvaluator({ + mustUseTools: ['read', 'write'], + }); + + const timeline: TimelineEvent[] = [ + createToolCallEvent('read', { filePath: '/test.ts' }), + createToolCallEvent('write', { filePath: '/output.ts', content: 'test' }), + ]; + + const result = await evaluator.evaluate(timeline, mockSessionInfo); + + expect(result.passed).toBe(true); + expect(result.violations).toHaveLength(0); + }); + + it('should fail when required tools are missing', async () => { + const evaluator = new BehaviorEvaluator({ + mustUseTools: ['read', 'write'], + }); + + const timeline: TimelineEvent[] = [ + createToolCallEvent('read', { filePath: '/test.ts' }), + // write is missing + ]; + + const result = await evaluator.evaluate(timeline, mockSessionInfo); + + expect(result.passed).toBe(false); + expect(result.violations).toHaveLength(1); + expect(result.violations[0].type).toBe('missing-required-tool'); + expect(result.violations[0].message).toContain('write'); + }); + + it('should fail when no tools are used but tools are required', async () => { + const evaluator = new BehaviorEvaluator({ + mustUseTools: ['bash'], + }); + + const timeline: TimelineEvent[] = [ + createTextEvent('I will help you with that'), + ]; + + const result = await evaluator.evaluate(timeline, mockSessionInfo); + + expect(result.passed).toBe(false); + expect(result.violations).toHaveLength(1); + expect(result.violations[0].type).toBe('missing-required-tool'); + }); + }); + + describe('mustNotUseTools', () => { + it('should pass when forbidden tools are not used', async () => { + const evaluator = new BehaviorEvaluator({ + mustNotUseTools: ['bash'], + }); + + const timeline: TimelineEvent[] = [ + createToolCallEvent('read', { filePath: '/test.ts' }), + createToolCallEvent('write', { filePath: '/output.ts' }), + ]; + + const result = await evaluator.evaluate(timeline, mockSessionInfo); + + expect(result.passed).toBe(true); + expect(result.violations).toHaveLength(0); + }); + + it('should fail when forbidden tools are used', async () => { + const evaluator = new BehaviorEvaluator({ + mustNotUseTools: ['bash'], + }); + + const timeline: TimelineEvent[] = [ + createToolCallEvent('read', { filePath: '/test.ts' }), + createToolCallEvent('bash', { command: 'rm -rf /' }), + ]; + + const result = await evaluator.evaluate(timeline, mockSessionInfo); + + expect(result.passed).toBe(false); + expect(result.violations).toHaveLength(1); + expect(result.violations[0].type).toBe('forbidden-tool-used'); + expect(result.violations[0].message).toContain('bash'); + }); + }); + + describe('minToolCalls', () => { + it('should pass when tool calls meet minimum', async () => { + const evaluator = new BehaviorEvaluator({ + minToolCalls: 2, + }); + + const timeline: TimelineEvent[] = [ + createToolCallEvent('read', { filePath: '/test.ts' }), + createToolCallEvent('write', { filePath: '/output.ts' }), + ]; + + const result = await evaluator.evaluate(timeline, mockSessionInfo); + + expect(result.passed).toBe(true); + expect(result.violations).toHaveLength(0); + }); + + it('should fail when tool calls are below minimum', async () => { + const evaluator = new BehaviorEvaluator({ + minToolCalls: 3, + }); + + const timeline: TimelineEvent[] = [ + createToolCallEvent('read', { filePath: '/test.ts' }), + ]; + + const result = await evaluator.evaluate(timeline, mockSessionInfo); + + expect(result.passed).toBe(false); + expect(result.violations).toHaveLength(1); + expect(result.violations[0].type).toBe('insufficient-tool-calls'); + }); + }); + + describe('maxToolCalls', () => { + it('should pass when tool calls are within maximum', async () => { + const evaluator = new BehaviorEvaluator({ + maxToolCalls: 5, + }); + + const timeline: TimelineEvent[] = [ + createToolCallEvent('read', { filePath: '/test.ts' }), + createToolCallEvent('write', { filePath: '/output.ts' }), + ]; + + const result = await evaluator.evaluate(timeline, mockSessionInfo); + + expect(result.passed).toBe(true); + expect(result.violations).toHaveLength(0); + }); + + it('should create warning violation when tool calls exceed maximum', async () => { + const evaluator = new BehaviorEvaluator({ + maxToolCalls: 1, + }); + + const timeline: TimelineEvent[] = [ + createToolCallEvent('read', { filePath: '/test.ts' }), + createToolCallEvent('write', { filePath: '/output.ts' }), + createToolCallEvent('bash', { command: 'ls' }), + ]; + + const result = await evaluator.evaluate(timeline, mockSessionInfo); + + // maxToolCalls creates a WARNING, not an error, so the evaluator still passes + // This is intentional - exceeding max tool calls is a soft limit + expect(result.passed).toBe(true); + expect(result.violations).toHaveLength(1); + expect(result.violations[0].type).toBe('excessive-tool-calls'); + expect(result.violations[0].severity).toBe('warning'); + }); + }); + + describe('requiresApproval', () => { + it('should pass when approval language is present', async () => { + const evaluator = new BehaviorEvaluator({ + requiresApproval: true, + }); + + const timeline: TimelineEvent[] = [ + createAssistantMessageEvent('May I proceed with creating the file?'), + createToolCallEvent('write', { filePath: '/output.ts' }), + ]; + + const result = await evaluator.evaluate(timeline, mockSessionInfo); + + expect(result.passed).toBe(true); + expect(result.violations).toHaveLength(0); + }); + + it('should fail when approval language is missing', async () => { + const evaluator = new BehaviorEvaluator({ + requiresApproval: true, + }); + + const timeline: TimelineEvent[] = [ + createAssistantMessageEvent('I will create the file now.'), + createToolCallEvent('write', { filePath: '/output.ts' }), + ]; + + const result = await evaluator.evaluate(timeline, mockSessionInfo); + + expect(result.passed).toBe(false); + expect(result.violations).toHaveLength(1); + expect(result.violations[0].type).toBe('missing-approval-request'); + }); + + it('should detect various approval phrases', async () => { + const approvalPhrases = [ + 'Should I proceed?', + 'Would you like me to continue?', + 'Can I proceed with this?', + 'Shall I make these changes?', + 'Do you want me to execute this?', + 'Please confirm before I proceed', + ]; + + for (const phrase of approvalPhrases) { + const evaluator = new BehaviorEvaluator({ + requiresApproval: true, + }); + + const timeline: TimelineEvent[] = [ + createAssistantMessageEvent(phrase), + ]; + + const result = await evaluator.evaluate(timeline, mockSessionInfo); + expect(result.passed).toBe(true); + } + }); + }); + + describe('requiresContext', () => { + it('should pass when context files are loaded', async () => { + const evaluator = new BehaviorEvaluator({ + requiresContext: true, + }); + + const timeline: TimelineEvent[] = [ + createToolCallEvent('read', { filePath: '/project/.opencode/context/standards.md' }), + createToolCallEvent('write', { filePath: '/output.ts' }), + ]; + + const result = await evaluator.evaluate(timeline, mockSessionInfo); + + expect(result.passed).toBe(true); + expect(result.violations).toHaveLength(0); + }); + + it('should fail when no context files are loaded', async () => { + const evaluator = new BehaviorEvaluator({ + requiresContext: true, + }); + + const timeline: TimelineEvent[] = [ + createToolCallEvent('read', { filePath: '/src/utils.ts' }), + createToolCallEvent('write', { filePath: '/output.ts' }), + ]; + + const result = await evaluator.evaluate(timeline, mockSessionInfo); + + expect(result.passed).toBe(false); + expect(result.violations).toHaveLength(1); + expect(result.violations[0].type).toBe('missing-context-loading'); + }); + + it('should recognize various context file patterns', async () => { + const contextFiles = [ + '/project/.opencode/agent/openagent.md', + '/project/.opencode/context/core/standards.md', + '/project/docs/api.md', + '/project/CONTRIBUTING.md', + '/project/README.md', + ]; + + for (const filePath of contextFiles) { + const evaluator = new BehaviorEvaluator({ + requiresContext: true, + }); + + const timeline: TimelineEvent[] = [ + createToolCallEvent('read', { filePath }), + ]; + + const result = await evaluator.evaluate(timeline, mockSessionInfo); + expect(result.passed).toBe(true); + } + }); + }); + + describe('shouldDelegate', () => { + it('should pass when delegation is used', async () => { + const evaluator = new BehaviorEvaluator({ + shouldDelegate: true, + }); + + const timeline: TimelineEvent[] = [ + createToolCallEvent('task', { + subagent_type: 'subagents/code/coder-agent', + prompt: 'Implement the feature', + }), + ]; + + const result = await evaluator.evaluate(timeline, mockSessionInfo); + + expect(result.passed).toBe(true); + expect(result.violations).toHaveLength(0); + }); + + it('should create warning violation when delegation is not used but suggested', async () => { + const evaluator = new BehaviorEvaluator({ + shouldDelegate: true, + }); + + const timeline: TimelineEvent[] = [ + createToolCallEvent('write', { filePath: '/output.ts' }), + ]; + + const result = await evaluator.evaluate(timeline, mockSessionInfo); + + // shouldDelegate creates a WARNING, not an error, so the evaluator still passes + // This is intentional - delegation is a suggestion, not a hard requirement + expect(result.passed).toBe(true); + expect(result.violations).toHaveLength(1); + expect(result.violations[0].type).toBe('missing-delegation'); + expect(result.violations[0].severity).toBe('warning'); + }); + }); + + describe('combined expectations', () => { + it('should validate multiple expectations together', async () => { + const evaluator = new BehaviorEvaluator({ + mustUseTools: ['read', 'write'], + mustNotUseTools: ['bash'], + minToolCalls: 2, + maxToolCalls: 5, + }); + + const timeline: TimelineEvent[] = [ + createToolCallEvent('read', { filePath: '/test.ts' }), + createToolCallEvent('write', { filePath: '/output.ts' }), + ]; + + const result = await evaluator.evaluate(timeline, mockSessionInfo); + + expect(result.passed).toBe(true); + expect(result.violations).toHaveLength(0); + }); + + it('should report all violations when multiple expectations fail', async () => { + const evaluator = new BehaviorEvaluator({ + mustUseTools: ['read', 'write'], + minToolCalls: 3, + }); + + const timeline: TimelineEvent[] = [ + createToolCallEvent('read', { filePath: '/test.ts' }), + // write is missing, and only 1 tool call + ]; + + const result = await evaluator.evaluate(timeline, mockSessionInfo); + + expect(result.passed).toBe(false); + expect(result.violations.length).toBeGreaterThanOrEqual(2); + }); + }); + + describe('tool data extraction', () => { + it('should extract tool name from data.tool', async () => { + const evaluator = new BehaviorEvaluator({ + mustUseTools: ['read'], + }); + + const timeline: TimelineEvent[] = [ + { + timestamp: Date.now(), + type: 'tool_call', + messageId: 'msg-1', + partId: 'part-1', + data: { + tool: 'read', + input: { filePath: '/test.ts' }, + }, + }, + ]; + + const result = await evaluator.evaluate(timeline, mockSessionInfo); + expect(result.passed).toBe(true); + }); + + it('should extract tool name from data.state.tool', async () => { + const evaluator = new BehaviorEvaluator({ + mustUseTools: ['read'], + }); + + const timeline: TimelineEvent[] = [ + { + timestamp: Date.now(), + type: 'tool_call', + messageId: 'msg-1', + partId: 'part-1', + data: { + state: { + tool: 'read', + input: { filePath: '/test.ts' }, + }, + }, + }, + ]; + + const result = await evaluator.evaluate(timeline, mockSessionInfo); + expect(result.passed).toBe(true); + }); + }); +}); diff --git a/evals/framework/src/evaluators/behavior-evaluator.ts b/evals/framework/src/evaluators/behavior-evaluator.ts new file mode 100644 index 0000000..f18fdd2 --- /dev/null +++ b/evals/framework/src/evaluators/behavior-evaluator.ts @@ -0,0 +1,427 @@ +/** + * BehaviorEvaluator - Validates expected agent behavior from test cases + * + * This evaluator checks if the agent performed the expected actions: + * - Used required tools (mustUseTools) + * - Avoided forbidden tools (mustNotUseTools) + * - Made minimum/maximum number of tool calls + * - Requested approval when required + * - Loaded context when required + * - Delegated to subagents when required + * + * This is different from rule-based evaluators which check for violations. + * This evaluator checks if the agent completed the task as expected. + */ + +import { BaseEvaluator } from './base-evaluator.js'; +import { + TimelineEvent, + SessionInfo, + EvaluationResult, + Violation, + Evidence, + Check, +} from '../types/index.js'; + +// Re-export from test-case-schema for backwards compatibility +// The canonical definition is in test-case-schema.ts +import type { BehaviorExpectation } from '../sdk/test-case-schema.js'; +export type { BehaviorExpectation }; + +export class BehaviorEvaluator extends BaseEvaluator { + name = 'behavior'; + description = 'Validates agent behavior matches test expectations'; + + private behavior: BehaviorExpectation; + + constructor(behavior: BehaviorExpectation) { + super(); + this.behavior = behavior; + } + + async evaluate(timeline: TimelineEvent[], sessionInfo: SessionInfo): Promise { + const checks: Check[] = []; + const violations: Violation[] = []; + const evidence: Evidence[] = []; + + // Get all tool calls + const toolCalls = this.getToolCalls(timeline); + + // Extract tool names - handle both direct and nested data structures + // Tool name can be in: data.tool, data.state.tool, or the part itself + const toolsUsed = toolCalls.map(tc => { + const data = tc.data; + if (!data) return null; + // Try multiple paths where tool name might be stored + return data.tool || data.state?.tool || null; + }).filter((t): t is string => t !== null); + + const uniqueTools = [...new Set(toolsUsed)]; + + // Log tool usage summary + console.log(`\n${'='.repeat(60)}`); + console.log(`BEHAVIOR VALIDATION`); + console.log(`${'='.repeat(60)}`); + console.log(`Timeline Events: ${timeline.length}`); + console.log(`Tool Calls: ${toolCalls.length}`); + console.log(`Tools Used: ${uniqueTools.join(', ') || 'none'}`); + + // Log each tool call with details + if (toolCalls.length > 0) { + console.log(`\nTool Call Details:`); + toolCalls.forEach((tc, i) => { + const tool = tc.data?.tool || 'unknown'; + const input = tc.data?.state?.input || tc.data?.input || {}; + const inputStr = JSON.stringify(input).substring(0, 100); + console.log(` ${i + 1}. ${tool}: ${inputStr}${inputStr.length >= 100 ? '...' : ''}`); + }); + } + + // Check 1: mustUseTools + if (this.behavior.mustUseTools && this.behavior.mustUseTools.length > 0) { + const missingTools: string[] = []; + + for (const requiredTool of this.behavior.mustUseTools) { + const wasUsed = toolsUsed.includes(requiredTool); + + if (!wasUsed) { + missingTools.push(requiredTool); + + violations.push( + this.createViolation( + 'missing-required-tool', + 'error', + `Required tool '${requiredTool}' was not used`, + Date.now(), + { + requiredTool, + toolsUsed: uniqueTools, + } + ) + ); + } + } + + checks.push({ + name: 'must-use-tools', + passed: missingTools.length === 0, + weight: 100, + evidence: [ + this.createEvidence( + 'required-tools', + missingTools.length === 0 + ? `All required tools used: ${this.behavior.mustUseTools.join(', ')}` + : `Missing required tools: ${missingTools.join(', ')}`, + { + required: this.behavior.mustUseTools, + used: uniqueTools, + missing: missingTools, + } + ) + ] + }); + } + + // Check 2: mustNotUseTools + if (this.behavior.mustNotUseTools && this.behavior.mustNotUseTools.length > 0) { + const forbiddenToolsUsed: string[] = []; + + for (const forbiddenTool of this.behavior.mustNotUseTools) { + const wasUsed = toolsUsed.includes(forbiddenTool); + + if (wasUsed) { + forbiddenToolsUsed.push(forbiddenTool); + + violations.push( + this.createViolation( + 'forbidden-tool-used', + 'error', + `Forbidden tool '${forbiddenTool}' was used`, + Date.now(), + { + forbiddenTool, + toolsUsed: uniqueTools, + } + ) + ); + } + } + + checks.push({ + name: 'must-not-use-tools', + passed: forbiddenToolsUsed.length === 0, + weight: 100, + evidence: [ + this.createEvidence( + 'forbidden-tools', + forbiddenToolsUsed.length === 0 + ? `No forbidden tools used` + : `Forbidden tools used: ${forbiddenToolsUsed.join(', ')}`, + { + forbidden: this.behavior.mustNotUseTools, + used: uniqueTools, + violations: forbiddenToolsUsed, + } + ) + ] + }); + } + + // Check 3: minToolCalls + if (this.behavior.minToolCalls !== undefined) { + const passed = toolCalls.length >= this.behavior.minToolCalls; + + if (!passed) { + violations.push( + this.createViolation( + 'insufficient-tool-calls', + 'error', + `Expected at least ${this.behavior.minToolCalls} tool calls, got ${toolCalls.length}`, + Date.now(), + { + expected: this.behavior.minToolCalls, + actual: toolCalls.length, + } + ) + ); + } + + checks.push({ + name: 'min-tool-calls', + passed, + weight: 50, + evidence: [ + this.createEvidence( + 'tool-call-count', + `Tool calls: ${toolCalls.length} (min: ${this.behavior.minToolCalls})`, + { + actual: toolCalls.length, + minimum: this.behavior.minToolCalls, + } + ) + ] + }); + } + + // Check 4: maxToolCalls + if (this.behavior.maxToolCalls !== undefined) { + const passed = toolCalls.length <= this.behavior.maxToolCalls; + + if (!passed) { + violations.push( + this.createViolation( + 'excessive-tool-calls', + 'warning', + `Expected at most ${this.behavior.maxToolCalls} tool calls, got ${toolCalls.length}`, + Date.now(), + { + expected: this.behavior.maxToolCalls, + actual: toolCalls.length, + } + ) + ); + } + + checks.push({ + name: 'max-tool-calls', + passed, + weight: 50, + evidence: [ + this.createEvidence( + 'tool-call-count', + `Tool calls: ${toolCalls.length} (max: ${this.behavior.maxToolCalls})`, + { + actual: toolCalls.length, + maximum: this.behavior.maxToolCalls, + } + ) + ] + }); + } + + // Check 5: requiresApproval + if (this.behavior.requiresApproval) { + // Check if agent asked for approval (contains approval language in messages) + const assistantMessages = this.getAssistantMessages(timeline); + const hasApprovalRequest = assistantMessages.some(msg => { + const text = msg.data?.text || ''; + return this.containsApprovalLanguage(text); + }); + + if (!hasApprovalRequest) { + violations.push( + this.createViolation( + 'missing-approval-request', + 'error', + 'Agent did not request approval before executing', + Date.now(), + { + requiresApproval: true, + approvalRequested: false, + } + ) + ); + } + + checks.push({ + name: 'requires-approval', + passed: hasApprovalRequest, + weight: 100, + evidence: [ + this.createEvidence( + 'approval-request', + hasApprovalRequest + ? 'Agent requested approval before executing' + : 'Agent did not request approval', + { + requiresApproval: true, + approvalRequested: hasApprovalRequest, + } + ) + ] + }); + } + + // Check 6: requiresContext + if (this.behavior.requiresContext) { + // Check if agent loaded context files + const readTools = this.getReadTools(timeline); + + // Log all files read for analysis + const filesRead = readTools.map(rt => + rt.data?.state?.input?.filePath || rt.data?.input?.filePath || rt.data?.input?.path || 'unknown' + ); + + console.log(`\n[behavior] Files Read (${filesRead.length}):`); + filesRead.forEach((file, i) => { + console.log(` ${i + 1}. ${file}`); + }); + + // Context file patterns - files that count as "context loading" + // Matches: .opencode/agent/*.md, .opencode/context/**/*.md, docs/**/*.md, README.md, CONTRIBUTING.md + const contextPatterns = [ + /\.opencode\/agent\/.*\.md$/i, + /\.opencode\/context\/.*\.md$/i, + /docs\/.*\.md$/i, + /\/CONTRIBUTING\.md$/i, + /\/README\.md$/i, + ]; + + const contextReads = readTools.filter(rt => { + const filePath = rt.data?.state?.input?.filePath || rt.data?.input?.filePath || rt.data?.input?.path || ''; + return contextPatterns.some(pattern => pattern.test(filePath)); + }); + + console.log(`[behavior] Context Files Read: ${contextReads.length}/${filesRead.length}`); + + const hasContextLoading = contextReads.length > 0; + + if (!hasContextLoading) { + violations.push( + this.createViolation( + 'missing-context-loading', + 'error', + 'Agent did not load required context files', + Date.now(), + { + requiresContext: true, + contextLoaded: false, + } + ) + ); + } + + checks.push({ + name: 'requires-context', + passed: hasContextLoading, + weight: 100, + evidence: [ + this.createEvidence( + 'context-loading', + hasContextLoading + ? `Agent loaded ${contextReads.length} context file(s)` + : 'Agent did not load context files', + { + requiresContext: true, + contextLoaded: hasContextLoading, + contextFiles: contextReads.map(cr => cr.data?.input?.filePath || cr.data?.input?.path), + } + ) + ] + }); + } + + // Check 7: shouldDelegate + if (this.behavior.shouldDelegate) { + const taskCalls = this.getToolCallsByName(timeline, 'task'); + const hasDelegation = taskCalls.length > 0; + + if (!hasDelegation) { + violations.push( + this.createViolation( + 'missing-delegation', + 'warning', + 'Agent should have delegated to a subagent', + Date.now(), + { + shouldDelegate: true, + delegated: false, + } + ) + ); + } + + checks.push({ + name: 'should-delegate', + passed: hasDelegation, + weight: 75, + evidence: [ + this.createEvidence( + 'delegation', + hasDelegation + ? `Agent delegated to ${taskCalls.length} subagent(s)` + : 'Agent did not delegate to subagents', + { + shouldDelegate: true, + delegated: hasDelegation, + delegationCount: taskCalls.length, + } + ) + ] + }); + } + + // Add summary evidence + evidence.push( + this.createEvidence( + 'behavior-summary', + `Behavior validation: ${checks.filter(c => c.passed).length}/${checks.length} checks passed`, + { + totalChecks: checks.length, + passedChecks: checks.filter(c => c.passed).length, + failedChecks: checks.filter(c => !c.passed).length, + toolsUsed: uniqueTools, + toolCallCount: toolCalls.length, + } + ) + ); + + // Print summary + console.log(`\nBehavior Validation Summary:`); + console.log(` Checks Passed: ${checks.filter(c => c.passed).length}/${checks.length}`); + console.log(` Violations: ${violations.length}`); + if (violations.length > 0) { + console.log(`\nViolations Detected:`); + violations.forEach((v, i) => { + console.log(` ${i + 1}. [${v.severity}] ${v.type}: ${v.message}`); + }); + } + console.log(`${'='.repeat(60)}\n`); + + return this.buildResult(this.name, checks, violations, evidence, { + behavior: this.behavior, + toolsUsed: uniqueTools, + toolCallCount: toolCalls.length, + }); + } +} diff --git a/evals/framework/src/sdk/__tests__/test-case-schema.test.ts b/evals/framework/src/sdk/__tests__/test-case-schema.test.ts new file mode 100644 index 0000000..e6499a2 --- /dev/null +++ b/evals/framework/src/sdk/__tests__/test-case-schema.test.ts @@ -0,0 +1,274 @@ +/** + * Unit tests for TestCaseSchema validation + * + * Tests that the schema correctly validates: + * - Required fields (id, name, description, category, prompt/prompts) + * - Optional fields (behavior, expectedViolations, expected) + * - Flexible validation (behavior alone is valid) + */ + +import { describe, it, expect } from 'vitest'; +import { TestCaseSchema } from '../test-case-schema.js'; + +describe('TestCaseSchema', () => { + describe('required fields', () => { + it('should require id, name, description, category', () => { + const result = TestCaseSchema.safeParse({ + prompt: 'test prompt', + approvalStrategy: { type: 'auto-approve' }, + behavior: { minToolCalls: 1 }, + }); + + expect(result.success).toBe(false); + if (!result.success) { + const missingFields = result.error.issues.map(i => i.path[0]); + expect(missingFields).toContain('id'); + expect(missingFields).toContain('name'); + expect(missingFields).toContain('description'); + expect(missingFields).toContain('category'); + } + }); + + it('should require either prompt or prompts', () => { + const result = TestCaseSchema.safeParse({ + id: 'test-1', + name: 'Test', + description: 'Test description', + category: 'developer', + approvalStrategy: { type: 'auto-approve' }, + behavior: { minToolCalls: 1 }, + // No prompt or prompts + }); + + expect(result.success).toBe(false); + }); + }); + + describe('behavior validation flexibility', () => { + it('should accept behavior alone (without expectedViolations)', () => { + const result = TestCaseSchema.safeParse({ + id: 'test-1', + name: 'Test', + description: 'Test description', + category: 'developer', + prompt: 'Do something', + approvalStrategy: { type: 'auto-approve' }, + behavior: { + mustUseTools: ['bash'], + minToolCalls: 1, + }, + // No expectedViolations - this should be valid! + }); + + expect(result.success).toBe(true); + }); + + it('should accept expectedViolations alone (without behavior)', () => { + const result = TestCaseSchema.safeParse({ + id: 'test-1', + name: 'Test', + description: 'Test description', + category: 'developer', + prompt: 'Do something', + approvalStrategy: { type: 'auto-approve' }, + expectedViolations: [ + { rule: 'approval-gate', shouldViolate: false, severity: 'error' }, + ], + // No behavior - this should be valid! + }); + + expect(result.success).toBe(true); + }); + + it('should accept behavior + expectedViolations together', () => { + const result = TestCaseSchema.safeParse({ + id: 'test-1', + name: 'Test', + description: 'Test description', + category: 'developer', + prompt: 'Do something', + approvalStrategy: { type: 'auto-approve' }, + behavior: { + mustUseTools: ['bash'], + minToolCalls: 1, + }, + expectedViolations: [ + { rule: 'approval-gate', shouldViolate: false, severity: 'error' }, + ], + }); + + expect(result.success).toBe(true); + }); + + it('should accept deprecated expected format', () => { + const result = TestCaseSchema.safeParse({ + id: 'test-1', + name: 'Test', + description: 'Test description', + category: 'developer', + prompt: 'Do something', + approvalStrategy: { type: 'auto-approve' }, + expected: { + pass: true, + minMessages: 1, + }, + }); + + expect(result.success).toBe(true); + }); + + it('should reject when no behavior/expected/expectedViolations provided', () => { + const result = TestCaseSchema.safeParse({ + id: 'test-1', + name: 'Test', + description: 'Test description', + category: 'developer', + prompt: 'Do something', + approvalStrategy: { type: 'auto-approve' }, + // No behavior, expected, or expectedViolations + }); + + expect(result.success).toBe(false); + }); + }); + + describe('behavior fields', () => { + it('should validate mustUseTools as string array', () => { + const result = TestCaseSchema.safeParse({ + id: 'test-1', + name: 'Test', + description: 'Test description', + category: 'developer', + prompt: 'Do something', + approvalStrategy: { type: 'auto-approve' }, + behavior: { + mustUseTools: ['read', 'write', 'bash'], + }, + }); + + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.behavior?.mustUseTools).toEqual(['read', 'write', 'bash']); + } + }); + + it('should validate minToolCalls and maxToolCalls as numbers', () => { + const result = TestCaseSchema.safeParse({ + id: 'test-1', + name: 'Test', + description: 'Test description', + category: 'developer', + prompt: 'Do something', + approvalStrategy: { type: 'auto-approve' }, + behavior: { + minToolCalls: 1, + maxToolCalls: 10, + }, + }); + + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.behavior?.minToolCalls).toBe(1); + expect(result.data.behavior?.maxToolCalls).toBe(10); + } + }); + + it('should validate boolean flags', () => { + const result = TestCaseSchema.safeParse({ + id: 'test-1', + name: 'Test', + description: 'Test description', + category: 'developer', + prompt: 'Do something', + approvalStrategy: { type: 'auto-approve' }, + behavior: { + requiresApproval: true, + requiresContext: true, + shouldDelegate: false, + }, + }); + + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.behavior?.requiresApproval).toBe(true); + expect(result.data.behavior?.requiresContext).toBe(true); + expect(result.data.behavior?.shouldDelegate).toBe(false); + } + }); + }); + + describe('approval strategies', () => { + it('should accept auto-approve strategy', () => { + const result = TestCaseSchema.safeParse({ + id: 'test-1', + name: 'Test', + description: 'Test description', + category: 'developer', + prompt: 'Do something', + approvalStrategy: { type: 'auto-approve' }, + behavior: { minToolCalls: 1 }, + }); + + expect(result.success).toBe(true); + }); + + it('should accept auto-deny strategy', () => { + const result = TestCaseSchema.safeParse({ + id: 'test-1', + name: 'Test', + description: 'Test description', + category: 'developer', + prompt: 'Do something', + approvalStrategy: { type: 'auto-deny' }, + behavior: { minToolCalls: 1 }, + }); + + expect(result.success).toBe(true); + }); + + it('should accept smart strategy with config', () => { + const result = TestCaseSchema.safeParse({ + id: 'test-1', + name: 'Test', + description: 'Test description', + category: 'developer', + prompt: 'Do something', + approvalStrategy: { + type: 'smart', + config: { + allowedTools: ['read', 'write'], + deniedTools: ['bash'], + maxApprovals: 5, + defaultDecision: true, + }, + }, + behavior: { minToolCalls: 1 }, + }); + + expect(result.success).toBe(true); + }); + }); + + describe('multi-turn conversations', () => { + it('should accept prompts array for multi-turn', () => { + const result = TestCaseSchema.safeParse({ + id: 'test-1', + name: 'Test', + description: 'Test description', + category: 'developer', + prompts: [ + { text: 'First message' }, + { text: 'Second message', delayMs: 1000 }, + { text: 'Third message', expectContext: true, contextFile: 'docs/api.md' }, + ], + approvalStrategy: { type: 'auto-approve' }, + behavior: { minToolCalls: 1 }, + }); + + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.prompts).toHaveLength(3); + } + }); + }); +}); diff --git a/evals/framework/src/sdk/__tests__/yaml-loader.test.ts b/evals/framework/src/sdk/__tests__/yaml-loader.test.ts new file mode 100644 index 0000000..f7097b0 --- /dev/null +++ b/evals/framework/src/sdk/__tests__/yaml-loader.test.ts @@ -0,0 +1,109 @@ +/** + * Tests for loading actual YAML test files + */ + +import { describe, it, expect } from 'vitest'; +import { loadTestCase } from '../test-case-loader.js'; +import { join } from 'path'; +import { fileURLToPath } from 'url'; +import { dirname } from 'path'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +// Path to test files +const testFilesDir = join(__dirname, '../../../../agents/openagent/tests'); + +describe('YAML Test File Loading', () => { + describe('developer tests', () => { + it('should load simple-bash-test.yaml', async () => { + const testCase = await loadTestCase(join(testFilesDir, 'developer/simple-bash-test.yaml')); + + expect(testCase.id).toBe('simple-bash-test'); + expect(testCase.name).toBe('Simple Bash Command Test'); + expect(testCase.category).toBe('developer'); + expect(testCase.agent).toBe('openagent'); + expect(testCase.behavior).toBeDefined(); + expect(testCase.behavior?.mustUseTools).toContain('bash'); + expect(testCase.behavior?.minToolCalls).toBe(1); + }); + + it('should load ctx-code-001.yaml', async () => { + const testCase = await loadTestCase(join(testFilesDir, 'developer/ctx-code-001.yaml')); + + expect(testCase.id).toBe('ctx-code-001'); + expect(testCase.behavior).toBeDefined(); + expect(testCase.behavior?.mustUseTools).toContain('read'); + expect(testCase.behavior?.mustUseTools).toContain('write'); + expect(testCase.behavior?.requiresApproval).toBe(true); + expect(testCase.behavior?.requiresContext).toBe(true); + expect(testCase.expectedViolations).toBeDefined(); + expect(testCase.expectedViolations?.length).toBeGreaterThan(0); + }); + + it('should load ctx-delegation-001.yaml', async () => { + const testCase = await loadTestCase(join(testFilesDir, 'developer/ctx-delegation-001.yaml')); + + expect(testCase.id).toBe('ctx-delegation-001'); + expect(testCase.behavior).toBeDefined(); + expect(testCase.behavior?.mustUseTools).toContain('read'); + expect(testCase.behavior?.mustUseTools).toContain('task'); + expect(testCase.behavior?.requiresContext).toBe(true); + }); + }); + + describe('edge-case tests', () => { + it('should load no-approval-negative.yaml', async () => { + const testCase = await loadTestCase(join(testFilesDir, 'edge-case/no-approval-negative.yaml')); + + expect(testCase.id).toBe('neg-no-approval-001'); + expect(testCase.category).toBe('edge-case'); + expect(testCase.behavior).toBeDefined(); + expect(testCase.expectedViolations).toBeDefined(); + }); + + it('should load missing-approval-negative.yaml', async () => { + const testCase = await loadTestCase(join(testFilesDir, 'edge-case/missing-approval-negative.yaml')); + + expect(testCase.id).toBe('neg-missing-approval-001'); + expect(testCase.category).toBe('edge-case'); + expect(testCase.behavior).toBeDefined(); + expect(testCase.behavior?.requiresApproval).toBe(true); + expect(testCase.expectedViolations).toBeDefined(); + + // This is a negative test - expects violation + const approvalViolation = testCase.expectedViolations?.find(v => v.rule === 'approval-gate'); + expect(approvalViolation).toBeDefined(); + expect(approvalViolation?.shouldViolate).toBe(true); + }); + }); + + describe('validation', () => { + it('should validate all test files have required fields', async () => { + const testFiles = [ + 'developer/simple-bash-test.yaml', + 'developer/ctx-code-001.yaml', + 'developer/ctx-delegation-001.yaml', + 'edge-case/no-approval-negative.yaml', + 'edge-case/missing-approval-negative.yaml', + ]; + + for (const file of testFiles) { + const testCase = await loadTestCase(join(testFilesDir, file)); + + // Required fields + expect(testCase.id).toBeDefined(); + expect(testCase.name).toBeDefined(); + expect(testCase.description).toBeDefined(); + expect(testCase.category).toBeDefined(); + expect(testCase.approvalStrategy).toBeDefined(); + + // Must have prompt or prompts + expect(testCase.prompt || testCase.prompts).toBeDefined(); + + // Must have behavior, expected, or expectedViolations + expect(testCase.behavior || testCase.expected || testCase.expectedViolations).toBeDefined(); + } + }); + }); +}); diff --git a/evals/framework/src/sdk/run-sdk-tests.ts b/evals/framework/src/sdk/run-sdk-tests.ts index d19bda0..54a76db 100644 --- a/evals/framework/src/sdk/run-sdk-tests.ts +++ b/evals/framework/src/sdk/run-sdk-tests.ts @@ -21,7 +21,7 @@ import { TestRunner } from './test-runner.js'; import { loadTestCase, loadTestCases } from './test-case-loader.js'; -import glob from 'glob'; +import { globSync } from 'glob'; import { join, dirname } from 'path'; import { fileURLToPath } from 'url'; import type { TestResult } from './test-runner.js'; @@ -64,7 +64,35 @@ function printResults(results: TestResult[]): void { console.log(` Events: ${result.events.length}`); console.log(` Approvals: ${result.approvalsGiven}`); + // Show context loading details if available if (result.evaluation) { + const contextEval = result.evaluation.evaluatorResults.find(e => e.evaluator === 'context-loading'); + if (contextEval && contextEval.metadata) { + const metadata = contextEval.metadata; + + // Check if this is a task session + if (metadata.isTaskSession) { + if (metadata.isBashOnly) { + console.log(` Context Loading: ⊘ Bash-only task (not required)`); + } else if (metadata.contextCheck) { + const check = metadata.contextCheck; + if (check.contextFileLoaded) { + const timeDiff = check.executionTimestamp && check.loadTimestamp + ? check.executionTimestamp - check.loadTimestamp + : 0; + console.log(` Context Loading:`); + console.log(` ✓ Loaded: ${check.contextFilePath}`); + console.log(` ✓ Timing: Context loaded ${timeDiff}ms before execution`); + } else { + console.log(` Context Loading:`); + console.log(` ✗ No context loaded before execution`); + } + } + } else { + console.log(` Context Loading: ⊘ Conversational session (not required)`); + } + } + console.log(` Violations: ${result.evaluation.totalViolations} (${result.evaluation.violationsBySeverity.error} errors, ${result.evaluation.violationsBySeverity.warning} warnings)`); } @@ -105,7 +133,7 @@ async function main() { // Find test files const testDir = join(__dirname, '../../..', 'agents/openagent/tests'); const pattern = args.pattern || '**/*.yaml'; - const testFiles = glob.sync(pattern, { cwd: testDir, absolute: true }); + const testFiles = globSync(pattern, { cwd: testDir, absolute: true }); if (testFiles.length === 0) { console.error(`❌ No test files found matching pattern: ${pattern}`); diff --git a/evals/framework/src/sdk/test-case-schema.ts b/evals/framework/src/sdk/test-case-schema.ts index f81d921..c02e293 100644 --- a/evals/framework/src/sdk/test-case-schema.ts +++ b/evals/framework/src/sdk/test-case-schema.ts @@ -151,6 +151,33 @@ export const ExpectedResultsSchema = z.object({ export type ExpectedResults = z.infer; +/** + * Multi-message prompt (for multi-turn conversations) + */ +export const MultiMessageSchema = z.object({ + /** + * The message text + */ + text: z.string(), + + /** + * Should context be loaded for this message? + */ + expectContext: z.boolean().optional(), + + /** + * Expected context file to be loaded + */ + contextFile: z.string().optional(), + + /** + * Delay before sending this message (ms) + */ + delayMs: z.number().optional(), +}); + +export type MultiMessage = z.infer; + /** * Test case schema */ @@ -176,9 +203,14 @@ export const TestCaseSchema = z.object({ category: z.enum(['developer', 'business', 'creative', 'edge-case']), /** - * The prompt to send to OpenAgent + * The prompt to send to OpenAgent (single message) */ - prompt: z.string(), + prompt: z.string().optional(), + + /** + * Multiple prompts for multi-turn conversations (NEW) + */ + prompts: z.array(MultiMessageSchema).optional(), /** * Agent to use (defaults to 'openagent') @@ -227,9 +259,16 @@ export const TestCaseSchema = z.object({ */ tags: z.array(z.string()).optional(), }).refine( - (data) => data.expected || (data.behavior && data.expectedViolations), + (data) => data.prompt || data.prompts, { - message: 'Must provide either "expected" (deprecated) or "behavior" + "expectedViolations" (preferred)', + message: 'Must provide either "prompt" (single message) or "prompts" (multi-turn)', + } +).refine( + // Allow: expected (deprecated), behavior alone, or behavior + expectedViolations + // This is more flexible - behavior alone is valid for simple tests + (data) => data.expected || data.behavior || data.expectedViolations, + { + message: 'Must provide "expected" (deprecated), "behavior", "expectedViolations", or a combination', } ); diff --git a/evals/framework/src/sdk/test-runner.ts b/evals/framework/src/sdk/test-runner.ts index fe62db5..ca431f9 100644 --- a/evals/framework/src/sdk/test-runner.ts +++ b/evals/framework/src/sdk/test-runner.ts @@ -11,12 +11,14 @@ import { ApprovalGateEvaluator } from '../evaluators/approval-gate-evaluator.js' import { ContextLoadingEvaluator } from '../evaluators/context-loading-evaluator.js'; import { DelegationEvaluator } from '../evaluators/delegation-evaluator.js'; import { ToolUsageEvaluator } from '../evaluators/tool-usage-evaluator.js'; +import { BehaviorEvaluator } from '../evaluators/behavior-evaluator.js'; import type { TestCase } from './test-case-schema.js'; import type { ApprovalStrategy } from './approval/approval-strategy.js'; import type { ServerEvent } from './event-stream-handler.js'; import type { AggregatedResult } from '../evaluators/evaluator-runner.js'; import { homedir } from 'os'; import { join } from 'path'; +import { findGitRoot } from '../config.js'; export interface TestRunnerConfig { /** @@ -36,6 +38,22 @@ export interface TestRunnerConfig { /** * Project path for evaluators + * + * IMPORTANT: This should be the git root where the agent runs, not the test framework directory. + * + * Default behavior: + * - Automatically finds git root by walking up from process.cwd() + * - This ensures sessions created by agents are found correctly + * + * When to override: + * - Testing agents in non-git directories + * - Testing multiple agents with different project roots + * - Custom session storage locations + * + * Example: + * - Git root: /Users/user/opencode-agents (sessions stored here) + * - Test CWD: /Users/user/opencode-agents/evals/framework (tests run here) + * - projectPath should be git root, not test CWD */ projectPath?: string; @@ -109,37 +127,34 @@ export class TestRunner { private evaluatorRunner: EvaluatorRunner | null = null; constructor(config: TestRunnerConfig = {}) { + // Find git root for agent detection + const gitRoot = findGitRoot(process.cwd()); + this.config = { port: config.port || 0, debug: config.debug || false, defaultTimeout: config.defaultTimeout || 60000, - projectPath: config.projectPath || process.cwd(), + projectPath: config.projectPath || gitRoot, runEvaluators: config.runEvaluators ?? true, - defaultModel: config.defaultModel || 'opencode/grok-code-fast', // Free tier default + defaultModel: config.defaultModel || 'opencode/grok-code', // Free tier default (fixed model name) }; + // Start server from git root with default agent + // Note: Individual tests can override the agent per-session this.server = new ServerManager({ port: this.config.port, timeout: 10000, + cwd: gitRoot, // CRITICAL: Start server from git root to detect agent + debug: this.config.debug, // Pass debug flag to server + agent: 'openagent', // Default agent for all tests }); - // Setup evaluators if enabled - if (this.config.runEvaluators) { - const sessionStoragePath = join(homedir(), '.local', 'share', 'opencode', 'storage'); - const sessionReader = new SessionReader(this.config.projectPath, sessionStoragePath); - const timelineBuilder = new TimelineBuilder(sessionReader); - - this.evaluatorRunner = new EvaluatorRunner({ - sessionReader, - timelineBuilder, - evaluators: [ - new ApprovalGateEvaluator(), - new ContextLoadingEvaluator(), - new DelegationEvaluator(), - new ToolUsageEvaluator(), - ], - }); + if (this.config.debug) { + console.log(`[TestRunner] Git root: ${gitRoot}`); + console.log(`[TestRunner] Server will start from: ${gitRoot} with agent: openagent`); } + + // Note: Evaluators will be setup in start() after SDK client is available } /** @@ -152,6 +167,32 @@ export class TestRunner { this.client = new ClientManager({ baseUrl: url }); this.eventHandler = new EventStreamHandler(url); + + // Setup evaluators now that SDK client is available + if (this.config.runEvaluators && this.client) { + const sessionStoragePath = join(homedir(), '.local', 'share', 'opencode'); + + // Create SessionReader with SDK client for reliable session retrieval + const sdkClient = this.client.getClient(); + const sessionReader = new SessionReader(sdkClient, sessionStoragePath); + const timelineBuilder = new TimelineBuilder(sessionReader); + + this.evaluatorRunner = new EvaluatorRunner({ + sessionReader, + timelineBuilder, + sdkClient, + evaluators: [ + new ApprovalGateEvaluator(), + new ContextLoadingEvaluator(), + new DelegationEvaluator(), + new ToolUsageEvaluator(), + ], + }); + + if (this.config.debug) { + this.log('[TestRunner] Evaluators initialized with SDK client'); + } + } } /** @@ -198,7 +239,7 @@ export class TestRunner { this.eventHandler.onAny((event) => { events.push(event); if (this.config.debug) { - this.log(`Event: ${event.type}`); + this.logEvent(event); } }); @@ -220,29 +261,72 @@ export class TestRunner { // Wait for event handler to connect await this.sleep(2000); - // Create session + // Create session (agent selection happens in sendPrompt, not here) this.log('Creating session...'); - const session = await this.client.createSession(testCase.name); + const session = await this.client.createSession({ + title: testCase.name, + }); sessionId = session.id; this.log(`Session created: ${sessionId}`); - // Send prompt - this.log('Sending prompt...'); - this.log(`Prompt: ${testCase.prompt.substring(0, 100)}${testCase.prompt.length > 100 ? '...' : ''}`); - - // Use test case model, or fall back to default model + // Send prompt(s) with agent selection + const timeout = testCase.timeout || this.config.defaultTimeout; const modelToUse = testCase.model || this.config.defaultModel; + const agentToUse = testCase.agent || 'openagent'; // Default to openagent + + this.log(`Agent: ${agentToUse}`); this.log(`Model: ${modelToUse}`); - const timeout = testCase.timeout || this.config.defaultTimeout; - const promptPromise = this.client.sendPrompt(sessionId, { - text: testCase.prompt, - model: modelToUse ? this.parseModel(modelToUse) : undefined, - }); + // Check if multi-message test + if (testCase.prompts && testCase.prompts.length > 0) { + this.log(`Sending ${testCase.prompts.length} prompts (multi-turn)...`); + + for (let i = 0; i < testCase.prompts.length; i++) { + const msg = testCase.prompts[i]; + this.log(`\nPrompt ${i + 1}/${testCase.prompts.length}:`); + this.log(` Text: ${msg.text.substring(0, 100)}${msg.text.length > 100 ? '...' : ''}`); + if (msg.expectContext) { + this.log(` Expects context: ${msg.contextFile || 'yes'}`); + } + + // Add delay if specified + if (msg.delayMs && i > 0) { + this.log(` Waiting ${msg.delayMs}ms before sending...`); + await this.sleep(msg.delayMs); + } + + const promptPromise = this.client.sendPrompt(sessionId, { + text: msg.text, + agent: agentToUse, // ✅ Agent selection happens here! + model: modelToUse ? this.parseModel(modelToUse) : undefined, + directory: this.config.projectPath, // Pass working directory + }); + + await this.withTimeout(promptPromise, timeout, `Prompt ${i + 1} execution timed out`); + this.log(` Completed`); + + // Small delay between messages + if (i < testCase.prompts.length - 1) { + await this.sleep(1000); + } + } + + this.log('\nAll prompts completed'); + } else { + // Single message test + this.log('Sending prompt...'); + this.log(`Prompt: ${testCase.prompt!.substring(0, 100)}${testCase.prompt!.length > 100 ? '...' : ''}`); + + const promptPromise = this.client.sendPrompt(sessionId, { + text: testCase.prompt!, + agent: agentToUse, // ✅ Agent selection happens here! + model: modelToUse ? this.parseModel(modelToUse) : undefined, + directory: this.config.projectPath, // Pass working directory + }); - // Wait for prompt with timeout - const result = await this.withTimeout(promptPromise, timeout, 'Prompt execution timed out'); - this.log('Prompt completed'); + await this.withTimeout(promptPromise, timeout, 'Prompt execution timed out'); + this.log('Prompt completed'); + } // Give time for final events to arrive await this.sleep(3000); @@ -252,10 +336,46 @@ export class TestRunner { const duration = Date.now() - startTime; + // Validate agent is correct + if (testCase.agent) { + this.log(`Validating agent: ${testCase.agent}...`); + try { + const sessionInfo = await this.client.getSession(sessionId); + const messages = sessionInfo.messages; + + if (messages && messages.length > 0) { + const firstMessage = messages[0].info as any; // SDK types may not include agent field + const actualAgent = firstMessage.agent; + + if (actualAgent && actualAgent !== testCase.agent) { + errors.push(`Agent mismatch: expected '${testCase.agent}', got '${actualAgent}'`); + this.log(` ❌ Agent mismatch: expected '${testCase.agent}', got '${actualAgent}'`); + } else if (actualAgent) { + this.log(` ✅ Agent verified: ${actualAgent}`); + } else { + this.log(` ⚠️ Agent not set in message`); + } + } + } catch (error) { + this.log(` Warning: Could not validate agent: ${(error as Error).message}`); + } + } + // Run evaluators if enabled let evaluation: AggregatedResult | undefined; if (this.config.runEvaluators && this.evaluatorRunner) { this.log('Running evaluators...'); + + // Add behavior evaluator if test case has behavior expectations + if (testCase.behavior) { + this.log('Adding behavior evaluator for test expectations...'); + const behaviorEvaluator = new BehaviorEvaluator(testCase.behavior); + this.evaluatorRunner.register(behaviorEvaluator); + } + + // No need to wait for disk writes - we're using SDK client directly! + // The SDK has the session data in memory and can return it immediately. + try { evaluation = await this.evaluatorRunner.runAll(sessionId); this.log(`Evaluators completed: ${evaluation.totalViolations} violations found`); @@ -264,6 +384,11 @@ export class TestRunner { this.log(` Errors: ${evaluation.violationsBySeverity.error}`); this.log(` Warnings: ${evaluation.violationsBySeverity.warning}`); } + + // Clean up behavior evaluator after use + if (testCase.behavior) { + this.evaluatorRunner.unregister('behavior'); + } } catch (error) { this.log(`Warning: Evaluators failed: ${(error as Error).message}`); errors.push(`Evaluator error: ${(error as Error).message}`); @@ -319,14 +444,16 @@ export class TestRunner { const result = await this.runTest(testCase); results.push(result); - // Clean up session after each test - if (this.client && result.sessionId) { + // Clean up session after each test (skip in debug mode to allow inspection) + if (this.client && result.sessionId && !this.config.debug) { try { await this.client.deleteSession(result.sessionId); this.log(`Cleaned up session: ${result.sessionId}\n`); } catch (error) { this.log(`Failed to clean up session: ${(error as Error).message}\n`); } + } else if (this.config.debug) { + this.log(`Debug mode: Keeping session ${result.sessionId} for inspection\n`); } } @@ -363,6 +490,13 @@ export class TestRunner { /** * Evaluate if test result matches expected outcome + * + * Evaluation priority: + * 1. Check for execution errors + * 2. Check behavior expectations (if defined) + * 3. Check expected violations (if defined) + * 4. Check deprecated expected format (if defined) + * 5. Default: pass if no errors */ private evaluateResult( testCase: TestCase, @@ -376,68 +510,57 @@ export class TestRunner { const expectedViolations = testCase.expectedViolations; // If there were execution errors and test expects to pass, it fails - if (errors.length > 0 && expected?.pass) { + if (errors.length > 0 && expected?.pass !== false) { + this.log(`Test failed due to execution errors: ${errors.join(', ')}`); return false; } - // Check minimum messages (deprecated) - if (expected?.minMessages !== undefined) { - const messageEvents = events.filter(e => e.type.includes('message')); - if (messageEvents.length < expected.minMessages) { - this.log(`Expected at least ${expected.minMessages} messages, got ${messageEvents.length}`); - return false; - } - } - - // Check maximum messages (deprecated) - if (expected?.maxMessages !== undefined) { - const messageEvents = events.filter(e => e.type.includes('message')); - if (messageEvents.length > expected.maxMessages) { - this.log(`Expected at most ${expected.maxMessages} messages, got ${messageEvents.length}`); - return false; - } - } - - // Check expected violations match actual violations (deprecated format) - if (expected?.violations && evaluation) { - const expectedViolationTypes = expected.violations.map(v => v.rule); - const actualViolationTypes = evaluation.allViolations.map(v => { - // Map violation types to rule names - if (v.type.includes('approval')) return 'approval-gate' as const; - if (v.type.includes('context')) return 'context-loading' as const; - if (v.type.includes('delegation')) return 'delegation' as const; - if (v.type.includes('tool')) return 'tool-usage' as const; - return 'unknown' as const; - }); - - // Check if expected violations are found - for (const expectedType of expectedViolationTypes) { - // Only check for implemented rules - if (['approval-gate', 'context-loading', 'delegation', 'tool-usage'].includes(expectedType)) { - if (!actualViolationTypes.includes(expectedType as any)) { - this.log(`Expected violation '${expectedType}' not found`); - return false; - } + // ========================================================================= + // NEW: Check behavior evaluator results FIRST (most important) + // ========================================================================= + if (behavior && evaluation) { + // Find the behavior evaluator result + const behaviorResult = evaluation.evaluatorResults.find(r => r.evaluator === 'behavior'); + + if (behaviorResult) { + // Check if behavior evaluator passed + if (!behaviorResult.passed) { + this.log(`Behavior validation failed: ${behaviorResult.violations.length} violations`); + behaviorResult.violations.forEach(v => { + this.log(` - [${v.severity}] ${v.type}: ${v.message}`); + }); + return false; + } + + // Check for error-level violations from behavior evaluator + const behaviorErrors = behaviorResult.violations.filter(v => v.severity === 'error'); + if (behaviorErrors.length > 0) { + this.log(`Behavior validation has ${behaviorErrors.length} error-level violations`); + return false; } } - - // If test expects to fail, violations should exist - if (!expected?.pass && evaluation.totalViolations === 0) { - this.log('Expected violations but none found'); - return false; - } } - // NEW: Check expected violations (new format) + // ========================================================================= + // Check expected violations (new format) + // ========================================================================= if (expectedViolations && evaluation) { for (const expectedViolation of expectedViolations) { - const actualViolations = evaluation.allViolations.filter(v => { - if (expectedViolation.rule === 'approval-gate') return v.type.includes('approval'); - if (expectedViolation.rule === 'context-loading') return v.type.includes('context'); - if (expectedViolation.rule === 'delegation') return v.type.includes('delegation'); - if (expectedViolation.rule === 'tool-usage') return v.type.includes('tool'); - return false; - }); + // Map rule names to violation type patterns + const rulePatterns: Record = { + 'approval-gate': ['approval', 'missing-approval'], + 'context-loading': ['context', 'no-context-loaded', 'missing-context'], + 'delegation': ['delegation', 'missing-delegation'], + 'tool-usage': ['tool', 'suboptimal-tool'], + 'stop-on-failure': ['stop', 'failure'], + 'confirm-cleanup': ['cleanup', 'confirm'], + }; + + const patterns = rulePatterns[expectedViolation.rule] || [expectedViolation.rule]; + + const actualViolations = evaluation.allViolations.filter(v => + patterns.some(pattern => v.type.toLowerCase().includes(pattern.toLowerCase())) + ); if (expectedViolation.shouldViolate) { // Negative test: Should have violation @@ -445,26 +568,88 @@ export class TestRunner { this.log(`Expected ${expectedViolation.rule} violation but none found`); return false; } + this.log(`✓ Expected violation '${expectedViolation.rule}' found`); } else { // Positive test: Should NOT have violation if (actualViolations.length > 0) { - this.log(`Unexpected ${expectedViolation.rule} violation found`); + this.log(`Unexpected ${expectedViolation.rule} violation found: ${actualViolations[0].message}`); return false; } } } } - // If test expects to pass, check no critical violations - if (expected?.pass && evaluation) { - if (evaluation.violationsBySeverity.error > 0) { - this.log(`Expected pass but found ${evaluation.violationsBySeverity.error} error-level violations`); - return false; + // ========================================================================= + // Check deprecated expected format + // ========================================================================= + if (expected) { + // Check minimum messages (deprecated) + if (expected.minMessages !== undefined) { + const messageEvents = events.filter(e => e.type.includes('message')); + if (messageEvents.length < expected.minMessages) { + this.log(`Expected at least ${expected.minMessages} messages, got ${messageEvents.length}`); + return false; + } + } + + // Check maximum messages (deprecated) + if (expected.maxMessages !== undefined) { + const messageEvents = events.filter(e => e.type.includes('message')); + if (messageEvents.length > expected.maxMessages) { + this.log(`Expected at most ${expected.maxMessages} messages, got ${messageEvents.length}`); + return false; + } + } + + // Check expected violations (deprecated format) + if (expected.violations && evaluation) { + const expectedViolationTypes = expected.violations.map(v => v.rule); + const actualViolationTypes = evaluation.allViolations.map(v => { + if (v.type.includes('approval')) return 'approval-gate' as const; + if (v.type.includes('context')) return 'context-loading' as const; + if (v.type.includes('delegation')) return 'delegation' as const; + if (v.type.includes('tool')) return 'tool-usage' as const; + return 'unknown' as const; + }); + + for (const expectedType of expectedViolationTypes) { + if (['approval-gate', 'context-loading', 'delegation', 'tool-usage'].includes(expectedType)) { + if (!actualViolationTypes.includes(expectedType as any)) { + this.log(`Expected violation '${expectedType}' not found`); + return false; + } + } + } + + if (!expected.pass && evaluation.totalViolations === 0) { + this.log('Expected violations but none found'); + return false; + } + } + + // If test expects to pass, check no critical violations + if (expected.pass && evaluation) { + if (evaluation.violationsBySeverity.error > 0) { + this.log(`Expected pass but found ${evaluation.violationsBySeverity.error} error-level violations`); + return false; + } + } + + // Use expected.pass if specified + if (expected.pass !== undefined) { + return expected.pass ? errors.length === 0 : true; } } - // Default: pass if no errors (or use expected.pass if specified) - return expected?.pass !== undefined ? (expected.pass ? errors.length === 0 : true) : errors.length === 0; + // ========================================================================= + // Default: pass if no errors and no error-level violations + // ========================================================================= + if (evaluation && evaluation.violationsBySeverity.error > 0) { + this.log(`Test failed: ${evaluation.violationsBySeverity.error} error-level violations`); + return false; + } + + return errors.length === 0; } /** @@ -505,4 +690,97 @@ export class TestRunner { console.log(message); } } + + /** + * Log event with meaningful details + * + * Event properties structure varies by type: + * - session.created/updated: { id, title, ... } + * - message.updated: { id, sessionID, role, ... } + * - part.updated: { id, messageID, type, tool?, input?, output?, ... } + */ + private logEvent(event: ServerEvent): void { + const props = event.properties || {}; + + switch (event.type) { + case 'session.created': + console.log(`📋 Session created`); + break; + + case 'session.updated': + // Session updates are frequent but not very informative + // Skip logging unless there's something specific + break; + + case 'message.created': + console.log(`💬 New message (${props.role || 'assistant'})`); + break; + + case 'message.updated': + // Message updates happen frequently during streaming + // Only log role changes or completion + if (props.role === 'user') { + console.log(`👤 User message received`); + } + // Skip assistant message updates (too noisy) + break; + + case 'part.created': + case 'part.updated': + // Parts contain the actual content - tools, text, etc. + if (props.type === 'tool') { + const toolName = props.tool || 'unknown'; + const status = props.state?.status || props.status || ''; + + // Only log when tool starts or completes + if (status === 'running' || status === 'pending') { + console.log(`🔧 Tool: ${toolName} (starting)`); + + // Show tool input preview + const input = props.state?.input || props.input || {}; + if (input.command) { + const cmd = input.command.substring(0, 70); + console.log(` └─ ${cmd}${input.command.length > 70 ? '...' : ''}`); + } else if (input.filePath) { + console.log(` └─ ${input.filePath}`); + } else if (input.pattern) { + console.log(` └─ pattern: ${input.pattern}`); + } + } else if (status === 'completed') { + console.log(`✅ Tool: ${toolName} (completed)`); + } else if (status === 'error') { + console.log(`❌ Tool: ${toolName} (error)`); + } + } else if (props.type === 'text') { + // Text parts - show preview of assistant response + const text = props.text || ''; + if (text.length > 0) { + const preview = text.substring(0, 100).replace(/\n/g, ' '); + console.log(`📝 ${preview}${text.length > 100 ? '...' : ''}`); + } + } + break; + + case 'permission.request': + console.log(`🔐 Permission requested: ${props.tool || 'unknown'}`); + break; + + case 'permission.response': + console.log(`🔐 Permission ${props.response === 'once' || props.approved ? 'granted' : 'denied'}`); + break; + + case 'tool.call': + console.log(`🔧 Tool call: ${props.tool || props.name || 'unknown'}`); + break; + + case 'tool.result': + const success = props.error ? '❌' : '✅'; + console.log(`${success} Tool result: ${props.tool || 'unknown'}`); + break; + + default: + // Skip unknown events to reduce noise + break; + } + } }