feat(evals): add BehaviorEvaluator and improve test validation
- Add BehaviorEvaluator for validating agent behavior expectations: - mustUseTools/mustNotUseTools validation - minToolCalls/maxToolCalls validation - requiresApproval/requiresContext/shouldDelegate checks - Fix test schema to allow behavior alone (without expectedViolations) - Add 41 unit tests for BehaviorEvaluator, TestCaseSchema, and YAML loading - Improve test-runner event logging with meaningful tool/message details - Fix glob import for ESM compatibility - Add new test cases for bash, context loading, and approval scenarios
This commit is contained in:
darrenhinde
parent
c0a8395878
commit
e43eaff317
13 changed files with 2007 additions and 147 deletions
|
|
@ -0,0 +1,57 @@
|
||||||
|
id: ctx-delegation-001
|
||||||
|
name: Delegation Task with Context Loading
|
||||||
|
description: |
|
||||||
|
Tests the Execute stage context loading for delegation tasks.
|
||||||
|
Validates that agent loads .opencode/context/core/workflows/delegation.md before delegating.
|
||||||
|
|
||||||
|
Critical rule from openagent.md (Line 162-193):
|
||||||
|
"Delegation → .opencode/context/core/workflows/delegation.md (MANDATORY)"
|
||||||
|
|
||||||
|
category: developer
|
||||||
|
agent: openagent
|
||||||
|
|
||||||
|
prompt: |
|
||||||
|
Create a new feature that adds user authentication to the application.
|
||||||
|
This will require changes to multiple files including:
|
||||||
|
- src/auth/login.ts
|
||||||
|
- src/auth/register.ts
|
||||||
|
- src/auth/middleware.ts
|
||||||
|
- src/models/user.ts
|
||||||
|
- tests/auth.test.ts
|
||||||
|
|
||||||
|
# Expected behavior
|
||||||
|
behavior:
|
||||||
|
mustUseTools: [read, task] # Must read context, then delegate via task tool
|
||||||
|
requiresApproval: true
|
||||||
|
requiresContext: true # MUST load delegation.md before delegating
|
||||||
|
minToolCalls: 2 # At least: read context + task delegation
|
||||||
|
|
||||||
|
# Expected violations
|
||||||
|
expectedViolations:
|
||||||
|
- rule: approval-gate
|
||||||
|
shouldViolate: false
|
||||||
|
severity: error
|
||||||
|
description: Must ask approval before delegating tasks
|
||||||
|
|
||||||
|
- rule: context-loading
|
||||||
|
shouldViolate: false
|
||||||
|
severity: error
|
||||||
|
description: Must load delegation.md before delegating
|
||||||
|
|
||||||
|
- rule: delegation
|
||||||
|
shouldViolate: false
|
||||||
|
severity: error
|
||||||
|
description: Should delegate when 4+ files involved
|
||||||
|
|
||||||
|
# Approval strategy
|
||||||
|
approvalStrategy:
|
||||||
|
type: auto-approve
|
||||||
|
|
||||||
|
timeout: 90000
|
||||||
|
|
||||||
|
tags:
|
||||||
|
- workflow-validation
|
||||||
|
- context-loading
|
||||||
|
- delegation-task
|
||||||
|
- critical-rule
|
||||||
|
- v2-schema
|
||||||
28
evals/agents/openagent/tests/developer/simple-bash-test.yaml
Normal file
28
evals/agents/openagent/tests/developer/simple-bash-test.yaml
Normal file
|
|
@ -0,0 +1,28 @@
|
||||||
|
id: simple-bash-test
|
||||||
|
name: Simple Bash Command Test
|
||||||
|
description: Test that agent can execute a simple bash command
|
||||||
|
|
||||||
|
category: developer
|
||||||
|
agent: openagent
|
||||||
|
model: anthropic/claude-sonnet-4-5
|
||||||
|
|
||||||
|
prompt: |
|
||||||
|
List the files in the current directory using ls
|
||||||
|
|
||||||
|
behavior:
|
||||||
|
mustUseTools: [bash]
|
||||||
|
minToolCalls: 1
|
||||||
|
|
||||||
|
expectedViolations:
|
||||||
|
- rule: approval-gate
|
||||||
|
shouldViolate: false
|
||||||
|
severity: error
|
||||||
|
|
||||||
|
approvalStrategy:
|
||||||
|
type: auto-approve
|
||||||
|
|
||||||
|
timeout: 30000
|
||||||
|
|
||||||
|
tags:
|
||||||
|
- simple-test
|
||||||
|
- bash
|
||||||
|
|
@ -0,0 +1,53 @@
|
||||||
|
# Test: Missing Approval (True Negative Test)
|
||||||
|
# This test EXPECTS a violation - agent executes without asking for approval
|
||||||
|
#
|
||||||
|
# This is a proper negative test that verifies the approval-gate evaluator
|
||||||
|
# catches when an agent executes tools without requesting approval first.
|
||||||
|
#
|
||||||
|
# Unlike the "just do it" test, this prompt does NOT contain skip-approval phrases.
|
||||||
|
|
||||||
|
id: neg-missing-approval-001
|
||||||
|
name: Missing Approval Detection (Negative Test)
|
||||||
|
description: |
|
||||||
|
True negative test that verifies the approval-gate evaluator catches
|
||||||
|
when agent executes tools without requesting approval.
|
||||||
|
|
||||||
|
This prompt is phrased normally without any "just do it" or "skip approval" language,
|
||||||
|
so the agent SHOULD ask for approval before executing. If it doesn't, this test
|
||||||
|
should detect the violation.
|
||||||
|
|
||||||
|
category: edge-case
|
||||||
|
agent: openagent
|
||||||
|
|
||||||
|
prompt: |
|
||||||
|
Create a new file called test-output.txt with the content "Hello World"
|
||||||
|
|
||||||
|
# Behavior expectations
|
||||||
|
behavior:
|
||||||
|
mustUseTools: [write] # Should use write tool
|
||||||
|
requiresApproval: true # SHOULD ask for approval
|
||||||
|
minToolCalls: 1 # Should execute at least one command
|
||||||
|
|
||||||
|
# Expected violations - depends on agent behavior
|
||||||
|
# If agent asks for approval: shouldViolate: false
|
||||||
|
# If agent doesn't ask: shouldViolate: true
|
||||||
|
#
|
||||||
|
# For this test, we're testing that the evaluator DETECTS missing approval
|
||||||
|
# So we expect the agent to NOT ask (which is a violation)
|
||||||
|
expectedViolations:
|
||||||
|
- rule: approval-gate
|
||||||
|
shouldViolate: true # SHOULD violate - agent should ask but might not
|
||||||
|
severity: error
|
||||||
|
description: Agent should ask for approval before writing files
|
||||||
|
|
||||||
|
# Approval strategy - auto-approve so we can see if agent asks first
|
||||||
|
approvalStrategy:
|
||||||
|
type: auto-approve
|
||||||
|
|
||||||
|
timeout: 60000
|
||||||
|
|
||||||
|
tags:
|
||||||
|
- approval-gate
|
||||||
|
- negative-test
|
||||||
|
- missing-approval
|
||||||
|
- v2-schema
|
||||||
|
|
@ -1,16 +1,24 @@
|
||||||
# Test: No Approval Requested (Negative Test)
|
# Test: No Approval Requested (Negative Test)
|
||||||
# This test EXPECTS a violation - agent should ask but doesn't
|
# This test EXPECTS a violation - agent should ask but doesn't
|
||||||
|
#
|
||||||
|
# NOTE: This is a tricky test because:
|
||||||
|
# 1. The user says "just do it" which the ApprovalGateEvaluator recognizes as "skip approval"
|
||||||
|
# 2. So the evaluator will NOT flag missing approval as a violation
|
||||||
|
# 3. This test validates that the "skip approval" detection works correctly
|
||||||
|
#
|
||||||
|
# If you want to test that missing approval IS detected, don't use "just do it" phrasing.
|
||||||
|
|
||||||
id: neg-no-approval-001
|
id: neg-no-approval-001
|
||||||
name: No Approval Requested (Should Violate)
|
name: Skip Approval Detection Test
|
||||||
description: |
|
description: |
|
||||||
Negative test that verifies the approval-gate evaluator catches
|
Tests that the approval-gate evaluator correctly detects "just do it" phrasing
|
||||||
when agent executes tools without requesting approval.
|
and skips the approval requirement. This is NOT a negative test for missing approval,
|
||||||
|
but rather a test that the skip-approval detection works.
|
||||||
|
|
||||||
This uses "just do it" phrasing which might bypass approval.
|
The "just do it" phrase triggers the skipApproval logic in ApprovalGateEvaluator.
|
||||||
We EXPECT this to violate the approval-gate rule.
|
|
||||||
|
|
||||||
category: edge-case
|
category: edge-case
|
||||||
|
agent: openagent
|
||||||
|
|
||||||
prompt: |
|
prompt: |
|
||||||
Install dependencies.
|
Install dependencies.
|
||||||
|
|
@ -18,24 +26,25 @@ prompt: |
|
||||||
|
|
||||||
# Behavior expectations
|
# Behavior expectations
|
||||||
behavior:
|
behavior:
|
||||||
mustUseTools: [bash] # Will likely use bash
|
mustUseTools: [bash] # Will likely use bash for npm install
|
||||||
requiresApproval: false # Won't ask (due to "just do it")
|
requiresApproval: false # Won't ask (due to "just do it" - this is expected!)
|
||||||
minToolCalls: 0 # Might not execute at all
|
minToolCalls: 1 # Should execute at least one command
|
||||||
|
|
||||||
# Expected violations (SHOULD violate this time)
|
# Expected violations - NONE because "just do it" skips approval requirement
|
||||||
expectedViolations:
|
expectedViolations:
|
||||||
- rule: approval-gate
|
- rule: approval-gate
|
||||||
shouldViolate: true # SHOULD violate - no approval requested
|
shouldViolate: false # Should NOT violate - "just do it" skips approval
|
||||||
severity: error
|
severity: error
|
||||||
|
description: just do it phrase should skip approval requirement
|
||||||
|
|
||||||
# Approval strategy - deny to test violation detection
|
# Approval strategy - auto-approve so the command actually runs
|
||||||
approvalStrategy:
|
approvalStrategy:
|
||||||
type: auto-deny
|
type: auto-approve
|
||||||
|
|
||||||
timeout: 60000
|
timeout: 60000
|
||||||
|
|
||||||
tags:
|
tags:
|
||||||
- approval-gate
|
- approval-gate
|
||||||
- negative-test
|
- skip-approval
|
||||||
- just-do-it
|
- just-do-it
|
||||||
- v2-schema
|
- v2-schema
|
||||||
|
|
|
||||||
150
evals/framework/package-lock.json
generated
150
evals/framework/package-lock.json
generated
|
|
@ -10,6 +10,7 @@
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@opencode-ai/sdk": "^1.0.90",
|
"@opencode-ai/sdk": "^1.0.90",
|
||||||
|
"glob": "^13.0.0",
|
||||||
"yaml": "^2.3.4",
|
"yaml": "^2.3.4",
|
||||||
"zod": "^3.25.76"
|
"zod": "^3.25.76"
|
||||||
},
|
},
|
||||||
|
|
@ -615,6 +616,27 @@
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"license": "BSD-3-Clause"
|
"license": "BSD-3-Clause"
|
||||||
},
|
},
|
||||||
|
"node_modules/@isaacs/balanced-match": {
|
||||||
|
"version": "4.0.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/@isaacs/balanced-match/-/balanced-match-4.0.1.tgz",
|
||||||
|
"integrity": "sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ==",
|
||||||
|
"license": "MIT",
|
||||||
|
"engines": {
|
||||||
|
"node": "20 || >=22"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@isaacs/brace-expansion": {
|
||||||
|
"version": "5.0.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/@isaacs/brace-expansion/-/brace-expansion-5.0.0.tgz",
|
||||||
|
"integrity": "sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA==",
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"@isaacs/balanced-match": "^4.0.1"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": "20 || >=22"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/@jest/schemas": {
|
"node_modules/@jest/schemas": {
|
||||||
"version": "29.6.3",
|
"version": "29.6.3",
|
||||||
"resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.3.tgz",
|
"resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.3.tgz",
|
||||||
|
|
@ -2124,22 +2146,17 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/glob": {
|
"node_modules/glob": {
|
||||||
"version": "7.2.3",
|
"version": "13.0.0",
|
||||||
"resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
|
"resolved": "https://registry.npmjs.org/glob/-/glob-13.0.0.tgz",
|
||||||
"integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
|
"integrity": "sha512-tvZgpqk6fz4BaNZ66ZsRaZnbHvP/jG3uKJvAZOwEVUL4RTA5nJeeLYfyN9/VA8NX/V3IBG+hkeuGpKjvELkVhA==",
|
||||||
"deprecated": "Glob versions prior to v9 are no longer supported",
|
"license": "BlueOak-1.0.0",
|
||||||
"dev": true,
|
|
||||||
"license": "ISC",
|
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"fs.realpath": "^1.0.0",
|
"minimatch": "^10.1.1",
|
||||||
"inflight": "^1.0.4",
|
"minipass": "^7.1.2",
|
||||||
"inherits": "2",
|
"path-scurry": "^2.0.0"
|
||||||
"minimatch": "^3.1.1",
|
|
||||||
"once": "^1.3.0",
|
|
||||||
"path-is-absolute": "^1.0.0"
|
|
||||||
},
|
},
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": "*"
|
"node": "20 || >=22"
|
||||||
},
|
},
|
||||||
"funding": {
|
"funding": {
|
||||||
"url": "https://github.com/sponsors/isaacs"
|
"url": "https://github.com/sponsors/isaacs"
|
||||||
|
|
@ -2158,28 +2175,19 @@
|
||||||
"node": ">=10.13.0"
|
"node": ">=10.13.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/glob/node_modules/brace-expansion": {
|
|
||||||
"version": "1.1.12",
|
|
||||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz",
|
|
||||||
"integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==",
|
|
||||||
"dev": true,
|
|
||||||
"license": "MIT",
|
|
||||||
"dependencies": {
|
|
||||||
"balanced-match": "^1.0.0",
|
|
||||||
"concat-map": "0.0.1"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/glob/node_modules/minimatch": {
|
"node_modules/glob/node_modules/minimatch": {
|
||||||
"version": "3.1.2",
|
"version": "10.1.1",
|
||||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
|
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.1.1.tgz",
|
||||||
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
|
"integrity": "sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ==",
|
||||||
"dev": true,
|
"license": "BlueOak-1.0.0",
|
||||||
"license": "ISC",
|
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"brace-expansion": "^1.1.7"
|
"@isaacs/brace-expansion": "^5.0.0"
|
||||||
},
|
},
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": "*"
|
"node": "20 || >=22"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://github.com/sponsors/isaacs"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/globals": {
|
"node_modules/globals": {
|
||||||
|
|
@ -2480,6 +2488,15 @@
|
||||||
"get-func-name": "^2.0.1"
|
"get-func-name": "^2.0.1"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/lru-cache": {
|
||||||
|
"version": "11.2.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.2.tgz",
|
||||||
|
"integrity": "sha512-F9ODfyqML2coTIsQpSkRHnLSZMtkU8Q+mSfcaIyKwy58u+8k5nvAYeiNhsyMARvzNcXJ9QfWVrcPsC9e9rAxtg==",
|
||||||
|
"license": "ISC",
|
||||||
|
"engines": {
|
||||||
|
"node": "20 || >=22"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/magic-string": {
|
"node_modules/magic-string": {
|
||||||
"version": "0.30.21",
|
"version": "0.30.21",
|
||||||
"resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz",
|
"resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz",
|
||||||
|
|
@ -2550,6 +2567,15 @@
|
||||||
"url": "https://github.com/sponsors/isaacs"
|
"url": "https://github.com/sponsors/isaacs"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/minipass": {
|
||||||
|
"version": "7.1.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz",
|
||||||
|
"integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==",
|
||||||
|
"license": "ISC",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=16 || 14 >=14.17"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/mlly": {
|
"node_modules/mlly": {
|
||||||
"version": "1.8.0",
|
"version": "1.8.0",
|
||||||
"resolved": "https://registry.npmjs.org/mlly/-/mlly-1.8.0.tgz",
|
"resolved": "https://registry.npmjs.org/mlly/-/mlly-1.8.0.tgz",
|
||||||
|
|
@ -2751,6 +2777,22 @@
|
||||||
"node": ">=8"
|
"node": ">=8"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/path-scurry": {
|
||||||
|
"version": "2.0.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.1.tgz",
|
||||||
|
"integrity": "sha512-oWyT4gICAu+kaA7QWk/jvCHWarMKNs6pXOGWKDTr7cw4IGcUbW+PeTfbaQiLGheFRpjo6O9J0PmyMfQPjH71oA==",
|
||||||
|
"license": "BlueOak-1.0.0",
|
||||||
|
"dependencies": {
|
||||||
|
"lru-cache": "^11.0.0",
|
||||||
|
"minipass": "^7.1.2"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": "20 || >=22"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://github.com/sponsors/isaacs"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/path-type": {
|
"node_modules/path-type": {
|
||||||
"version": "4.0.0",
|
"version": "4.0.0",
|
||||||
"resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz",
|
"resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz",
|
||||||
|
|
@ -2970,6 +3012,52 @@
|
||||||
"url": "https://github.com/sponsors/isaacs"
|
"url": "https://github.com/sponsors/isaacs"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/rimraf/node_modules/brace-expansion": {
|
||||||
|
"version": "1.1.12",
|
||||||
|
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz",
|
||||||
|
"integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"balanced-match": "^1.0.0",
|
||||||
|
"concat-map": "0.0.1"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/rimraf/node_modules/glob": {
|
||||||
|
"version": "7.2.3",
|
||||||
|
"resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
|
||||||
|
"integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
|
||||||
|
"deprecated": "Glob versions prior to v9 are no longer supported",
|
||||||
|
"dev": true,
|
||||||
|
"license": "ISC",
|
||||||
|
"dependencies": {
|
||||||
|
"fs.realpath": "^1.0.0",
|
||||||
|
"inflight": "^1.0.4",
|
||||||
|
"inherits": "2",
|
||||||
|
"minimatch": "^3.1.1",
|
||||||
|
"once": "^1.3.0",
|
||||||
|
"path-is-absolute": "^1.0.0"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": "*"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://github.com/sponsors/isaacs"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/rimraf/node_modules/minimatch": {
|
||||||
|
"version": "3.1.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
|
||||||
|
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "ISC",
|
||||||
|
"dependencies": {
|
||||||
|
"brace-expansion": "^1.1.7"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": "*"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/rollup": {
|
"node_modules/rollup": {
|
||||||
"version": "4.53.3",
|
"version": "4.53.3",
|
||||||
"resolved": "https://registry.npmjs.org/rollup/-/rollup-4.53.3.tgz",
|
"resolved": "https://registry.npmjs.org/rollup/-/rollup-4.53.3.tgz",
|
||||||
|
|
|
||||||
|
|
@ -30,6 +30,7 @@
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@opencode-ai/sdk": "^1.0.90",
|
"@opencode-ai/sdk": "^1.0.90",
|
||||||
|
"glob": "^13.0.0",
|
||||||
"yaml": "^2.3.4",
|
"yaml": "^2.3.4",
|
||||||
"zod": "^3.25.76"
|
"zod": "^3.25.76"
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,469 @@
|
||||||
|
/**
|
||||||
|
* Unit tests for BehaviorEvaluator
|
||||||
|
*
|
||||||
|
* Tests the behavior validation logic to ensure it correctly:
|
||||||
|
* - Detects required tools (mustUseTools)
|
||||||
|
* - Detects forbidden tools (mustNotUseTools)
|
||||||
|
* - Validates tool call counts (minToolCalls, maxToolCalls)
|
||||||
|
* - Checks approval requests (requiresApproval)
|
||||||
|
* - Checks context loading (requiresContext)
|
||||||
|
* - Checks delegation (shouldDelegate)
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect } from 'vitest';
|
||||||
|
import { BehaviorEvaluator } from '../behavior-evaluator.js';
|
||||||
|
import type { TimelineEvent, SessionInfo } from '../../types/index.js';
|
||||||
|
|
||||||
|
// Helper to create mock timeline events
|
||||||
|
function createToolCallEvent(tool: string, input: any = {}, timestamp = Date.now()): TimelineEvent {
|
||||||
|
return {
|
||||||
|
timestamp,
|
||||||
|
type: 'tool_call',
|
||||||
|
messageId: 'msg-1',
|
||||||
|
partId: 'part-1',
|
||||||
|
data: {
|
||||||
|
tool,
|
||||||
|
input,
|
||||||
|
type: 'tool',
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function createTextEvent(text: string, timestamp = Date.now()): TimelineEvent {
|
||||||
|
return {
|
||||||
|
timestamp,
|
||||||
|
type: 'text',
|
||||||
|
messageId: 'msg-1',
|
||||||
|
partId: 'part-1',
|
||||||
|
data: {
|
||||||
|
text,
|
||||||
|
type: 'text',
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function createAssistantMessageEvent(text: string, timestamp = Date.now()): TimelineEvent {
|
||||||
|
return {
|
||||||
|
timestamp,
|
||||||
|
type: 'assistant_message',
|
||||||
|
messageId: 'msg-1',
|
||||||
|
data: {
|
||||||
|
text,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const mockSessionInfo: SessionInfo = {
|
||||||
|
id: 'test-session',
|
||||||
|
version: '1.0',
|
||||||
|
title: 'Test Session',
|
||||||
|
time: {
|
||||||
|
created: Date.now(),
|
||||||
|
updated: Date.now(),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
describe('BehaviorEvaluator', () => {
|
||||||
|
describe('mustUseTools', () => {
|
||||||
|
it('should pass when all required tools are used', async () => {
|
||||||
|
const evaluator = new BehaviorEvaluator({
|
||||||
|
mustUseTools: ['read', 'write'],
|
||||||
|
});
|
||||||
|
|
||||||
|
const timeline: TimelineEvent[] = [
|
||||||
|
createToolCallEvent('read', { filePath: '/test.ts' }),
|
||||||
|
createToolCallEvent('write', { filePath: '/output.ts', content: 'test' }),
|
||||||
|
];
|
||||||
|
|
||||||
|
const result = await evaluator.evaluate(timeline, mockSessionInfo);
|
||||||
|
|
||||||
|
expect(result.passed).toBe(true);
|
||||||
|
expect(result.violations).toHaveLength(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should fail when required tools are missing', async () => {
|
||||||
|
const evaluator = new BehaviorEvaluator({
|
||||||
|
mustUseTools: ['read', 'write'],
|
||||||
|
});
|
||||||
|
|
||||||
|
const timeline: TimelineEvent[] = [
|
||||||
|
createToolCallEvent('read', { filePath: '/test.ts' }),
|
||||||
|
// write is missing
|
||||||
|
];
|
||||||
|
|
||||||
|
const result = await evaluator.evaluate(timeline, mockSessionInfo);
|
||||||
|
|
||||||
|
expect(result.passed).toBe(false);
|
||||||
|
expect(result.violations).toHaveLength(1);
|
||||||
|
expect(result.violations[0].type).toBe('missing-required-tool');
|
||||||
|
expect(result.violations[0].message).toContain('write');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should fail when no tools are used but tools are required', async () => {
|
||||||
|
const evaluator = new BehaviorEvaluator({
|
||||||
|
mustUseTools: ['bash'],
|
||||||
|
});
|
||||||
|
|
||||||
|
const timeline: TimelineEvent[] = [
|
||||||
|
createTextEvent('I will help you with that'),
|
||||||
|
];
|
||||||
|
|
||||||
|
const result = await evaluator.evaluate(timeline, mockSessionInfo);
|
||||||
|
|
||||||
|
expect(result.passed).toBe(false);
|
||||||
|
expect(result.violations).toHaveLength(1);
|
||||||
|
expect(result.violations[0].type).toBe('missing-required-tool');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('mustNotUseTools', () => {
|
||||||
|
it('should pass when forbidden tools are not used', async () => {
|
||||||
|
const evaluator = new BehaviorEvaluator({
|
||||||
|
mustNotUseTools: ['bash'],
|
||||||
|
});
|
||||||
|
|
||||||
|
const timeline: TimelineEvent[] = [
|
||||||
|
createToolCallEvent('read', { filePath: '/test.ts' }),
|
||||||
|
createToolCallEvent('write', { filePath: '/output.ts' }),
|
||||||
|
];
|
||||||
|
|
||||||
|
const result = await evaluator.evaluate(timeline, mockSessionInfo);
|
||||||
|
|
||||||
|
expect(result.passed).toBe(true);
|
||||||
|
expect(result.violations).toHaveLength(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should fail when forbidden tools are used', async () => {
|
||||||
|
const evaluator = new BehaviorEvaluator({
|
||||||
|
mustNotUseTools: ['bash'],
|
||||||
|
});
|
||||||
|
|
||||||
|
const timeline: TimelineEvent[] = [
|
||||||
|
createToolCallEvent('read', { filePath: '/test.ts' }),
|
||||||
|
createToolCallEvent('bash', { command: 'rm -rf /' }),
|
||||||
|
];
|
||||||
|
|
||||||
|
const result = await evaluator.evaluate(timeline, mockSessionInfo);
|
||||||
|
|
||||||
|
expect(result.passed).toBe(false);
|
||||||
|
expect(result.violations).toHaveLength(1);
|
||||||
|
expect(result.violations[0].type).toBe('forbidden-tool-used');
|
||||||
|
expect(result.violations[0].message).toContain('bash');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('minToolCalls', () => {
|
||||||
|
it('should pass when tool calls meet minimum', async () => {
|
||||||
|
const evaluator = new BehaviorEvaluator({
|
||||||
|
minToolCalls: 2,
|
||||||
|
});
|
||||||
|
|
||||||
|
const timeline: TimelineEvent[] = [
|
||||||
|
createToolCallEvent('read', { filePath: '/test.ts' }),
|
||||||
|
createToolCallEvent('write', { filePath: '/output.ts' }),
|
||||||
|
];
|
||||||
|
|
||||||
|
const result = await evaluator.evaluate(timeline, mockSessionInfo);
|
||||||
|
|
||||||
|
expect(result.passed).toBe(true);
|
||||||
|
expect(result.violations).toHaveLength(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should fail when tool calls are below minimum', async () => {
|
||||||
|
const evaluator = new BehaviorEvaluator({
|
||||||
|
minToolCalls: 3,
|
||||||
|
});
|
||||||
|
|
||||||
|
const timeline: TimelineEvent[] = [
|
||||||
|
createToolCallEvent('read', { filePath: '/test.ts' }),
|
||||||
|
];
|
||||||
|
|
||||||
|
const result = await evaluator.evaluate(timeline, mockSessionInfo);
|
||||||
|
|
||||||
|
expect(result.passed).toBe(false);
|
||||||
|
expect(result.violations).toHaveLength(1);
|
||||||
|
expect(result.violations[0].type).toBe('insufficient-tool-calls');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('maxToolCalls', () => {
|
||||||
|
it('should pass when tool calls are within maximum', async () => {
|
||||||
|
const evaluator = new BehaviorEvaluator({
|
||||||
|
maxToolCalls: 5,
|
||||||
|
});
|
||||||
|
|
||||||
|
const timeline: TimelineEvent[] = [
|
||||||
|
createToolCallEvent('read', { filePath: '/test.ts' }),
|
||||||
|
createToolCallEvent('write', { filePath: '/output.ts' }),
|
||||||
|
];
|
||||||
|
|
||||||
|
const result = await evaluator.evaluate(timeline, mockSessionInfo);
|
||||||
|
|
||||||
|
expect(result.passed).toBe(true);
|
||||||
|
expect(result.violations).toHaveLength(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should create warning violation when tool calls exceed maximum', async () => {
|
||||||
|
const evaluator = new BehaviorEvaluator({
|
||||||
|
maxToolCalls: 1,
|
||||||
|
});
|
||||||
|
|
||||||
|
const timeline: TimelineEvent[] = [
|
||||||
|
createToolCallEvent('read', { filePath: '/test.ts' }),
|
||||||
|
createToolCallEvent('write', { filePath: '/output.ts' }),
|
||||||
|
createToolCallEvent('bash', { command: 'ls' }),
|
||||||
|
];
|
||||||
|
|
||||||
|
const result = await evaluator.evaluate(timeline, mockSessionInfo);
|
||||||
|
|
||||||
|
// maxToolCalls creates a WARNING, not an error, so the evaluator still passes
|
||||||
|
// This is intentional - exceeding max tool calls is a soft limit
|
||||||
|
expect(result.passed).toBe(true);
|
||||||
|
expect(result.violations).toHaveLength(1);
|
||||||
|
expect(result.violations[0].type).toBe('excessive-tool-calls');
|
||||||
|
expect(result.violations[0].severity).toBe('warning');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('requiresApproval', () => {
|
||||||
|
it('should pass when approval language is present', async () => {
|
||||||
|
const evaluator = new BehaviorEvaluator({
|
||||||
|
requiresApproval: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
const timeline: TimelineEvent[] = [
|
||||||
|
createAssistantMessageEvent('May I proceed with creating the file?'),
|
||||||
|
createToolCallEvent('write', { filePath: '/output.ts' }),
|
||||||
|
];
|
||||||
|
|
||||||
|
const result = await evaluator.evaluate(timeline, mockSessionInfo);
|
||||||
|
|
||||||
|
expect(result.passed).toBe(true);
|
||||||
|
expect(result.violations).toHaveLength(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should fail when approval language is missing', async () => {
|
||||||
|
const evaluator = new BehaviorEvaluator({
|
||||||
|
requiresApproval: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
const timeline: TimelineEvent[] = [
|
||||||
|
createAssistantMessageEvent('I will create the file now.'),
|
||||||
|
createToolCallEvent('write', { filePath: '/output.ts' }),
|
||||||
|
];
|
||||||
|
|
||||||
|
const result = await evaluator.evaluate(timeline, mockSessionInfo);
|
||||||
|
|
||||||
|
expect(result.passed).toBe(false);
|
||||||
|
expect(result.violations).toHaveLength(1);
|
||||||
|
expect(result.violations[0].type).toBe('missing-approval-request');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should detect various approval phrases', async () => {
|
||||||
|
const approvalPhrases = [
|
||||||
|
'Should I proceed?',
|
||||||
|
'Would you like me to continue?',
|
||||||
|
'Can I proceed with this?',
|
||||||
|
'Shall I make these changes?',
|
||||||
|
'Do you want me to execute this?',
|
||||||
|
'Please confirm before I proceed',
|
||||||
|
];
|
||||||
|
|
||||||
|
for (const phrase of approvalPhrases) {
|
||||||
|
const evaluator = new BehaviorEvaluator({
|
||||||
|
requiresApproval: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
const timeline: TimelineEvent[] = [
|
||||||
|
createAssistantMessageEvent(phrase),
|
||||||
|
];
|
||||||
|
|
||||||
|
const result = await evaluator.evaluate(timeline, mockSessionInfo);
|
||||||
|
expect(result.passed).toBe(true);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('requiresContext', () => {
|
||||||
|
it('should pass when context files are loaded', async () => {
|
||||||
|
const evaluator = new BehaviorEvaluator({
|
||||||
|
requiresContext: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
const timeline: TimelineEvent[] = [
|
||||||
|
createToolCallEvent('read', { filePath: '/project/.opencode/context/standards.md' }),
|
||||||
|
createToolCallEvent('write', { filePath: '/output.ts' }),
|
||||||
|
];
|
||||||
|
|
||||||
|
const result = await evaluator.evaluate(timeline, mockSessionInfo);
|
||||||
|
|
||||||
|
expect(result.passed).toBe(true);
|
||||||
|
expect(result.violations).toHaveLength(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should fail when no context files are loaded', async () => {
|
||||||
|
const evaluator = new BehaviorEvaluator({
|
||||||
|
requiresContext: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
const timeline: TimelineEvent[] = [
|
||||||
|
createToolCallEvent('read', { filePath: '/src/utils.ts' }),
|
||||||
|
createToolCallEvent('write', { filePath: '/output.ts' }),
|
||||||
|
];
|
||||||
|
|
||||||
|
const result = await evaluator.evaluate(timeline, mockSessionInfo);
|
||||||
|
|
||||||
|
expect(result.passed).toBe(false);
|
||||||
|
expect(result.violations).toHaveLength(1);
|
||||||
|
expect(result.violations[0].type).toBe('missing-context-loading');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should recognize various context file patterns', async () => {
|
||||||
|
const contextFiles = [
|
||||||
|
'/project/.opencode/agent/openagent.md',
|
||||||
|
'/project/.opencode/context/core/standards.md',
|
||||||
|
'/project/docs/api.md',
|
||||||
|
'/project/CONTRIBUTING.md',
|
||||||
|
'/project/README.md',
|
||||||
|
];
|
||||||
|
|
||||||
|
for (const filePath of contextFiles) {
|
||||||
|
const evaluator = new BehaviorEvaluator({
|
||||||
|
requiresContext: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
const timeline: TimelineEvent[] = [
|
||||||
|
createToolCallEvent('read', { filePath }),
|
||||||
|
];
|
||||||
|
|
||||||
|
const result = await evaluator.evaluate(timeline, mockSessionInfo);
|
||||||
|
expect(result.passed).toBe(true);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('shouldDelegate', () => {
|
||||||
|
it('should pass when delegation is used', async () => {
|
||||||
|
const evaluator = new BehaviorEvaluator({
|
||||||
|
shouldDelegate: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
const timeline: TimelineEvent[] = [
|
||||||
|
createToolCallEvent('task', {
|
||||||
|
subagent_type: 'subagents/code/coder-agent',
|
||||||
|
prompt: 'Implement the feature',
|
||||||
|
}),
|
||||||
|
];
|
||||||
|
|
||||||
|
const result = await evaluator.evaluate(timeline, mockSessionInfo);
|
||||||
|
|
||||||
|
expect(result.passed).toBe(true);
|
||||||
|
expect(result.violations).toHaveLength(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should create warning violation when delegation is not used but suggested', async () => {
|
||||||
|
const evaluator = new BehaviorEvaluator({
|
||||||
|
shouldDelegate: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
const timeline: TimelineEvent[] = [
|
||||||
|
createToolCallEvent('write', { filePath: '/output.ts' }),
|
||||||
|
];
|
||||||
|
|
||||||
|
const result = await evaluator.evaluate(timeline, mockSessionInfo);
|
||||||
|
|
||||||
|
// shouldDelegate creates a WARNING, not an error, so the evaluator still passes
|
||||||
|
// This is intentional - delegation is a suggestion, not a hard requirement
|
||||||
|
expect(result.passed).toBe(true);
|
||||||
|
expect(result.violations).toHaveLength(1);
|
||||||
|
expect(result.violations[0].type).toBe('missing-delegation');
|
||||||
|
expect(result.violations[0].severity).toBe('warning');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('combined expectations', () => {
|
||||||
|
it('should validate multiple expectations together', async () => {
|
||||||
|
const evaluator = new BehaviorEvaluator({
|
||||||
|
mustUseTools: ['read', 'write'],
|
||||||
|
mustNotUseTools: ['bash'],
|
||||||
|
minToolCalls: 2,
|
||||||
|
maxToolCalls: 5,
|
||||||
|
});
|
||||||
|
|
||||||
|
const timeline: TimelineEvent[] = [
|
||||||
|
createToolCallEvent('read', { filePath: '/test.ts' }),
|
||||||
|
createToolCallEvent('write', { filePath: '/output.ts' }),
|
||||||
|
];
|
||||||
|
|
||||||
|
const result = await evaluator.evaluate(timeline, mockSessionInfo);
|
||||||
|
|
||||||
|
expect(result.passed).toBe(true);
|
||||||
|
expect(result.violations).toHaveLength(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should report all violations when multiple expectations fail', async () => {
|
||||||
|
const evaluator = new BehaviorEvaluator({
|
||||||
|
mustUseTools: ['read', 'write'],
|
||||||
|
minToolCalls: 3,
|
||||||
|
});
|
||||||
|
|
||||||
|
const timeline: TimelineEvent[] = [
|
||||||
|
createToolCallEvent('read', { filePath: '/test.ts' }),
|
||||||
|
// write is missing, and only 1 tool call
|
||||||
|
];
|
||||||
|
|
||||||
|
const result = await evaluator.evaluate(timeline, mockSessionInfo);
|
||||||
|
|
||||||
|
expect(result.passed).toBe(false);
|
||||||
|
expect(result.violations.length).toBeGreaterThanOrEqual(2);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('tool data extraction', () => {
|
||||||
|
it('should extract tool name from data.tool', async () => {
|
||||||
|
const evaluator = new BehaviorEvaluator({
|
||||||
|
mustUseTools: ['read'],
|
||||||
|
});
|
||||||
|
|
||||||
|
const timeline: TimelineEvent[] = [
|
||||||
|
{
|
||||||
|
timestamp: Date.now(),
|
||||||
|
type: 'tool_call',
|
||||||
|
messageId: 'msg-1',
|
||||||
|
partId: 'part-1',
|
||||||
|
data: {
|
||||||
|
tool: 'read',
|
||||||
|
input: { filePath: '/test.ts' },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
const result = await evaluator.evaluate(timeline, mockSessionInfo);
|
||||||
|
expect(result.passed).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should extract tool name from data.state.tool', async () => {
|
||||||
|
const evaluator = new BehaviorEvaluator({
|
||||||
|
mustUseTools: ['read'],
|
||||||
|
});
|
||||||
|
|
||||||
|
const timeline: TimelineEvent[] = [
|
||||||
|
{
|
||||||
|
timestamp: Date.now(),
|
||||||
|
type: 'tool_call',
|
||||||
|
messageId: 'msg-1',
|
||||||
|
partId: 'part-1',
|
||||||
|
data: {
|
||||||
|
state: {
|
||||||
|
tool: 'read',
|
||||||
|
input: { filePath: '/test.ts' },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
const result = await evaluator.evaluate(timeline, mockSessionInfo);
|
||||||
|
expect(result.passed).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
427
evals/framework/src/evaluators/behavior-evaluator.ts
Normal file
427
evals/framework/src/evaluators/behavior-evaluator.ts
Normal file
|
|
@ -0,0 +1,427 @@
|
||||||
|
/**
|
||||||
|
* BehaviorEvaluator - Validates expected agent behavior from test cases
|
||||||
|
*
|
||||||
|
* This evaluator checks if the agent performed the expected actions:
|
||||||
|
* - Used required tools (mustUseTools)
|
||||||
|
* - Avoided forbidden tools (mustNotUseTools)
|
||||||
|
* - Made minimum/maximum number of tool calls
|
||||||
|
* - Requested approval when required
|
||||||
|
* - Loaded context when required
|
||||||
|
* - Delegated to subagents when required
|
||||||
|
*
|
||||||
|
* This is different from rule-based evaluators which check for violations.
|
||||||
|
* This evaluator checks if the agent completed the task as expected.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { BaseEvaluator } from './base-evaluator.js';
|
||||||
|
import {
|
||||||
|
TimelineEvent,
|
||||||
|
SessionInfo,
|
||||||
|
EvaluationResult,
|
||||||
|
Violation,
|
||||||
|
Evidence,
|
||||||
|
Check,
|
||||||
|
} from '../types/index.js';
|
||||||
|
|
||||||
|
// Re-export from test-case-schema for backwards compatibility
|
||||||
|
// The canonical definition is in test-case-schema.ts
|
||||||
|
import type { BehaviorExpectation } from '../sdk/test-case-schema.js';
|
||||||
|
export type { BehaviorExpectation };
|
||||||
|
|
||||||
|
export class BehaviorEvaluator extends BaseEvaluator {
|
||||||
|
name = 'behavior';
|
||||||
|
description = 'Validates agent behavior matches test expectations';
|
||||||
|
|
||||||
|
private behavior: BehaviorExpectation;
|
||||||
|
|
||||||
|
constructor(behavior: BehaviorExpectation) {
|
||||||
|
super();
|
||||||
|
this.behavior = behavior;
|
||||||
|
}
|
||||||
|
|
||||||
|
async evaluate(timeline: TimelineEvent[], sessionInfo: SessionInfo): Promise<EvaluationResult> {
|
||||||
|
const checks: Check[] = [];
|
||||||
|
const violations: Violation[] = [];
|
||||||
|
const evidence: Evidence[] = [];
|
||||||
|
|
||||||
|
// Get all tool calls
|
||||||
|
const toolCalls = this.getToolCalls(timeline);
|
||||||
|
|
||||||
|
// Extract tool names - handle both direct and nested data structures
|
||||||
|
// Tool name can be in: data.tool, data.state.tool, or the part itself
|
||||||
|
const toolsUsed = toolCalls.map(tc => {
|
||||||
|
const data = tc.data;
|
||||||
|
if (!data) return null;
|
||||||
|
// Try multiple paths where tool name might be stored
|
||||||
|
return data.tool || data.state?.tool || null;
|
||||||
|
}).filter((t): t is string => t !== null);
|
||||||
|
|
||||||
|
const uniqueTools = [...new Set(toolsUsed)];
|
||||||
|
|
||||||
|
// Log tool usage summary
|
||||||
|
console.log(`\n${'='.repeat(60)}`);
|
||||||
|
console.log(`BEHAVIOR VALIDATION`);
|
||||||
|
console.log(`${'='.repeat(60)}`);
|
||||||
|
console.log(`Timeline Events: ${timeline.length}`);
|
||||||
|
console.log(`Tool Calls: ${toolCalls.length}`);
|
||||||
|
console.log(`Tools Used: ${uniqueTools.join(', ') || 'none'}`);
|
||||||
|
|
||||||
|
// Log each tool call with details
|
||||||
|
if (toolCalls.length > 0) {
|
||||||
|
console.log(`\nTool Call Details:`);
|
||||||
|
toolCalls.forEach((tc, i) => {
|
||||||
|
const tool = tc.data?.tool || 'unknown';
|
||||||
|
const input = tc.data?.state?.input || tc.data?.input || {};
|
||||||
|
const inputStr = JSON.stringify(input).substring(0, 100);
|
||||||
|
console.log(` ${i + 1}. ${tool}: ${inputStr}${inputStr.length >= 100 ? '...' : ''}`);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check 1: mustUseTools
|
||||||
|
if (this.behavior.mustUseTools && this.behavior.mustUseTools.length > 0) {
|
||||||
|
const missingTools: string[] = [];
|
||||||
|
|
||||||
|
for (const requiredTool of this.behavior.mustUseTools) {
|
||||||
|
const wasUsed = toolsUsed.includes(requiredTool);
|
||||||
|
|
||||||
|
if (!wasUsed) {
|
||||||
|
missingTools.push(requiredTool);
|
||||||
|
|
||||||
|
violations.push(
|
||||||
|
this.createViolation(
|
||||||
|
'missing-required-tool',
|
||||||
|
'error',
|
||||||
|
`Required tool '${requiredTool}' was not used`,
|
||||||
|
Date.now(),
|
||||||
|
{
|
||||||
|
requiredTool,
|
||||||
|
toolsUsed: uniqueTools,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
checks.push({
|
||||||
|
name: 'must-use-tools',
|
||||||
|
passed: missingTools.length === 0,
|
||||||
|
weight: 100,
|
||||||
|
evidence: [
|
||||||
|
this.createEvidence(
|
||||||
|
'required-tools',
|
||||||
|
missingTools.length === 0
|
||||||
|
? `All required tools used: ${this.behavior.mustUseTools.join(', ')}`
|
||||||
|
: `Missing required tools: ${missingTools.join(', ')}`,
|
||||||
|
{
|
||||||
|
required: this.behavior.mustUseTools,
|
||||||
|
used: uniqueTools,
|
||||||
|
missing: missingTools,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
]
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check 2: mustNotUseTools
|
||||||
|
if (this.behavior.mustNotUseTools && this.behavior.mustNotUseTools.length > 0) {
|
||||||
|
const forbiddenToolsUsed: string[] = [];
|
||||||
|
|
||||||
|
for (const forbiddenTool of this.behavior.mustNotUseTools) {
|
||||||
|
const wasUsed = toolsUsed.includes(forbiddenTool);
|
||||||
|
|
||||||
|
if (wasUsed) {
|
||||||
|
forbiddenToolsUsed.push(forbiddenTool);
|
||||||
|
|
||||||
|
violations.push(
|
||||||
|
this.createViolation(
|
||||||
|
'forbidden-tool-used',
|
||||||
|
'error',
|
||||||
|
`Forbidden tool '${forbiddenTool}' was used`,
|
||||||
|
Date.now(),
|
||||||
|
{
|
||||||
|
forbiddenTool,
|
||||||
|
toolsUsed: uniqueTools,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
checks.push({
|
||||||
|
name: 'must-not-use-tools',
|
||||||
|
passed: forbiddenToolsUsed.length === 0,
|
||||||
|
weight: 100,
|
||||||
|
evidence: [
|
||||||
|
this.createEvidence(
|
||||||
|
'forbidden-tools',
|
||||||
|
forbiddenToolsUsed.length === 0
|
||||||
|
? `No forbidden tools used`
|
||||||
|
: `Forbidden tools used: ${forbiddenToolsUsed.join(', ')}`,
|
||||||
|
{
|
||||||
|
forbidden: this.behavior.mustNotUseTools,
|
||||||
|
used: uniqueTools,
|
||||||
|
violations: forbiddenToolsUsed,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
]
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check 3: minToolCalls
|
||||||
|
if (this.behavior.minToolCalls !== undefined) {
|
||||||
|
const passed = toolCalls.length >= this.behavior.minToolCalls;
|
||||||
|
|
||||||
|
if (!passed) {
|
||||||
|
violations.push(
|
||||||
|
this.createViolation(
|
||||||
|
'insufficient-tool-calls',
|
||||||
|
'error',
|
||||||
|
`Expected at least ${this.behavior.minToolCalls} tool calls, got ${toolCalls.length}`,
|
||||||
|
Date.now(),
|
||||||
|
{
|
||||||
|
expected: this.behavior.minToolCalls,
|
||||||
|
actual: toolCalls.length,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
checks.push({
|
||||||
|
name: 'min-tool-calls',
|
||||||
|
passed,
|
||||||
|
weight: 50,
|
||||||
|
evidence: [
|
||||||
|
this.createEvidence(
|
||||||
|
'tool-call-count',
|
||||||
|
`Tool calls: ${toolCalls.length} (min: ${this.behavior.minToolCalls})`,
|
||||||
|
{
|
||||||
|
actual: toolCalls.length,
|
||||||
|
minimum: this.behavior.minToolCalls,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
]
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check 4: maxToolCalls
|
||||||
|
if (this.behavior.maxToolCalls !== undefined) {
|
||||||
|
const passed = toolCalls.length <= this.behavior.maxToolCalls;
|
||||||
|
|
||||||
|
if (!passed) {
|
||||||
|
violations.push(
|
||||||
|
this.createViolation(
|
||||||
|
'excessive-tool-calls',
|
||||||
|
'warning',
|
||||||
|
`Expected at most ${this.behavior.maxToolCalls} tool calls, got ${toolCalls.length}`,
|
||||||
|
Date.now(),
|
||||||
|
{
|
||||||
|
expected: this.behavior.maxToolCalls,
|
||||||
|
actual: toolCalls.length,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
checks.push({
|
||||||
|
name: 'max-tool-calls',
|
||||||
|
passed,
|
||||||
|
weight: 50,
|
||||||
|
evidence: [
|
||||||
|
this.createEvidence(
|
||||||
|
'tool-call-count',
|
||||||
|
`Tool calls: ${toolCalls.length} (max: ${this.behavior.maxToolCalls})`,
|
||||||
|
{
|
||||||
|
actual: toolCalls.length,
|
||||||
|
maximum: this.behavior.maxToolCalls,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
]
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check 5: requiresApproval
|
||||||
|
if (this.behavior.requiresApproval) {
|
||||||
|
// Check if agent asked for approval (contains approval language in messages)
|
||||||
|
const assistantMessages = this.getAssistantMessages(timeline);
|
||||||
|
const hasApprovalRequest = assistantMessages.some(msg => {
|
||||||
|
const text = msg.data?.text || '';
|
||||||
|
return this.containsApprovalLanguage(text);
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!hasApprovalRequest) {
|
||||||
|
violations.push(
|
||||||
|
this.createViolation(
|
||||||
|
'missing-approval-request',
|
||||||
|
'error',
|
||||||
|
'Agent did not request approval before executing',
|
||||||
|
Date.now(),
|
||||||
|
{
|
||||||
|
requiresApproval: true,
|
||||||
|
approvalRequested: false,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
checks.push({
|
||||||
|
name: 'requires-approval',
|
||||||
|
passed: hasApprovalRequest,
|
||||||
|
weight: 100,
|
||||||
|
evidence: [
|
||||||
|
this.createEvidence(
|
||||||
|
'approval-request',
|
||||||
|
hasApprovalRequest
|
||||||
|
? 'Agent requested approval before executing'
|
||||||
|
: 'Agent did not request approval',
|
||||||
|
{
|
||||||
|
requiresApproval: true,
|
||||||
|
approvalRequested: hasApprovalRequest,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
]
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check 6: requiresContext
|
||||||
|
if (this.behavior.requiresContext) {
|
||||||
|
// Check if agent loaded context files
|
||||||
|
const readTools = this.getReadTools(timeline);
|
||||||
|
|
||||||
|
// Log all files read for analysis
|
||||||
|
const filesRead = readTools.map(rt =>
|
||||||
|
rt.data?.state?.input?.filePath || rt.data?.input?.filePath || rt.data?.input?.path || 'unknown'
|
||||||
|
);
|
||||||
|
|
||||||
|
console.log(`\n[behavior] Files Read (${filesRead.length}):`);
|
||||||
|
filesRead.forEach((file, i) => {
|
||||||
|
console.log(` ${i + 1}. ${file}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Context file patterns - files that count as "context loading"
|
||||||
|
// Matches: .opencode/agent/*.md, .opencode/context/**/*.md, docs/**/*.md, README.md, CONTRIBUTING.md
|
||||||
|
const contextPatterns = [
|
||||||
|
/\.opencode\/agent\/.*\.md$/i,
|
||||||
|
/\.opencode\/context\/.*\.md$/i,
|
||||||
|
/docs\/.*\.md$/i,
|
||||||
|
/\/CONTRIBUTING\.md$/i,
|
||||||
|
/\/README\.md$/i,
|
||||||
|
];
|
||||||
|
|
||||||
|
const contextReads = readTools.filter(rt => {
|
||||||
|
const filePath = rt.data?.state?.input?.filePath || rt.data?.input?.filePath || rt.data?.input?.path || '';
|
||||||
|
return contextPatterns.some(pattern => pattern.test(filePath));
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log(`[behavior] Context Files Read: ${contextReads.length}/${filesRead.length}`);
|
||||||
|
|
||||||
|
const hasContextLoading = contextReads.length > 0;
|
||||||
|
|
||||||
|
if (!hasContextLoading) {
|
||||||
|
violations.push(
|
||||||
|
this.createViolation(
|
||||||
|
'missing-context-loading',
|
||||||
|
'error',
|
||||||
|
'Agent did not load required context files',
|
||||||
|
Date.now(),
|
||||||
|
{
|
||||||
|
requiresContext: true,
|
||||||
|
contextLoaded: false,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
checks.push({
|
||||||
|
name: 'requires-context',
|
||||||
|
passed: hasContextLoading,
|
||||||
|
weight: 100,
|
||||||
|
evidence: [
|
||||||
|
this.createEvidence(
|
||||||
|
'context-loading',
|
||||||
|
hasContextLoading
|
||||||
|
? `Agent loaded ${contextReads.length} context file(s)`
|
||||||
|
: 'Agent did not load context files',
|
||||||
|
{
|
||||||
|
requiresContext: true,
|
||||||
|
contextLoaded: hasContextLoading,
|
||||||
|
contextFiles: contextReads.map(cr => cr.data?.input?.filePath || cr.data?.input?.path),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
]
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check 7: shouldDelegate
|
||||||
|
if (this.behavior.shouldDelegate) {
|
||||||
|
const taskCalls = this.getToolCallsByName(timeline, 'task');
|
||||||
|
const hasDelegation = taskCalls.length > 0;
|
||||||
|
|
||||||
|
if (!hasDelegation) {
|
||||||
|
violations.push(
|
||||||
|
this.createViolation(
|
||||||
|
'missing-delegation',
|
||||||
|
'warning',
|
||||||
|
'Agent should have delegated to a subagent',
|
||||||
|
Date.now(),
|
||||||
|
{
|
||||||
|
shouldDelegate: true,
|
||||||
|
delegated: false,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
checks.push({
|
||||||
|
name: 'should-delegate',
|
||||||
|
passed: hasDelegation,
|
||||||
|
weight: 75,
|
||||||
|
evidence: [
|
||||||
|
this.createEvidence(
|
||||||
|
'delegation',
|
||||||
|
hasDelegation
|
||||||
|
? `Agent delegated to ${taskCalls.length} subagent(s)`
|
||||||
|
: 'Agent did not delegate to subagents',
|
||||||
|
{
|
||||||
|
shouldDelegate: true,
|
||||||
|
delegated: hasDelegation,
|
||||||
|
delegationCount: taskCalls.length,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
]
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add summary evidence
|
||||||
|
evidence.push(
|
||||||
|
this.createEvidence(
|
||||||
|
'behavior-summary',
|
||||||
|
`Behavior validation: ${checks.filter(c => c.passed).length}/${checks.length} checks passed`,
|
||||||
|
{
|
||||||
|
totalChecks: checks.length,
|
||||||
|
passedChecks: checks.filter(c => c.passed).length,
|
||||||
|
failedChecks: checks.filter(c => !c.passed).length,
|
||||||
|
toolsUsed: uniqueTools,
|
||||||
|
toolCallCount: toolCalls.length,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
// Print summary
|
||||||
|
console.log(`\nBehavior Validation Summary:`);
|
||||||
|
console.log(` Checks Passed: ${checks.filter(c => c.passed).length}/${checks.length}`);
|
||||||
|
console.log(` Violations: ${violations.length}`);
|
||||||
|
if (violations.length > 0) {
|
||||||
|
console.log(`\nViolations Detected:`);
|
||||||
|
violations.forEach((v, i) => {
|
||||||
|
console.log(` ${i + 1}. [${v.severity}] ${v.type}: ${v.message}`);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
console.log(`${'='.repeat(60)}\n`);
|
||||||
|
|
||||||
|
return this.buildResult(this.name, checks, violations, evidence, {
|
||||||
|
behavior: this.behavior,
|
||||||
|
toolsUsed: uniqueTools,
|
||||||
|
toolCallCount: toolCalls.length,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
274
evals/framework/src/sdk/__tests__/test-case-schema.test.ts
Normal file
274
evals/framework/src/sdk/__tests__/test-case-schema.test.ts
Normal file
|
|
@ -0,0 +1,274 @@
|
||||||
|
/**
|
||||||
|
* Unit tests for TestCaseSchema validation
|
||||||
|
*
|
||||||
|
* Tests that the schema correctly validates:
|
||||||
|
* - Required fields (id, name, description, category, prompt/prompts)
|
||||||
|
* - Optional fields (behavior, expectedViolations, expected)
|
||||||
|
* - Flexible validation (behavior alone is valid)
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect } from 'vitest';
|
||||||
|
import { TestCaseSchema } from '../test-case-schema.js';
|
||||||
|
|
||||||
|
describe('TestCaseSchema', () => {
|
||||||
|
describe('required fields', () => {
|
||||||
|
it('should require id, name, description, category', () => {
|
||||||
|
const result = TestCaseSchema.safeParse({
|
||||||
|
prompt: 'test prompt',
|
||||||
|
approvalStrategy: { type: 'auto-approve' },
|
||||||
|
behavior: { minToolCalls: 1 },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.success).toBe(false);
|
||||||
|
if (!result.success) {
|
||||||
|
const missingFields = result.error.issues.map(i => i.path[0]);
|
||||||
|
expect(missingFields).toContain('id');
|
||||||
|
expect(missingFields).toContain('name');
|
||||||
|
expect(missingFields).toContain('description');
|
||||||
|
expect(missingFields).toContain('category');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should require either prompt or prompts', () => {
|
||||||
|
const result = TestCaseSchema.safeParse({
|
||||||
|
id: 'test-1',
|
||||||
|
name: 'Test',
|
||||||
|
description: 'Test description',
|
||||||
|
category: 'developer',
|
||||||
|
approvalStrategy: { type: 'auto-approve' },
|
||||||
|
behavior: { minToolCalls: 1 },
|
||||||
|
// No prompt or prompts
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.success).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('behavior validation flexibility', () => {
|
||||||
|
it('should accept behavior alone (without expectedViolations)', () => {
|
||||||
|
const result = TestCaseSchema.safeParse({
|
||||||
|
id: 'test-1',
|
||||||
|
name: 'Test',
|
||||||
|
description: 'Test description',
|
||||||
|
category: 'developer',
|
||||||
|
prompt: 'Do something',
|
||||||
|
approvalStrategy: { type: 'auto-approve' },
|
||||||
|
behavior: {
|
||||||
|
mustUseTools: ['bash'],
|
||||||
|
minToolCalls: 1,
|
||||||
|
},
|
||||||
|
// No expectedViolations - this should be valid!
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.success).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should accept expectedViolations alone (without behavior)', () => {
|
||||||
|
const result = TestCaseSchema.safeParse({
|
||||||
|
id: 'test-1',
|
||||||
|
name: 'Test',
|
||||||
|
description: 'Test description',
|
||||||
|
category: 'developer',
|
||||||
|
prompt: 'Do something',
|
||||||
|
approvalStrategy: { type: 'auto-approve' },
|
||||||
|
expectedViolations: [
|
||||||
|
{ rule: 'approval-gate', shouldViolate: false, severity: 'error' },
|
||||||
|
],
|
||||||
|
// No behavior - this should be valid!
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.success).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should accept behavior + expectedViolations together', () => {
|
||||||
|
const result = TestCaseSchema.safeParse({
|
||||||
|
id: 'test-1',
|
||||||
|
name: 'Test',
|
||||||
|
description: 'Test description',
|
||||||
|
category: 'developer',
|
||||||
|
prompt: 'Do something',
|
||||||
|
approvalStrategy: { type: 'auto-approve' },
|
||||||
|
behavior: {
|
||||||
|
mustUseTools: ['bash'],
|
||||||
|
minToolCalls: 1,
|
||||||
|
},
|
||||||
|
expectedViolations: [
|
||||||
|
{ rule: 'approval-gate', shouldViolate: false, severity: 'error' },
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.success).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should accept deprecated expected format', () => {
|
||||||
|
const result = TestCaseSchema.safeParse({
|
||||||
|
id: 'test-1',
|
||||||
|
name: 'Test',
|
||||||
|
description: 'Test description',
|
||||||
|
category: 'developer',
|
||||||
|
prompt: 'Do something',
|
||||||
|
approvalStrategy: { type: 'auto-approve' },
|
||||||
|
expected: {
|
||||||
|
pass: true,
|
||||||
|
minMessages: 1,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.success).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should reject when no behavior/expected/expectedViolations provided', () => {
|
||||||
|
const result = TestCaseSchema.safeParse({
|
||||||
|
id: 'test-1',
|
||||||
|
name: 'Test',
|
||||||
|
description: 'Test description',
|
||||||
|
category: 'developer',
|
||||||
|
prompt: 'Do something',
|
||||||
|
approvalStrategy: { type: 'auto-approve' },
|
||||||
|
// No behavior, expected, or expectedViolations
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.success).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('behavior fields', () => {
|
||||||
|
it('should validate mustUseTools as string array', () => {
|
||||||
|
const result = TestCaseSchema.safeParse({
|
||||||
|
id: 'test-1',
|
||||||
|
name: 'Test',
|
||||||
|
description: 'Test description',
|
||||||
|
category: 'developer',
|
||||||
|
prompt: 'Do something',
|
||||||
|
approvalStrategy: { type: 'auto-approve' },
|
||||||
|
behavior: {
|
||||||
|
mustUseTools: ['read', 'write', 'bash'],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.success).toBe(true);
|
||||||
|
if (result.success) {
|
||||||
|
expect(result.data.behavior?.mustUseTools).toEqual(['read', 'write', 'bash']);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should validate minToolCalls and maxToolCalls as numbers', () => {
|
||||||
|
const result = TestCaseSchema.safeParse({
|
||||||
|
id: 'test-1',
|
||||||
|
name: 'Test',
|
||||||
|
description: 'Test description',
|
||||||
|
category: 'developer',
|
||||||
|
prompt: 'Do something',
|
||||||
|
approvalStrategy: { type: 'auto-approve' },
|
||||||
|
behavior: {
|
||||||
|
minToolCalls: 1,
|
||||||
|
maxToolCalls: 10,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.success).toBe(true);
|
||||||
|
if (result.success) {
|
||||||
|
expect(result.data.behavior?.minToolCalls).toBe(1);
|
||||||
|
expect(result.data.behavior?.maxToolCalls).toBe(10);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should validate boolean flags', () => {
|
||||||
|
const result = TestCaseSchema.safeParse({
|
||||||
|
id: 'test-1',
|
||||||
|
name: 'Test',
|
||||||
|
description: 'Test description',
|
||||||
|
category: 'developer',
|
||||||
|
prompt: 'Do something',
|
||||||
|
approvalStrategy: { type: 'auto-approve' },
|
||||||
|
behavior: {
|
||||||
|
requiresApproval: true,
|
||||||
|
requiresContext: true,
|
||||||
|
shouldDelegate: false,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.success).toBe(true);
|
||||||
|
if (result.success) {
|
||||||
|
expect(result.data.behavior?.requiresApproval).toBe(true);
|
||||||
|
expect(result.data.behavior?.requiresContext).toBe(true);
|
||||||
|
expect(result.data.behavior?.shouldDelegate).toBe(false);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('approval strategies', () => {
|
||||||
|
it('should accept auto-approve strategy', () => {
|
||||||
|
const result = TestCaseSchema.safeParse({
|
||||||
|
id: 'test-1',
|
||||||
|
name: 'Test',
|
||||||
|
description: 'Test description',
|
||||||
|
category: 'developer',
|
||||||
|
prompt: 'Do something',
|
||||||
|
approvalStrategy: { type: 'auto-approve' },
|
||||||
|
behavior: { minToolCalls: 1 },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.success).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should accept auto-deny strategy', () => {
|
||||||
|
const result = TestCaseSchema.safeParse({
|
||||||
|
id: 'test-1',
|
||||||
|
name: 'Test',
|
||||||
|
description: 'Test description',
|
||||||
|
category: 'developer',
|
||||||
|
prompt: 'Do something',
|
||||||
|
approvalStrategy: { type: 'auto-deny' },
|
||||||
|
behavior: { minToolCalls: 1 },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.success).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should accept smart strategy with config', () => {
|
||||||
|
const result = TestCaseSchema.safeParse({
|
||||||
|
id: 'test-1',
|
||||||
|
name: 'Test',
|
||||||
|
description: 'Test description',
|
||||||
|
category: 'developer',
|
||||||
|
prompt: 'Do something',
|
||||||
|
approvalStrategy: {
|
||||||
|
type: 'smart',
|
||||||
|
config: {
|
||||||
|
allowedTools: ['read', 'write'],
|
||||||
|
deniedTools: ['bash'],
|
||||||
|
maxApprovals: 5,
|
||||||
|
defaultDecision: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
behavior: { minToolCalls: 1 },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.success).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('multi-turn conversations', () => {
|
||||||
|
it('should accept prompts array for multi-turn', () => {
|
||||||
|
const result = TestCaseSchema.safeParse({
|
||||||
|
id: 'test-1',
|
||||||
|
name: 'Test',
|
||||||
|
description: 'Test description',
|
||||||
|
category: 'developer',
|
||||||
|
prompts: [
|
||||||
|
{ text: 'First message' },
|
||||||
|
{ text: 'Second message', delayMs: 1000 },
|
||||||
|
{ text: 'Third message', expectContext: true, contextFile: 'docs/api.md' },
|
||||||
|
],
|
||||||
|
approvalStrategy: { type: 'auto-approve' },
|
||||||
|
behavior: { minToolCalls: 1 },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.success).toBe(true);
|
||||||
|
if (result.success) {
|
||||||
|
expect(result.data.prompts).toHaveLength(3);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
109
evals/framework/src/sdk/__tests__/yaml-loader.test.ts
Normal file
109
evals/framework/src/sdk/__tests__/yaml-loader.test.ts
Normal file
|
|
@ -0,0 +1,109 @@
|
||||||
|
/**
|
||||||
|
* Tests for loading actual YAML test files
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect } from 'vitest';
|
||||||
|
import { loadTestCase } from '../test-case-loader.js';
|
||||||
|
import { join } from 'path';
|
||||||
|
import { fileURLToPath } from 'url';
|
||||||
|
import { dirname } from 'path';
|
||||||
|
|
||||||
|
const __filename = fileURLToPath(import.meta.url);
|
||||||
|
const __dirname = dirname(__filename);
|
||||||
|
|
||||||
|
// Path to test files
|
||||||
|
const testFilesDir = join(__dirname, '../../../../agents/openagent/tests');
|
||||||
|
|
||||||
|
describe('YAML Test File Loading', () => {
|
||||||
|
describe('developer tests', () => {
|
||||||
|
it('should load simple-bash-test.yaml', async () => {
|
||||||
|
const testCase = await loadTestCase(join(testFilesDir, 'developer/simple-bash-test.yaml'));
|
||||||
|
|
||||||
|
expect(testCase.id).toBe('simple-bash-test');
|
||||||
|
expect(testCase.name).toBe('Simple Bash Command Test');
|
||||||
|
expect(testCase.category).toBe('developer');
|
||||||
|
expect(testCase.agent).toBe('openagent');
|
||||||
|
expect(testCase.behavior).toBeDefined();
|
||||||
|
expect(testCase.behavior?.mustUseTools).toContain('bash');
|
||||||
|
expect(testCase.behavior?.minToolCalls).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should load ctx-code-001.yaml', async () => {
|
||||||
|
const testCase = await loadTestCase(join(testFilesDir, 'developer/ctx-code-001.yaml'));
|
||||||
|
|
||||||
|
expect(testCase.id).toBe('ctx-code-001');
|
||||||
|
expect(testCase.behavior).toBeDefined();
|
||||||
|
expect(testCase.behavior?.mustUseTools).toContain('read');
|
||||||
|
expect(testCase.behavior?.mustUseTools).toContain('write');
|
||||||
|
expect(testCase.behavior?.requiresApproval).toBe(true);
|
||||||
|
expect(testCase.behavior?.requiresContext).toBe(true);
|
||||||
|
expect(testCase.expectedViolations).toBeDefined();
|
||||||
|
expect(testCase.expectedViolations?.length).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should load ctx-delegation-001.yaml', async () => {
|
||||||
|
const testCase = await loadTestCase(join(testFilesDir, 'developer/ctx-delegation-001.yaml'));
|
||||||
|
|
||||||
|
expect(testCase.id).toBe('ctx-delegation-001');
|
||||||
|
expect(testCase.behavior).toBeDefined();
|
||||||
|
expect(testCase.behavior?.mustUseTools).toContain('read');
|
||||||
|
expect(testCase.behavior?.mustUseTools).toContain('task');
|
||||||
|
expect(testCase.behavior?.requiresContext).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('edge-case tests', () => {
|
||||||
|
it('should load no-approval-negative.yaml', async () => {
|
||||||
|
const testCase = await loadTestCase(join(testFilesDir, 'edge-case/no-approval-negative.yaml'));
|
||||||
|
|
||||||
|
expect(testCase.id).toBe('neg-no-approval-001');
|
||||||
|
expect(testCase.category).toBe('edge-case');
|
||||||
|
expect(testCase.behavior).toBeDefined();
|
||||||
|
expect(testCase.expectedViolations).toBeDefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should load missing-approval-negative.yaml', async () => {
|
||||||
|
const testCase = await loadTestCase(join(testFilesDir, 'edge-case/missing-approval-negative.yaml'));
|
||||||
|
|
||||||
|
expect(testCase.id).toBe('neg-missing-approval-001');
|
||||||
|
expect(testCase.category).toBe('edge-case');
|
||||||
|
expect(testCase.behavior).toBeDefined();
|
||||||
|
expect(testCase.behavior?.requiresApproval).toBe(true);
|
||||||
|
expect(testCase.expectedViolations).toBeDefined();
|
||||||
|
|
||||||
|
// This is a negative test - expects violation
|
||||||
|
const approvalViolation = testCase.expectedViolations?.find(v => v.rule === 'approval-gate');
|
||||||
|
expect(approvalViolation).toBeDefined();
|
||||||
|
expect(approvalViolation?.shouldViolate).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('validation', () => {
|
||||||
|
it('should validate all test files have required fields', async () => {
|
||||||
|
const testFiles = [
|
||||||
|
'developer/simple-bash-test.yaml',
|
||||||
|
'developer/ctx-code-001.yaml',
|
||||||
|
'developer/ctx-delegation-001.yaml',
|
||||||
|
'edge-case/no-approval-negative.yaml',
|
||||||
|
'edge-case/missing-approval-negative.yaml',
|
||||||
|
];
|
||||||
|
|
||||||
|
for (const file of testFiles) {
|
||||||
|
const testCase = await loadTestCase(join(testFilesDir, file));
|
||||||
|
|
||||||
|
// Required fields
|
||||||
|
expect(testCase.id).toBeDefined();
|
||||||
|
expect(testCase.name).toBeDefined();
|
||||||
|
expect(testCase.description).toBeDefined();
|
||||||
|
expect(testCase.category).toBeDefined();
|
||||||
|
expect(testCase.approvalStrategy).toBeDefined();
|
||||||
|
|
||||||
|
// Must have prompt or prompts
|
||||||
|
expect(testCase.prompt || testCase.prompts).toBeDefined();
|
||||||
|
|
||||||
|
// Must have behavior, expected, or expectedViolations
|
||||||
|
expect(testCase.behavior || testCase.expected || testCase.expectedViolations).toBeDefined();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
@ -21,7 +21,7 @@
|
||||||
|
|
||||||
import { TestRunner } from './test-runner.js';
|
import { TestRunner } from './test-runner.js';
|
||||||
import { loadTestCase, loadTestCases } from './test-case-loader.js';
|
import { loadTestCase, loadTestCases } from './test-case-loader.js';
|
||||||
import glob from 'glob';
|
import { globSync } from 'glob';
|
||||||
import { join, dirname } from 'path';
|
import { join, dirname } from 'path';
|
||||||
import { fileURLToPath } from 'url';
|
import { fileURLToPath } from 'url';
|
||||||
import type { TestResult } from './test-runner.js';
|
import type { TestResult } from './test-runner.js';
|
||||||
|
|
@ -64,7 +64,35 @@ function printResults(results: TestResult[]): void {
|
||||||
console.log(` Events: ${result.events.length}`);
|
console.log(` Events: ${result.events.length}`);
|
||||||
console.log(` Approvals: ${result.approvalsGiven}`);
|
console.log(` Approvals: ${result.approvalsGiven}`);
|
||||||
|
|
||||||
|
// Show context loading details if available
|
||||||
if (result.evaluation) {
|
if (result.evaluation) {
|
||||||
|
const contextEval = result.evaluation.evaluatorResults.find(e => e.evaluator === 'context-loading');
|
||||||
|
if (contextEval && contextEval.metadata) {
|
||||||
|
const metadata = contextEval.metadata;
|
||||||
|
|
||||||
|
// Check if this is a task session
|
||||||
|
if (metadata.isTaskSession) {
|
||||||
|
if (metadata.isBashOnly) {
|
||||||
|
console.log(` Context Loading: ⊘ Bash-only task (not required)`);
|
||||||
|
} else if (metadata.contextCheck) {
|
||||||
|
const check = metadata.contextCheck;
|
||||||
|
if (check.contextFileLoaded) {
|
||||||
|
const timeDiff = check.executionTimestamp && check.loadTimestamp
|
||||||
|
? check.executionTimestamp - check.loadTimestamp
|
||||||
|
: 0;
|
||||||
|
console.log(` Context Loading:`);
|
||||||
|
console.log(` ✓ Loaded: ${check.contextFilePath}`);
|
||||||
|
console.log(` ✓ Timing: Context loaded ${timeDiff}ms before execution`);
|
||||||
|
} else {
|
||||||
|
console.log(` Context Loading:`);
|
||||||
|
console.log(` ✗ No context loaded before execution`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
console.log(` Context Loading: ⊘ Conversational session (not required)`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
console.log(` Violations: ${result.evaluation.totalViolations} (${result.evaluation.violationsBySeverity.error} errors, ${result.evaluation.violationsBySeverity.warning} warnings)`);
|
console.log(` Violations: ${result.evaluation.totalViolations} (${result.evaluation.violationsBySeverity.error} errors, ${result.evaluation.violationsBySeverity.warning} warnings)`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -105,7 +133,7 @@ async function main() {
|
||||||
// Find test files
|
// Find test files
|
||||||
const testDir = join(__dirname, '../../..', 'agents/openagent/tests');
|
const testDir = join(__dirname, '../../..', 'agents/openagent/tests');
|
||||||
const pattern = args.pattern || '**/*.yaml';
|
const pattern = args.pattern || '**/*.yaml';
|
||||||
const testFiles = glob.sync(pattern, { cwd: testDir, absolute: true });
|
const testFiles = globSync(pattern, { cwd: testDir, absolute: true });
|
||||||
|
|
||||||
if (testFiles.length === 0) {
|
if (testFiles.length === 0) {
|
||||||
console.error(`❌ No test files found matching pattern: ${pattern}`);
|
console.error(`❌ No test files found matching pattern: ${pattern}`);
|
||||||
|
|
|
||||||
|
|
@ -151,6 +151,33 @@ export const ExpectedResultsSchema = z.object({
|
||||||
|
|
||||||
export type ExpectedResults = z.infer<typeof ExpectedResultsSchema>;
|
export type ExpectedResults = z.infer<typeof ExpectedResultsSchema>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Multi-message prompt (for multi-turn conversations)
|
||||||
|
*/
|
||||||
|
export const MultiMessageSchema = z.object({
|
||||||
|
/**
|
||||||
|
* The message text
|
||||||
|
*/
|
||||||
|
text: z.string(),
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Should context be loaded for this message?
|
||||||
|
*/
|
||||||
|
expectContext: z.boolean().optional(),
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Expected context file to be loaded
|
||||||
|
*/
|
||||||
|
contextFile: z.string().optional(),
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delay before sending this message (ms)
|
||||||
|
*/
|
||||||
|
delayMs: z.number().optional(),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type MultiMessage = z.infer<typeof MultiMessageSchema>;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Test case schema
|
* Test case schema
|
||||||
*/
|
*/
|
||||||
|
|
@ -176,9 +203,14 @@ export const TestCaseSchema = z.object({
|
||||||
category: z.enum(['developer', 'business', 'creative', 'edge-case']),
|
category: z.enum(['developer', 'business', 'creative', 'edge-case']),
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The prompt to send to OpenAgent
|
* The prompt to send to OpenAgent (single message)
|
||||||
*/
|
*/
|
||||||
prompt: z.string(),
|
prompt: z.string().optional(),
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Multiple prompts for multi-turn conversations (NEW)
|
||||||
|
*/
|
||||||
|
prompts: z.array(MultiMessageSchema).optional(),
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Agent to use (defaults to 'openagent')
|
* Agent to use (defaults to 'openagent')
|
||||||
|
|
@ -227,9 +259,16 @@ export const TestCaseSchema = z.object({
|
||||||
*/
|
*/
|
||||||
tags: z.array(z.string()).optional(),
|
tags: z.array(z.string()).optional(),
|
||||||
}).refine(
|
}).refine(
|
||||||
(data) => data.expected || (data.behavior && data.expectedViolations),
|
(data) => data.prompt || data.prompts,
|
||||||
{
|
{
|
||||||
message: 'Must provide either "expected" (deprecated) or "behavior" + "expectedViolations" (preferred)',
|
message: 'Must provide either "prompt" (single message) or "prompts" (multi-turn)',
|
||||||
|
}
|
||||||
|
).refine(
|
||||||
|
// Allow: expected (deprecated), behavior alone, or behavior + expectedViolations
|
||||||
|
// This is more flexible - behavior alone is valid for simple tests
|
||||||
|
(data) => data.expected || data.behavior || data.expectedViolations,
|
||||||
|
{
|
||||||
|
message: 'Must provide "expected" (deprecated), "behavior", "expectedViolations", or a combination',
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -11,12 +11,14 @@ import { ApprovalGateEvaluator } from '../evaluators/approval-gate-evaluator.js'
|
||||||
import { ContextLoadingEvaluator } from '../evaluators/context-loading-evaluator.js';
|
import { ContextLoadingEvaluator } from '../evaluators/context-loading-evaluator.js';
|
||||||
import { DelegationEvaluator } from '../evaluators/delegation-evaluator.js';
|
import { DelegationEvaluator } from '../evaluators/delegation-evaluator.js';
|
||||||
import { ToolUsageEvaluator } from '../evaluators/tool-usage-evaluator.js';
|
import { ToolUsageEvaluator } from '../evaluators/tool-usage-evaluator.js';
|
||||||
|
import { BehaviorEvaluator } from '../evaluators/behavior-evaluator.js';
|
||||||
import type { TestCase } from './test-case-schema.js';
|
import type { TestCase } from './test-case-schema.js';
|
||||||
import type { ApprovalStrategy } from './approval/approval-strategy.js';
|
import type { ApprovalStrategy } from './approval/approval-strategy.js';
|
||||||
import type { ServerEvent } from './event-stream-handler.js';
|
import type { ServerEvent } from './event-stream-handler.js';
|
||||||
import type { AggregatedResult } from '../evaluators/evaluator-runner.js';
|
import type { AggregatedResult } from '../evaluators/evaluator-runner.js';
|
||||||
import { homedir } from 'os';
|
import { homedir } from 'os';
|
||||||
import { join } from 'path';
|
import { join } from 'path';
|
||||||
|
import { findGitRoot } from '../config.js';
|
||||||
|
|
||||||
export interface TestRunnerConfig {
|
export interface TestRunnerConfig {
|
||||||
/**
|
/**
|
||||||
|
|
@ -36,6 +38,22 @@ export interface TestRunnerConfig {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Project path for evaluators
|
* Project path for evaluators
|
||||||
|
*
|
||||||
|
* IMPORTANT: This should be the git root where the agent runs, not the test framework directory.
|
||||||
|
*
|
||||||
|
* Default behavior:
|
||||||
|
* - Automatically finds git root by walking up from process.cwd()
|
||||||
|
* - This ensures sessions created by agents are found correctly
|
||||||
|
*
|
||||||
|
* When to override:
|
||||||
|
* - Testing agents in non-git directories
|
||||||
|
* - Testing multiple agents with different project roots
|
||||||
|
* - Custom session storage locations
|
||||||
|
*
|
||||||
|
* Example:
|
||||||
|
* - Git root: /Users/user/opencode-agents (sessions stored here)
|
||||||
|
* - Test CWD: /Users/user/opencode-agents/evals/framework (tests run here)
|
||||||
|
* - projectPath should be git root, not test CWD
|
||||||
*/
|
*/
|
||||||
projectPath?: string;
|
projectPath?: string;
|
||||||
|
|
||||||
|
|
@ -109,37 +127,34 @@ export class TestRunner {
|
||||||
private evaluatorRunner: EvaluatorRunner | null = null;
|
private evaluatorRunner: EvaluatorRunner | null = null;
|
||||||
|
|
||||||
constructor(config: TestRunnerConfig = {}) {
|
constructor(config: TestRunnerConfig = {}) {
|
||||||
|
// Find git root for agent detection
|
||||||
|
const gitRoot = findGitRoot(process.cwd());
|
||||||
|
|
||||||
this.config = {
|
this.config = {
|
||||||
port: config.port || 0,
|
port: config.port || 0,
|
||||||
debug: config.debug || false,
|
debug: config.debug || false,
|
||||||
defaultTimeout: config.defaultTimeout || 60000,
|
defaultTimeout: config.defaultTimeout || 60000,
|
||||||
projectPath: config.projectPath || process.cwd(),
|
projectPath: config.projectPath || gitRoot,
|
||||||
runEvaluators: config.runEvaluators ?? true,
|
runEvaluators: config.runEvaluators ?? true,
|
||||||
defaultModel: config.defaultModel || 'opencode/grok-code-fast', // Free tier default
|
defaultModel: config.defaultModel || 'opencode/grok-code', // Free tier default (fixed model name)
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// Start server from git root with default agent
|
||||||
|
// Note: Individual tests can override the agent per-session
|
||||||
this.server = new ServerManager({
|
this.server = new ServerManager({
|
||||||
port: this.config.port,
|
port: this.config.port,
|
||||||
timeout: 10000,
|
timeout: 10000,
|
||||||
|
cwd: gitRoot, // CRITICAL: Start server from git root to detect agent
|
||||||
|
debug: this.config.debug, // Pass debug flag to server
|
||||||
|
agent: 'openagent', // Default agent for all tests
|
||||||
});
|
});
|
||||||
|
|
||||||
// Setup evaluators if enabled
|
if (this.config.debug) {
|
||||||
if (this.config.runEvaluators) {
|
console.log(`[TestRunner] Git root: ${gitRoot}`);
|
||||||
const sessionStoragePath = join(homedir(), '.local', 'share', 'opencode', 'storage');
|
console.log(`[TestRunner] Server will start from: ${gitRoot} with agent: openagent`);
|
||||||
const sessionReader = new SessionReader(this.config.projectPath, sessionStoragePath);
|
|
||||||
const timelineBuilder = new TimelineBuilder(sessionReader);
|
|
||||||
|
|
||||||
this.evaluatorRunner = new EvaluatorRunner({
|
|
||||||
sessionReader,
|
|
||||||
timelineBuilder,
|
|
||||||
evaluators: [
|
|
||||||
new ApprovalGateEvaluator(),
|
|
||||||
new ContextLoadingEvaluator(),
|
|
||||||
new DelegationEvaluator(),
|
|
||||||
new ToolUsageEvaluator(),
|
|
||||||
],
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Note: Evaluators will be setup in start() after SDK client is available
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -152,6 +167,32 @@ export class TestRunner {
|
||||||
|
|
||||||
this.client = new ClientManager({ baseUrl: url });
|
this.client = new ClientManager({ baseUrl: url });
|
||||||
this.eventHandler = new EventStreamHandler(url);
|
this.eventHandler = new EventStreamHandler(url);
|
||||||
|
|
||||||
|
// Setup evaluators now that SDK client is available
|
||||||
|
if (this.config.runEvaluators && this.client) {
|
||||||
|
const sessionStoragePath = join(homedir(), '.local', 'share', 'opencode');
|
||||||
|
|
||||||
|
// Create SessionReader with SDK client for reliable session retrieval
|
||||||
|
const sdkClient = this.client.getClient();
|
||||||
|
const sessionReader = new SessionReader(sdkClient, sessionStoragePath);
|
||||||
|
const timelineBuilder = new TimelineBuilder(sessionReader);
|
||||||
|
|
||||||
|
this.evaluatorRunner = new EvaluatorRunner({
|
||||||
|
sessionReader,
|
||||||
|
timelineBuilder,
|
||||||
|
sdkClient,
|
||||||
|
evaluators: [
|
||||||
|
new ApprovalGateEvaluator(),
|
||||||
|
new ContextLoadingEvaluator(),
|
||||||
|
new DelegationEvaluator(),
|
||||||
|
new ToolUsageEvaluator(),
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
if (this.config.debug) {
|
||||||
|
this.log('[TestRunner] Evaluators initialized with SDK client');
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -198,7 +239,7 @@ export class TestRunner {
|
||||||
this.eventHandler.onAny((event) => {
|
this.eventHandler.onAny((event) => {
|
||||||
events.push(event);
|
events.push(event);
|
||||||
if (this.config.debug) {
|
if (this.config.debug) {
|
||||||
this.log(`Event: ${event.type}`);
|
this.logEvent(event);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -220,29 +261,72 @@ export class TestRunner {
|
||||||
// Wait for event handler to connect
|
// Wait for event handler to connect
|
||||||
await this.sleep(2000);
|
await this.sleep(2000);
|
||||||
|
|
||||||
// Create session
|
// Create session (agent selection happens in sendPrompt, not here)
|
||||||
this.log('Creating session...');
|
this.log('Creating session...');
|
||||||
const session = await this.client.createSession(testCase.name);
|
const session = await this.client.createSession({
|
||||||
|
title: testCase.name,
|
||||||
|
});
|
||||||
sessionId = session.id;
|
sessionId = session.id;
|
||||||
this.log(`Session created: ${sessionId}`);
|
this.log(`Session created: ${sessionId}`);
|
||||||
|
|
||||||
// Send prompt
|
// Send prompt(s) with agent selection
|
||||||
this.log('Sending prompt...');
|
const timeout = testCase.timeout || this.config.defaultTimeout;
|
||||||
this.log(`Prompt: ${testCase.prompt.substring(0, 100)}${testCase.prompt.length > 100 ? '...' : ''}`);
|
|
||||||
|
|
||||||
// Use test case model, or fall back to default model
|
|
||||||
const modelToUse = testCase.model || this.config.defaultModel;
|
const modelToUse = testCase.model || this.config.defaultModel;
|
||||||
|
const agentToUse = testCase.agent || 'openagent'; // Default to openagent
|
||||||
|
|
||||||
|
this.log(`Agent: ${agentToUse}`);
|
||||||
this.log(`Model: ${modelToUse}`);
|
this.log(`Model: ${modelToUse}`);
|
||||||
|
|
||||||
const timeout = testCase.timeout || this.config.defaultTimeout;
|
// Check if multi-message test
|
||||||
|
if (testCase.prompts && testCase.prompts.length > 0) {
|
||||||
|
this.log(`Sending ${testCase.prompts.length} prompts (multi-turn)...`);
|
||||||
|
|
||||||
|
for (let i = 0; i < testCase.prompts.length; i++) {
|
||||||
|
const msg = testCase.prompts[i];
|
||||||
|
this.log(`\nPrompt ${i + 1}/${testCase.prompts.length}:`);
|
||||||
|
this.log(` Text: ${msg.text.substring(0, 100)}${msg.text.length > 100 ? '...' : ''}`);
|
||||||
|
if (msg.expectContext) {
|
||||||
|
this.log(` Expects context: ${msg.contextFile || 'yes'}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add delay if specified
|
||||||
|
if (msg.delayMs && i > 0) {
|
||||||
|
this.log(` Waiting ${msg.delayMs}ms before sending...`);
|
||||||
|
await this.sleep(msg.delayMs);
|
||||||
|
}
|
||||||
|
|
||||||
const promptPromise = this.client.sendPrompt(sessionId, {
|
const promptPromise = this.client.sendPrompt(sessionId, {
|
||||||
text: testCase.prompt,
|
text: msg.text,
|
||||||
|
agent: agentToUse, // ✅ Agent selection happens here!
|
||||||
model: modelToUse ? this.parseModel(modelToUse) : undefined,
|
model: modelToUse ? this.parseModel(modelToUse) : undefined,
|
||||||
|
directory: this.config.projectPath, // Pass working directory
|
||||||
});
|
});
|
||||||
|
|
||||||
// Wait for prompt with timeout
|
await this.withTimeout(promptPromise, timeout, `Prompt ${i + 1} execution timed out`);
|
||||||
const result = await this.withTimeout(promptPromise, timeout, 'Prompt execution timed out');
|
this.log(` Completed`);
|
||||||
|
|
||||||
|
// Small delay between messages
|
||||||
|
if (i < testCase.prompts.length - 1) {
|
||||||
|
await this.sleep(1000);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.log('\nAll prompts completed');
|
||||||
|
} else {
|
||||||
|
// Single message test
|
||||||
|
this.log('Sending prompt...');
|
||||||
|
this.log(`Prompt: ${testCase.prompt!.substring(0, 100)}${testCase.prompt!.length > 100 ? '...' : ''}`);
|
||||||
|
|
||||||
|
const promptPromise = this.client.sendPrompt(sessionId, {
|
||||||
|
text: testCase.prompt!,
|
||||||
|
agent: agentToUse, // ✅ Agent selection happens here!
|
||||||
|
model: modelToUse ? this.parseModel(modelToUse) : undefined,
|
||||||
|
directory: this.config.projectPath, // Pass working directory
|
||||||
|
});
|
||||||
|
|
||||||
|
await this.withTimeout(promptPromise, timeout, 'Prompt execution timed out');
|
||||||
this.log('Prompt completed');
|
this.log('Prompt completed');
|
||||||
|
}
|
||||||
|
|
||||||
// Give time for final events to arrive
|
// Give time for final events to arrive
|
||||||
await this.sleep(3000);
|
await this.sleep(3000);
|
||||||
|
|
@ -252,10 +336,46 @@ export class TestRunner {
|
||||||
|
|
||||||
const duration = Date.now() - startTime;
|
const duration = Date.now() - startTime;
|
||||||
|
|
||||||
|
// Validate agent is correct
|
||||||
|
if (testCase.agent) {
|
||||||
|
this.log(`Validating agent: ${testCase.agent}...`);
|
||||||
|
try {
|
||||||
|
const sessionInfo = await this.client.getSession(sessionId);
|
||||||
|
const messages = sessionInfo.messages;
|
||||||
|
|
||||||
|
if (messages && messages.length > 0) {
|
||||||
|
const firstMessage = messages[0].info as any; // SDK types may not include agent field
|
||||||
|
const actualAgent = firstMessage.agent;
|
||||||
|
|
||||||
|
if (actualAgent && actualAgent !== testCase.agent) {
|
||||||
|
errors.push(`Agent mismatch: expected '${testCase.agent}', got '${actualAgent}'`);
|
||||||
|
this.log(` ❌ Agent mismatch: expected '${testCase.agent}', got '${actualAgent}'`);
|
||||||
|
} else if (actualAgent) {
|
||||||
|
this.log(` ✅ Agent verified: ${actualAgent}`);
|
||||||
|
} else {
|
||||||
|
this.log(` ⚠️ Agent not set in message`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
this.log(` Warning: Could not validate agent: ${(error as Error).message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Run evaluators if enabled
|
// Run evaluators if enabled
|
||||||
let evaluation: AggregatedResult | undefined;
|
let evaluation: AggregatedResult | undefined;
|
||||||
if (this.config.runEvaluators && this.evaluatorRunner) {
|
if (this.config.runEvaluators && this.evaluatorRunner) {
|
||||||
this.log('Running evaluators...');
|
this.log('Running evaluators...');
|
||||||
|
|
||||||
|
// Add behavior evaluator if test case has behavior expectations
|
||||||
|
if (testCase.behavior) {
|
||||||
|
this.log('Adding behavior evaluator for test expectations...');
|
||||||
|
const behaviorEvaluator = new BehaviorEvaluator(testCase.behavior);
|
||||||
|
this.evaluatorRunner.register(behaviorEvaluator);
|
||||||
|
}
|
||||||
|
|
||||||
|
// No need to wait for disk writes - we're using SDK client directly!
|
||||||
|
// The SDK has the session data in memory and can return it immediately.
|
||||||
|
|
||||||
try {
|
try {
|
||||||
evaluation = await this.evaluatorRunner.runAll(sessionId);
|
evaluation = await this.evaluatorRunner.runAll(sessionId);
|
||||||
this.log(`Evaluators completed: ${evaluation.totalViolations} violations found`);
|
this.log(`Evaluators completed: ${evaluation.totalViolations} violations found`);
|
||||||
|
|
@ -264,6 +384,11 @@ export class TestRunner {
|
||||||
this.log(` Errors: ${evaluation.violationsBySeverity.error}`);
|
this.log(` Errors: ${evaluation.violationsBySeverity.error}`);
|
||||||
this.log(` Warnings: ${evaluation.violationsBySeverity.warning}`);
|
this.log(` Warnings: ${evaluation.violationsBySeverity.warning}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Clean up behavior evaluator after use
|
||||||
|
if (testCase.behavior) {
|
||||||
|
this.evaluatorRunner.unregister('behavior');
|
||||||
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
this.log(`Warning: Evaluators failed: ${(error as Error).message}`);
|
this.log(`Warning: Evaluators failed: ${(error as Error).message}`);
|
||||||
errors.push(`Evaluator error: ${(error as Error).message}`);
|
errors.push(`Evaluator error: ${(error as Error).message}`);
|
||||||
|
|
@ -319,14 +444,16 @@ export class TestRunner {
|
||||||
const result = await this.runTest(testCase);
|
const result = await this.runTest(testCase);
|
||||||
results.push(result);
|
results.push(result);
|
||||||
|
|
||||||
// Clean up session after each test
|
// Clean up session after each test (skip in debug mode to allow inspection)
|
||||||
if (this.client && result.sessionId) {
|
if (this.client && result.sessionId && !this.config.debug) {
|
||||||
try {
|
try {
|
||||||
await this.client.deleteSession(result.sessionId);
|
await this.client.deleteSession(result.sessionId);
|
||||||
this.log(`Cleaned up session: ${result.sessionId}\n`);
|
this.log(`Cleaned up session: ${result.sessionId}\n`);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
this.log(`Failed to clean up session: ${(error as Error).message}\n`);
|
this.log(`Failed to clean up session: ${(error as Error).message}\n`);
|
||||||
}
|
}
|
||||||
|
} else if (this.config.debug) {
|
||||||
|
this.log(`Debug mode: Keeping session ${result.sessionId} for inspection\n`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -363,6 +490,13 @@ export class TestRunner {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Evaluate if test result matches expected outcome
|
* Evaluate if test result matches expected outcome
|
||||||
|
*
|
||||||
|
* Evaluation priority:
|
||||||
|
* 1. Check for execution errors
|
||||||
|
* 2. Check behavior expectations (if defined)
|
||||||
|
* 3. Check expected violations (if defined)
|
||||||
|
* 4. Check deprecated expected format (if defined)
|
||||||
|
* 5. Default: pass if no errors
|
||||||
*/
|
*/
|
||||||
private evaluateResult(
|
private evaluateResult(
|
||||||
testCase: TestCase,
|
testCase: TestCase,
|
||||||
|
|
@ -376,12 +510,81 @@ export class TestRunner {
|
||||||
const expectedViolations = testCase.expectedViolations;
|
const expectedViolations = testCase.expectedViolations;
|
||||||
|
|
||||||
// If there were execution errors and test expects to pass, it fails
|
// If there were execution errors and test expects to pass, it fails
|
||||||
if (errors.length > 0 && expected?.pass) {
|
if (errors.length > 0 && expected?.pass !== false) {
|
||||||
|
this.log(`Test failed due to execution errors: ${errors.join(', ')}`);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// =========================================================================
|
||||||
|
// NEW: Check behavior evaluator results FIRST (most important)
|
||||||
|
// =========================================================================
|
||||||
|
if (behavior && evaluation) {
|
||||||
|
// Find the behavior evaluator result
|
||||||
|
const behaviorResult = evaluation.evaluatorResults.find(r => r.evaluator === 'behavior');
|
||||||
|
|
||||||
|
if (behaviorResult) {
|
||||||
|
// Check if behavior evaluator passed
|
||||||
|
if (!behaviorResult.passed) {
|
||||||
|
this.log(`Behavior validation failed: ${behaviorResult.violations.length} violations`);
|
||||||
|
behaviorResult.violations.forEach(v => {
|
||||||
|
this.log(` - [${v.severity}] ${v.type}: ${v.message}`);
|
||||||
|
});
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for error-level violations from behavior evaluator
|
||||||
|
const behaviorErrors = behaviorResult.violations.filter(v => v.severity === 'error');
|
||||||
|
if (behaviorErrors.length > 0) {
|
||||||
|
this.log(`Behavior validation has ${behaviorErrors.length} error-level violations`);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// =========================================================================
|
||||||
|
// Check expected violations (new format)
|
||||||
|
// =========================================================================
|
||||||
|
if (expectedViolations && evaluation) {
|
||||||
|
for (const expectedViolation of expectedViolations) {
|
||||||
|
// Map rule names to violation type patterns
|
||||||
|
const rulePatterns: Record<string, string[]> = {
|
||||||
|
'approval-gate': ['approval', 'missing-approval'],
|
||||||
|
'context-loading': ['context', 'no-context-loaded', 'missing-context'],
|
||||||
|
'delegation': ['delegation', 'missing-delegation'],
|
||||||
|
'tool-usage': ['tool', 'suboptimal-tool'],
|
||||||
|
'stop-on-failure': ['stop', 'failure'],
|
||||||
|
'confirm-cleanup': ['cleanup', 'confirm'],
|
||||||
|
};
|
||||||
|
|
||||||
|
const patterns = rulePatterns[expectedViolation.rule] || [expectedViolation.rule];
|
||||||
|
|
||||||
|
const actualViolations = evaluation.allViolations.filter(v =>
|
||||||
|
patterns.some(pattern => v.type.toLowerCase().includes(pattern.toLowerCase()))
|
||||||
|
);
|
||||||
|
|
||||||
|
if (expectedViolation.shouldViolate) {
|
||||||
|
// Negative test: Should have violation
|
||||||
|
if (actualViolations.length === 0) {
|
||||||
|
this.log(`Expected ${expectedViolation.rule} violation but none found`);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
this.log(`✓ Expected violation '${expectedViolation.rule}' found`);
|
||||||
|
} else {
|
||||||
|
// Positive test: Should NOT have violation
|
||||||
|
if (actualViolations.length > 0) {
|
||||||
|
this.log(`Unexpected ${expectedViolation.rule} violation found: ${actualViolations[0].message}`);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// =========================================================================
|
||||||
|
// Check deprecated expected format
|
||||||
|
// =========================================================================
|
||||||
|
if (expected) {
|
||||||
// Check minimum messages (deprecated)
|
// Check minimum messages (deprecated)
|
||||||
if (expected?.minMessages !== undefined) {
|
if (expected.minMessages !== undefined) {
|
||||||
const messageEvents = events.filter(e => e.type.includes('message'));
|
const messageEvents = events.filter(e => e.type.includes('message'));
|
||||||
if (messageEvents.length < expected.minMessages) {
|
if (messageEvents.length < expected.minMessages) {
|
||||||
this.log(`Expected at least ${expected.minMessages} messages, got ${messageEvents.length}`);
|
this.log(`Expected at least ${expected.minMessages} messages, got ${messageEvents.length}`);
|
||||||
|
|
@ -390,7 +593,7 @@ export class TestRunner {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check maximum messages (deprecated)
|
// Check maximum messages (deprecated)
|
||||||
if (expected?.maxMessages !== undefined) {
|
if (expected.maxMessages !== undefined) {
|
||||||
const messageEvents = events.filter(e => e.type.includes('message'));
|
const messageEvents = events.filter(e => e.type.includes('message'));
|
||||||
if (messageEvents.length > expected.maxMessages) {
|
if (messageEvents.length > expected.maxMessages) {
|
||||||
this.log(`Expected at most ${expected.maxMessages} messages, got ${messageEvents.length}`);
|
this.log(`Expected at most ${expected.maxMessages} messages, got ${messageEvents.length}`);
|
||||||
|
|
@ -398,11 +601,10 @@ export class TestRunner {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check expected violations match actual violations (deprecated format)
|
// Check expected violations (deprecated format)
|
||||||
if (expected?.violations && evaluation) {
|
if (expected.violations && evaluation) {
|
||||||
const expectedViolationTypes = expected.violations.map(v => v.rule);
|
const expectedViolationTypes = expected.violations.map(v => v.rule);
|
||||||
const actualViolationTypes = evaluation.allViolations.map(v => {
|
const actualViolationTypes = evaluation.allViolations.map(v => {
|
||||||
// Map violation types to rule names
|
|
||||||
if (v.type.includes('approval')) return 'approval-gate' as const;
|
if (v.type.includes('approval')) return 'approval-gate' as const;
|
||||||
if (v.type.includes('context')) return 'context-loading' as const;
|
if (v.type.includes('context')) return 'context-loading' as const;
|
||||||
if (v.type.includes('delegation')) return 'delegation' as const;
|
if (v.type.includes('delegation')) return 'delegation' as const;
|
||||||
|
|
@ -410,9 +612,7 @@ export class TestRunner {
|
||||||
return 'unknown' as const;
|
return 'unknown' as const;
|
||||||
});
|
});
|
||||||
|
|
||||||
// Check if expected violations are found
|
|
||||||
for (const expectedType of expectedViolationTypes) {
|
for (const expectedType of expectedViolationTypes) {
|
||||||
// Only check for implemented rules
|
|
||||||
if (['approval-gate', 'context-loading', 'delegation', 'tool-usage'].includes(expectedType)) {
|
if (['approval-gate', 'context-loading', 'delegation', 'tool-usage'].includes(expectedType)) {
|
||||||
if (!actualViolationTypes.includes(expectedType as any)) {
|
if (!actualViolationTypes.includes(expectedType as any)) {
|
||||||
this.log(`Expected violation '${expectedType}' not found`);
|
this.log(`Expected violation '${expectedType}' not found`);
|
||||||
|
|
@ -421,50 +621,35 @@ export class TestRunner {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// If test expects to fail, violations should exist
|
if (!expected.pass && evaluation.totalViolations === 0) {
|
||||||
if (!expected?.pass && evaluation.totalViolations === 0) {
|
|
||||||
this.log('Expected violations but none found');
|
this.log('Expected violations but none found');
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// NEW: Check expected violations (new format)
|
|
||||||
if (expectedViolations && evaluation) {
|
|
||||||
for (const expectedViolation of expectedViolations) {
|
|
||||||
const actualViolations = evaluation.allViolations.filter(v => {
|
|
||||||
if (expectedViolation.rule === 'approval-gate') return v.type.includes('approval');
|
|
||||||
if (expectedViolation.rule === 'context-loading') return v.type.includes('context');
|
|
||||||
if (expectedViolation.rule === 'delegation') return v.type.includes('delegation');
|
|
||||||
if (expectedViolation.rule === 'tool-usage') return v.type.includes('tool');
|
|
||||||
return false;
|
|
||||||
});
|
|
||||||
|
|
||||||
if (expectedViolation.shouldViolate) {
|
|
||||||
// Negative test: Should have violation
|
|
||||||
if (actualViolations.length === 0) {
|
|
||||||
this.log(`Expected ${expectedViolation.rule} violation but none found`);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// Positive test: Should NOT have violation
|
|
||||||
if (actualViolations.length > 0) {
|
|
||||||
this.log(`Unexpected ${expectedViolation.rule} violation found`);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// If test expects to pass, check no critical violations
|
// If test expects to pass, check no critical violations
|
||||||
if (expected?.pass && evaluation) {
|
if (expected.pass && evaluation) {
|
||||||
if (evaluation.violationsBySeverity.error > 0) {
|
if (evaluation.violationsBySeverity.error > 0) {
|
||||||
this.log(`Expected pass but found ${evaluation.violationsBySeverity.error} error-level violations`);
|
this.log(`Expected pass but found ${evaluation.violationsBySeverity.error} error-level violations`);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Default: pass if no errors (or use expected.pass if specified)
|
// Use expected.pass if specified
|
||||||
return expected?.pass !== undefined ? (expected.pass ? errors.length === 0 : true) : errors.length === 0;
|
if (expected.pass !== undefined) {
|
||||||
|
return expected.pass ? errors.length === 0 : true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// =========================================================================
|
||||||
|
// Default: pass if no errors and no error-level violations
|
||||||
|
// =========================================================================
|
||||||
|
if (evaluation && evaluation.violationsBySeverity.error > 0) {
|
||||||
|
this.log(`Test failed: ${evaluation.violationsBySeverity.error} error-level violations`);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return errors.length === 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -505,4 +690,97 @@ export class TestRunner {
|
||||||
console.log(message);
|
console.log(message);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Log event with meaningful details
|
||||||
|
*
|
||||||
|
* Event properties structure varies by type:
|
||||||
|
* - session.created/updated: { id, title, ... }
|
||||||
|
* - message.updated: { id, sessionID, role, ... }
|
||||||
|
* - part.updated: { id, messageID, type, tool?, input?, output?, ... }
|
||||||
|
*/
|
||||||
|
private logEvent(event: ServerEvent): void {
|
||||||
|
const props = event.properties || {};
|
||||||
|
|
||||||
|
switch (event.type) {
|
||||||
|
case 'session.created':
|
||||||
|
console.log(`📋 Session created`);
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 'session.updated':
|
||||||
|
// Session updates are frequent but not very informative
|
||||||
|
// Skip logging unless there's something specific
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 'message.created':
|
||||||
|
console.log(`💬 New message (${props.role || 'assistant'})`);
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 'message.updated':
|
||||||
|
// Message updates happen frequently during streaming
|
||||||
|
// Only log role changes or completion
|
||||||
|
if (props.role === 'user') {
|
||||||
|
console.log(`👤 User message received`);
|
||||||
|
}
|
||||||
|
// Skip assistant message updates (too noisy)
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 'part.created':
|
||||||
|
case 'part.updated':
|
||||||
|
// Parts contain the actual content - tools, text, etc.
|
||||||
|
if (props.type === 'tool') {
|
||||||
|
const toolName = props.tool || 'unknown';
|
||||||
|
const status = props.state?.status || props.status || '';
|
||||||
|
|
||||||
|
// Only log when tool starts or completes
|
||||||
|
if (status === 'running' || status === 'pending') {
|
||||||
|
console.log(`🔧 Tool: ${toolName} (starting)`);
|
||||||
|
|
||||||
|
// Show tool input preview
|
||||||
|
const input = props.state?.input || props.input || {};
|
||||||
|
if (input.command) {
|
||||||
|
const cmd = input.command.substring(0, 70);
|
||||||
|
console.log(` └─ ${cmd}${input.command.length > 70 ? '...' : ''}`);
|
||||||
|
} else if (input.filePath) {
|
||||||
|
console.log(` └─ ${input.filePath}`);
|
||||||
|
} else if (input.pattern) {
|
||||||
|
console.log(` └─ pattern: ${input.pattern}`);
|
||||||
|
}
|
||||||
|
} else if (status === 'completed') {
|
||||||
|
console.log(`✅ Tool: ${toolName} (completed)`);
|
||||||
|
} else if (status === 'error') {
|
||||||
|
console.log(`❌ Tool: ${toolName} (error)`);
|
||||||
|
}
|
||||||
|
} else if (props.type === 'text') {
|
||||||
|
// Text parts - show preview of assistant response
|
||||||
|
const text = props.text || '';
|
||||||
|
if (text.length > 0) {
|
||||||
|
const preview = text.substring(0, 100).replace(/\n/g, ' ');
|
||||||
|
console.log(`📝 ${preview}${text.length > 100 ? '...' : ''}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 'permission.request':
|
||||||
|
console.log(`🔐 Permission requested: ${props.tool || 'unknown'}`);
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 'permission.response':
|
||||||
|
console.log(`🔐 Permission ${props.response === 'once' || props.approved ? 'granted' : 'denied'}`);
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 'tool.call':
|
||||||
|
console.log(`🔧 Tool call: ${props.tool || props.name || 'unknown'}`);
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 'tool.result':
|
||||||
|
const success = props.error ? '❌' : '✅';
|
||||||
|
console.log(`${success} Tool result: ${props.tool || 'unknown'}`);
|
||||||
|
break;
|
||||||
|
|
||||||
|
default:
|
||||||
|
// Skip unknown events to reduce noise
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue