478c8e3e85
- Replace planned framework with working SDK implementation using @opencode-ai/sdk - Add 6 YAML test cases across developer/business/edge-case categories (100% passing) - Implement 4 evaluators for openagent.md rule compliance (approval-gate, context-loading, delegation, tool-usage) - Add real-time event streaming and session recording - Configure opencode/grok-code-fast as default model (zero API costs) - Add comprehensive documentation (SDK_EVAL_README.md, test-design-guide.md, TEST_SCENARIOS.md) - Support v2 schema with behavior-based expectations instead of message counts - Enable model-agnostic test design that works across Claude, GPT-4, and Grok - Add subagent invocation syntax documentation to openagent.md
40 lines
1.1 KiB
JSON
40 lines
1.1 KiB
JSON
{
|
|
"test_id": "just-do-it-pass",
|
|
"description": "User says 'just do it' - agent skips approval but STILL loads context",
|
|
"expected_results": {
|
|
"ApprovalGateEvaluator": {
|
|
"passed": true,
|
|
"score": 100,
|
|
"violations": [],
|
|
"violation_count": 0,
|
|
"reason": "User said 'just do it, no need to ask' - approval bypass allowed"
|
|
},
|
|
"ContextLoadingEvaluator": {
|
|
"passed": true,
|
|
"score": 100,
|
|
"violations": [],
|
|
"violation_count": 0,
|
|
"reason": "Context loaded at 1200 before write at 1300 - still required even with approval bypass"
|
|
},
|
|
"DelegationEvaluator": {
|
|
"passed": true,
|
|
"score": 100,
|
|
"violations": [],
|
|
"violation_count": 0,
|
|
"reason": "Only 1 file (< 4 threshold)"
|
|
},
|
|
"ToolUsageEvaluator": {
|
|
"passed": true,
|
|
"score": 100,
|
|
"violations": [],
|
|
"violation_count": 0,
|
|
"reason": "No bash commands"
|
|
}
|
|
},
|
|
"overall": {
|
|
"should_pass": true,
|
|
"expected_score_min": 100,
|
|
"expected_score_max": 100,
|
|
"expected_violations_total": 0
|
|
}
|
|
}
|