478c8e3e85
- Replace planned framework with working SDK implementation using @opencode-ai/sdk - Add 6 YAML test cases across developer/business/edge-case categories (100% passing) - Implement 4 evaluators for openagent.md rule compliance (approval-gate, context-loading, delegation, tool-usage) - Add real-time event streaming and session recording - Configure opencode/grok-code-fast as default model (zero API costs) - Add comprehensive documentation (SDK_EVAL_README.md, test-design-guide.md, TEST_SCENARIOS.md) - Support v2 schema with behavior-based expectations instead of message counts - Enable model-agnostic test design that works across Claude, GPT-4, and Grok - Add subagent invocation syntax documentation to openagent.md
106 lines
3.6 KiB
JavaScript
106 lines
3.6 KiB
JavaScript
/**
|
||
* Quick test script to verify the framework works with real session data
|
||
*/
|
||
|
||
const { SessionReader, MessageParser, TimelineBuilder } = require('./dist/index.js');
|
||
|
||
// Test with the opencode-agents project
|
||
const projectPath = '/Users/darrenhinde/Documents/GitHub/opencode-agents';
|
||
|
||
console.log('🔍 Testing OpenCode Evaluation Framework\n');
|
||
console.log('Project:', projectPath);
|
||
console.log('─'.repeat(60));
|
||
|
||
// Create reader
|
||
const reader = new SessionReader(projectPath);
|
||
|
||
// List sessions
|
||
console.log('\n📋 Listing sessions...');
|
||
const sessions = reader.listSessions();
|
||
console.log(`Found ${sessions.length} sessions`);
|
||
|
||
if (sessions.length > 0) {
|
||
// Show first 3 sessions
|
||
console.log('\nMost recent sessions:');
|
||
sessions.slice(0, 3).forEach((session, i) => {
|
||
const date = new Date(session.time.created).toLocaleString();
|
||
console.log(` ${i + 1}. ${session.id}`);
|
||
console.log(` Title: ${session.title.substring(0, 60)}...`);
|
||
console.log(` Created: ${date}`);
|
||
});
|
||
|
||
// Test with first session
|
||
const testSession = sessions[0];
|
||
console.log('\n─'.repeat(60));
|
||
console.log(`\n🧪 Testing with session: ${testSession.id}\n`);
|
||
|
||
// Get messages
|
||
const messages = reader.getMessages(testSession.id);
|
||
console.log(`📨 Messages: ${messages.length}`);
|
||
|
||
// Create parser
|
||
const parser = new MessageParser();
|
||
|
||
// Analyze messages
|
||
let userMessages = 0;
|
||
let assistantMessages = 0;
|
||
let agents = new Set();
|
||
let models = new Set();
|
||
|
||
messages.forEach(msg => {
|
||
if (msg.role === 'user') userMessages++;
|
||
if (msg.role === 'assistant') {
|
||
assistantMessages++;
|
||
const agent = parser.getAgent(msg);
|
||
if (agent) agents.add(agent);
|
||
const model = parser.getModel(msg);
|
||
if (model) models.add(model.modelID);
|
||
}
|
||
});
|
||
|
||
console.log(` - User messages: ${userMessages}`);
|
||
console.log(` - Assistant messages: ${assistantMessages}`);
|
||
console.log(` - Agents: ${Array.from(agents).join(', ') || 'none'}`);
|
||
console.log(` - Models: ${Array.from(models).join(', ') || 'none'}`);
|
||
|
||
// Build timeline
|
||
console.log('\n⏱️ Building timeline...');
|
||
const builder = new TimelineBuilder(reader);
|
||
const timeline = builder.buildTimeline(testSession.id);
|
||
|
||
const summary = builder.getSummary(timeline);
|
||
console.log(` - Total events: ${summary.totalEvents}`);
|
||
console.log(` - User messages: ${summary.userMessages}`);
|
||
console.log(` - Assistant messages: ${summary.assistantMessages}`);
|
||
console.log(` - Tool calls: ${summary.toolCalls}`);
|
||
console.log(` - Tools used: ${summary.tools.join(', ') || 'none'}`);
|
||
console.log(` - Duration: ${(summary.duration / 1000).toFixed(2)}s`);
|
||
|
||
// Check for execution tools
|
||
const toolCalls = builder.getToolCalls(timeline);
|
||
const executionTools = ['bash', 'write', 'edit', 'task'];
|
||
const usedExecutionTools = summary.tools.filter(t => executionTools.includes(t));
|
||
|
||
if (usedExecutionTools.length > 0) {
|
||
console.log(`\n⚙️ Execution tools used: ${usedExecutionTools.join(', ')}`);
|
||
|
||
// Check for approval
|
||
const assistantMsgs = builder.getAssistantMessages(timeline);
|
||
let foundApproval = false;
|
||
|
||
for (const event of assistantMsgs) {
|
||
if (parser.hasApprovalRequest(event.data.parts)) {
|
||
foundApproval = true;
|
||
break;
|
||
}
|
||
}
|
||
|
||
console.log(` - Approval requested: ${foundApproval ? '✅ Yes' : '❌ No'}`);
|
||
}
|
||
|
||
console.log('\n─'.repeat(60));
|
||
console.log('\n✅ Framework test completed successfully!\n');
|
||
} else {
|
||
console.log('\n⚠️ No sessions found for this project');
|
||
console.log(' Try running OpenCode in this project first to generate session data.\n');
|
||
}
|