OpenWriterControle/evals/framework/src/sdk/test-runner.ts

/**
 * TestRunner - Orchestrates test execution
 * 
 * This is a thin orchestrator that coordinates:
 * - Server lifecycle management
 * - Test execution via TestExecutor
 * - Result validation via ResultValidator
 * - Evaluator management
 * 
 * The actual execution and validation logic has been extracted to:
 * - test-executor.ts - Core test execution
 * - result-validator.ts - Result validation
 * - event-logger.ts - Event logging utilities
 */

import { ServerManager } from './server-manager.js';
import { ClientManager } from './client-manager.js';
import { EventStreamHandler } from './event-stream-handler.js';
import { AutoApproveStrategy } from './approval/auto-approve-strategy.js';
import { AutoDenyStrategy } from './approval/auto-deny-strategy.js';
import { SmartApprovalStrategy } from './approval/smart-approval-strategy.js';
import { SessionReader } from '../collector/session-reader.js';
import { TimelineBuilder } from '../collector/timeline-builder.js';
import { EvaluatorRunner } from '../evaluators/evaluator-runner.js';
import { ApprovalGateEvaluator } from '../evaluators/approval-gate-evaluator.js';
import { ContextLoadingEvaluator } from '../evaluators/context-loading-evaluator.js';
import { DelegationEvaluator } from '../evaluators/delegation-evaluator.js';
import { ToolUsageEvaluator } from '../evaluators/tool-usage-evaluator.js';
import { StopOnFailureEvaluator } from '../evaluators/stop-on-failure-evaluator.js';
import { ReportFirstEvaluator } from '../evaluators/report-first-evaluator.js';
import { CleanupConfirmationEvaluator } from '../evaluators/cleanup-confirmation-evaluator.js';
import { BehaviorEvaluator } from '../evaluators/behavior-evaluator.js';
import { TestExecutor } from './test-executor.js';
import { ResultValidator } from './result-validator.js';
import { createLogger } from './event-logger.js';
import type { TestCase } from './test-case-schema.js';
import type { ApprovalStrategy } from './approval/approval-strategy.js';
import type { ServerEvent } from './event-stream-handler.js';
import type { AggregatedResult } from '../evaluators/evaluator-runner.js';
import { homedir } from 'os';
import { join } from 'path';
import { findGitRoot } from '../config.js';

export interface TestRunnerConfig {
  /**
   * Port for opencode server (0 = random)
   */
  port?: number;

  /**
   * Enable debug logging
   */
  debug?: boolean;

  /**
   * Default timeout for tests (ms)
   */
  defaultTimeout?: number;

  /**
   * Project path for evaluators
   * 
   * IMPORTANT: This should be the git root where the agent runs, not the test framework directory.
   * 
   * Default behavior:
   * - Automatically finds git root by walking up from process.cwd()
   * - This ensures sessions created by agents are found correctly
   * 
   * When to override:
   * - Testing agents in non-git directories
   * - Testing multiple agents with different project roots
   * - Custom session storage locations
   * 
   * Example:
   * - Git root: /Users/user/opencode-agents (sessions stored here)
   * - Test CWD: /Users/user/opencode-agents/evals/framework (tests run here)
   * - projectPath should be git root, not test CWD
   */
  projectPath?: string;

  /**
   * Run evaluators after test execution
   */
  runEvaluators?: boolean;

  /**
   * Default model to use for tests (format: provider/model)
   * Examples:
   * - "opencode/grok-code-fast" (free tier)
   * - "anthropic/claude-3-5-sonnet-20241022"
   * - "openai/gpt-4-turbo"
   */
  defaultModel?: string;
}

export interface TestResult {
  /**
   * Test case that was run
   */
  testCase: TestCase;

  /**
   * Session ID created for this test
   */
  sessionId: string;

  /**
   * Whether the test passed
   */
  passed: boolean;

  /**
   * Errors encountered during test execution
   */
  errors: string[];

  /**
   * Events captured during test
   */
  events: ServerEvent[];

  /**
   * Duration of test execution (ms)
   */
  duration: number;

  /**
   * Number of approvals given
   */
  approvalsGiven: number;

  /**
   * Path to recorded session data
   */
  sessionPath?: string;

  /**
   * Evaluation results from evaluators (if runEvaluators = true)
   */
  evaluation?: AggregatedResult;
}

/**
 * TestRunner orchestrates the test execution process
 */
export class TestRunner {
  private server: ServerManager;
  private client: ClientManager | null = null;
  private eventHandler: EventStreamHandler | null = null;
  private config: Required<TestRunnerConfig>;
  private evaluatorRunner: EvaluatorRunner | null = null;
  private executor: TestExecutor | null = null;
  private validator: ResultValidator;
  private logger: ReturnType<typeof createLogger>;

  constructor(config: TestRunnerConfig = {}) {
    // Find git root for agent detection
    const gitRoot = findGitRoot(process.cwd());
    
    this.config = {
      port: config.port || 0,
      debug: config.debug || false,
      defaultTimeout: config.defaultTimeout || 60000,
      projectPath: config.projectPath || gitRoot,
      runEvaluators: config.runEvaluators ?? true,
      defaultModel: config.defaultModel || 'opencode/grok-code',
    };

    // Create logger
    this.logger = createLogger(this.config.debug);

    // Create validator
    this.validator = new ResultValidator(this.logger);

    // Start server from git root with default agent
    this.server = new ServerManager({
      port: this.config.port,
      timeout: 10000,
      cwd: gitRoot,
      debug: this.config.debug,
      agent: 'openagent',
    });

    if (this.config.debug) {
      console.log(`[TestRunner] Git root: ${gitRoot}`);
      console.log(`[TestRunner] Server will start from: ${gitRoot} with agent: openagent`);
    }
  }

  /**
   * Start the test runner (starts opencode server)
   */
  async start(): Promise<void> {
    this.logger.log('Starting opencode server...');
    const { url } = await this.server.start();
    this.logger.log(`Server started at ${url}`);

    this.client = new ClientManager({ baseUrl: url });
    this.eventHandler = new EventStreamHandler(url);

    // Create executor
    this.executor = new TestExecutor(
      this.client,
      this.eventHandler,
      {
        defaultTimeout: this.config.defaultTimeout,
        projectPath: this.config.projectPath,
        defaultModel: this.config.defaultModel,
        debug: this.config.debug,
      },
      this.logger
    );

    // Setup evaluators
    if (this.config.runEvaluators && this.client) {
      this.setupEvaluators();
    }
  }

  /**
   * Setup evaluators with SDK client
   */
  private setupEvaluators(): void {
    if (!this.client) return;

    const sessionStoragePath = join(homedir(), '.local', 'share', 'opencode');
    const sdkClient = this.client.getClient();
    const sessionReader = new SessionReader(sdkClient, sessionStoragePath);
    const timelineBuilder = new TimelineBuilder(sessionReader);

    this.evaluatorRunner = new EvaluatorRunner({
      sessionReader,
      timelineBuilder,
      sdkClient,
      evaluators: [
        new ApprovalGateEvaluator(),
        new ContextLoadingEvaluator(),
        new DelegationEvaluator(),
        new ToolUsageEvaluator(),
        new StopOnFailureEvaluator(),
        new ReportFirstEvaluator(),
        new CleanupConfirmationEvaluator(),
      ],
    });

    if (this.config.debug) {
      this.logger.log('[TestRunner] Evaluators initialized with SDK client');
    }
  }

  /**
   * Stop the test runner (stops server)
   */
  async stop(): Promise<void> {
    this.logger.log('Stopping event handler...');
    if (this.eventHandler) {
      this.eventHandler.stopListening();
      this.eventHandler = null;
    }

    this.logger.log('Stopping server...');
    await this.server.stop();
    this.client = null;
    this.executor = null;
  }

  /**
   * Run a single test case
   */
  async runTest(testCase: TestCase): Promise<TestResult> {
    if (!this.client || !this.eventHandler || !this.executor) {
      throw new Error('Test runner not started. Call start() first.');
    }

    // Stop event handler if it's still listening from previous test
    if (this.eventHandler.listening()) {
      this.eventHandler.stopListening();
      // Wait a bit for cleanup
      await new Promise(resolve => setTimeout(resolve, 500));
    }

    // Create approval strategy
    const approvalStrategy = this.createApprovalStrategy(testCase);

    // Execute test
    const executionResult = await this.executor.execute(testCase, approvalStrategy);

    // Run evaluators if enabled
    let evaluation: AggregatedResult | undefined;
    if (this.config.runEvaluators && this.evaluatorRunner && executionResult.sessionId) {
      evaluation = await this.runEvaluators(testCase, executionResult.sessionId);
    }

    // Validate result
    const passed = this.validator.validate(
      testCase,
      executionResult.events,
      executionResult.errors,
      evaluation
    );

    // Log summary
    this.logTestSummary(passed, executionResult, evaluation);

    return {
      testCase,
      sessionId: executionResult.sessionId,
      passed,
      errors: executionResult.errors,
      events: executionResult.events,
      duration: executionResult.duration,
      approvalsGiven: executionResult.approvalsGiven,
      evaluation,
    };
  }

  /**
   * Run evaluators for a test
   */
  private async runEvaluators(
    testCase: TestCase,
    sessionId: string
  ): Promise<AggregatedResult | undefined> {
    if (!this.evaluatorRunner) return undefined;

    this.logger.log('Running evaluators...');
    
    // Add behavior evaluator if test case has behavior expectations
    if (testCase.behavior) {
      this.logger.log('Adding behavior evaluator for test expectations...');
      const behaviorEvaluator = new BehaviorEvaluator(testCase.behavior);
      this.evaluatorRunner.register(behaviorEvaluator);
    }
    
    try {
      const evaluation = await this.evaluatorRunner.runAll(sessionId);
      this.logger.log(`Evaluators completed: ${evaluation.totalViolations} violations found`);
      
      if (evaluation && evaluation.totalViolations > 0) {
        this.logger.log(`  Errors: ${evaluation.violationsBySeverity.error}`);
        this.logger.log(`  Warnings: ${evaluation.violationsBySeverity.warning}`);
      }
      
      // Clean up behavior evaluator after use
      if (testCase.behavior) {
        this.evaluatorRunner.unregister('behavior');
      }

      return evaluation;
    } catch (error) {
      this.logger.log(`Warning: Evaluators failed: ${(error as Error).message}`);
      return undefined;
    }
  }

  /**
   * Log test summary
   */
  private logTestSummary(
    passed: boolean,
    executionResult: { duration: number; events: ServerEvent[]; approvalsGiven: number; errors: string[] },
    evaluation?: AggregatedResult
  ): void {
    this.logger.log(`\nTest ${passed ? 'PASSED' : 'FAILED'}`);
    this.logger.log(`Duration: ${executionResult.duration}ms`);
    this.logger.log(`Events captured: ${executionResult.events.length}`);
    this.logger.log(`Approvals given: ${executionResult.approvalsGiven}`);
    this.logger.log(`Errors: ${executionResult.errors.length}`);
    
    if (evaluation) {
      this.logger.log(`Evaluation Score: ${evaluation.overallScore}/100`);
      this.logger.log(`Violations: ${evaluation.totalViolations} (E:${evaluation.violationsBySeverity.error} W:${evaluation.violationsBySeverity.warning})`);
    }
  }

  /**
   * Run multiple test cases
   */
  async runTests(testCases: TestCase[]): Promise<TestResult[]> {
    const results: TestResult[] = [];

    for (let i = 0; i < testCases.length; i++) {
      const testCase = testCases[i];
      
      // Add delay between tests to avoid rate limiting (except for first test)
      if (i > 0) {
        const delayMs = 3000; // 3 second delay between tests
        this.logger.log(`⏳ Waiting ${delayMs}ms before next test to avoid rate limiting...\n`);
        await new Promise(resolve => setTimeout(resolve, delayMs));
      }
      
      const result = await this.runTest(testCase);
      results.push(result);

      // Clean up session after each test (skip in debug mode to allow inspection)
      if (this.client && result.sessionId && !this.config.debug) {
        try {
          await this.client.deleteSession(result.sessionId);
          this.logger.log(`Cleaned up session: ${result.sessionId}\n`);
        } catch (error) {
          this.logger.log(`Failed to clean up session: ${(error as Error).message}\n`);
        }
      } else if (this.config.debug) {
        this.logger.log(`Debug mode: Keeping session ${result.sessionId} for inspection\n`);
      }
    }

    return results;
  }

  /**
   * Create approval strategy from test case config
   */
  private createApprovalStrategy(testCase: TestCase): ApprovalStrategy {
    const strategy = testCase.approvalStrategy;

    switch (strategy.type) {
      case 'auto-approve':
        return new AutoApproveStrategy();

      case 'auto-deny':
        return new AutoDenyStrategy();

      case 'smart':
        return new SmartApprovalStrategy({
          allowedTools: strategy.config?.allowedTools,
          deniedTools: strategy.config?.deniedTools,
          approvePatterns: strategy.config?.approvePatterns?.map(p => new RegExp(p)),
          denyPatterns: strategy.config?.denyPatterns?.map(p => new RegExp(p)),
          maxApprovals: strategy.config?.maxApprovals,
          defaultDecision: strategy.config?.defaultDecision,
        });

      default:
        throw new Error(`Unknown approval strategy: ${(strategy as any).type}`);
    }
  }
}