f773b290ce
Documentation: - Add ARCHITECTURE.md with comprehensive system review - Add GETTING_STARTED.md for new users - Add DOCUMENTATION_CLEANUP.md tracking cleanup changes - Add SCRIPTS_ORGANIZATION.md documenting script reorganization - Remove outdated docs (TESTING_CONFIDENCE.md, TEST_REVIEW.md, SESSION_STORAGE_FIX.md) - Update README files with current information Script Organization: - Move 12 scripts from framework root to organized directories: - scripts/debug/ (4 files): Session debugging tools - scripts/test/ (6 files): Test execution scripts - scripts/utils/ (2 files): Utility scripts - Add comprehensive scripts/README.md with usage examples Test Infrastructure: - Add 5 new context loading tests for openagent - Add result-saver.ts for test result persistence - Add interactive results dashboard (index.html) - Add error-handling.test.ts for framework testing - Add result-saver.test.ts with comprehensive unit tests Evaluator Improvements: - Enhance context-loading-evaluator with better detection - Update run-sdk-tests.ts with improved result handling Agent Configuration: - Add .opencode/agent/AGENT.md for agent definition Test Utilities: - Add test-agent-manual.mjs for manual testing - Update test_tmp/README.md with usage guidelines This commit consolidates multiple improvements: - Better organization (scripts, docs) - Improved testing infrastructure - Comprehensive documentation - Cleaner codebase structure
98 lines
2.6 KiB
Bash
Executable file
98 lines
2.6 KiB
Bash
Executable file
#!/bin/bash
|
|
|
|
# Batch Test Runner for OpenCode Agents
|
|
# Runs tests in smaller batches to avoid API timeouts and rate limits
|
|
|
|
set -e
|
|
|
|
# Colors for output
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
RED='\033[0;31m'
|
|
NC='\033[0m' # No Color
|
|
|
|
# Configuration
|
|
AGENT="${1:-openagent}"
|
|
BATCH_SIZE="${2:-3}"
|
|
DELAY_BETWEEN_BATCHES="${3:-10}"
|
|
|
|
echo -e "${GREEN}🚀 Batch Test Runner${NC}"
|
|
echo "Agent: $AGENT"
|
|
echo "Batch size: $BATCH_SIZE tests"
|
|
echo "Delay between batches: ${DELAY_BETWEEN_BATCHES}s"
|
|
echo ""
|
|
|
|
# Get all test files
|
|
TEST_DIR="../agents/$AGENT/tests"
|
|
|
|
if [ ! -d "$TEST_DIR" ]; then
|
|
echo -e "${RED}❌ Test directory not found: $TEST_DIR${NC}"
|
|
exit 1
|
|
fi
|
|
|
|
# Find all test files
|
|
TEST_FILES=$(find "$TEST_DIR" -name "*.yaml" -type f | sort)
|
|
TOTAL_TESTS=$(echo "$TEST_FILES" | wc -l | tr -d ' ')
|
|
|
|
echo -e "${GREEN}Found $TOTAL_TESTS test files${NC}"
|
|
echo ""
|
|
|
|
# Split into batches
|
|
BATCH_NUM=1
|
|
CURRENT_BATCH=()
|
|
BATCH_COUNT=0
|
|
|
|
for TEST_FILE in $TEST_FILES; do
|
|
CURRENT_BATCH+=("$TEST_FILE")
|
|
BATCH_COUNT=$((BATCH_COUNT + 1))
|
|
|
|
# Run batch when it reaches BATCH_SIZE or is the last file
|
|
if [ $BATCH_COUNT -eq $BATCH_SIZE ] || [ $(echo "$TEST_FILES" | grep -c "$TEST_FILE") -eq $TOTAL_TESTS ]; then
|
|
echo -e "${YELLOW}📦 Running Batch $BATCH_NUM (${#CURRENT_BATCH[@]} tests)${NC}"
|
|
echo "----------------------------------------"
|
|
|
|
# Build pattern for this batch
|
|
PATTERNS=""
|
|
for FILE in "${CURRENT_BATCH[@]}"; do
|
|
# Extract relative path from test directory
|
|
REL_PATH=$(echo "$FILE" | sed "s|$TEST_DIR/||")
|
|
echo " - $REL_PATH"
|
|
|
|
# Add to pattern (run each test individually to avoid conflicts)
|
|
if [ -z "$PATTERNS" ]; then
|
|
PATTERNS="$REL_PATH"
|
|
else
|
|
PATTERNS="$PATTERNS|$REL_PATH"
|
|
fi
|
|
done
|
|
|
|
echo ""
|
|
|
|
# Run tests in this batch
|
|
for FILE in "${CURRENT_BATCH[@]}"; do
|
|
REL_PATH=$(echo "$FILE" | sed "s|$TEST_DIR/||")
|
|
echo -e "${GREEN}▶ Running: $REL_PATH${NC}"
|
|
|
|
npm run eval:sdk -- --agent="$AGENT" --pattern="$REL_PATH" 2>&1 | grep -E "(PASSED|FAILED|Duration|Violations)" || true
|
|
|
|
echo ""
|
|
done
|
|
|
|
# Reset batch
|
|
CURRENT_BATCH=()
|
|
BATCH_COUNT=0
|
|
BATCH_NUM=$((BATCH_NUM + 1))
|
|
|
|
# Delay between batches (except for last batch)
|
|
if [ $BATCH_NUM -le $((TOTAL_TESTS / BATCH_SIZE + 1)) ]; then
|
|
echo -e "${YELLOW}⏳ Waiting ${DELAY_BETWEEN_BATCHES}s before next batch...${NC}"
|
|
echo ""
|
|
sleep $DELAY_BETWEEN_BATCHES
|
|
fi
|
|
fi
|
|
done
|
|
|
|
echo -e "${GREEN}✅ All batches completed!${NC}"
|
|
echo ""
|
|
echo "View results:"
|
|
echo " cd ../results && ./serve.sh"
|