880396d37e
* Improve LLM integration tests: replace 'always pass' tests with meaningful validation - Replace old llm-agent-behavior.test.ts (14 tests that always passed) with new llm-integration.test.ts (10 tests that can actually fail) - Remove redundant tests already covered by unit tests (bash antipatterns, auto-fix detection) - Add behavior-based validation using framework's built-in expectations (requiresApproval, mustUseDedicatedTools, requiresContext) - Improve test resilience with graceful timeout handling for LLM unpredictability - Add comprehensive validation report (LLM_INTEGRATION_VALIDATION.md) - Remove outdated documentation (LLM_AGENT_TESTING.md, LLM_TEST_SUITE.md, PHASE_3_4_5_SUMMARY.md, COMPREHENSIVE_TEST_REPORT.md) Test Results: - All 10 LLM integration tests passing (42s execution time, 25% faster) - Total test suite: 317 tests, 316 passing (99.7%) - Tests proven to catch real issues (failed during development) - Reviewer approved: Production ready (9.5/10 confidence) What Changed: - Tests can now actually fail (vs always passing) - Use behavior expectations instead of forcing violations - Focus on integration testing, not violation detection - Honest about purpose and limitations * fix(registry): add missing agents to installation profiles (#64) - Add development agents (frontend-specialist, backend-specialist, devops-specialist, codebase-agent) to developer profile - Add content agents (copywriter, technical-writer) and data-analyst to business profile - Add all new agents to full and advanced profiles - Add eval-runner and repo-manager to appropriate profiles - Add context-retriever subagent to advanced profile Create validation and documentation: - Add profile coverage validation script (scripts/registry/validate-profile-coverage.sh) - Add profile validation guide (.opencode/context/openagents-repo/guides/profile-validation.md) - Add subagent invocation guide (.opencode/context/openagents-repo/guides/subagent-invocation.md) - Document issue resolution (ISSUE_64_RESOLUTION.md) Fixes #64 - Users installing with profiles now receive all agents added in v0.5.0
81 lines
2.7 KiB
Bash
Executable file
81 lines
2.7 KiB
Bash
Executable file
#!/bin/bash
|
|
# Check if all agents are in appropriate profiles
|
|
|
|
set -e
|
|
|
|
echo "🔍 Checking profile coverage..."
|
|
echo ""
|
|
|
|
# Get all agent IDs
|
|
agents=$(cat registry.json | jq -r '.components.agents[].id')
|
|
|
|
errors=0
|
|
|
|
for agent in $agents; do
|
|
# Get agent category
|
|
category=$(cat registry.json | jq -r ".components.agents[] | select(.id == \"$agent\") | .category")
|
|
|
|
# Check which profiles include this agent
|
|
in_essential=$(cat registry.json | jq -r ".profiles.essential.components[] | select(. == \"agent:$agent\")" 2>/dev/null || echo "")
|
|
in_developer=$(cat registry.json | jq -r ".profiles.developer.components[] | select(. == \"agent:$agent\")" 2>/dev/null || echo "")
|
|
in_business=$(cat registry.json | jq -r ".profiles.business.components[] | select(. == \"agent:$agent\")" 2>/dev/null || echo "")
|
|
in_full=$(cat registry.json | jq -r ".profiles.full.components[] | select(. == \"agent:$agent\")" 2>/dev/null || echo "")
|
|
in_advanced=$(cat registry.json | jq -r ".profiles.advanced.components[] | select(. == \"agent:$agent\")" 2>/dev/null || echo "")
|
|
|
|
# Validate based on category
|
|
case $category in
|
|
"development")
|
|
if [[ -z "$in_developer" ]]; then
|
|
echo "❌ $agent (development) missing from developer profile"
|
|
errors=$((errors + 1))
|
|
fi
|
|
if [[ -z "$in_full" ]]; then
|
|
echo "❌ $agent (development) missing from full profile"
|
|
errors=$((errors + 1))
|
|
fi
|
|
if [[ -z "$in_advanced" ]]; then
|
|
echo "❌ $agent (development) missing from advanced profile"
|
|
errors=$((errors + 1))
|
|
fi
|
|
;;
|
|
"content"|"data")
|
|
if [[ -z "$in_business" ]]; then
|
|
echo "❌ $agent ($category) missing from business profile"
|
|
errors=$((errors + 1))
|
|
fi
|
|
if [[ -z "$in_full" ]]; then
|
|
echo "❌ $agent ($category) missing from full profile"
|
|
errors=$((errors + 1))
|
|
fi
|
|
if [[ -z "$in_advanced" ]]; then
|
|
echo "❌ $agent ($category) missing from advanced profile"
|
|
errors=$((errors + 1))
|
|
fi
|
|
;;
|
|
"meta")
|
|
if [[ -z "$in_advanced" ]]; then
|
|
echo "❌ $agent (meta) missing from advanced profile"
|
|
errors=$((errors + 1))
|
|
fi
|
|
;;
|
|
"essential"|"standard")
|
|
if [[ -z "$in_full" ]]; then
|
|
echo "❌ $agent ($category) missing from full profile"
|
|
errors=$((errors + 1))
|
|
fi
|
|
if [[ -z "$in_advanced" ]]; then
|
|
echo "❌ $agent ($category) missing from advanced profile"
|
|
errors=$((errors + 1))
|
|
fi
|
|
;;
|
|
esac
|
|
done
|
|
|
|
echo ""
|
|
if [[ $errors -eq 0 ]]; then
|
|
echo "✅ Profile coverage check complete - no issues found"
|
|
exit 0
|
|
else
|
|
echo "❌ Profile coverage check found $errors issue(s)"
|
|
exit 1
|
|
fi
|