#!/usr/bin/env node /** * Aggregates E2E test results and benchmark data from CI runs * into a unified worlds-status.json for the dashboard. * * Usage: * node scripts/aggregate-worlds-data.mjs [results-dir] [--output path/to/output.json] * * Input files expected: * - e2e-results-{world}.json: Vitest JSON output for E2E tests * - bench-results-{app}-{world}.json: Vitest benchmark output * - bench-timings-{app}-{world}.json: Custom timing data * * Output: * - worlds-status.json: Combined status of all worlds */ import fs from 'fs'; import path from 'path'; import { fileURLToPath } from 'url'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const rootDir = path.join(__dirname, '..'); // Parse command line arguments const args = process.argv.slice(2); let resultsDir = '.'; let outputPath = path.join(rootDir, 'docs/public/data/worlds-status.json'); for (let i = 0; i < args.length; i++) { if (args[i] === '--output' && args[i + 1]) { outputPath = args[i + 1]; i++; } else if (!args[i].startsWith('--')) { resultsDir = args[i]; } } // Load worlds manifest const manifestPath = path.join(rootDir, 'worlds-manifest.json'); let manifest; try { manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8')); } catch (e) { console.error(`Error: Could not load worlds manifest: ${e.message}`); process.exit(1); } // Get all worlds from manifest (now a flat array with type field) function getAllWorlds() { return manifest.worlds || []; } // Find all E2E result files function findE2EResultFiles(dir) { const files = []; try { const entries = fs.readdirSync(dir, { withFileTypes: true }); for (const entry of entries) { const fullPath = path.join(dir, entry.name); if (entry.isDirectory()) { files.push(...findE2EResultFiles(fullPath)); } else if ( entry.name.startsWith('e2e-results-') && entry.name.endsWith('.json') ) { files.push(fullPath); } } } catch (_e) { // Directory may not exist } return files; } // Find all benchmark result files function findBenchmarkFiles(dir) { const files = []; try { const entries = fs.readdirSync(dir, { withFileTypes: true }); for (const entry of entries) { const fullPath = path.join(dir, entry.name); if (entry.isDirectory()) { files.push(...findBenchmarkFiles(fullPath)); } else if ( entry.name.startsWith('bench-timings-') && entry.name.endsWith('.json') ) { files.push(fullPath); } } } catch (_e) { // Directory may not exist } return files; } // Parse E2E result file (vitest JSON output) function parseE2EResults(filePath) { try { const data = JSON.parse(fs.readFileSync(filePath, 'utf-8')); let total = 0; let passed = 0; let failed = 0; let skipped = 0; const tests = []; // Vitest JSON format for (const file of data.testResults || []) { for (const test of file.assertionResults || []) { total++; const status = test.status; if (status === 'passed') passed++; else if (status === 'failed') failed++; else if (status === 'skipped' || status === 'pending') skipped++; tests.push({ name: test.fullName || test.title, status: status === 'pending' ? 'skipped' : status, duration: test.duration || 0, }); } } // Alternative format (direct vitest output) if (total === 0 && data.numTotalTests) { total = data.numTotalTests; passed = data.numPassedTests || 0; failed = data.numFailedTests || 0; skipped = (data.numPendingTests || 0) + (data.numTodoTests || 0); } return { total, passed, failed, skipped, tests }; } catch (e) { console.error( `Warning: Could not parse E2E results ${filePath}: ${e.message}` ); return null; } } // Parse benchmark timing file function parseBenchmarkTimings(filePath) { try { const data = JSON.parse(fs.readFileSync(filePath, 'utf-8')); const metrics = {}; if (data.summary) { for (const [benchName, stats] of Object.entries(data.summary)) { metrics[benchName] = { mean: stats.avgExecutionTimeMs, min: stats.minExecutionTimeMs, max: stats.maxExecutionTimeMs, samples: stats.samples, }; // Add TTFB for stream benchmarks if (stats.avgFirstByteTimeMs !== undefined) { metrics[benchName].ttfb = { mean: stats.avgFirstByteTimeMs, min: stats.minFirstByteTimeMs, max: stats.maxFirstByteTimeMs, }; } } } return metrics; } catch (e) { console.error( `Warning: Could not parse benchmark timings ${filePath}: ${e.message}` ); return null; } } // Extract world ID from filename function extractWorldFromFilename(filename, prefix) { // e2e-results-{world}.json -> world // bench-timings-{app}-{world}.json -> world const basename = path.basename(filename, '.json'); const withoutPrefix = basename.replace(prefix, ''); // For bench files, format is {app}-{world}, we want the last part const parts = withoutPrefix.split('-'); return parts[parts.length - 1]; } // Aggregate all data function aggregateWorldsData() { const allWorlds = getAllWorlds(); const timestamp = new Date().toISOString(); // Initialize worlds status const worldsStatus = {}; for (const world of allWorlds) { worldsStatus[world.id] = { type: world.type, name: world.name, package: world.package, description: world.description, docs: world.docs, repository: world.repository, e2e: null, benchmark: null, }; } // Process E2E results const e2eFiles = findE2EResultFiles(resultsDir); for (const file of e2eFiles) { const worldId = extractWorldFromFilename(file, 'e2e-results-'); if (worldsStatus[worldId]) { const results = parseE2EResults(file); if (results) { const progress = results.total > 0 ? Math.round((results.passed / results.total) * 1000) / 10 : 0; worldsStatus[worldId].e2e = { status: results.failed === 0 ? 'passing' : results.passed > 0 ? 'partial' : 'failing', total: results.total, passed: results.passed, failed: results.failed, skipped: results.skipped, progress, tests: results.tests, lastRun: timestamp, }; } } } // Process benchmark results const benchFiles = findBenchmarkFiles(resultsDir); const benchmarksByWorld = {}; for (const file of benchFiles) { const worldId = extractWorldFromFilename(file, 'bench-timings-'); if (!benchmarksByWorld[worldId]) { benchmarksByWorld[worldId] = {}; } const metrics = parseBenchmarkTimings(file); if (metrics) { // Merge metrics (could have multiple apps) Object.assign(benchmarksByWorld[worldId], metrics); } } // Assign benchmarks to worlds for (const [worldId, metrics] of Object.entries(benchmarksByWorld)) { if (worldsStatus[worldId]) { worldsStatus[worldId].benchmark = { status: Object.keys(metrics).length > 0 ? 'measured' : 'pending', metrics, lastRun: timestamp, }; } } return { $schema: './worlds-status.schema.json', lastUpdated: timestamp, commit: process.env.GITHUB_SHA || null, branch: process.env.GITHUB_REF_NAME || null, worlds: worldsStatus, }; } // Generate test matrix (detailed per-test breakdown) function generateTestMatrix(worldsStatus) { const allTests = new Map(); // testName -> { world -> status } // Collect all tests from all worlds for (const [worldId, world] of Object.entries(worldsStatus)) { if (world.e2e?.tests) { for (const test of world.e2e.tests) { if (!allTests.has(test.name)) { allTests.set(test.name, {}); } allTests.get(test.name)[worldId] = test.status; } } } // Convert to array format const tests = []; for (const [name, results] of allTests) { tests.push({ name, results }); } // Sort by test name tests.sort((a, b) => a.name.localeCompare(b.name)); return { lastUpdated: new Date().toISOString(), tests, }; } // Main console.log('Aggregating worlds data...'); console.log(` Results directory: ${resultsDir}`); console.log(` Output path: ${outputPath}`); const status = aggregateWorldsData(); const testMatrix = generateTestMatrix(status.worlds); // Ensure output directory exists const outputDir = path.dirname(outputPath); if (!fs.existsSync(outputDir)) { fs.mkdirSync(outputDir, { recursive: true }); } // Write worlds-status.json fs.writeFileSync(outputPath, JSON.stringify(status, null, 2)); console.log(`\nGenerated ${outputPath}`); // Write test-matrix.json const testMatrixPath = path.join(path.dirname(outputPath), 'test-matrix.json'); fs.writeFileSync(testMatrixPath, JSON.stringify(testMatrix, null, 2)); console.log(`Generated ${testMatrixPath}`); // Summary const worldCount = Object.keys(status.worlds).length; const withE2E = Object.values(status.worlds).filter((w) => w.e2e).length; const withBenchmarks = Object.values(status.worlds).filter( (w) => w.benchmark ).length; console.log(`\nSummary:`); console.log(` Total worlds: ${worldCount}`); console.log(` With E2E data: ${withE2E}`); console.log(` With benchmark data: ${withBenchmarks}`); for (const [_id, world] of Object.entries(status.worlds)) { const e2eStatus = world.e2e ? `${world.e2e.passed}/${world.e2e.total} (${world.e2e.progress}%)` : 'no data'; const benchStatus = world.benchmark ? `${Object.keys(world.benchmark.metrics).length} benchmarks` : 'no data'; console.log(` - ${world.name}: E2E ${e2eStatus}, Benchmarks ${benchStatus}`); }