import { existsSync, readFileSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import fastify, { type FastifyInstance } from 'fastify'; import { describe, expect, it } from 'vitest'; import { copyFixturesToTempDir, execThymianRaw, execThymianRawAsync, fixturesDir, useTempDir, } from './helpers.js'; import { getAvailablePort } from './port-utils.js'; /** * Helper: copy a fixture to temp dir and start a test server. Returns the server * and its port so tests can use `--target-url`. * * Nothing is generated first, deliberately: the sampler projects request * samples in memory on every run, so `thymian test` needs no setup step and * nothing committed. */ async function setupTestEnvironment( fixtureName: string, tempDir: string, handler: (server: FastifyInstance) => void = addDefaultHelloHandler, ) { copyFixturesToTempDir(join(fixturesDir, fixtureName), tempDir); const port = await getAvailablePort(); const server = fastify(); handler(server); await server.listen({ port, host: '0.0.0.0' }); return { server, port, targetUrl: `http://localhost:${port}` }; } function addDefaultHelloHandler(server: FastifyInstance) { server.get<{ Querystring: { name: string } }>('/api/hello', async (req) => { const { name } = req.query; return { content: `Hello ${name}` }; }); } describe('thymian test', () => { const getTempDir = useTempDir(); // ─── Happy Path ────────────────────────────────────────────────────── it('should run a dynamic API test with --target-url and exit 0 on clean run', async () => { const { server, targetUrl } = await setupTestEnvironment( 'dynamic-test', getTempDir(), ); try { const result = await execThymianRawAsync( ['test', '--target-url', targetUrl], { cwd: getTempDir() }, ); expect(result.exitCode).toBe(0); expect(result.stdout).toContain('@thymian/plugin-http-tester'); expect(result.stdout).toMatch( /Summary: 0 errors, 0 warnings, 0 hints, \d+ infos?\./, ); } finally { await server.close(); } }, 180_000); it('should include test rule results in output', async () => { const { server, targetUrl } = await setupTestEnvironment( 'dynamic-test', getTempDir(), ); try { const result = await execThymianRawAsync( ['test', '--target-url', targetUrl], { cwd: getTempDir() }, ); expect(result.output).toContain('@thymian/plugin-http-tester'); expect(result.output).toMatch( /Summary: 0 errors, 0 warnings, 0 hints, \d+ infos?\./, ); } finally { await server.close(); } }, 180_000); // ─── --target-url Overrides Spec Servers ────────────────────────── it('should override spec server URL with --target-url', async () => { const { server, targetUrl } = await setupTestEnvironment( 'dynamic-test', getTempDir(), ); try { // The spec says localhost:3000, but --target-url should redirect requests const result = await execThymianRawAsync( ['test', '--target-url', targetUrl], { cwd: getTempDir() }, ); // If target-url wasn't applied, requests would go to port 3000 and fail expect(result.exitCode).toBe(0); } finally { await server.close(); } }, 180_000); it('should use --target-url with a spec that has no servers defined', async () => { const { server, targetUrl } = await setupTestEnvironment( 'dynamic-test-no-servers', getTempDir(), ); try { const result = await execThymianRawAsync( ['test', '--target-url', targetUrl], { cwd: getTempDir() }, ); expect(result.exitCode).toBe(0); } finally { await server.close(); } }, 180_000); // ─── Exit Code Semantics ────────────────────────────────────────── it('should report skipped test cases when server returns unexpected status', async () => { const { server, targetUrl } = await setupTestEnvironment( 'dynamic-test', getTempDir(), (s) => { // Return 501 Not Implemented — rules skip when the initial response // does not match the expected status from the OpenAPI spec. s.get('/api/hello', async (_req, reply) => { await reply.status(501).send({ error: 'Not Implemented' }); }); }, ); try { const result = await execThymianRawAsync( ['test', '--target-url', targetUrl], { cwd: getTempDir(), allowFailure: true }, ); expect(result.exitCode).toBe(0); expect(result.stdout).toContain('@thymian/plugin-http-tester'); expect(result.stdout).toContain('Summary:'); } finally { await server.close(); } }, 180_000); it('should run with nothing on disk and materialize no samples', async () => { const { server, targetUrl } = await setupTestEnvironment( 'dynamic-test', getTempDir(), ); try { // Async, not `execThymianRaw`: the server answering this request lives in // this process, and `spawnSync` blocks the event loop it needs. const result = await execThymianRawAsync( ['test', '--target-url', targetUrl], { cwd: getTempDir() }, ); expect(result.exitCode).toBe(0); // The whole point of the virtual model: a clean checkout runs, and the // *sampler* leaves nothing behind that a later spec edit could strand. // // Named precisely rather than asserting `.thymian` is absent: this // fixture enables the reporter, which legitimately writes // `.thymian/reports/report.md`. The broader assertion was wrong, and it // never ran to say so, because the blocking spawn above it hung first. expect(existsSync(join(getTempDir(), '.thymian', 'sampler'))).toBe(false); expect(existsSync(join(getTempDir(), '.thymian', 'samples'))).toBe(false); } finally { await server.close(); } }, 180_000); it('should exit 2 when server is unreachable (tool error)', async () => { copyFixturesToTempDir(join(fixturesDir, 'dynamic-test'), getTempDir()); // Use a port that has nothing listening — no server in this test, so // spawnSync is fine (no event-loop contention). const port = await getAvailablePort(); const result = execThymianRaw( ['test', '--target-url', `http://localhost:${port}`], { cwd: getTempDir(), allowFailure: true }, ); // ECONNREFUSED causes a tool-error → exit code 2 expect(result.exitCode).toBe(2); }, 180_000); it('should exit 1 when the live response violates a rule', async () => { const { server, targetUrl } = await setupTestEnvironment( 'dynamic-test', getTempDir(), (s) => { // Send content without a Content-Type header, so // rfc9110/sender-should-generate-content-type-for-message-with-content // is violated by the live response. Hijacking bypasses fastify, which // would otherwise add the header itself. s.get('/api/hello', (_req, reply) => { reply.hijack(); reply.raw.writeHead(200, { 'content-length': '2' }); reply.raw.end('{}'); }); }, ); try { const result = await execThymianRawAsync( [ 'test', '--target-url', targetUrl, '--rule-severity', 'warn', '--rule-set', '@thymian/rules-rfc-9110', ], { cwd: getTempDir(), allowFailure: true }, ); expect(result.exitCode).toBe(1); expect(result.stdout).toContain( 'rfc9110/sender-should-generate-content-type-for-message-with-content', ); expect(result.stdout).toContain('Summary:'); } finally { await server.close(); } }, 180_000); // ─── --spec Flag ────────────────────────────────────────────────── it('should accept specification via --spec flag', async () => { const { server, targetUrl } = await setupTestEnvironment( 'dynamic-test', getTempDir(), ); try { const result = await execThymianRawAsync( [ 'test', '--spec', 'openapi:test.openapi.yaml', '--target-url', targetUrl, ], { cwd: getTempDir() }, ); expect(result.exitCode).toBe(0); } finally { await server.close(); } }, 180_000); // ─── --rule-set Flag ────────────────────────────────────────────── it('should accept rule sets via --rule-set flag', async () => { const { server, targetUrl } = await setupTestEnvironment( 'dynamic-test', getTempDir(), ); try { const result = await execThymianRawAsync( [ 'test', '--rule-set', '@thymian/rules-rfc-9110', '--target-url', targetUrl, ], { cwd: getTempDir() }, ); expect(result.output).toMatch( /Summary: 0 errors, 0 warnings, 0 hints, \d+ infos?\./, ); } finally { await server.close(); } }, 180_000); // ─── Output Format ──────────────────────────────────────────────── it('should produce structured output with reporter', async () => { const { server, targetUrl } = await setupTestEnvironment( 'dynamic-test', getTempDir(), ); try { const result = await execThymianRawAsync( ['test', '--target-url', targetUrl], { cwd: getTempDir() }, ); // Reporter output should be on stdout expect(result.stdout.length).toBeGreaterThan(0); } finally { await server.close(); } }, 180_000); // ─── Stderr vs Stdout Separation ────────────────────────────────── it('should write report output to stdout, not stderr', async () => { const { server, targetUrl } = await setupTestEnvironment( 'dynamic-test', getTempDir(), ); try { const result = await execThymianRawAsync( ['test', '--target-url', targetUrl], { cwd: getTempDir() }, ); expect(result.stdout).toMatch( /Summary: 0 errors, 0 warnings, 0 hints, \d+ infos?\./, ); } finally { await server.close(); } }, 180_000); // ─── Deterministic Output ───────────────────────────────────────── it('should produce deterministic output across consecutive runs', async () => { const { server, targetUrl } = await setupTestEnvironment( 'dynamic-test', getTempDir(), ); try { const run1 = await execThymianRawAsync( ['test', '--target-url', targetUrl, '--suppress-feedback'], { cwd: getTempDir() }, ); const run2 = await execThymianRawAsync( ['test', '--target-url', targetUrl, '--suppress-feedback'], { cwd: getTempDir() }, ); expect(run1.exitCode).toBe(run2.exitCode); const normalize = (value: string) => value .replace( /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z/g, '', ) // Real execution durations legitimately vary between runs. .replace(/\(\d+(?:\.\d+)?ms\)/g, '()'); expect(normalize(run1.stdout)).toBe(normalize(run2.stdout)); } finally { await server.close(); } }, 180_000); // ─── Missing Specification ──────────────────────────────────────── it('should exit 2 when no specification is configured', async () => { const result = execThymianRaw(['test'], { cwd: getTempDir(), allowFailure: true, }); expect(result.exitCode).toBe(2); expect(result.output).toMatch(/No specification/i); }, 180_000); // ─── Config-based targetUrl ─────────────────────────────────────── it('should read targetUrl from config file', async () => { const port = await getAvailablePort(); const server = fastify(); addDefaultHelloHandler(server); await server.listen({ port, host: '0.0.0.0' }); try { copyFixturesToTempDir(join(fixturesDir, 'dynamic-test'), getTempDir()); // Write config with targetUrl baked in const configPath = join(getTempDir(), 'thymian.config.yaml'); const configContent = readFileSync(configPath, 'utf-8'); writeFileSync( configPath, configContent + `targetUrl: 'http://localhost:${port}'\n`, ); const result = await execThymianRawAsync(['test'], { cwd: getTempDir(), }); expect(result.exitCode).toBe(0); expect(result.stdout).toMatch( /Summary: 0 errors, 0 warnings, 0 hints, \d+ infos?\./, ); } finally { await server.close(); } }, 180_000); });