#!/usr/bin/env bun /** * Stable CPU + memory bench for Zod version bumps (no z.compile). * * Measures existing validation paths only: * - production `.is` predicate * - zod-compiler AOT `.is` * - Zod runtime `safeParse` * - representative backend org schema `safeParse` * * Usage: * bun scripts/bench_zod_stable_cpu.ts * BENCH_RUNS=5 bun scripts/bench_zod_stable_cpu.ts /workspace */ import { spawnSync } from 'node:child_process' import { resolve } from 'node:path' import { fileURLToPath, pathToFileURL } from 'node:url' import { performance } from 'node:perf_hooks' const SCRIPT_PATH = fileURLToPath(import.meta.url) const cliArgs = process.argv.slice(2) const memWorkerCase = cliArgs[0] === '--mem-worker' ? cliArgs[1] : undefined const ROOT = resolve(memWorkerCase ? cliArgs[2] : cliArgs[0] ?? resolve(import.meta.dirname, '..')) const RUNS = Math.max(1, Number.parseInt(process.env.BENCH_RUNS ?? '1', 10) || 1) const ITERATIONS = 80_000 interface CpuRow { name: string runs: number iterations: number nsCpuPerOp: number nsCpuPerOpStd: number } interface MemRow { name: string runs: number heapUsedDeltaMB: number rssDeltaMB: number } function forceGc() { if (typeof Bun !== 'undefined' && 'gc' in Bun && typeof Bun.gc === 'function') Bun.gc(true) else if (globalThis.gc) globalThis.gc() } async function readZodVersion(root: string) { const mod = await import(pathToFileURL(resolve(root, 'node_modules/zod/package.json')).href) as { default?: { version?: string } version?: string } return mod.default?.version ?? mod.version ?? 'unknown' } async function importZodFromRoot(root: string) { return import(pathToFileURL(resolve(root, 'node_modules/zod/index.js')).href) } function ensureCompiledZod() { const out = resolve(ROOT, 'scripts/bench/validation/plugin_schemas.zod.compiled.ts') const src = resolve(ROOT, 'scripts/bench/validation/plugin_schemas.zod.ts') const result = spawnSync(process.execPath, ['x', 'zod-compiler', 'generate', src, '-o', out, '--emit', 'bag'], { cwd: ROOT, encoding: 'utf8', }) if (result.status === null) throw new Error(`zod-compiler generate failed to start: ${result.error?.message ?? 'unknown error'}`) if (result.status !== 0) throw new Error(`zod-compiler generate failed:\n${result.stdout}\n${result.stderr}`) } function validUpdatePayload() { return { app_id: 'com.demo.app', device_id: 'a1b2c3d4-e5f6-7890-abcd-ef1234567890', version_name: '1.2.3', version_build: '1.2.3', is_emulator: false, is_prod: true, platform: 'ios' as const, plugin_version: '6.8.1', defaultChannel: 'production', key_id: 'key_1', } } function invalidUpdatePayload() { return { ...validUpdatePayload(), app_id: 'not a domain', device_id: 'bad', plugin_version: 'nope', } } function validOrgPayload() { return { id: '11111111-1111-4111-8111-111111111111', created_by: '22222222-2222-4222-8222-222222222222', created_at: '2026-01-01T00:00:00.000Z', updated_at: '2026-01-01T00:00:00.000Z', logo: null, name: 'Example Org', management_email: 'org-admin@example.com', customer_id: null, website: null, } } function mixedUpdateInputs(validRatio = 0.75) { const valid = validUpdatePayload() const invalid = invalidUpdatePayload() const invalidEvery = Math.max(1, Math.round(1 / (1 - validRatio))) return Array.from({ length: 16 }, (_, i) => (i % invalidEvery === 0 ? invalid : valid)) } function benchCpuOnce(name: string, iterations: number, fn: (input: unknown) => boolean, inputs: unknown[]) { for (let i = 0; i < Math.min(2000, iterations); i++) fn(inputs[i % inputs.length]) const cpu0 = process.cpuUsage() const t0 = performance.now() for (let i = 0; i < iterations; i++) fn(inputs[i % inputs.length]) const wallMs = performance.now() - t0 const cpu = process.cpuUsage(cpu0) const cpuMs = (cpu.user + cpu.system) / 1000 return { name, iterations, wallMs, cpuMs, nsCpuPerOp: (cpuMs * 1e6) / iterations, } } function summarizeCpu(name: string, iterations: number, samples: number[]) { const mean = samples.reduce((sum, value) => sum + value, 0) / samples.length const variance = samples.reduce((sum, value) => sum + (value - mean) ** 2, 0) / samples.length return { name, runs: samples.length, iterations, nsCpuPerOp: mean, nsCpuPerOpStd: Math.sqrt(variance), } } function summarizeMem(name: string, samples: Array<{ heapUsedDeltaMB: number, rssDeltaMB: number }>) { const heapUsedDeltaMB = samples.reduce((sum, row) => sum + row.heapUsedDeltaMB, 0) / samples.length const rssDeltaMB = samples.reduce((sum, row) => sum + row.rssDeltaMB, 0) / samples.length return { name, runs: samples.length, heapUsedDeltaMB, rssDeltaMB, } } function measureMemInChild(caseName: string) { const result = spawnSync(process.execPath, [SCRIPT_PATH, '--mem-worker', caseName, ROOT], { cwd: ROOT, encoding: 'utf8', env: process.env, }) if (result.status === null) throw new Error(`memory worker failed to start for ${caseName}: ${result.error?.message ?? 'unknown error'}`) if (result.status !== 0) throw new Error(`memory worker ${caseName} failed:\n${result.stdout}\n${result.stderr}`) return JSON.parse(result.stdout.trim()) as { heapUsedDeltaMB: number, rssDeltaMB: number } } async function runMemWorker(caseName: string, root: string) { const zodRuntime = await import(pathToFileURL(resolve(root, 'scripts/bench/validation/plugin_schemas.zod.ts')).href) const { z } = await importZodFromRoot(root) const valid = validUpdatePayload() const retained: unknown[] = [] forceGc() const before = process.memoryUsage() if (caseName === '80k_runtime_parse_heap_rss') { for (let i = 0; i < ITERATIONS; i++) retained.push(zodRuntime.updateRequestSchemaZod.safeParse(valid)) } else if (caseName === '100x_z_string_heap') { for (let i = 0; i < 100; i++) retained.push(z.string()) } else { throw new Error(`unknown memory worker case: ${caseName}`) } forceGc() const after = process.memoryUsage() void retained.length console.log(JSON.stringify({ heapUsedDeltaMB: (after.heapUsed - before.heapUsed) / 1024 / 1024, rssDeltaMB: (after.rss - before.rss) / 1024 / 1024, })) } async function main() { ensureCompiledZod() const prodIs = await import(pathToFileURL(resolve(ROOT, 'supabase/functions/_backend/plugin_runtime/utils/plugin_schemas/update_request.is.ts')).href) const zodRuntime = await import(pathToFileURL(resolve(ROOT, 'scripts/bench/validation/plugin_schemas.zod.ts')).href) const zodCompiled = await import(pathToFileURL(resolve(ROOT, 'scripts/bench/validation/plugin_schemas.zod.compiled.ts')).href) const { z } = await importZodFromRoot(ROOT) const orgSchema = z.object({ id: z.uuid(), created_by: z.uuid(), created_at: z.union([z.string(), z.date()]), updated_at: z.union([z.string(), z.date()]), logo: z.string().nullable(), name: z.string(), management_email: z.email(), customer_id: z.string().nullable(), website: z.string().nullable(), }) const valid = validUpdatePayload() const mixed = mixedUpdateInputs(0.75) const orgValid = validOrgPayload() const cpuCases: Array<{ name: string, fn: (input: unknown) => boolean, inputs: unknown[] }> = [ { name: 'plugin_is_predicate_valid', fn: input => prodIs.isUpdateRequestBody(input), inputs: [valid], }, { name: 'zod_compiler_is_valid', fn: input => zodCompiled.updateRequestSchemaZod.is(input), inputs: [valid], }, { name: 'zod_runtime_safeParse_valid', fn: input => zodRuntime.updateRequestSchemaZod.safeParse(input).success, inputs: [valid], }, { name: 'zod_runtime_safeParse_mixed_75pct_valid', fn: input => zodRuntime.updateRequestSchemaZod.safeParse(input).success, inputs: mixed, }, { name: 'backend_org_schema_safeParse_valid', fn: input => orgSchema.safeParse(input).success, inputs: [orgValid], }, ] const cpuRows: CpuRow[] = [] for (const testCase of cpuCases) { const samples: number[] = [] for (let run = 0; run < RUNS; run++) samples.push(benchCpuOnce(testCase.name, ITERATIONS, testCase.fn, testCase.inputs).nsCpuPerOp) cpuRows.push(summarizeCpu(testCase.name, ITERATIONS, samples)) } const memCaseNames = ['80k_runtime_parse_heap_rss', '100x_z_string_heap'] as const const memRows: MemRow[] = [] for (const caseName of memCaseNames) { const samples = [] for (let run = 0; run < RUNS; run++) samples.push(measureMemInChild(caseName)) memRows.push(summarizeMem(caseName, samples)) } const zodVersion = await readZodVersion(ROOT) console.log(`\n=== Zod stable CPU + memory bench (zod@${zodVersion}, runs=${RUNS}) ===`) console.log('\nCPU (lower ns/op is better):') for (const row of cpuRows) { console.log( `${row.name.padEnd(40)} ${row.nsCpuPerOp.toFixed(1).padStart(10)} ns/op ± ${row.nsCpuPerOpStd.toFixed(1)}`, ) } console.log('\nMemory (lower delta is better; each case runs in a fresh child process):') for (const row of memRows) { console.log( `${row.name.padEnd(40)} heap Δ ${row.heapUsedDeltaMB.toFixed(3).padStart(8)} MB | rss Δ ${row.rssDeltaMB.toFixed(3).padStart(8)} MB`, ) } } if (memWorkerCase) { runMemWorker(memWorkerCase, ROOT).catch((error) => { console.error(error) process.exit(1) }) } else { main().catch((error) => { console.error(error) process.exit(1) }) }