import type { OutputEntry } from './test-utils.ts' import crypto from 'node:crypto' import fsp from 'node:fs/promises' import path from 'node:path' import process from 'node:process' import { fileURLToPath } from 'node:url' import { promisify } from 'node:util' import { imageSize } from 'image-size' import rreaddir from 'recursive-readdir' import { describe, expect, it } from 'vitest' import { create as assembliesCreate } from '../../../src/cli/commands/assemblies.ts' import OutputCtl from './OutputCtl.ts' import { hasTransloaditCredentials, testCase } from './test-utils.ts' const rreaddirAsync = promisify(rreaddir) const describeLive = hasTransloaditCredentials ? describe : describe.skip describeLive('assemblies', { retry: 1 }, () => { describe('create', () => { const genericImg = 'https://demos.transloadit.com/66/01604e7d0248109df8c7cc0f8daef8/snowflake.jpg' const fixtureImg = path.resolve( path.dirname(fileURLToPath(import.meta.url)), '../fixtures/sample.jpg', ) async function imgPromise(fname = 'in.jpg'): Promise { await fsp.copyFile(fixtureImg, fname) return fname } const genericSteps = { resize: { robot: '/image/resize', use: ':original', result: true, width: 130, height: 130, }, } async function stepsPromise( _fname = 'steps.json', steps: Record = genericSteps, ): Promise { await fsp.writeFile('steps.json', JSON.stringify(steps)) return 'steps.json' } it( 'should transcode a file', testCase(async (client) => { const infile = await imgPromise() const steps = await stepsPromise() const output = new OutputCtl() await assembliesCreate(output, client, { steps, inputs: [infile], output: 'out.jpg', }) const result = output.get(true) as OutputEntry[] expect(result.length).to.be.at.least(3) const msgs = result.map((r) => r.msg) expect(msgs).to.include('GOT JOB in.jpg out.jpg') expect(msgs).to.include('DOWNLOADING') expect(msgs).to.include('COMPLETED in.jpg out.jpg') const imgBuffer = await fsp.readFile('out.jpg') const dim = imageSize(new Uint8Array(imgBuffer)) expect(dim).to.have.property('width').that.equals(130) expect(dim).to.have.property('height').that.equals(130) }), ) it( 'should download file with correct md5 hash', testCase(async (client) => { const infile = await imgPromise() const steps = await stepsPromise() const output = new OutputCtl() const { results } = await assembliesCreate(output, client, { steps, inputs: [infile], output: 'out-md5.jpg', }) // Get the assembly result to find the expected md5hash // The results array contains assembly statuses const assemblyResult = results[0] as { results?: Record> } expect(assemblyResult).to.have.property('results') const resultSteps = Object.values(assemblyResult.results ?? {}) expect(resultSteps.length).to.be.greaterThan(0) const firstResult = resultSteps[0]?.[0] expect(firstResult).to.have.property('md5hash') const expectedMd5 = firstResult?.md5hash // Calculate md5 of downloaded file const downloadedBuffer = await fsp.readFile('out-md5.jpg') const actualMd5 = crypto.createHash('md5').update(downloadedBuffer).digest('hex') expect(actualMd5).to.equal(expectedMd5) }), ) it( 'should handle multiple inputs', testCase(async (client) => { const infiles = await Promise.all(['in1.jpg', 'in2.jpg', 'in3.jpg'].map(imgPromise)) const steps = await stepsPromise() await fsp.mkdir('out') const output = new OutputCtl() await assembliesCreate(output, client, { steps, inputs: infiles, output: 'out', }) const outs = await fsp.readdir('out') expect(outs[0]).to.equal('in1.jpg') expect(outs[1]).to.equal('in2.jpg') expect(outs[2]).to.equal('in3.jpg') expect(outs).to.have.lengthOf(3) }), ) it( 'should not output outside outdir', testCase(async (client) => { await fsp.mkdir('sub') process.chdir('sub') const infile = await imgPromise('../in.jpg') await fsp.mkdir('out') const steps = await stepsPromise() const output = new OutputCtl() await assembliesCreate(output, client, { steps, inputs: [infile], output: 'out', }) const outs = await fsp.readdir('out') expect(outs[0]).to.equal('in.jpg') expect(outs).to.have.lengthOf(1) const ls = await fsp.readdir('.') expect(ls).to.not.contain('in.jpg') }), ) it( 'should structure output directory correctly', testCase(async (client) => { await fsp.mkdir('in') await fsp.mkdir('in/sub') await Promise.all(['1.jpg', 'in/2.jpg', 'in/sub/3.jpg'].map(imgPromise)) await fsp.mkdir('out') const steps = await stepsPromise() const output = new OutputCtl() await assembliesCreate(output, client, { recursive: true, steps, inputs: ['1.jpg', 'in'], output: 'out', }) const outs = await rreaddirAsync('out') expect(outs).to.include('out/1.jpg') expect(outs).to.include('out/2.jpg') expect(outs).to.include('out/sub/3.jpg') expect(outs).to.have.lengthOf(3) }), 180_000, ) it( 'should not be recursive by default', testCase(async (client) => { await fsp.mkdir('in') await fsp.mkdir('in/sub') await Promise.all(['in/2.jpg', 'in/sub/3.jpg'].map(imgPromise)) await fsp.mkdir('out') const steps = await stepsPromise() const output = new OutputCtl() await assembliesCreate(output, client, { steps, inputs: ['in'], output: 'out', }) const outs = await rreaddirAsync('out') expect(outs).to.include('out/2.jpg') expect(outs).to.not.include('out/sub/3.jpg') expect(outs).to.have.lengthOf(1) }), ) it( 'should be able to handle directories recursively', testCase(async (client) => { await fsp.mkdir('in') await fsp.mkdir('in/sub') await Promise.all(['in/2.jpg', 'in/sub/3.jpg'].map(imgPromise)) await fsp.mkdir('out') const steps = await stepsPromise() const output = new OutputCtl() await assembliesCreate(output, client, { recursive: true, steps, inputs: ['in'], output: 'out', }) const outs = await rreaddirAsync('out') expect(outs).to.include('out/2.jpg') expect(outs).to.include('out/sub/3.jpg') expect(outs).to.have.lengthOf(2) }), 180_000, ) it( 'should disambiguate colliding input names in directory outputs', testCase(async (client) => { await fsp.mkdir('in') await Promise.all(['1.jpg', 'in/1.jpg'].map(imgPromise)) await fsp.mkdir('out') const steps = await stepsPromise() const output = new OutputCtl() await assembliesCreate(output, client, { steps, inputs: ['1.jpg', 'in'], output: 'out', }) expect(await fsp.readdir('out')).to.deep.equal(['1.jpg', '1__1.jpg']) }), ) it( 'should not download the result if no output is specified', testCase(async (client) => { const infile = await imgPromise() const steps = await stepsPromise() const output = new OutputCtl() await assembliesCreate(output, client, { steps, inputs: [infile], output: null, }) const result = output.get(true) as OutputEntry[] // When no output is specified, we might still get debug messages but no actual downloads const downloadingMsgs = result.filter((line) => String(line.msg) === 'DOWNLOADING') expect(downloadingMsgs.length).to.be.lessThanOrEqual(1) }), ) it( 'should accept invocations with no inputs', testCase(async (client) => { await imgPromise() const steps = await stepsPromise('steps.json', { import: { robot: '/http/import', url: genericImg, }, resize: { robot: '/image/resize', use: 'import', result: true, width: 130, height: 130, }, }) const output = new OutputCtl() await assembliesCreate(output, client, { steps, inputs: [], output: 'out.jpg', }) await fsp.access('out.jpg') }), ) it( 'should allow output directory for no-input workspace templates', testCase(async (client) => { await fsp.mkdir('out') const template = await client.createTemplate({ name: `node-sdk-no-input-${crypto.randomUUID()}`, template: { steps: { import: { robot: '/http/import', url: genericImg, }, resize: { robot: '/image/resize', use: 'import', result: true, width: 256, height: 256, format: 'png', }, }, }, }) try { const output = new OutputCtl() await assembliesCreate(output, client, { template: template.id, inputs: [], output: 'out', }) const files = await rreaddirAsync('out') expect(files.length).to.be.greaterThan(0) // Ensure at least one output file is a valid image. const first = files[0] expect(first).to.be.a('string') const buf = await fsp.readFile(first) const dim = imageSize(new Uint8Array(buf)) expect(dim.width).to.be.greaterThan(0) expect(dim.height).to.be.greaterThan(0) } finally { await client.deleteTemplate(template.id) } }), 180_000, ) it( 'should download all results for multi-output templates into a directory', testCase(async (client) => { const fixtureMp4 = path.resolve( path.dirname(fileURLToPath(import.meta.url)), '../fixtures/testsrc.mp4', ) await fsp.copyFile(fixtureMp4, 'in.mp4') await fsp.mkdir('out') const output = new OutputCtl() await assembliesCreate(output, client, { template: 'builtin/encode-hls-video@0.0.1', inputs: ['in.mp4'], output: 'out', }) const files = await rreaddirAsync('out') const hasM3u8 = files.some((p) => p.endsWith('.m3u8')) const hasTs = files.some((p) => p.endsWith('.ts')) expect(hasM3u8).to.equal(true) expect(hasTs).to.equal(true) }), 180_000, ) it( 'should allow deleting inputs after processing', testCase(async (client) => { const infile = await imgPromise() const steps = await stepsPromise() const output = new OutputCtl() await assembliesCreate(output, client, { steps, inputs: [infile], output: null, del: true, }) try { await fsp.access(infile) throw new Error('File should have been deleted') } catch (err) { expect((err as NodeJS.ErrnoException).code).to.equal('ENOENT') } }), ) it( 'should not reprocess inputs that are older than their directory output', testCase(async (client) => { const infiles = await Promise.all(['in1.jpg', 'in2.jpg', 'in3.jpg'].map(imgPromise)) const steps = await stepsPromise() await fsp.mkdir('out') const output1 = new OutputCtl() await assembliesCreate(output1, client, { steps, inputs: [infiles[0] as string], output: 'out', }) const inputTime = new Date(Date.now() - 60_000) await fsp.utimes(infiles[0] as string, inputTime, inputTime) const output2 = new OutputCtl() const { results } = await assembliesCreate(output2, client, { steps, inputs: infiles, output: 'out', }) expect(results).to.have.lengthOf(2) expect(await fsp.readdir('out')).to.deep.equal(['in1.jpg', 'in2.jpg', 'in3.jpg']) }), 180_000, ) it( 'should skip an explicit file output newer than its input', testCase(async (client) => { const infile = await imgPromise() const steps = await stepsPromise() const inputTime = new Date(Date.now() - 60_000) await fsp.utimes(infile, inputTime, inputTime) await assembliesCreate(new OutputCtl(), client, { steps, inputs: [infile], output: 'out.jpg', }) const originalOutput = await fsp.readFile('out.jpg') const { results } = await assembliesCreate(new OutputCtl(), client, { steps, inputs: [infile], output: 'out.jpg', }) expect(results).to.have.lengthOf(0) expect(await fsp.readFile('out.jpg')).to.deep.equal(originalOutput) }), 180_000, ) it( 'should process many files with concurrency limiting', testCase(async (client) => { // Create 6 input files const fileCount = 6 const infiles = await Promise.all( Array.from({ length: fileCount }, (_, i) => imgPromise(`in${i}.jpg`)), ) const steps = await stepsPromise() await fsp.mkdir('out') const output = new OutputCtl() await assembliesCreate(output, client, { steps, inputs: infiles, output: 'out', concurrency: 2, // Only process 2 at a time }) // Verify all files were processed successfully const outs = await fsp.readdir('out') expect(outs).to.have.lengthOf(fileCount) // Analyze debug output to verify concurrency limiting was applied. // The fixed code emits "PROCESSING JOB" when jobs start (up to concurrency limit). // The unfixed code has no such limiting - all jobs start at once with "GOT JOB". const debugOutput = output.get(true) as OutputEntry[] const messages = debugOutput.map((e) => String(e.msg)) // Check that "PROCESSING JOB" messages exist (added by the fix) const processingMessages = messages.filter((m) => m.startsWith('PROCESSING JOB')) expect( processingMessages.length, 'Expected "PROCESSING JOB" debug messages from concurrency-limited processing', ).to.be.greaterThan(0) // Track max concurrent jobs by counting "PROCESSING JOB" vs "COMPLETED" messages let activeJobs = 0 let maxActiveJobs = 0 for (const msg of messages) { if (msg.startsWith('PROCESSING JOB')) { activeJobs++ maxActiveJobs = Math.max(maxActiveJobs, activeJobs) } else if (msg.startsWith('COMPLETED')) { activeJobs-- } } // With concurrency=2, we should never have more than 2 jobs processing at once expect( maxActiveJobs, 'Max concurrent jobs should not exceed concurrency limit', ).to.be.at.most(2) }), 300_000, ) }) })