import { describe, expect } from "bun:test" import { NodeFileSystem } from "@effect/platform-node" import { Effect, Ref, Schema } from "effect" import { FileSystem } from "effect" import { HttpClientRequest } from "effect/http" import { ai, AIError, Media, Message, ToolCallPart } from "../src/effect.js" import { OpenAI } from "../src/providers.js" import { compileRequest } from "../src/route/client.js" import { it } from "./lib/effect.js" import { dynamicResponse, scriptedResponses } from "./lib/http.js" const PNG = Uint8Array.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0]) const JPEG = Uint8Array.from([0xff, 0xd8, 0xff, 0xe0, 0, 0]) const GIF = Uint8Array.from([0x47, 0x49, 0x46, 0x38, 0x39, 0x61]) const WEBP = Uint8Array.from([0x52, 0x49, 0x46, 0x46, 0, 0, 0, 0, 0x57, 0x45, 0x42, 0x50]) const PDF = Uint8Array.from([0x25, 0x50, 0x44, 0x46, 0x2d, 0x31]) describe("Media", () => { it.effect("round-trips every source through Media.from(asset.source)", () => Effect.sync(() => { const assets = [ Media.bytes(PNG), Media.base64("AQID", "image/png"), Media.url("https://example.test/a.png", { mediaType: "image/png", expiresAt: 123, headers: { a: "b" } }), Media.ref("openai", "file_123", "image/webp"), ] for (const asset of assets) { const copy = Media.from(asset.source) expect(copy.source).toEqual(asset.source) expect(copy.mediaType).toBe(asset.mediaType) expect(copy.kind).toBe(asset.kind) expect(copy.expiresAt).toBe(asset.expiresAt) } expect(assets.map((asset) => asset.kind)).toEqual(["image", "image", "image", "image"]) expect(Media.url("https://example.test/unknown").mediaType).toBe("application/octet-stream") expect(Media.url("https://example.test/unknown").kind).toBe("other") expect(Media.base64("AQID", "application/pdf").kind).toBe("document") expect(Media.base64("AQID", "video/mp4").kind).toBe("video") expect(Media.base64("AQID", "audio/wav").kind).toBe("audio") }), ) it.effect("rejects local file references as request errors without breaking construction or decoding", () => Effect.gen(function* () { const model = OpenAI.configure({ apiKey: "test" }).responses("video-model") const restored = yield* Schema.decodeUnknownEffect(Media.AssetSchema)({ source: { type: "url", url: " FILE:/C:/clips/video.mp4", mediaType: "video/mp4" }, }) for (const asset of [ Media.url("file:///Users/example/clip.mp4", { mediaType: "video/mp4" }), Media.ref("openai", "file:///Users/example/clip.mp4", "video/mp4"), restored, ]) { const error = yield* compileRequest(ai.request({ model, prompt: [Message.media(asset)] })).pipe(Effect.flip) expect(error.reason._tag).toBe("InvalidRequest") expect(error.message).toContain("Media.file(path)") } const error = yield* compileRequest( ai.request({ model, messages: [ Message.assistant([ToolCallPart.make({ id: "read", name: "read", input: {} })]), Message.tool({ id: "read", name: "read", resultType: "content", result: [{ type: "file", uri: "file:///Users/example/clip.mp4", mime: "video/mp4" }], }), ], }), ).pipe(Effect.flip) expect(error.reason._tag).toBe("InvalidRequest") expect(error.message).toContain("Media.file(path)") expect(Media.ref("google", "gs://bucket/clip.mp4").source).toMatchObject({ id: "gs://bucket/clip.mp4" }) expect(Media.url("https://example.test/clip.mp4").source).toMatchObject({ url: "https://example.test/clip.mp4", }) }), ) it.effect("parses data URLs and rejects malformed ones", () => Effect.sync(() => { const asset = Media.fromDataUrl("data:image/jpeg;base64,/9j/") expect(asset.source).toEqual({ type: "base64", data: "/9j/", mediaType: "image/jpeg" }) expect(asset.mediaType).toBe("image/jpeg") expect(Media.fromDataUrl("data:text/plain;charset=utf-8;base64,aGk=").source).toEqual({ type: "base64", data: "aGk=", mediaType: "text/plain", }) expect(() => Media.fromDataUrl("https://example.test/a.png")).toThrow( "Media data URLs must contain a MIME type and base64 data", ) expect(() => Media.fromDataUrl("data:image/png,rawtext")).toThrow() }), ) it.effect("detects media types from magic bytes", () => Effect.sync(() => { expect(Media.detectMediaType(PNG)).toBe("image/png") expect(Media.detectMediaType(JPEG)).toBe("image/jpeg") expect(Media.detectMediaType(GIF)).toBe("image/gif") expect(Media.detectMediaType(WEBP)).toBe("image/webp") expect(Media.detectMediaType(PDF)).toBe("application/pdf") expect(Media.detectMediaType(Uint8Array.from([1, 2, 3]))).toBeUndefined() expect(Media.detectMediaType(new TextEncoder().encode("ID3\x04"))).toBe("audio/mpeg") expect(Media.detectMediaType(Uint8Array.from([0xff, 0xfb, 0x90, 0x64]))).toBe("audio/mpeg") expect(Media.detectMediaType(Uint8Array.from([0xff, 0xf3, 0x90, 0x64]))).toBe("audio/mpeg") expect(Media.detectMediaType(Uint8Array.from([0xff, 0xf1, 0x50, 0x80]))).toBe("audio/aac") expect(Media.detectMediaType(new TextEncoder().encode("RIFF\0\0\0\0WAVEfmt "))).toBe("audio/wav") expect(Media.detectMediaType(new TextEncoder().encode("OggS\0\x02"))).toBe("audio/ogg") expect(Media.detectMediaType(new TextEncoder().encode("fLaC\0\0\0\x22"))).toBe("audio/flac") expect(Media.bytes(PNG).mediaType).toBe("image/png") expect(Media.bytes(Uint8Array.from([1, 2, 3])).mediaType).toBe("application/octet-stream") expect(Media.bytes(Uint8Array.from([1, 2, 3]), "image/x-custom").mediaType).toBe("image/x-custom") }), ) it.effect("tells ISO-BMFF brands apart", () => Effect.sync(() => { const ftyp = (hex: string) => Uint8Array.from(Buffer.from(hex, "hex")) expect(Media.detectMediaType(ftyp("000000206674797061766966000000004d695072617669666d6961666d696631"))).toBe( "image/avif", ) expect(Media.detectMediaType(ftyp("000000186674797061766973000000006d73663161766973"))).toBe("image/avif") expect(Media.detectMediaType(ftyp("000000246674797068656963000000006d6966314d6950726d6961664d694842"))).toBe( "image/heic", ) expect(Media.detectMediaType(ftyp("00000018667479706d69663100000000617669666d69663100000000"))).toBe("image/avif") expect(Media.detectMediaType(ftyp("00000018667479706d69663100000000686569636d69663100000000"))).toBe("image/heic") expect(Media.detectMediaType(ftyp("000000206674797069736f6d0000020069736f6d69736f32617663316d703431"))).toBe( "video/mp4", ) expect(Media.detectMediaType(ftyp("00000014667479707174202000000200717420200000000877696465"))).toBe( "video/quicktime", ) expect(Media.detectMediaType(ftyp("0000001c667479704d344120000002004d34412069736f6d69736f3200000008"))).toBe( "audio/mp4", ) expect(Media.bytes(ftyp("000000206674797061766966000000004d695072617669666d6961666d696631")).kind).toBe("image") }), ) it.effect("materializes url assets through the request executor once", () => Effect.gen(function* () { const requests = yield* Ref.make>([]) const asset = Media.url("https://cdn.example.test/generated", { expiresAt: 42, headers: { authorization: "Bearer media" }, providerMetadata: { example: { id: "gen_1" } }, }) const program = Effect.gen(function* () { const first = yield* asset.bytes() const second = yield* asset.bytes() expect(second).toBe(first) expect(first).toEqual(PNG) expect(yield* asset.base64()).toBe(Buffer.from(PNG).toString("base64")) expect(yield* asset.dataUrl()).toBe( `data:application/octet-stream;base64,${Buffer.from(PNG).toString("base64")}`, ) const owned = yield* asset.materialize() expect(owned.source).toEqual({ type: "bytes", data: PNG, mediaType: "image/png" }) expect(owned.mediaType).toBe("image/png") expect(owned.kind).toBe("image") expect(owned.expiresAt).toBeUndefined() expect(owned.providerMetadata).toEqual({ example: { id: "gen_1" } }) expect(yield* owned.materialize()).toBe(owned) const seen = yield* Ref.get(requests) expect(seen).toHaveLength(1) expect(seen[0].url).toBe("https://cdn.example.test/generated") expect(seen[0].headers.authorization).toBe("Bearer media") }) yield* program.pipe( Effect.provide( dynamicResponse((input) => Ref.update(requests, (all) => [...all, input.request]).pipe( Effect.map(() => input.respond(PNG, { headers: { "content-type": "image/png" } })), ), ), ), ) }), ) it.effect("caches decoded base64 bytes and encodes owned bytes lazily", () => Effect.gen(function* () { const fromBase64 = Media.base64("AQID", "image/png") const decoded = yield* fromBase64.bytes() expect(decoded).toEqual(Uint8Array.from([1, 2, 3])) expect(yield* fromBase64.bytes()).toBe(decoded) expect(yield* fromBase64.base64()).toBe("AQID") const fromBytes = Media.bytes(Uint8Array.from([1, 2, 3]), "image/png") const encoded = yield* fromBytes.base64() expect(encoded).toBe("AQID") expect(yield* fromBytes.base64()).toBe(encoded) expect(yield* fromBytes.dataUrl()).toBe("data:image/png;base64,AQID") const invalid = yield* Media.base64("not base64!", "image/png").bytes().pipe(Effect.flip) expect(invalid.reason._tag).toBe("InvalidRequest") const ref = yield* Media.ref("openai", "file_1").bytes().pipe(Effect.flip) expect(ref.reason._tag).toBe("InvalidRequest") }).pipe(Effect.provide(scriptedResponses(["unused"]))), ) it.effect("serializes assets inside messages and restores them as Media.Asset", () => Effect.sync(() => { const codec = Schema.fromJsonString(Message) const message = Message.user([ Message.media(Media.base64("AQID", "image/png", { info: { width: 1, height: 1 } }), { filename: "a.png" }), Message.media(Media.url("https://example.test/b.pdf", { mediaType: "application/pdf", expiresAt: 7 })), ]) const json = Schema.encodeSync(codec)(message) expect(json).not.toContain("null") const restored = Schema.decodeSync(codec)(json) expect(restored.role).toBe("user") const parts = restored.role === "user" ? restored.content.filter((part) => part.type === "media") : [] expect(parts).toHaveLength(2) expect(parts[0].media).toBeInstanceOf(Media.Asset) expect(parts[0].media.source).toEqual({ type: "base64", data: "AQID", mediaType: "image/png" }) expect(parts[0].media.info).toEqual({ width: 1, height: 1 }) expect(parts[0].filename).toBe("a.png") expect(parts[1].media.kind).toBe("document") expect(parts[1].media.expiresAt).toBe(7) }), ) it.effect("keeps transient url download headers out of toJSON and AssetSchema encoding", () => Effect.sync(() => { const asset = Media.url("https://cdn.example.test/video.mp4", { mediaType: "video/mp4", expiresAt: 42, headers: { "x-goog-api-key": "secret" }, }) expect(asset.headers).toEqual({ "x-goog-api-key": "secret" }) const source = { type: "url", url: "https://cdn.example.test/video.mp4", mediaType: "video/mp4", expiresAt: 42 } expect(asset.toJSON()).not.toHaveProperty("headers") expect(JSON.stringify(asset)).not.toContain("secret") expect(asset.toJSON().source).toEqual(source) const encoded = Schema.encodeSync(Media.AssetSchema)(asset) expect(encoded).not.toHaveProperty("headers") expect(encoded.source).toEqual(source) const codec = Schema.fromJsonString(Media.AssetSchema) const json = Schema.encodeSync(codec)(asset) expect(json).not.toContain("secret") const restored = Schema.decodeSync(codec)(json) expect(restored).toBeInstanceOf(Media.Asset) expect(restored.source).toEqual(source) expect(restored.expiresAt).toBe(42) expect(restored.headers).toBeUndefined() }), ) it.effect("fails url downloads with non-2xx status as a typed AIError keeping http and body", () => Effect.gen(function* () { const body = JSON.stringify({ error: { message: "file expired" } }) const error = yield* Media.url("https://cdn.example.test/expired.png") .bytes() .pipe( Effect.flip, Effect.provide( dynamicResponse((input) => Effect.succeed(input.respond(body, { status: 404, headers: { "content-type": "application/json" } })), ), ), ) expect(error).toBeInstanceOf(AIError) expect(error.message).toContain("file expired") expect(error.reason.http?.status).toBe(404) expect(error.reason.http?.url).toBe("https://cdn.example.test/expired.png") expect(error.reason.body).toBe(body) }), ) it.effect("reads files with sniffed media types and writes materialized assets", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem const dir = yield* fs.makeTempDirectoryScoped() const source = `${dir}/source.bin` yield* fs.writeFile(source, PNG) const asset = yield* Media.file(source) expect(asset.mediaType).toBe("image/png") expect(asset.source).toEqual({ type: "bytes", data: PNG, mediaType: "image/png" }) const plain = `${dir}/notes.md` yield* fs.writeFile(plain, new TextEncoder().encode("# hi")) expect((yield* Media.file(plain)).mediaType).toBe("text/markdown") for (const [name, mediaType] of [ ["photo.avif", "image/avif"], ["photo.HEIC", "image/heic"], ["photo.heif", "image/heif"], ["clip.mov", "video/quicktime"], ]) { yield* fs.writeFile(`${dir}/${name}`, Uint8Array.from([1, 2, 3])) expect((yield* Media.file(`${dir}/${name}`)).mediaType).toBe(mediaType) } const target = `${dir}/copy.png` yield* Media.write(Media.base64("AQID", "image/png"), target) expect(yield* fs.readFile(target)).toEqual(Uint8Array.from([1, 2, 3])) const missing = yield* Media.file(`${dir}/missing.png`).pipe(Effect.flip) expect(missing.reason._tag).toBe("InvalidRequest") }).pipe(Effect.provide(NodeFileSystem.layer), Effect.provide(scriptedResponses(["unused"]))), ) })