import { describe, expect, test } from "bun:test" import { Schema } from "effect" import { ai, type LLMRequest, Message, ToolCallPart, ToolDefinition } from "../src/effect.js" import { OpenAIChat } from "../src/protocols.js" import { Auth } from "../src/route.js" import { sanitizeSurrogates } from "../src/utils/sanitize.js" const model = OpenAIChat.route .with({ endpoint: { baseURL: "https://api.openai.test/v1/" }, auth: Auth.bearer("test") }) .model({ id: "gpt-4o-mini" }) const tool = new ToolDefinition({ name: "lookup", description: "Look something up", inputSchema: { type: "object", properties: { query: { type: "string" } } }, }) // Mirrors request preparation: sanitize everything but the model, then normalize the copies back into classes. const sanitizeRequest = (request: LLMRequest) => ai.request({ ...request, ...sanitizeSurrogates({ ...request, model: undefined }), model: request.model }) describe("sanitizeSurrogates", () => { test("replaces lone surrogates in nested message text, tool input, and tool output", () => { const sanitized = sanitizeRequest( ai.request({ model, system: "system \uD800", tools: [tool], messages: [ Message.user("user \uDC00 \u{1F600}"), Message.assistant([ ToolCallPart.make({ id: "call_1", name: "lookup", input: { "q\uD800": "input \uDC00" } }), ]), Message.tool({ id: "call_1", name: "lookup", result: { output: ["result \uD800", "valid"] } }), ], }), ) expect(sanitized.messages.every((message) => Schema.is(Message)(message))).toBe(true) expect(sanitized.tools[0]).toBe(tool) expect(sanitized.system).toEqual([{ type: "text", text: "system \uFFFD" }]) expect(sanitized.messages).toEqual([ Message.user("user \uFFFD \u{1F600}"), Message.assistant([ToolCallPart.make({ id: "call_1", name: "lookup", input: { "q\uFFFD": "input \uFFFD" } })]), Message.tool({ id: "call_1", name: "lookup", result: { output: ["result \uFFFD", "valid"] } }), ]) }) test("keeps clean requests equal and shares unchanged nested values", () => { const messages = [Message.user("hello \u{1F600}"), Message.assistant("hi")] const request = ai.request({ model, system: "system", tools: [tool], messages: [...messages, Message.user("bad \uD800")], maxTokens: 10, providerOptions: { nested: { list: ["a", "b"] } }, }) const sanitized = sanitizeRequest(request) expect(sanitized.messages.slice(0, 2)).toEqual(request.messages.slice(0, 2)) expect(sanitized.messages[0]).toBe(request.messages[0]) expect(sanitized.messages[1]).toBe(request.messages[1]) expect(sanitized.messages[2]).not.toBe(request.messages[2]) expect(sanitized.messages[2]).toEqual(Message.user("bad \uFFFD")) expect(sanitized.tools[0]).toBe(request.tools[0]) expect(sanitized.maxTokens).toBe(10) const input = request const output = sanitizeSurrogates(input) expect(output.system).toBe(input.system) expect(output.tools).toBe(input.tools) expect(output.providerOptions).toBe(input.providerOptions) expect(output.messages).not.toBe(input.messages) const clean = ai.request({ model, system: "system", tools: [tool], messages }) expect(sanitizeRequest(clean)).toEqual(clean) expect(sanitizeSurrogates(clean).messages).toBe(clean.messages) }) })