import Foundation import NIOCore import NIOHTTP1 import NIOPosix import SwiftletCore // Loopback OpenAI-compatible Chat Completions server (modeled on // TurboFieldfare's). One warm model, requests serialized. No auth/TLS: keep it // on 127.0.0.1. // // swiftlet-server --model [--port 8080] [--cache-gb 2] /// Message content per the OpenAI Chat Completions spec: a plain string, an /// array of parts ([{type:"text",text:"..."}]), or null on tool-call turns. /// Text parts are joined; non-text parts are dropped; null decodes as "". struct ChatContent: Decodable { struct Part: Decodable { let text: String? } let text: String init(from decoder: Decoder) throws { let c = try decoder.singleValueContainer() if try c.decodeNil() { text = "" } else if let s = try? c.decode(String.self) { text = s } else { text = try c.decode([Part].self).compactMap(\.text).joined() } } } /// OpenAI-compatible Chat Completions request body. struct ChatRequest: Decodable { struct Message: Decodable { let role: String; let content: ChatContent } let messages: [Message] let stream: Bool? let max_tokens: Int? let max_completion_tokens: Int? let temperature: Float? let top_p: Float? } let cliArgs = CommandLine.arguments func flag(_ name: String) -> String? { guard let i = cliArgs.firstIndex(of: name), i + 1 < cliArgs.count else { return nil } return cliArgs[i + 1] } guard let modelPath = flag("--model") else { print("usage: swiftlet-server --model [--port 8080] [--cache-gb 2]") exit(2) } let port = Int(flag("--port") ?? "8080") ?? 8080 let cacheGB = Double(flag("--cache-gb") ?? "2") ?? 2 let modelURL = URL(fileURLWithPath: modelPath) FileHandle.standardError.write(Data("loading model + tokenizer...\n".utf8)) // The same Metal streaming engine as `swiftlet chat`: routed experts stream // from the .qpack blobs, so a 35B/80B container serves in a few GB of RAM. // (The CPU model reader cannot serve containers: their experts are not in // model.safetensors.) let session = try await SwiftletSession(modelDir: modelURL, cacheBudgetGB: cacheGB) let modelName: String = { let configURL = modelURL.appendingPathComponent("config.json") if let cfg = try? JSONSerialization.jsonObject(with: Data(contentsOf: configURL)) as? [String: Any], let t = cfg["model_type"] as? String { return t } return "swiftlet" }() // One request at a time: the session mutates shared conversation state. let generationQueue = DispatchQueue(label: "swiftlet.generation") @Sendable func jsonData(_ obj: [String: Any]) -> Data { (try? JSONSerialization.data(withJSONObject: obj)) ?? Data("{}".utf8) } @Sendable func completionPayload( id: String, text: String?, delta: String?, finish: String?, usage: (prompt: Int, completion: Int)? = nil ) -> [String: Any] { var choice: [String: Any] = ["index": 0] if let text { choice["message"] = ["role": "assistant", "content": text] } if let delta { choice["delta"] = ["content": delta] } if let finish { choice["finish_reason"] = finish } var payload: [String: Any] = [ "id": id, "object": delta != nil ? "chat.completion.chunk" : "chat.completion", "created": Int(Date().timeIntervalSince1970), "model": modelName, "choices": [choice], ] if let usage { payload["usage"] = [ "prompt_tokens": usage.prompt, "completion_tokens": usage.completion, "total_tokens": usage.prompt + usage.completion, ] } return payload } final class HTTPHandler: ChannelInboundHandler { typealias InboundIn = HTTPServerRequestPart typealias OutboundOut = HTTPServerResponsePart private var requestHead: HTTPRequestHead? private var body = ByteBuffer() func channelRead(context: ChannelHandlerContext, data: NIOAny) { switch unwrapInboundIn(data) { case .head(let head): requestHead = head body = ByteBuffer() case .body(var chunk): body.writeBuffer(&chunk) case .end: guard let head = requestHead else { return } route(context: context, head: head, body: body) requestHead = nil } } private func route(context: ChannelHandlerContext, head: HTTPRequestHead, body: ByteBuffer) { switch (head.method, head.uri) { case (.GET, "/v1/models"): respondJSON(context, status: .ok, data: jsonData([ "object": "list", "data": [["id": modelName, "object": "model", "owned_by": "swiftlet"]], ])) case (.POST, "/v1/chat/completions"): handleChat(context: context, body: body) default: respondJSON(context, status: .notFound, data: jsonData(["error": "not found"])) } } private func handleChat(context: ChannelHandlerContext, body: ByteBuffer) { let bytes: [UInt8] = body.getBytes(at: 0, length: body.readableBytes) ?? [] let data = Data(bytes) guard let request = try? JSONDecoder().decode(ChatRequest.self, from: data) else { respondJSON(context, status: .badRequest, data: jsonData(["error": "malformed request"])) return } let messages = request.messages.map { ["role": $0.role, "content": $0.content.text] } let maxNew = request.max_tokens ?? request.max_completion_tokens ?? 512 let streaming = request.stream ?? false let id = "chatcmpl-\(UUID().uuidString.prefix(8))" let eventLoop = context.eventLoop let channel = context.channel let options: SwiftletSession.GenerationOptions = { var o = SwiftletSession.GenerationOptions() if let t = request.temperature { if t <= 0 { o = .greedy } else { o.temperature = t } } if let p = request.top_p { o.topP = p } return o }() // Agent clients (OpenCode etc.) send multi-thousand-token prompts, so // prefill can run minutes with no output. SSE comment heartbeats keep // the idle connection from being dropped; parsers ignore them. var heartbeat: RepeatedTask? = nil if streaming { var head = HTTPResponseHead(version: .http1_1, status: .ok) head.headers.add(name: "Content-Type", value: "text/event-stream") head.headers.add(name: "Cache-Control", value: "no-cache") head.headers.add(name: "Transfer-Encoding", value: "chunked") context.writeAndFlush(wrapOutboundOut(.head(head)), promise: nil) heartbeat = eventLoop.scheduleRepeatedTask( initialDelay: .seconds(15), delay: .seconds(15) ) { _ in var buf = channel.allocator.buffer(capacity: 16) buf.writeString(": ping\n\n") _ = channel.writeAndFlush(HTTPServerResponsePart.body(.byteBuffer(buf))) } } let stopHeartbeat = heartbeat @Sendable func writeSSE(_ obj: [String: Any]) { let payload = "data: " + (String(data: jsonData(obj), encoding: .utf8) ?? "{}") + "\n\n" eventLoop.execute { var buf = channel.allocator.buffer(capacity: payload.utf8.count) buf.writeString(payload) _ = channel.writeAndFlush(HTTPServerResponsePart.body(.byteBuffer(buf))) } } generationQueue.async { let done = DispatchSemaphore(value: 0) Task { defer { done.signal() } do { var fullText = "" for try await delta in session.streamChat( messages: messages, maxNew: maxNew, options: options ) { fullText += delta if streaming { writeSSE(completionPayload(id: id, text: nil, delta: delta, finish: nil)) } } let m = session.lastMetrics FileHandle.standardError.write(Data(String( format: "[%@] %d prompt + %d generated, ttft %.1fs, %.2f tok/s\n", id as NSString, m.promptTokens, m.generatedTokens, m.timeToFirstToken, m.tokensPerSecond ).utf8)) eventLoop.execute { stopHeartbeat?.cancel() if streaming { // The finish chunk, [DONE], and the HTTP end must // land in this order on the wire. Write them here // directly: routing the finish chunk through // writeSSE would re-enqueue it on the event loop // and it would land after .end and be dropped // (strict clients then wait for it forever). let finish = jsonData(completionPayload( id: id, text: nil, delta: "", finish: "stop", usage: (m.promptTokens, m.generatedTokens))) let tail = "data: " + (String(data: finish, encoding: .utf8) ?? "{}") + "\n\ndata: [DONE]\n\n" var buf = channel.allocator.buffer(capacity: tail.utf8.count) buf.writeString(tail) _ = channel.write(HTTPServerResponsePart.body(.byteBuffer(buf))) _ = channel.writeAndFlush(HTTPServerResponsePart.end(nil)) } else { let data = jsonData(completionPayload( id: id, text: fullText, delta: nil, finish: "stop", usage: (m.promptTokens, m.generatedTokens))) var head = HTTPResponseHead(version: .http1_1, status: .ok) head.headers.add(name: "Content-Type", value: "application/json") head.headers.add(name: "Content-Length", value: String(data.count)) var buf = channel.allocator.buffer(capacity: data.count) buf.writeBytes(data) _ = channel.write(HTTPServerResponsePart.head(head)) _ = channel.write(HTTPServerResponsePart.body(.byteBuffer(buf))) _ = channel.writeAndFlush(HTTPServerResponsePart.end(nil)) } } } catch { eventLoop.execute { stopHeartbeat?.cancel() if streaming { // The 200 + SSE head is already on the wire; a // second head would be a protocol error. Report // the failure as an SSE event and end the stream. let payload = "data: " + (String( data: jsonData(["error": "\(error)"]), encoding: .utf8 ) ?? "{}") + "\n\ndata: [DONE]\n\n" var buf = channel.allocator.buffer(capacity: payload.utf8.count) buf.writeString(payload) _ = channel.write(HTTPServerResponsePart.body(.byteBuffer(buf))) _ = channel.writeAndFlush(HTTPServerResponsePart.end(nil)) } else { let data = jsonData(["error": "\(error)"]) var head = HTTPResponseHead(version: .http1_1, status: .internalServerError) head.headers.add(name: "Content-Length", value: String(data.count)) var buf = channel.allocator.buffer(capacity: data.count) buf.writeBytes(data) _ = channel.write(HTTPServerResponsePart.head(head)) _ = channel.write(HTTPServerResponsePart.body(.byteBuffer(buf))) _ = channel.writeAndFlush(HTTPServerResponsePart.end(nil)) } } } } done.wait() } } private func respondJSON(_ context: ChannelHandlerContext, status: HTTPResponseStatus, data: Data) { var head = HTTPResponseHead(version: .http1_1, status: status) head.headers.add(name: "Content-Type", value: "application/json") head.headers.add(name: "Content-Length", value: String(data.count)) var buf = context.channel.allocator.buffer(capacity: data.count) buf.writeBytes(data) context.write(wrapOutboundOut(.head(head)), promise: nil) context.write(wrapOutboundOut(.body(.byteBuffer(buf))), promise: nil) context.writeAndFlush(wrapOutboundOut(.end(nil)), promise: nil) } } let group = MultiThreadedEventLoopGroup(numberOfThreads: 2) let bootstrap = ServerBootstrap(group: group) .serverChannelOption(ChannelOptions.socketOption(.so_reuseaddr), value: 1) .childChannelInitializer { channel in channel.pipeline.configureHTTPServerPipeline().flatMap { channel.pipeline.addHandler(HTTPHandler()) } } let channel = try await bootstrap.bind(host: "127.0.0.1", port: port).get() print("swiftlet-server listening on http://127.0.0.1:\(port)/v1 (model: \(modelName))") try await channel.closeFuture.get()