/** * Vision analysis bridge — HTTP transport and context formatting for * coordinate-based region detection (0–1000 normalised). * * Faithful port of the `pi-accurate-vision` vision core (itself a TypeScript * extraction of DeepSeek-TUI's `crates/tui/src/vision/bridge.rs`). * Provider-agnostic: it speaks any OpenAI-compatible vision * `chat/completions` endpoint. The {@link ./index.ts} host plugin wires it to * the DeepSeek Harness tool registry; the pure parsing core lives in * {@link ./parse.ts} and is re-exported here for a stable public surface. * @module @deepseek-ai/dsh-tool-accurate-vision/bridge */ import { analysisUserMessage, noteOnlyPrompt, normalizeContent, parseAnalysisResponse, primitivesAnalysisPrompt, stripMarkdownFences, } from './parse.ts' import type { ImageNote, VisionAnalysis } from './parse.ts' // Re-export the parsing core so consumers of this module (and the package's // `export *`) keep a single import path. export { NORM_MAX, createBBox, bboxEdgeDistance, primitivesAnalysisPrompt, parseAnalysisResponse, normalizeContent, stripMarkdownFences, } from './parse.ts' export type { BBox, VisualPrimitive, ImageNote, VisionAnalysis } from './parse.ts' // -- Constants -------------------------------------------------------------- /** Maximum accepted image size in bytes (20 MB). */ export const MAX_IMAGE_BYTES = 20_000_000 /** User-Agent sent to the vision provider; matches the homepage in package.json. */ export const USER_AGENT = 'dsh-tool-accurate-vision (+https://github.com/deepseek-ai/deepseek-harness)' // -- Types ------------------------------------------------------------------ /** Parameters for the vision analysis HTTP call. */ export interface VisionAnalysisParams { apiKey: string baseUrl: string model: string maxTokens: number temperature: number timeoutSecs: number imageDataUrl: string userQuestion?: string /** Whether to request bounding-box primitives. Defaults to true. */ primitives: boolean /** * Disable the provider's thinking/reasoning phase (MiniMax: sends * `thinking: {type: "disabled"}`). Faster, cheaper, and empirically more * stable boxes; ignored by providers without such a switch. */ disableThinking?: boolean /** Optional abort signal from the caller (e.g. the tool execution signal). */ signal?: AbortSignal } // -- Image helpers ---------------------------------------------------------- /** MIME types accepted by the vision bridge. */ const MIME_BY_EXT: Readonly> = { png: 'image/png', jpg: 'image/jpeg', jpeg: 'image/jpeg', gif: 'image/gif', webp: 'image/webp', } /** Detect the MIME type from a file extension. Returns `undefined` for unsupported formats. */ export function mimeTypeForPath(path: string): string | undefined { const ext = path.split('.').pop()?.toLowerCase() return ext === undefined ? undefined : MIME_BY_EXT[ext] } /** Convert image bytes to a base64 data URI. */ export function buildDataUrl(mime: string, bytes: Uint8Array | Buffer): string { const base64 = typeof Buffer !== 'undefined' ? Buffer.from(bytes).toString('base64') : fallbackBase64(bytes) return `data:${mime};base64,${base64}` } /** Chunked base64 for runtimes without Buffer; one huge spread overflows the stack. */ function fallbackBase64(bytes: Uint8Array): string { let binary = '' const CHUNK = 0x8000 for (let i = 0; i < bytes.length; i += CHUNK) { binary += String.fromCharCode(...bytes.subarray(i, i + CHUNK)) } return btoa(binary) } // -- HTTP vision analysis --------------------------------------------------- /** * Run vision analysis against an OpenAI-compatible vision model API. * * POSTs to `{baseUrl}/chat/completions` with the image as an `image_url` * content part. Returns a parsed {@link VisionAnalysis}; on a parse failure it * falls back to a note-only analysis carrying the raw model text. * @param params - connection facts, the image data URL, and the prompt toggles. * @returns the parsed analysis (possibly a note-only fallback). * @throws on transport failure, non-2xx, or timeout. */ export async function runVisionAnalysis(params: VisionAnalysisParams): Promise { const systemPrompt = params.primitives ? primitivesAnalysisPrompt() : noteOnlyPrompt() const userText = `${systemPrompt}\n\n${analysisUserMessage('see attached image', params.userQuestion)}` const body: Record = { model: params.model, messages: [ { role: 'user', content: [ { type: 'text', text: userText }, { type: 'image_url', image_url: { url: params.imageDataUrl } }, ], }, ], } if (params.maxTokens > 0) { body.max_tokens = params.maxTokens } const temp = Math.round(params.temperature * 10) / 10 if (temp > 0) { body.temperature = temp } if (params.disableThinking === true) { // MiniMax-compatible switch; providers without it ignore unknown fields // (OpenAI rejects unknown top-level args, so only set it when requested). body.thinking = { type: 'disabled' } } const url = `${params.baseUrl.replace(/\/+$/, '')}/chat/completions` // Combine the request timeout with the caller's cancellation signal (Node 22 // supports AbortSignal.any). Without it, forward the caller's abort into the // timeout controller so cancellation still interrupts the in-flight fetch. const timeoutController = new AbortController() const timeout = setTimeout(() => timeoutController.abort(), params.timeoutSecs * 1000) let combined = timeoutController.signal let unlinkCallerAbort: (() => void) | undefined if (params.signal !== undefined) { const callerSignal = params.signal if (typeof AbortSignal.any === 'function') { combined = AbortSignal.any([timeoutController.signal, callerSignal]) } else { const onCallerAbort = () => timeoutController.abort() callerSignal.addEventListener('abort', onCallerAbort) unlinkCallerAbort = () => callerSignal.removeEventListener('abort', onCallerAbort) } } let responseText: string try { const resp = await fetch(url, { method: 'POST', headers: { Authorization: `Bearer ${params.apiKey}`, 'Content-Type': 'application/json', 'User-Agent': USER_AGENT, }, body: JSON.stringify(body), signal: combined, }) responseText = await resp.text() if (!resp.ok) { throw new Error(`Vision API HTTP ${resp.status}: ${responseText}`) } } catch (error) { const aborted = timeoutController.signal.aborted || (params.signal?.aborted ?? false) || (error instanceof Error && error.name === 'AbortError') if (aborted && !params.signal?.aborted) { throw new Error(`Vision request timed out after ${params.timeoutSecs}s`) } throw error } finally { clearTimeout(timeout) unlinkCallerAbort?.() } // Extract content from the OpenAI-format response let content: string try { const parsed = JSON.parse(responseText) as { choices?: Array<{ message?: { content?: unknown } }> } content = normalizeContent(parsed.choices?.[0]?.message?.content) } catch { throw new Error(`Failed to parse vision API response: ${responseText.slice(0, 200)}`) } // Try structured parse; fall back to raw text (matches bridge.rs behavior) try { return parseAnalysisResponse(content) } catch { const fallbackNote: ImageNote = { imageOverview: content.length > 0 ? stripMarkdownFences(content) : 'Image processed but vision model returned empty.', visibleText: '', objectsAndLayout: '', } return { note: fallbackNote, primitives: [] } } } // -- Context formatting ----------------------------------------------------- /** Format a {@link VisionAnalysis} into structured XML-like text for a downstream model. */ export function formatVisionContext(analysis: VisionAnalysis): string { const parts: string[] = [] parts.push(`\n\n${analysis.note.imageOverview}\n`) if (analysis.note.visibleText) { parts.push(`\n${analysis.note.visibleText}\n`) } if (analysis.note.objectsAndLayout) { parts.push(`\n${analysis.note.objectsAndLayout}\n`) } pushOptTag(parts, 'charts_or_data', analysis.note.chartsOrData) pushOptTag(parts, 'user_request', analysis.note.userRequest) pushOptTag(parts, 'user_request_answer', analysis.note.userRequestAnswer) pushOptTag(parts, 'evidence', analysis.note.evidence) parts.push('') if (analysis.primitives.length === 0) { parts.push('- unavailable | reason: no valid coordinates') } else { for (const prim of analysis.primitives) { const box = prim.box parts.push( `- ${prim.id} | type: ${prim.type}` + ` | box: [${box.x1},${box.y1},${box.x2},${box.y2}]` + ` | ref: ${prim.label} | confidence: ${prim.confidence.toFixed(2)}`, ) } } parts.push('') pushOptTag(parts, 'uncertainty', analysis.note.uncertainty) parts.push('') return parts.join('\n') } function pushOptTag(parts: string[], tag: string, val: string | undefined): void { if (val && val.length > 0) { parts.push(`<${tag}>\n${val}\n`) } }