/** * Model-facing `accurate_vision` tool: precise spatial reasoning over an image * file via an OpenAI-compatible vision model. The model returns a structured * note plus bounding-box primitives (normalised 0–1000); this tool formats them * as a `` block the next model turn reads. * * Ported from `pi-accurate-vision` into the DeepSeek Harness tool registry. The * pure vision core lives in {@link ./bridge.ts}; this file owns the Cordis * plugin: schemastery config, credential resolution, and the registered tool. * @module @deepseek-ai/dsh-tool-accurate-vision */ import { mkdirSync, readFileSync, writeFileSync } from 'node:fs' import { createHash } from 'node:crypto' import { tmpdir } from 'node:os' import { basename, join, resolve } from 'node:path' import type { Context } from '@deepseek-ai/cordis' import z from '@deepseek-ai/schemastery' import { defineTool } from '@deepseek-ai/dsh-tools' import { credentialRef } from '@deepseek-ai/dsh-credentials' import { buildDataUrl, formatVisionContext, MAX_IMAGE_BYTES, mimeTypeForPath, runVisionAnalysis, } from './bridge.ts' import type { VisionAnalysisParams } from './bridge.ts' import { buildAnnotatedSvg, imageSize } from './render.ts' import type { VisualPrimitive } from './parse.ts' export * from './bridge.ts' /** Cordis plugin name used by loader diagnostics. */ export const name = 'tool-accurate-vision' /** The tool registry is the only hard service dependency. */ export const inject = ['tools'] /** Default OpenAI-compatible vision endpoint. */ const DEFAULT_BASE_URL = 'https://api.openai.com/v1' /** Default vision model identifier. */ const DEFAULT_MODEL = 'gpt-4o' /** Default credential reference resolved through the credentials seam. */ const DEFAULT_API_KEY_ENV = 'VISION_API_KEY' /** Plugin configuration for the accurate-vision tool. */ export interface Config { /** Vision model identifier (any OpenAI-compatible multimodal model). */ model: string /** OpenAI-compatible base URL; `/chat/completions` is appended. */ baseURL: string /** Credential reference (env-var name) resolved per call via `ctx.credentials`. */ apiKeyEnv: string /** Whether to request bounding-box primitives. */ primitives: boolean /** Write a self-contained SVG copy of the image with the boxes drawn on it. */ annotate: boolean /** Per-call output cap forwarded as `max_tokens`. */ maxTokens: number /** Request timeout in seconds. */ timeoutSecs: number /** Sampling temperature forwarded to the provider. */ temperature: number /** Disable the provider's thinking phase (MiniMax `thinking:{type:disabled}`). */ disableThinking: boolean } export const Config: z = z.object({ model: z.string().default(DEFAULT_MODEL), baseURL: z.string().default(DEFAULT_BASE_URL), apiKeyEnv: z.string().role('credential-ref').default(DEFAULT_API_KEY_ENV), primitives: z.boolean().default(true), annotate: z.boolean().default(true), maxTokens: z.number().step(1).min(1).default(8192), timeoutSecs: z.number().step(1).min(1).default(120), temperature: z.number().min(0).default(0), disableThinking: z.boolean().default(true), }) /** The canonical tool result: the formatted context plus provenance facts. */ export interface AccurateVisionResult { /** Structured `` text for the next model turn. */ context: string /** The model identifier that produced the analysis. */ model: string /** Number of bounding-box primitives extracted. */ primitiveCount: number /** Path of the annotated SVG (original image + drawn boxes), when produced. */ annotatedImage?: string } /** * Resolve the vision API key for one call. * * Mirrors the LLM adapter's per-call resolution: prefer the credentials seam * (which ranks env/file/project/user layers), then fall back to the ambient * environment. Never cache across calls so a rotated key reaches the next call. * @param ctx - registrant context (may lack the credentials seam). * @param ref - the credential reference to resolve. * @throws when no key is resolvable. */ async function resolveApiKey(ctx: Context, ref: string): Promise { const credentials = ctx.get('credentials') if (credentials !== undefined) { const hit = await credentials.resolve(credentialRef(ref)) if (hit !== undefined && hit.value.length > 0) return hit.value } const ambient = process.env[ref] if (ambient !== undefined && ambient.length > 0) return ambient throw new Error( `accurate_vision: no API key for credential ref "${ref}". Set ${ref} in the` + ' environment or store it through the credentials service (the web Models page),' + ' or set config.apiKeyEnv to a configured reference.', ) } /** * Write the annotated SVG next to the other temp artifacts: one file per * source image (path-hashed), overwritten on every call so it always mirrors * the latest primitives. * @returns the written path, or `undefined` when the image size cannot be * sniffed (no SVG can be dimensioned without it). */ function writeAnnotatedSvg( imageDataUrl: string, sourcePath: string, primitives: VisualPrimitive[], ): string | undefined { const comma = imageDataUrl.indexOf(',') const b64 = comma >= 0 ? imageDataUrl.slice(comma + 1) : '' const size = imageSize(Buffer.from(b64, 'base64')) if (size === undefined) return undefined const svg = buildAnnotatedSvg({ imageDataUrl, width: size.width, height: size.height, primitives, }) const hash = createHash('sha1').update(sourcePath).digest('hex').slice(0, 10) const dir = join(tmpdir(), 'dsh-accurate-vision') mkdirSync(dir, { recursive: true }) const outPath = join(dir, `${basename(sourcePath).replace(/\.[^.]+$/, '')}-${hash}.svg`) writeFileSync(outPath, svg) return outPath } /** * Register the `accurate_vision` tool on `ctx.tools`. Registration is an * effect: disposing this plugin fiber unregisters the tool. * @param ctx - registrant context carrying the tool registry. * @param config - deployment configuration (schemastery has applied defaults). */ export function apply(ctx: Context, config: Config): void { ctx.tools.register(defineTool({ name: 'accurate_vision', description: 'Analyze an image file with a vision model. Returns structured spatial context with' + ' bounding-box primitives (normalised 0–1000 coordinates) plus an annotated SVG copy' + ' (path in annotatedImage) with every box drawn on the original image. Use when you' + ' need precise object positions, distances, or layout info from an image.', // Slightly above the HTTP timeout so the bridge's descriptive // "timed out after Xs" error wins the race against the framework timeout. timeoutMs: config.timeoutSecs * 1000 + 5000, parameters: { image_path: { type: 'string', required: true, description: 'Absolute or relative path to the image file (png/jpg/jpeg/gif/webp).', }, question: { type: 'string', description: 'Optional question about the image.', }, }, output: { schema: { type: 'object', additionalProperties: false, properties: { context: { type: 'string', required: true, description: 'Structured block: image overview, OCR, layout,' + ' and normalised bounding-box primitives.', }, model: { type: 'string', required: true }, primitiveCount: { type: 'integer', required: true }, annotatedImage: { type: 'string', description: 'Path of a self-contained SVG: the original image with every bounding box and' + ' label drawn on it. Present only when annotation succeeded.', }, }, }, render: (_args, value) => [{ type: 'text', text: value.annotatedImage !== undefined ? `${value.context}\n${value.annotatedImage}` : value.context, }], }, async execute(args, exec) { const apiKey = await resolveApiKey(ctx, config.apiKeyEnv) const resolvedPath = resolve(args.image_path) const mime = mimeTypeForPath(resolvedPath) if (mime === undefined) { throw new Error(`Unsupported image format: ${resolvedPath}`) } const bytes = new Uint8Array(readFileSync(resolvedPath)) if (bytes.length > MAX_IMAGE_BYTES) { throw new Error( `Image too large: ${bytes.length} bytes (limit ${MAX_IMAGE_BYTES}).` + ' Reduce the image dimensions or compress it.', ) } const visionParams: VisionAnalysisParams = { apiKey, baseUrl: config.baseURL, model: config.model, maxTokens: config.maxTokens, temperature: config.temperature, timeoutSecs: config.timeoutSecs, imageDataUrl: buildDataUrl(mime, bytes), primitives: config.primitives, disableThinking: config.disableThinking, signal: exec.signal, } if (args.question !== undefined) visionParams.userQuestion = args.question const analysis = await runVisionAnalysis(visionParams) const result: AccurateVisionResult = { context: formatVisionContext(analysis), model: config.model, primitiveCount: analysis.primitives.length, } if (config.annotate && analysis.primitives.length > 0) { const annotated = writeAnnotatedSvg(visionParams.imageDataUrl, resolvedPath, analysis.primitives) if (annotated !== undefined) result.annotatedImage = annotated } return result }, presentCall: (args) => ({ card: 'generic', title: 'Analyze image', kind: 'other', rawInput: args, locations: [{ path: args.image_path }], }), })) }