/* eslint-disable unicorn/prefer-module */ // This is a demo utilising Google Gemini through Google Generative Language API. // Google Gemini allows many frame images to be supplied because of its huge context length. // Video frame images are sent through Google Generative Language API directly. // // This script can be executed with a command line like this from the project root directory: // export GEMINI_API_KEY=... // ENABLE_DEBUG=true npx ts-node test/integration/gemini-json.ts import { HarmBlockThreshold, HarmCategory } from '@google/generative-ai'; import { consoleWithColour } from '@handy-common-utils/misc-utils'; /* eslint-disable node/no-unpublished-import */ import chalk from 'chalk'; import path from 'node:path'; import readline from 'node:readline'; import { ChatAboutVideo, ConversationWithGemini } from '../../src'; async function demo() { const chat = new ChatAboutVideo( { credential: { key: process.env.GEMINI_API_KEY!, }, clientSettings: { modelParams: { model: 'gemini-2.5-flash', }, }, extractVideoFrames: { limit: 100, interval: 0.5, }, completionOptions: { safetySettings: [ { category: 'HARM_CATEGORY_HATE_SPEECH' as any, threshold: 'BLOCK_NONE' as any, }, ], }, }, consoleWithColour({ debug: process.env.ENABLE_DEBUG === 'true' }, chalk), ); const conversation = (await chat.startConversation( path.resolve(__dirname, '../sample-media-files/engine-start.h264.aac.mp4'), )) as ConversationWithGemini; const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); const prompt = (question: string) => new Promise((resolve) => rl.question(question, resolve)); while (true) { const question = await prompt(chalk.red('\nUser: ')); if (!question) { continue; } if (['exit', 'quit', 'q', 'end'].includes(question)) { await conversation.end(); break; } const answer = await conversation.say(question, { safetySettings: [{ category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT, threshold: HarmBlockThreshold.BLOCK_NONE }], }); console.log(chalk.blue('\nAI:' + answer)); // Below are for showing how to mandate JSON response from Gemini. const explanation = await conversation.say( 'Explain your answer. The response should be in JSON like this: {"referencedFrames": [1, 5], "why": "Reason for giving this response."}', { jsonResponse: true }, ); console.log(chalk.grey("\nAI's Explanation: " + JSON.stringify(JSON.parse(explanation as string), null, 2))); const detailedExplanation = await conversation.say('Explain your answer in detail. The response should be in JSON.', { jsonResponse: { name: 'DetailedExplanation', schema: { type: 'object', properties: { referencedFrames: { type: 'array', items: { type: 'integer' }, }, understandingOfTheQuestion: { type: 'string' }, reasoningSteps: { type: 'array', items: { type: 'string' } }, }, required: ['referencedFrames', 'understandingOfTheQuestion', 'reasoningSteps'], }, }, }); console.log(chalk.grey("\nAI's detailed explanation: " + JSON.stringify(JSON.parse(detailedExplanation as string), null, 2))); } console.log('Demo finished'); rl.close(); } // eslint-disable-next-line unicorn/prefer-top-level-await demo().catch((error) => console.log(chalk.red(JSON.stringify(error, null, 2)), error));