/* eslint-disable unicorn/prefer-module */ // This is a test verifying that ChatGPT style tool calling works with Gemini. // // This script can be executed with a command line like this from the project root directory: // export GEMINI_API_KEY=... // npx ts-node test/integration/gemini-chatgpt-style-tools.ts import { consoleWithColour } from '@handy-common-utils/misc-utils'; /* eslint-disable node/no-unpublished-import */ import chalk from 'chalk'; import path from 'node:path'; import readline from 'node:readline'; import { ChatAboutVideo, ConversationResponse, ConversationWithGemini, ToolCallResult } from '../../src'; async function demo() { const chat = new ChatAboutVideo( { credential: { key: process.env.GEMINI_API_KEY!, }, clientSettings: { modelParams: { model: 'gemini-2.5-flash', }, }, extractVideoFrames: { limit: 100, interval: 0.5, }, completionOptions: { safetySettings: [ { category: 'HARM_CATEGORY_HATE_SPEECH' as any, threshold: 'BLOCK_NONE' as any, }, ], }, }, consoleWithColour({ debug: process.env.ENABLE_DEBUG === 'true' }, chalk), ); const conversation = (await chat.startConversation( path.resolve(__dirname, '../sample-media-files/engine-start.h264.aac.mp4'), )) as ConversationWithGemini; // ChatGPT style tools const tools: any[] = [ { type: 'function', function: { name: 'get_weather_forecast', description: 'Get the current weather for a location', parameters: { type: 'object', properties: { location: { type: 'string', description: 'The city and state, e.g. San Francisco, CA', }, }, required: ['location'], }, }, }, ...exampleTools, ]; const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); console.log(chalk.cyan('Testing ChatGPT style tools with Gemini...')); const question = 'What is the weather in San Francisco?'; console.log(chalk.red('\nUser: ') + question); let response = await conversation.say(question, { tools, }); if (typeof response !== 'string' && response?.toolCalls) { const toolResults: ToolCallResult[] = []; for (const call of response.toolCalls) { console.log(chalk.yellow(`AI requests tool: ${call.name}(${JSON.stringify(call.arguments)})`)); const result = call.name === 'get_weather_forecast' ? { forecast: 'Sunny', temperature: '25C' } : { error: 'Unknown tool' }; toolResults.push({ name: call.name, result, }); } response = await conversation.submitToolCallResults(toolResults); } console.log(chalk.blue('\nAI: ' + response)); await conversation.end(); console.log('Demo finished'); rl.close(); } // eslint-disable-next-line unicorn/prefer-top-level-await demo().catch((error) => console.log(chalk.red(JSON.stringify(error, null, 2)), error)); const exampleTools = [ { type: 'function', function: { name: 'start_meeting', description: 'Start a new group meeting with the specified attendees. The first attendee is the host.', parameters: { type: 'object', properties: { attendees: { type: 'array', items: { type: 'string', }, description: 'Email addresses of meeting attendees. First one is the host.', }, }, required: ['attendees'], }, }, }, { type: 'function', function: { name: 'finish_meeting', description: 'End an in-progress meeting.', parameters: { type: 'object', properties: { meetingId: { type: 'string', description: 'The ID of the meeting to finish.', }, }, required: ['meetingId'], }, }, }, { type: 'function', function: { name: 'get_meeting_content', description: 'Get the transcript/content of an in-progress or finished meeting by its ID.', parameters: { type: 'object', properties: { meetingId: { type: 'string', description: 'The ID of the meeting to retrieve.', }, }, required: ['meetingId'], }, }, }, { type: 'function', function: { name: 'say_in_meeting', description: 'Say something in an ongoing meeting. The speaker must be an attendee of the meeting.', parameters: { type: 'object', properties: { meetingId: { type: 'string', description: 'The ID of the meeting.', }, speaker: { type: 'string', description: 'The email address of the speaker.', }, content: { type: 'string', description: 'What the speaker said.', }, }, required: ['meetingId', 'speaker', 'content'], }, }, }, { type: 'function', function: { name: 'send_email', description: "Send an email. Uses the agent's configured email account.", parameters: { type: 'object', properties: { to: { anyOf: [ { type: 'string', }, { type: 'array', items: { type: 'string', }, }, ], description: 'Recipient(s) - email address or array of addresses.', }, cc: { anyOf: [ { type: 'string', }, { type: 'array', items: { type: 'string', }, }, ], description: 'CC recipient(s).', }, subject: { type: 'string', description: 'Email subject.', }, bodyText: { type: 'string', description: 'Plain text body of the email.', }, }, required: ['to', 'subject', 'bodyText'], }, }, }, { type: 'function', function: { name: 'add_email_private_comment', description: 'Add a private comment/note to an email. These comments are only visible to the agent and are persisted in the email index. This is a good way to keep track of important information or status of an email across different work sessions. Identify the email by its Message-ID (e.g. ) from the email content.', parameters: { type: 'object', properties: { messageId: { type: 'string', description: 'The Message-ID (e.g. ) of the email to comment on.', }, comment: { type: 'string', description: 'The private comment to add to the email.', }, }, required: ['messageId', 'comment'], }, }, }, { type: 'function', function: { name: 'archive_email', description: 'Archive an email to hide it from future work sessions. Use this for emails that are handled, read, or no longer relevant, to reduce noise. Identify the email by its Message-ID (e.g. ) from the email content.', parameters: { type: 'object', properties: { messageId: { type: 'string', description: 'The Message-ID (e.g. ) of the email to archive.', }, }, required: ['messageId'], }, }, }, { type: 'function', function: { name: 'update_memory', description: 'Update your long-term memory. Use this to record important information, unfinished plans, and background context for future work.', parameters: { type: 'object', properties: { content: { type: 'string', description: 'The full markdown content of your memory. This will overwrite the existing memory.', }, }, required: ['content'], }, }, }, { type: 'function', function: { name: 'tavily_search', description: 'Search the web for current information on any topic. Use for news, facts, or data beyond your knowledge cutoff. Returns snippets and source URLs.', parameters: { type: 'object', properties: { query: { type: 'string', description: 'Search query', }, search_depth: { type: 'string', enum: ['basic', 'advanced', 'fast', 'ultra-fast'], description: "The depth of the search. 'basic' for generic results, 'advanced' for more thorough search, 'fast' for optimized low latency with high relevance, 'ultra-fast' for prioritizing latency above all else", default: 'basic', }, topic: { type: 'string', enum: ['general'], description: 'The category of the search. This will determine which of our agents will be used for the search', default: 'general', }, time_range: { type: 'string', description: 'The time range back from the current date to include in the search results', enum: ['day', 'week', 'month', 'year'], }, start_date: { type: 'string', description: 'Will return all results after the specified start date. Required to be written in the format YYYY-MM-DD.', default: '', }, end_date: { type: 'string', description: 'Will return all results before the specified end date. Required to be written in the format YYYY-MM-DD', default: '', }, max_results: { type: 'number', description: 'The maximum number of search results to return', default: 5, minimum: 5, maximum: 20, }, include_images: { type: 'boolean', description: 'Include a list of query-related images in the response', default: false, }, include_image_descriptions: { type: 'boolean', description: 'Include a list of query-related images and their descriptions in the response', default: false, }, include_raw_content: { type: 'boolean', description: 'Include the cleaned and parsed HTML content of each search result', default: false, }, include_domains: { type: 'array', items: { type: 'string', }, description: 'A list of domains to specifically include in the search results, if the user asks to search on specific sites set this to the domain of the site', default: [], }, exclude_domains: { type: 'array', items: { type: 'string', }, description: 'List of domains to specifically exclude, if the user asks to exclude a domain set this to the domain of the site', default: [], }, country: { type: 'string', description: 'Boost search results from a specific country. This will prioritize content from the selected country in the search results. Available only if topic is general.', default: '', }, include_favicon: { type: 'boolean', description: 'Whether to include the favicon URL for each result', default: false, }, }, required: ['query'], }, }, }, { type: 'function', function: { name: 'tavily_extract', description: 'Extract content from URLs. Returns raw page content in markdown or text format.', parameters: { type: 'object', properties: { urls: { type: 'array', items: { type: 'string', }, description: 'List of URLs to extract content from', }, extract_depth: { type: 'string', enum: ['basic', 'advanced'], description: "Use 'advanced' for LinkedIn, protected sites, or tables/embedded content", default: 'basic', }, include_images: { type: 'boolean', description: 'Include images from pages', default: false, }, format: { type: 'string', enum: ['markdown', 'text'], description: 'Output format', default: 'markdown', }, include_favicon: { type: 'boolean', description: 'Include favicon URLs', default: false, }, query: { type: 'string', description: 'Query to rerank content chunks by relevance', }, }, required: ['urls'], }, }, }, { type: 'function', function: { name: 'tavily_crawl', description: 'Crawl a website starting from a URL. Extracts content from pages with configurable depth and breadth.', parameters: { type: 'object', properties: { url: { type: 'string', description: 'The root URL to begin the crawl', }, max_depth: { type: 'integer', description: 'Max depth of the crawl. Defines how far from the base URL the crawler can explore.', default: 1, minimum: 1, }, max_breadth: { type: 'integer', description: 'Max number of links to follow per level of the tree (i.e., per page)', default: 20, minimum: 1, }, limit: { type: 'integer', description: 'Total number of links the crawler will process before stopping', default: 50, minimum: 1, }, instructions: { type: 'string', description: 'Natural language instructions for the crawler. Instructions specify which types of pages the crawler should return.', }, select_paths: { type: 'array', items: { type: 'string', }, description: 'Regex patterns to select only URLs with specific path patterns (e.g., /docs/.*, /api/v1.*)', default: [], }, select_domains: { type: 'array', items: { type: 'string', }, description: String.raw`Regex patterns to restrict crawling to specific domains or subdomains (e.g., ^docs\.example\.com$)`, default: [], }, allow_external: { type: 'boolean', description: 'Whether to return external links in the final response', default: true, }, extract_depth: { type: 'string', enum: ['basic', 'advanced'], description: 'Advanced extraction retrieves more data, including tables and embedded content, with higher success but may increase latency', default: 'basic', }, format: { type: 'string', enum: ['markdown', 'text'], description: 'The format of the extracted web page content. markdown returns content in markdown format. text returns plain text and may increase latency.', default: 'markdown', }, include_favicon: { type: 'boolean', description: 'Whether to include the favicon URL for each result', default: false, }, }, required: ['url'], }, }, }, { type: 'function', function: { name: 'tavily_map', description: "Map a website's structure. Returns a list of URLs found starting from the base URL.", parameters: { type: 'object', properties: { url: { type: 'string', description: 'The root URL to begin the mapping', }, max_depth: { type: 'integer', description: 'Max depth of the mapping. Defines how far from the base URL the crawler can explore', default: 1, minimum: 1, }, max_breadth: { type: 'integer', description: 'Max number of links to follow per level of the tree (i.e., per page)', default: 20, minimum: 1, }, limit: { type: 'integer', description: 'Total number of links the crawler will process before stopping', default: 50, minimum: 1, }, instructions: { type: 'string', description: 'Natural language instructions for the crawler', }, select_paths: { type: 'array', items: { type: 'string', }, description: 'Regex patterns to select only URLs with specific path patterns (e.g., /docs/.*, /api/v1.*)', default: [], }, select_domains: { type: 'array', items: { type: 'string', }, description: String.raw`Regex patterns to restrict crawling to specific domains or subdomains (e.g., ^docs\.example\.com$)`, default: [], }, allow_external: { type: 'boolean', description: 'Whether to return external links in the final response', default: true, }, }, required: ['url'], }, }, }, { type: 'function', function: { name: 'tavily_research', description: 'Perform comprehensive research on a given topic or question. Use this tool when you need to gather information from multiple sources to answer a question or complete a task. Returns a detailed response based on the research findings.', parameters: { type: 'object', properties: { input: { type: 'string', description: 'A comprehensive description of the research task', }, model: { type: 'string', enum: ['mini', 'pro', 'auto'], description: "Defines the degree of depth of the research. 'mini' is good for narrow tasks with few subtopics. 'pro' is good for broad tasks with many subtopics. 'auto' automatically selects the best model.", default: 'auto', }, }, required: ['input'], }, }, }, { type: 'function', function: { name: 'fetch_url', description: 'Retrieve web page content from a specified URL', parameters: { type: 'object', properties: { url: { type: 'string', description: 'URL to fetch. Make sure to include the schema (http:// or https:// if not defined, preferring https for most cases)', }, timeout: { type: 'number', description: 'Page loading timeout in milliseconds, default is 30000 (30 seconds)', }, waitUntil: { type: 'string', description: "Specifies when navigation is considered complete, options: 'load', 'domcontentloaded', 'networkidle', 'commit', default is 'load'", }, extractContent: { type: 'boolean', description: 'Whether to intelligently extract the main content, default is true', }, maxLength: { type: 'number', description: 'Maximum length of returned content (in characters), default is no limit', }, returnHtml: { type: 'boolean', description: 'Whether to return HTML content instead of Markdown, default is false', }, waitForNavigation: { type: 'boolean', description: 'Whether to wait for additional navigation after initial page load (useful for sites with anti-bot verification), default is false', }, navigationTimeout: { type: 'number', description: 'Maximum time to wait for additional navigation in milliseconds, default is 10000 (10 seconds)', }, disableMedia: { type: 'boolean', description: 'Whether to disable media resources (images, stylesheets, fonts, media), default is true', }, debug: { type: 'boolean', description: 'Whether to enable debug mode (showing browser window), overrides the --debug command line flag if specified', }, }, required: ['url'], }, }, }, { type: 'function', function: { name: 'fetch_urls', description: 'Retrieve web page content from multiple specified URLs', parameters: { type: 'object', properties: { urls: { type: 'array', items: { type: 'string', }, description: 'Array of URLs to fetch', }, timeout: { type: 'number', description: 'Page loading timeout in milliseconds, default is 30000 (30 seconds)', }, waitUntil: { type: 'string', description: "Specifies when navigation is considered complete, options: 'load', 'domcontentloaded', 'networkidle', 'commit', default is 'load'", }, extractContent: { type: 'boolean', description: 'Whether to intelligently extract the main content, default is true', }, maxLength: { type: 'number', description: 'Maximum length of returned content (in characters), default is no limit', }, returnHtml: { type: 'boolean', description: 'Whether to return HTML content instead of Markdown, default is false', }, waitForNavigation: { type: 'boolean', description: 'Whether to wait for additional navigation after initial page load (useful for sites with anti-bot verification), default is false', }, navigationTimeout: { type: 'number', description: 'Maximum time to wait for additional navigation in milliseconds, default is 10000 (10 seconds)', }, disableMedia: { type: 'boolean', description: 'Whether to disable media resources (images, stylesheets, fonts, media), default is true', }, debug: { type: 'boolean', description: 'Whether to enable debug mode (showing browser window), overrides the --debug command line flag if specified', }, }, required: ['urls'], }, }, }, { type: 'function', function: { name: 'browser_install', description: 'Install Playwright Chromium browser binary. Call this if you get an error about the browser not being installed.', parameters: { type: 'object', properties: { withDeps: { type: 'boolean', description: 'Install system dependencies required by Chromium browser. Default is false', default: false, }, force: { type: 'boolean', description: 'Force installation even if Chromium is already installed. Default is false', default: false, }, }, required: [], }, }, }, { type: 'function', function: { name: 'click', description: 'Clicks on the provided element', parameters: { type: 'object', properties: { uid: { type: 'string', description: 'The uid of an element on the page from the page content snapshot', }, dblClick: { type: 'boolean', description: 'Set to true for double clicks. Default is false.', }, includeSnapshot: { type: 'boolean', description: 'Whether to include a snapshot in the response. Default is false.', }, }, required: ['uid'], }, }, }, { type: 'function', function: { name: 'close_page', description: 'Closes the page by its index. The last open page cannot be closed.', parameters: { type: 'object', properties: { pageId: { type: 'number', description: 'The ID of the page to close. Call list_pages to list pages.', }, }, required: ['pageId'], }, }, }, { type: 'function', function: { name: 'drag', description: 'Drag an element onto another element', parameters: { type: 'object', properties: { from_uid: { type: 'string', description: 'The uid of the element to drag', }, to_uid: { type: 'string', description: 'The uid of the element to drop into', }, includeSnapshot: { type: 'boolean', description: 'Whether to include a snapshot in the response. Default is false.', }, }, required: ['from_uid', 'to_uid'], }, }, }, { type: 'function', function: { name: 'emulate', description: 'Emulates various features on the selected page.', parameters: { type: 'object', properties: { networkConditions: { type: 'string', enum: ['No emulation', 'Offline', 'Slow 3G', 'Fast 3G', 'Slow 4G', 'Fast 4G'], description: 'Throttle network. Set to "No emulation" to disable. If omitted, conditions remain unchanged.', }, cpuThrottlingRate: { type: 'number', minimum: 1, maximum: 20, description: 'Represents the CPU slowdown factor. Set the rate to 1 to disable throttling. If omitted, throttling remains unchanged.', }, geolocation: { anyOf: [ { type: 'object', properties: { latitude: { type: 'number', minimum: -90, maximum: 90, description: 'Latitude between -90 and 90.', }, longitude: { type: 'number', minimum: -180, maximum: 180, description: 'Longitude between -180 and 180.', }, }, required: ['latitude', 'longitude'], additionalProperties: false, }, { type: 'null', }, ], description: 'Geolocation to emulate. Set to null to clear the geolocation override.', }, userAgent: { type: ['string', 'null'], description: 'User agent to emulate. Set to null to clear the user agent override.', }, colorScheme: { type: 'string', enum: ['dark', 'light', 'auto'], description: 'Emulate the dark or the light mode. Set to "auto" to reset to the default.', }, viewport: { anyOf: [ { type: 'object', properties: { width: { type: 'integer', minimum: 0, description: 'Page width in pixels.', }, height: { type: 'integer', minimum: 0, description: 'Page height in pixels.', }, deviceScaleFactor: { type: 'number', minimum: 0, description: 'Specify device scale factor (can be thought of as dpr).', }, isMobile: { type: 'boolean', description: 'Whether the meta viewport tag is taken into account. Defaults to false.', }, hasTouch: { type: 'boolean', description: 'Specifies if viewport supports touch events. This should be set to true for mobile devices.', }, isLandscape: { type: 'boolean', description: 'Specifies if viewport is in landscape mode. Defaults to false.', }, }, required: ['width', 'height'], additionalProperties: false, }, { type: 'null', }, ], description: 'Viewport to emulate. Set to null to reset to the default viewport.', }, }, required: [], }, }, }, { type: 'function', function: { name: 'evaluate_script', description: 'Evaluate a JavaScript function inside the currently selected page. Returns the response as JSON,\nso returned values have to be JSON-serializable.', parameters: { type: 'object', properties: { function: { type: 'string', description: 'A JavaScript function declaration to be executed by the tool in the currently selected page.\nExample without arguments: `() => {\n return document.title\n}` or `async () => {\n return await fetch("example.com")\n}`.\nExample with arguments: `(el) => {\n return el.innerText;\n}`\n', }, args: { type: 'array', items: { type: 'object', properties: { uid: { type: 'string', description: 'The uid of an element on the page from the page content snapshot', }, }, required: ['uid'], additionalProperties: false, }, description: 'An optional list of arguments to pass to the function.', }, }, required: ['function'], }, }, }, { type: 'function', function: { name: 'fill', description: 'Type text into a input, text area or select an option from a