{ "kind": "open-flow-workflow-bundle", "version": 1, "entryFlowId": "batch-webpage-markdown-links", "workflows": [ { "flowId": "batch-webpage-markdown-links", "file": { "kind": "open-flow-workflow", "version": 1, "name": "Batch Webpage Markdown and Links", "content": { "modelVersion": 14, "document": { "bindings": {}, "graph": { "nodes": { "n_37edc14f0887c896": { "name": "Page URL list", "kind": "start", "inputDefs": [ { "jsonSchema": { "type": "string" }, "nullable": false, "handle": "urlsJson" } ] }, "n_b9acd046972331ce": { "inputs": { "urlsJson": { "kind": "sources", "sources": [ { "kind": "node", "nodeId": "n_37edc14f0887c896", "output": "urlsJson" } ] } }, "name": "Select and validate first 40 URLs", "kind": "task", "task": { "inputs": [ { "jsonSchema": { "type": "string" }, "nullable": false, "handle": "urlsJson" } ], "outputs": [ { "jsonSchema": { "type": "array", "items": { "type": "string", "minLength": 1 }, "maxItems": 40 }, "nullable": false, "handle": "urls" }, { "jsonSchema": { "type": "integer", "minimum": 0 }, "nullable": false, "handle": "inputCount" }, { "jsonSchema": { "type": "integer", "minimum": 0 }, "nullable": false, "handle": "skippedCount" } ], "name": "Select and validate first 40 URLs", "moduleId": "n_8b9389e04eb389ff", "capabilities": [ { "kind": "connector", "mode": "independent", "actions": [] } ] } }, "n_cc8f30f10d31eb82": { "name": "Page records and skipped count", "kind": "outputs", "outputDefs": [ { "jsonSchema": { "type": "array", "items": { "type": "object", "properties": { "url": { "type": "string" }, "title": { "type": "string" }, "description": { "type": "string" }, "content": { "type": "string" }, "links": { "type": "array", "items": { "type": "string" } }, "pageStatus": { "type": [ "integer", "null" ], "minimum": 100, "maximum": 599 } }, "required": [ "url", "title", "description", "content", "links", "pageStatus" ], "additionalProperties": false }, "maxItems": 40 }, "nullable": false, "handle": "pages" }, { "jsonSchema": { "type": "integer", "minimum": 0 }, "nullable": false, "handle": "processedCount" }, { "jsonSchema": { "type": "integer", "minimum": 0 }, "nullable": false, "handle": "inputCount" }, { "jsonSchema": { "type": "integer", "minimum": 0 }, "nullable": false, "handle": "skippedCount" } ], "outputMappings": { "inputCount": { "kind": "sources", "sources": [ { "kind": "node", "nodeId": "n_b9acd046972331ce", "output": "inputCount" } ] }, "skippedCount": { "kind": "sources", "sources": [ { "kind": "node", "nodeId": "n_b9acd046972331ce", "output": "skippedCount" } ] }, "pages": { "kind": "sources", "sources": [ { "kind": "node", "nodeId": "pjvkw7dp62", "output": "pages" } ] }, "processedCount": { "kind": "sources", "sources": [ { "kind": "node", "nodeId": "pjvkw7dp62", "output": "processedCount" } ] } } }, "pjvkw7dp62": { "inputs": { "urls": { "kind": "sources", "sources": [ { "kind": "node", "nodeId": "n_b9acd046972331ce", "output": "urls" } ] } }, "name": "Scrape ten URLs, pause one minute", "timeoutMs": 7200000, "kind": "task", "task": { "inputs": [ { "jsonSchema": { "type": "array", "items": { "type": "string", "minLength": 1 }, "maxItems": 40 }, "nullable": false, "handle": "urls" } ], "outputs": [ { "jsonSchema": { "type": "array", "items": { "type": "object", "properties": { "url": { "type": "string" }, "title": { "type": "string" }, "description": { "type": "string" }, "content": { "type": "string" }, "links": { "type": "array", "items": { "type": "string" } }, "pageStatus": { "type": [ "integer", "null" ], "minimum": 100, "maximum": 599 } }, "required": [ "url", "title", "description", "content", "links", "pageStatus" ], "additionalProperties": false }, "maxItems": 40 }, "nullable": false, "handle": "pages" }, { "jsonSchema": { "type": "integer", "minimum": 0 }, "nullable": false, "handle": "processedCount" } ], "name": "Scrape ten URLs, pause one minute", "moduleId": "zzztft2d3q", "capabilities": [ { "kind": "workflow", "name": "scrape-one-page", "flowId": "webpage-markdown-links-scraper", "revisionId": "", "inputs": [ { "jsonSchema": { "type": "string" }, "nullable": false, "handle": "url" } ], "outputs": [ { "jsonSchema": { "type": "object", "properties": { "url": { "type": "string" }, "title": { "type": "string" }, "description": { "type": "string" }, "content": { "type": "string" }, "links": { "type": "array", "items": { "type": "string" } }, "pageStatus": { "type": [ "integer", "null" ], "minimum": 100, "maximum": 599 } }, "required": [ "url", "title", "description", "content", "links", "pageStatus" ], "additionalProperties": false }, "nullable": false, "handle": "page" } ] } ] } } }, "edges": [ { "source": "n_37edc14f0887c896", "target": "n_b9acd046972331ce" }, { "source": "n_b9acd046972331ce", "target": "pjvkw7dp62" }, { "source": "pjvkw7dp62", "target": "n_cc8f30f10d31eb82" } ] } }, "modules": { "n_8b9389e04eb389ff": { "name": "Select and validate first 40 URLs", "imports": [], "source": "export default function prepareUrls({ urlsJson }) {\n let raw;\n try { raw = JSON.parse(urlsJson); } catch { throw new Error('Page URLs must be a JSON array.'); }\n if (!Array.isArray(raw)) throw new Error('Page URLs must be a JSON array.');\n const urls = raw.slice(0, 40).map(url => {\n if (typeof url !== 'string' || !/^https?:\\/\\/[^\\s/]+(?:\\/[^\\s]*)?$/i.test(url.trim())) throw new Error('Each selected page needs a nonempty HTTP or HTTPS URL.');\n return url.trim();\n });\n return { urls, inputCount: raw.length, skippedCount: Math.max(raw.length - 40, 0) };\n}\n" }, "zzztft2d3q": { "name": "Scrape ten URLs, pause one minute", "imports": [], "source": "import { setTimeout as delay } from 'node:timers/promises';\n\nexport async function scrapePages({ urls }, context, wait = delay) {\n const pages = [];\n for (let index = 0; index < urls.length; index++) {\n context.signal?.throwIfAborted();\n const result = await context.workflows.call('scrape-one-page', { url: urls[index] });\n if (result?.page?.url !== urls[index]) throw new Error('The page result belongs to a different source URL.');\n pages.push(result.page);\n if ((index + 1) % 10 === 0 && index + 1 < urls.length) await wait(60000, undefined, { signal: context.signal });\n }\n return { pages, processedCount: pages.length };\n}\nexport default async function run(input, context) { return scrapePages(input, context); }\n" } } }, "presentation": { "designer": { "version": 1, "flow": { "direction": "vertical" } } } } }, { "flowId": "webpage-markdown-links-scraper", "file": { "kind": "open-flow-workflow", "version": 1, "name": "Webpage Markdown and Links Scraper", "content": { "modelVersion": 14, "document": { "bindings": {}, "graph": { "nodes": { "n_ec67bff54e70ea1a": { "name": "One page URL", "kind": "start", "inputDefs": [ { "jsonSchema": { "type": "string" }, "nullable": false, "handle": "url" } ] }, "n_baff777eeb1bb6e1": { "inputs": { "url": { "kind": "sources", "sources": [ { "kind": "node", "nodeId": "n_ec67bff54e70ea1a", "output": "url" } ] }, "formats": { "kind": "value", "value": [ "markdown", "links" ] }, "maxAge": { "kind": "value", "value": 0 }, "onlyMainContent": { "kind": "value", "value": true }, "timeout": { "kind": "value", "value": 30000 } }, "name": "Read Markdown and links", "timeoutMs": 150000, "retry": { "maxRetries": 2, "intervalMs": 5000 }, "kind": "task", "task": { "inputs": [ { "description": "The URL to scrape.", "jsonSchema": { "type": "string", "description": "The URL to scrape." }, "nullable": true, "handle": "url", "value": null }, { "description": "One Alexandria provider-tool call or an array of up to ten calls.", "jsonSchema": { "description": "One Alexandria provider-tool call or an array of up to ten calls." }, "nullable": true, "handle": "alexandria", "value": null }, { "description": "Whether to return tool contracts discovered for the scraped domain.", "jsonSchema": { "type": "boolean", "description": "Whether to return tool contracts discovered for the scraped domain." }, "nullable": true, "handle": "domainTools", "value": null }, { "description": "The request origin accepted with Alexandria calls.", "jsonSchema": { "type": "string", "description": "The request origin accepted with Alexandria calls." }, "nullable": true, "handle": "origin", "value": null }, { "description": "The integration identifier accepted with Alexandria calls.", "jsonSchema": { "type": "string", "description": "The integration identifier accepted with Alexandria calls." }, "nullable": true, "handle": "integration", "value": null }, { "description": "The browser actions to run before scraping.", "jsonSchema": { "type": "array", "items": { "type": "object", "properties": { "type": { "type": "string", "description": "The browser action type, such as click, write, wait, or press." }, "selector": { "type": "string", "description": "The CSS selector targeted by the action." }, "text": { "type": "string", "description": "The text to type for write actions." }, "key": { "type": "string", "description": "The key to press for press actions." }, "milliseconds": { "type": "integer", "description": "The duration in milliseconds used by wait-style actions." } }, "required": [ "type" ], "additionalProperties": true, "description": "A browser action to execute." }, "description": "The browser actions to run before scraping." }, "nullable": true, "handle": "actions", "value": null }, { "description": "The output formats to return.", "jsonSchema": { "type": "array", "items": { "anyOf": [ { "type": "string", "enum": [ "markdown", "html", "rawHtml", "rawBase64", "links", "images", "screenshot", "screenshot@fullPage", "json", "changeTracking", "summary", "branding", "product", "menu", "audio", "video" ], "description": "A built-in Firecrawl output format." }, { "type": "object", "properties": { "type": { "type": "string", "enum": [ "markdown", "summary", "html", "rawHtml", "rawBase64", "links", "images", "screenshot", "json", "changeTracking", "branding", "product", "menu", "audio", "video", "question", "highlights" ], "description": "The structured Firecrawl output format type." }, "schema": { "type": "object", "additionalProperties": { "description": "A record value." }, "description": "The JSON Schema used for structured output." }, "prompt": { "type": "string", "description": "The prompt used for structured output." }, "checkPromptInjection": { "type": "boolean", "description": "Whether to reject prompt injection before structured extraction." }, "fullPage": { "type": "boolean", "description": "Whether a screenshot should capture the full page." }, "quality": { "type": "integer", "minimum": 1, "maximum": 100, "description": "The screenshot quality from 1 through 100." }, "viewport": { "type": "object", "additionalProperties": { "description": "A record value." }, "description": "The screenshot viewport dimensions." }, "modes": { "type": "array", "items": { "type": "string", "enum": [ "git-diff", "json" ], "description": "A change-tracking comparison mode." }, "description": "The change-tracking comparison modes." }, "tag": { "type": "string", "description": "The change-tracking history tag." }, "question": { "type": "string", "description": "The natural-language question to answer about the page." }, "query": { "type": "string", "description": "The query used to select relevant page highlights." } }, "required": [ "type" ], "additionalProperties": false, "description": "A structured Firecrawl format descriptor." } ], "description": "A requested Firecrawl output format." }, "description": "The output formats to return." }, "nullable": true, "handle": "formats", "value": null }, { "description": "Custom HTTP headers to send with the request.", "jsonSchema": { "type": "object", "additionalProperties": { "type": "string", "description": "A record value." }, "description": "Custom HTTP headers to send with the request." }, "nullable": true, "handle": "headers", "value": null }, { "description": "Location settings for the request.", "jsonSchema": { "type": "object", "properties": { "country": { "type": "string", "description": "The ISO 3166-1 alpha-2 country code to request from." }, "languages": { "type": "array", "items": { "type": "string", "description": "A preferred locale such as en-US or fr-CA." }, "description": "The preferred locales for the request." } }, "additionalProperties": true, "description": "Location settings for the request." }, "nullable": true, "handle": "location", "value": null }, { "description": "The request timeout in milliseconds.", "jsonSchema": { "type": "integer", "minimum": 1000, "maximum": 300000, "description": "The request timeout in milliseconds." }, "nullable": true, "handle": "timeout", "value": null }, { "description": "The delay before scraping starts.", "jsonSchema": { "type": "integer", "description": "The delay before scraping starts." }, "nullable": true, "handle": "waitFor", "value": null }, { "description": "The cache max age in milliseconds.", "jsonSchema": { "type": "integer", "description": "The cache max age in milliseconds." }, "nullable": true, "handle": "maxAge", "value": null }, { "description": "The minimum acceptable cache age without triggering a fresh scrape.", "jsonSchema": { "type": "integer", "description": "The minimum acceptable cache age without triggering a fresh scrape." }, "nullable": true, "handle": "minAge", "value": null }, { "description": "Whether to keep only the main content of the page.", "jsonSchema": { "type": "boolean", "description": "Whether to keep only the main content of the page." }, "nullable": true, "handle": "onlyMainContent", "value": null }, { "description": "Whether to run Firecrawl's additional LLM-based content cleanup.", "jsonSchema": { "type": "boolean", "description": "Whether to run Firecrawl's additional LLM-based content cleanup." }, "nullable": true, "handle": "onlyCleanContent", "value": null }, { "description": "The HTML tags that should be prioritized in the extracted content.", "jsonSchema": { "type": "array", "items": { "type": "string", "description": "An HTML tag name to keep." }, "description": "The HTML tags that should be prioritized in the extracted content." }, "nullable": true, "handle": "includeTags", "value": null }, { "description": "The HTML tags that should be removed from the extracted content.", "jsonSchema": { "type": "array", "items": { "type": "string", "description": "An HTML tag name to remove." }, "description": "The HTML tags that should be removed from the extracted content." }, "nullable": true, "handle": "excludeTags", "value": null }, { "description": "Whether to emulate a mobile device.", "jsonSchema": { "type": "boolean", "description": "Whether to emulate a mobile device." }, "nullable": true, "handle": "mobile", "value": null }, { "description": "The proxy mode to use for the request.", "jsonSchema": { "type": "string", "enum": [ "basic", "enhanced", "auto" ], "description": "The proxy mode to use for the request." }, "nullable": true, "handle": "proxy", "value": null }, { "description": "The parser configurations to enable for the request.", "jsonSchema": { "type": "array", "items": { "anyOf": [ { "const": "pdf", "description": "The default PDF parser." }, { "type": "object", "additionalProperties": { "description": "A record value." }, "description": "A configured PDF parser." } ], "description": "A parser name or configuration." }, "description": "The parser configurations to enable for the request." }, "nullable": true, "handle": "parsers", "value": null }, { "description": "Whether ad resources should be blocked.", "jsonSchema": { "type": "boolean", "description": "Whether ad resources should be blocked." }, "nullable": true, "handle": "blockAds", "value": null }, { "description": "Whether Firecrawl should store the result in cache.", "jsonSchema": { "type": "boolean", "description": "Whether Firecrawl should store the result in cache." }, "nullable": true, "handle": "storeInCache", "value": null }, { "description": "Whether base64-encoded images should be removed from the output.", "jsonSchema": { "type": "boolean", "description": "Whether base64-encoded images should be removed from the output." }, "nullable": true, "handle": "removeBase64Images", "value": null }, { "description": "Whether TLS verification should be skipped.", "jsonSchema": { "type": "boolean", "description": "Whether TLS verification should be skipped." }, "nullable": true, "handle": "skipTlsVerification", "value": null }, { "description": "Whether the request should opt into zero data retention.", "jsonSchema": { "type": "boolean", "description": "Whether the request should opt into zero data retention." }, "nullable": true, "handle": "zeroDataRetention", "value": null }, { "description": "Whether to enable Firecrawl lockdown mode.", "jsonSchema": { "type": "boolean", "description": "Whether to enable Firecrawl lockdown mode." }, "nullable": true, "handle": "lockdown", "value": null }, { "description": "Whether and how to redact personally identifiable information.", "jsonSchema": { "anyOf": [ { "type": "boolean", "description": "Whether to enable default PII redaction." }, { "type": "object", "additionalProperties": { "description": "A record value." }, "description": "Custom PII redaction settings." } ], "description": "Whether and how to redact personally identifiable information." }, "nullable": true, "handle": "redactPII", "value": null }, { "description": "The browser profile settings.", "jsonSchema": { "type": "object", "additionalProperties": { "description": "A record value." }, "description": "The browser profile settings." }, "nullable": true, "handle": "profile", "value": null }, { "description": "The threat-protection override settings.", "jsonSchema": { "type": "object", "additionalProperties": { "description": "A record value." }, "description": "The threat-protection override settings." }, "nullable": true, "handle": "threatProtection", "value": null }, { "description": "Metadata attached to the Firecrawl audit event.", "jsonSchema": { "type": "object", "additionalProperties": { "description": "A record value." }, "description": "Metadata attached to the Firecrawl audit event." }, "nullable": true, "handle": "auditMetadata", "value": null } ], "outputs": [ { "description": "The output payload for this action.", "jsonSchema": { "type": "object", "properties": { "success": { "type": "boolean", "description": "Whether the scrape request succeeded." }, "data": { "type": "object", "additionalProperties": { "description": "A record value." }, "description": "The extracted page payload." }, "warning": { "type": "string", "description": "A warning returned by Firecrawl." }, "error": { "type": "string", "description": "An error message returned by Firecrawl." }, "code": { "anyOf": [ { "type": "string", "description": "A union variant." }, { "type": "number", "description": "A union variant." } ], "description": "The Firecrawl response code." } }, "required": [ "success" ], "additionalProperties": true, "description": "The output payload for this action." }, "nullable": false, "handle": "output" } ], "name": "Read Markdown and links", "executor": { "kind": "connector", "action": "firecrawl.scrape", "outputMode": "value" } } }, "n_e219c1be93955c70": { "inputs": { "url": { "kind": "sources", "sources": [ { "kind": "node", "nodeId": "n_ec67bff54e70ea1a", "output": "url" } ] }, "result": { "kind": "sources", "sources": [ { "kind": "node", "nodeId": "n_baff777eeb1bb6e1", "output": "output" } ] } }, "name": "Keep page fields and source URL", "kind": "task", "task": { "inputs": [ { "jsonSchema": { "type": "string" }, "nullable": false, "handle": "url" }, { "jsonSchema": { "type": "object" }, "nullable": false, "handle": "result" } ], "outputs": [ { "jsonSchema": { "type": "object", "properties": { "url": { "type": "string" }, "title": { "type": "string" }, "description": { "type": "string" }, "content": { "type": "string" }, "links": { "type": "array", "items": { "type": "string" } }, "pageStatus": { "type": [ "integer", "null" ], "minimum": 100, "maximum": 599 } }, "required": [ "url", "title", "description", "content", "links", "pageStatus" ], "additionalProperties": false }, "nullable": false, "handle": "page" } ], "name": "Keep page fields and source URL", "moduleId": "n_711597d8171b76f8", "capabilities": [ { "kind": "connector", "mode": "independent", "actions": [] } ] } }, "n_5ef77887c45db2cf": { "name": "One page record", "kind": "outputs", "outputDefs": [ { "jsonSchema": { "type": "object", "properties": { "url": { "type": "string" }, "title": { "type": "string" }, "description": { "type": "string" }, "content": { "type": "string" }, "links": { "type": "array", "items": { "type": "string" } }, "pageStatus": { "type": [ "integer", "null" ], "minimum": 100, "maximum": 599 } }, "required": [ "url", "title", "description", "content", "links", "pageStatus" ], "additionalProperties": false }, "nullable": false, "handle": "page" } ], "outputMappings": { "page": { "kind": "sources", "sources": [ { "kind": "node", "nodeId": "n_e219c1be93955c70", "output": "page" } ] } } } }, "edges": [ { "source": "n_ec67bff54e70ea1a", "target": "n_baff777eeb1bb6e1" }, { "source": "n_baff777eeb1bb6e1", "target": "n_e219c1be93955c70" }, { "source": "n_e219c1be93955c70", "target": "n_5ef77887c45db2cf" } ] } }, "modules": { "n_711597d8171b76f8": { "name": "Keep page fields and source URL", "imports": [], "source": "export default function readPage({ url, result }) {\n if (!result || result.success !== true || !result.data || typeof result.data !== 'object' || Array.isArray(result.data)) throw new Error('Firecrawl did not return a successful page scrape.');\n const data = result.data;\n if (typeof data.markdown !== 'string' || !Array.isArray(data.links) || data.links.some(link => typeof link !== 'string')) throw new Error('Firecrawl omitted the requested Markdown or link array.');\n const text = value => value == null ? '' : typeof value === 'string' ? value : (() => { throw new Error('Page metadata must be text when present.'); })();\n const status = data.metadata?.statusCode;\n return { page: { url, title: text(data.metadata?.title), description: text(data.metadata?.description), content: data.markdown, links: data.links, pageStatus: Number.isInteger(status) && status >= 100 && status <= 599 ? status : null } };\n}\n" } } }, "presentation": { "designer": { "version": 1, "flow": { "direction": "vertical" } } } } } ] }