#!/usr/bin/env node /** * German Bundesanzeiger Company Register Scraper * * Searches the German Federal Gazette (Bundesanzeiger) for company financial reports * and extracts revenue, assets, and earnings using AI. * * Usage: * node de-bundesanzeiger.js search "Company Name" * node de-bundesanzeiger.js analyze "Company Name" [--index N] */ const https = require('https'); const http = require('http'); const fs = require('fs'); const os = require('os'); const path = require('path'); const { spawnSync, spawn } = require('child_process'); // The two LLM steps, reading the Bundesanzeiger CAPTCHA (vision) and extracting the // financial figures from the report text, run on Claude via the local `claude` CLI // (headless, native OAuth from ~/.claude/.credentials.json). No OpenRouter, no API key. const HOME = process.env.HOME || process.env.USERPROFILE || ''; const CLAUDE_BIN = process.env.CLAUDE_BIN || (HOME && fs.existsSync(path.join(HOME, '.local/bin/claude')) ? path.join(HOME, '.local/bin/claude') : 'claude'); // Cheap vision model for the CAPTCHA; a stronger model for the financial extraction. // Override via env if the model ids change. const CAPTCHA_MODEL = process.env.BA_CAPTCHA_MODEL || 'claude-haiku-4-5-20251001'; const EXTRACT_MODEL = process.env.BA_EXTRACT_MODEL || 'claude-sonnet-5'; const TMP = os.tmpdir(); // Session state let sessionCookies = {}; /** * Call the local `claude` CLI (headless, native OAuth, no API key). Returns trimmed * stdout, or null on failure. Never inherits ANTHROPIC_* (that would override the * subscription auth and hang claude). * opts.stdin text piped to claude on stdin (used as context) * opts.allowRead allow the Read tool (needed for image/vision prompts) * opts.timeoutMs hard timeout (default 120s) */ function runClaude(prompt, model, opts = {}) { const args = ['-p', prompt, '--model', model, '--output-format', 'text']; if (opts.allowRead) { args.push('--allowedTools', 'Read', '--max-turns', '6'); } else { args.push('--max-turns', '1'); } const env = { ...process.env }; delete env.ANTHROPIC_API_KEY; delete env.ANTHROPIC_BASE_URL; const res = spawnSync(CLAUDE_BIN, args, { input: opts.stdin || undefined, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024, timeout: opts.timeoutMs || 120000, env }); if (res.error) { console.error('claude CLI error:', res.error.message); return null; } if (res.status !== 0) { console.error('claude CLI exited', res.status, (res.stderr || '').trim().slice(0, 300)); return null; } return (res.stdout || '').trim(); } /** * Solve a CAPTCHA image with Claude (Haiku vision) via the local claude CLI. * Writes the image to a temp file and asks Claude to read the characters. */ function solveCaptcha(imageBuffer) { const imgPath = path.join(TMP, `ba_captcha_${process.pid}.png`); fs.writeFileSync(imgPath, imageBuffer); const prompt = `Read the CAPTCHA in the image file ${imgPath} . It contains about 6 ` + `distorted letters/digits. Output ONLY those characters, no words, no sentence, ` + `no punctuation, no quotes, no explanation. If unsure, give your single best guess.`; const out = runClaude(prompt, CAPTCHA_MODEL, { allowRead: true, timeoutMs: 90000 }); try { fs.unlinkSync(imgPath); } catch (e) { /* ignore */ } if (!out) return null; // Pick a captcha-like token: alphanumeric runs of length 4-8; prefer the last run // that has a digit or is all-caps (real codes look like "VF22ER"/"BMUFFG", not a // narrated word like "Please"), else the last run, else the whole stripped output. // Bundesanzeiger CAPTCHAs are ~6 chars. const runs = out.match(/[A-Za-z0-9]{4,8}/g) || []; let pick = null; for (const r of runs) { if (/[0-9]/.test(r) || r === r.toUpperCase()) pick = r; } if (!pick) pick = runs.length ? runs[runs.length - 1] : out.replace(/[^A-Za-z0-9]/g, ''); let content = pick.trim(); if (content.length > 6) content = content.substring(0, 6); return content || null; } /** * Make an HTTPS request with proper headers and redirect following. * Retries on transient network resets, the Bundesanzeiger intermittently drops the * connection ("socket hang up" / ECONNRESET), especially on the CAPTCHA POST; a short * retry clears it. */ async function makeRequest(url, options = {}, redirectCount = 0) { const MAX_NET_RETRIES = 5; let lastErr; for (let attempt = 0; attempt <= MAX_NET_RETRIES; attempt++) { try { return await _doRequest(url, options, redirectCount); } catch (e) { lastErr = e; const msg = (e && e.message) || ''; if (/socket hang up|ECONNRESET|EPIPE|ETIMEDOUT|EAI_AGAIN|ECONNREFUSED/i.test(msg) && attempt < MAX_NET_RETRIES) { await new Promise(r => setTimeout(r, 500 * (attempt + 1))); continue; } throw e; } } throw lastErr; } function _doRequest(url, options = {}, redirectCount = 0) { const MAX_REDIRECTS = 10; return new Promise((resolve, reject) => { if (redirectCount >= MAX_REDIRECTS) { reject(new Error('Too many redirects')); return; } const urlObj = new URL(url); const isHttps = urlObj.protocol === 'https:'; const lib = isHttps ? https : http; const cookieString = Object.entries(sessionCookies) .map(([k, v]) => `${k}=${v}`) .join('; '); const reqOptions = { hostname: urlObj.hostname, port: urlObj.port || (isHttps ? 443 : 80), path: urlObj.pathname + urlObj.search, method: options.method || 'GET', headers: { 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Encoding': 'identity', 'Accept-Language': 'de-DE,de;q=0.9,en-US;q=0.8', 'Cache-Control': 'no-cache', 'Connection': 'keep-alive', 'DNT': '1', 'Host': urlObj.hostname, 'Referer': 'https://www.bundesanzeiger.de/', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', ...(cookieString && { 'Cookie': cookieString }), ...options.headers } }; const req = lib.request(reqOptions, (res) => { // Store cookies from response const setCookies = res.headers['set-cookie'] || []; for (const cookie of setCookies) { const [nameValue] = cookie.split(';'); const [name, value] = nameValue.split('='); if (name && value) { sessionCookies[name.trim()] = value.trim(); } } // Handle redirects if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { let redirectUrl = res.headers.location; // Handle relative URLs if (redirectUrl.startsWith('/')) { redirectUrl = `${urlObj.protocol}//${urlObj.host}${redirectUrl}`; } else if (!redirectUrl.startsWith('http')) { redirectUrl = new URL(redirectUrl, url).href; } // Follow redirect resolve(makeRequest(redirectUrl, options, redirectCount + 1)); return; } let data = ''; res.on('data', chunk => data += chunk); res.on('end', () => resolve({ status: res.statusCode, headers: res.headers, body: data })); }); req.on('error', reject); if (options.body) { req.write(options.body); } req.end(); }); } /** * Initialize session with Bundesanzeiger */ async function initSession() { sessionCookies = { 'cc': '1628606977-805e172265bfdbde-10' }; // Get JSESSIONID await makeRequest('https://www.bundesanzeiger.de'); await makeRequest('https://www.bundesanzeiger.de/pub/de/start?0'); } /** * Parse search results HTML */ function parseSearchResults(html) { const results = []; // Find result container - new structure uses "result_container global-search" const containerMatch = html.match(/