//! Web fetch with real SSRF protection: every hop's host is resolved and all //! addresses must be public. Redirects are followed manually so each hop is //! re-validated. Bodies are streamed and capped, never fully buffered. use super::{arg_str, arg_u64, Tool, ToolContext}; use crate::error::{NurError, Result}; use serde_json::Value; use std::io::Read; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, ToSocketAddrs}; pub struct WebFetch; const MAX_REDIRECTS: usize = 5; impl Tool for WebFetch { fn name(&self) -> &str { "web_fetch" } fn description(&self) -> &str { "Fetch a public HTTP(S) URL and return text content (HTML/JSON/plain). \ Max 500KB. Use for docs and APIs, not for authenticated/private resources. \ With TREG_TOKEN or TINYFISH_API_KEY set, a page that refuses the request or needs \ JavaScript is read through treg or TinyFish instead and comes back as markdown." } fn parameters_schema(&self) -> Value { serde_json::json!({ "type": "object", "properties": { "url": {"type": "string"}, "max_bytes": {"type": "integer", "default": 200000} }, "required": ["url"] }) } fn execute(&self, args: &Value, _ctx: &ToolContext) -> Result { let url = arg_str(args, "url")?; let max = arg_u64(args, "max_bytes").unwrap_or(200_000) as usize; let max = max.min(500_000); let mut current = url.clone(); for _hop in 0..=MAX_REDIRECTS { let (parsed, pin) = validate_public_url(¤t)?; // Pin the validated IP so DNS rebinding between the check and the // connect cannot redirect the request to a private address. let mut builder = reqwest::blocking::Client::builder() .timeout(std::time::Duration::from_secs(30)) .user_agent(format!("nur-cli/{}", env!("CARGO_PKG_VERSION"))) // Redirects are followed manually below so each hop is re-checked. .redirect(reqwest::redirect::Policy::none()); if let Some((host, addr)) = &pin { builder = builder.resolve(host, *addr); } let client = builder.build().map_err(|e| NurError::Tool(e.to_string()))?; let resp = match client.get(parsed).send() { Ok(resp) => resp, Err(e) => { let failure = format!("fetch failed: {e}"); // Keep nur's own error: when the fallback fails too, a // dead host must not read as a quota problem. return match rendered_fallback(¤t, max) { None => Err(NurError::Tool(failure)), Some(Ok(out)) => Ok(out), Some(Err(f)) => { Err(NurError::Tool(format!("{failure}\nrendered fallback: {f}"))) } }; } }; let status = resp.status(); if status.is_redirection() { let loc = resp .headers() .get(reqwest::header::LOCATION) .and_then(|v| v.to_str().ok()) .ok_or_else(|| NurError::Tool("redirect without Location".into()))?; // Resolve relative redirects against the current URL. let base = reqwest::Url::parse(¤t) .map_err(|e| NurError::Tool(format!("bad url: {e}")))?; let next = base .join(loc) .map_err(|e| NurError::Tool(format!("bad redirect: {e}")))?; current = next.to_string(); continue; } let ctype = resp .headers() .get(reqwest::header::CONTENT_TYPE) .and_then(|v| v.to_str().ok()) .unwrap_or("") .to_string(); // Stream-capped read: never buffer more than max+1 bytes. let mut buf = Vec::with_capacity(max.min(65_536)); resp.take(max as u64 + 1) .read_to_end(&mut buf) .map_err(|e| NurError::Tool(format!("read body: {e}")))?; let truncated = buf.len() > max; let slice = if truncated { &buf[..max] } else { &buf[..] }; let text = String::from_utf8_lossy(slice); let mut fallback_failed = None; if super::treg::should_fall_back(status.as_u16(), &ctype, &text) { match rendered_fallback(¤t, max) { Some(Ok(out)) => return Ok(out), Some(Err(e)) => fallback_failed = Some(e), None => {} } } let mut out = format!("url: {current}\nstatus: {status}\ncontent-type: {ctype}\n\n"); out.push_str(&text); if truncated { out.push_str(&format!("\n\n[truncated at {max} bytes]")); } if let Some(e) = fallback_failed { out.push_str(&format!("\n\n[rendered fallback failed: {e}]")); } return Ok(out); } Err(NurError::Tool(format!( "too many redirects (>{MAX_REDIRECTS}): {url}" ))) } } /// Read a page nur's own request could not: treg first, then TinyFish Fetch. /// `None` when neither is configured, so callers keep their own answer; when /// both fail, both errors. fn rendered_fallback(url: &str, max: usize) -> Option> { let treg = through_treg(url, max).map(|r| { r.map_err(|e| match e { NurError::Tool(m) => format!("treg: {m}"), other => format!("treg: {other}"), }) }); if let Some(Ok(out)) = treg { return Some(Ok(out)); } match (treg, through_tinyfish(url, max)) { (_, Some(Ok(out))) => Some(Ok(out)), (Some(Err(t)), Some(Err(f))) => Some(Err(format!("{t}; {f}"))), (treg, tinyfish) => tinyfish.or(treg), } } /// Read `url` through TinyFish Fetch when a key is set (Chromium-rendered, /// returned as markdown). fn through_tinyfish(url: &str, max: usize) -> Option> { let key = super::tinyfish::key()?; Some( super::tinyfish::fetch(&key, url) .map(|(text, final_url)| super::tinyfish::fetched(&final_url, &text, max)) .map_err(|e| e.account_notice().unwrap_or(e.message)), ) } /// Read `url` through treg when a token is configured: `None` without one, so /// callers keep their own answer. fn through_treg(url: &str, max: usize) -> Option> { let token = super::treg::token()?; Some( super::treg::extract(&token, url) .map(|(text, served)| super::treg::fetched(url, &text, served.as_deref(), max)) .map_err(NurError::Tool), ) } /// Parse the URL, resolve its host, and require every address to be public. /// For hostname URLs, also returns `(host, addr)` to pin for the connection. fn validate_public_url( url: &str, ) -> Result<(reqwest::Url, Option<(String, std::net::SocketAddr)>)> { let parsed = reqwest::Url::parse(url).map_err(|e| NurError::Tool(format!("bad url: {e}")))?; match parsed.scheme() { "http" | "https" => {} s => return Err(NurError::Tool(format!("refused scheme: {s}"))), } let host: String = parsed .host_str() .ok_or_else(|| NurError::Tool("url has no host".into()))? .to_string(); let host = host.as_str(); // Cloud metadata hostnames. let hl = host.to_ascii_lowercase(); if hl == "localhost" || hl.ends_with(".localhost") || hl.ends_with(".local") || hl.ends_with(".internal") || hl.contains("metadata") { return Err(NurError::Tool(format!( "refused local/metadata host: {host}" ))); } // Literal IP → check directly (no pin needed); hostname → resolve, check // every record, and pin the first validated address for the connection. // Handle obfuscated forms: decimal (2130706433), octal (0177.0.0.1), hex (0x7f.0.0.1) if let Some(ip) = parse_obfuscated_ip(&hl) { if !ip_is_public(ip) { return Err(NurError::Tool(format!( "refused non-public address: {host} → {ip} (obfuscated)" ))); } // If obfuscation resolved to public, still treat as literal (no DNS pin needed) return Ok((parsed, None)); } if let Ok(ip) = hl.trim_matches(['[', ']']).parse::() { if !ip_is_public(ip) { return Err(NurError::Tool(format!( "refused non-public address: {host} → {ip}" ))); } // Extra: block 0.0.0.0 explicitly even though is_unspecified covers it if ip.is_unspecified() { return Err(NurError::Tool(format!( "refused unspecified address: {host}" ))); } return Ok((parsed, None)); } let port = parsed.port_or_known_default().unwrap_or(443); let sockaddrs: Vec = (host, port) .to_socket_addrs() .map_err(|e| NurError::Tool(format!("dns resolve {host}: {e}")))? .collect(); if sockaddrs.is_empty() { return Err(NurError::Tool(format!("dns: no addresses for {host}"))); } for sa in &sockaddrs { if !ip_is_public(sa.ip()) { return Err(NurError::Tool(format!( "refused non-public address: {host} → {}", sa.ip() ))); } } Ok((parsed, Some((host.to_string(), sockaddrs[0])))) } fn parse_obfuscated_ip(host: &str) -> Option { let h = host.trim().trim_matches(['[', ']']); // Single decimal number e.g. 2130706433 or 0x7f000001 or 0o... if h.chars().all(|c| c.is_ascii_digit()) { if let Ok(n) = h.parse::() { return Some(IpAddr::V4(Ipv4Addr::from(n))); } } if h.to_ascii_lowercase().starts_with("0x") { let hex = h.trim_start_matches("0x").trim_start_matches("0X"); if let Ok(n) = u32::from_str_radix(hex, 16) { return Some(IpAddr::V4(Ipv4Addr::from(n))); } } // Dotted form with possible octal/hex octets: e.g. 0177.0.0.01 or 0x7f.0.0.1 or 127.0.0.0x01 if h.contains('.') { let parts: Vec<&str> = h.split('.').collect(); if parts.len() == 4 { let mut octets = [0u8; 4]; for (i, p) in parts.iter().enumerate() { let p = p.trim(); if p.is_empty() { return None; } let val = if p.to_ascii_lowercase().starts_with("0x") { u32::from_str_radix(p.trim_start_matches("0x").trim_start_matches("0X"), 16) .ok()? } else if p.len() > 1 && p.starts_with('0') && p.chars().skip(1).all(|c| c.is_ascii_digit()) { // octal: 0177 -> 127 // Ensure all chars are octal digits if p.chars().all(|c| ('0'..='7').contains(&c)) || p.starts_with("0") { // If any digit 8 or 9, treat as decimal octal invalid -> fallback decimal if p.chars().any(|c| c == '8' || c == '9') { p.parse::().ok()? } else { u32::from_str_radix(p, 8).ok()? } } else { p.parse::().ok()? } } else { p.parse::().ok()? }; if val > 255 { // Could be large value in last octet representing 2 octets? Simplified: reject unless it's single-part handling return None; } octets[i] = val as u8; } return Some(IpAddr::V4(Ipv4Addr::from(octets))); } } None } fn ip_is_public(ip: IpAddr) -> bool { match ip { IpAddr::V4(v4) => ipv4_is_public(v4), IpAddr::V6(v6) => ipv6_is_public(v6), } } fn ipv4_is_public(ip: Ipv4Addr) -> bool { let o = ip.octets(); !(ip.is_loopback() || ip.is_private() || ip.is_link_local() || ip.is_unspecified() || ip.is_broadcast() || ip.is_multicast() || ip.is_documentation() || o[0] == 100 && (o[1] & 0xC0) == 64 // CGNAT 100.64.0.0/10 || o[0] == 192 && o[1] == 0 && o[2] == 0 // IETF protocol 192.0.0.0/24 || o[0] == 198 && (o[1] & 0xFE) == 18 // benchmarking 198.18.0.0/15 || o[0] >= 240) // reserved 240.0.0.0/4 } fn ipv6_is_public(ip: Ipv6Addr) -> bool { // IPv4-mapped/compatible → judge the embedded v4. if let Some(v4) = ip.to_ipv4_mapped() { return ipv4_is_public(v4); } let seg = ip.segments(); !(ip.is_loopback() || ip.is_unspecified() || ip.is_multicast() || (seg[0] & 0xFE00) == 0xFC00 // unique-local fc00::/7 || (seg[0] & 0xFFC0) == 0xFE80 // link-local fe80::/10 || (seg[0] == 0x2001 && seg[1] == 0x0DB8) // documentation || (seg[0] == 0x0064 && seg[1] == 0xFF9B)) // NAT64 well-known prefix } #[cfg(test)] mod tests { use super::*; #[test] fn blocks_private_and_local() { for bad in [ "http://127.0.0.1/x", "http://127.1/x", "http://localhost:8080/", "http://10.0.0.5/", "http://172.16.3.4/", "http://192.168.1.1/admin", "http://169.254.169.254/latest/meta-data/", "http://100.64.0.1/", "http://[::1]/", "http://[fd00::1]/", "http://[fe80::1]/", "http://[::ffff:10.0.0.1]/", "http://2130706433/", // decimal 127.0.0.1 "http://0x7f.0.0.1/", // hex "http://0177.0.0.1/", // octal "http://0x7f000001/", // hex single "http://0.0.0.0/", "file:///etc/passwd", "http://metadata.google.internal/", ] { assert!(validate_public_url(bad).is_err(), "should refuse {bad}"); } } #[test] fn allows_public_literals() { assert!(validate_public_url("https://1.1.1.1/").is_ok()); assert!(validate_public_url("http://93.184.216.34/").is_ok()); } }