// SPDX-License-Identifier: Apache-2.0 package server import ( "context" "crypto/tls" "errors" "fmt" "log" "net" "net/http" "net/url" "strings" "sync" "time" "github.com/X4Applegate/caddyui/internal/models" ) // appHealthEntry is the cached result of a single HTTPS GET against a proxy // host's public domain. Unlike the TCP/port health (which asks Caddy whether // it can open a socket to the upstream), this probe goes all the way through // — DNS → Caddy → upstream — so it catches "port open but app wedged" cases // that the TCP probe can't see. type appHealthEntry struct { Status string // "ok" / "degraded" / "down" / "unknown" / "disabled" Code int // HTTP status code, 0 if no response received Error string // short error detail LatencyMS int64 // wall-clock round trip CheckedAt time.Time // when the cache entry was written } // App-poller tunables. Kept conservative so a dashboard full of hosts doesn't // hammer the public edge: 60s between cycles, 5s per probe, max 3 redirects, // at most 8 probes in flight concurrently. const ( appHealthInterval = 60 * time.Second appHealthProbeTO = 5 * time.Second appHealthMaxRedirect = 3 appHealthMaxParallel = 8 ) // StartAppHealthPoller launches a background goroutine that periodically // probes every enabled proxy host's public URL and caches the result. The // cached result is read by /api/upstream-health and rendered as the "App" // dot next to the existing "Port" dot on the dashboard + proxy hosts pages. // // The poller shares cadence with (but runs independently of) the server // health poller — a stuck HTTP probe shouldn't delay the Caddy admin pings. func (s *Server) StartAppHealthPoller(ctx context.Context) { go func() { // Fire once right away so the UI is populated on first load after boot // without waiting a full interval. s.pollAllApps(ctx) ticker := time.NewTicker(appHealthInterval) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: s.pollAllApps(ctx) } } }() } // pollAllApps iterates every proxy host across every server and probes each // in a bounded worker pool. We don't scope to a single server here because // the dashboard can show hosts from the active server but the user may // switch servers — keeping all hosts warm means the "App" dot is instantly // correct after a server switch. func (s *Server) pollAllApps(ctx context.Context) { caddyServers, err := models.ListCaddyServers(s.DB) if err != nil { log.Printf("app-health poller: list servers: %v", err) return } // Collect every proxy host across every server. We deliberately ignore // ownership here (isAdmin=true, viewer=0) because the poller is a // system-level background job, not user-facing. var allHosts []models.ProxyHost for _, cs := range caddyServers { hosts, err := models.ListProxyHosts(s.DB, cs.ID, 0, true, nil) if err != nil { log.Printf("app-health poller: list hosts (server %d): %v", cs.ID, err) continue } allHosts = append(allHosts, hosts...) } // Worker pool to cap concurrent outbound probes. sem := make(chan struct{}, appHealthMaxParallel) var wg sync.WaitGroup for _, h := range allHosts { wg.Add(1) sem <- struct{}{} // v2.28.0 (issue #39): skip hosts whose custom interval hasn't elapsed // yet. The ticker still fires every appHealthInterval; a host asking // for a longer gap simply sits out the intervening cycles, so the // effective interval rounds up to a multiple of the tick. Checked // before spawning the goroutine so a mostly-idle fleet doesn't // allocate workers it will immediately discard. if !s.appProbeDue(h) { wg.Done() <-sem continue } go func(h models.ProxyHost) { defer wg.Done() defer func() { <-sem }() entry := s.probeApp(ctx, h) s.appHealthMu.Lock() s.appHealth[h.ID] = entry s.appHealthMu.Unlock() }(h) } wg.Wait() // Garbage-collect cache entries for hosts that no longer exist, so the // map doesn't grow forever as hosts are created and deleted. live := make(map[int64]struct{}, len(allHosts)) for _, h := range allHosts { live[h.ID] = struct{}{} } s.appHealthMu.Lock() for id := range s.appHealth { if _, ok := live[id]; !ok { delete(s.appHealth, id) } } s.appHealthMu.Unlock() } // probeApp performs one HTTPS GET against the host's primary domain and // classifies the result. Classification rules (designed to match what a // human reading the dashboard would actually call "up" or "down"): // // ok — 2xx / 3xx / 401 / 403 (app responded with something sensible) // degraded — 5xx or slow (>appHealthProbeTO) // down — connection refused / TLS error / timeout // unknown — domain doesn't resolve publicly (WG/Tailscale-only edge), // resolves only to private/RFC1918 IPs (split-horizon DNS, // /etc/hosts override, Docker embedded DNS — caddyui's view // of the world isn't the public internet's view, so refusing // to judge is the right call), or wildcard domain. // // Redirects are followed up to appHealthMaxRedirect hops so "/ → /login" // ends up as ok (200) rather than showing the 302. Cert validation is // skipped — an expired or self-signed cert shouldn't mask the fact that the // upstream *is* responding; the cert-expiry notifier handles that concern. // appProbeDue reports whether enough time has passed since this host's last // cached probe to warrant another one. Hosts on the default interval are // always due, because the poller tick already paces them. // // v2.28.0 (issue #39). func (s *Server) appProbeDue(h models.ProxyHost) bool { _, _, _, interval, _ := h.MonitorSettings(appHealthInterval, appHealthProbeTO) if interval <= appHealthInterval { return true } s.appHealthMu.Lock() prev, ok := s.appHealth[h.ID] s.appHealthMu.Unlock() if !ok || prev.CheckedAt.IsZero() { return true } // Subtract half a tick so a host configured at exactly 2× the interval // isn't pushed out to 3× by scheduling jitter. return time.Since(prev.CheckedAt) >= interval-(appHealthInterval/2) } func (s *Server) probeApp(ctx context.Context, h models.ProxyHost) appHealthEntry { now := time.Now() if !h.Enabled { return appHealthEntry{Status: "disabled", CheckedAt: now} } // v2.28.0 (issue #39): the operator has switched CaddyUI's own probes off // for this host. Emit no outbound request at all — the whole point is to // stop generating traffic toward a backend that is deliberately // unreachable from here, not merely to hide the dot. if h.MonitoringDisabled() { return appHealthEntry{Status: "off", Error: "monitoring disabled for this host", CheckedAt: now} } domains := h.DomainList() if len(domains) == 0 { return appHealthEntry{Status: "unknown", Error: "no domain configured", CheckedAt: now} } primary := domains[0] // Wildcard domains (e.g. "*.example.com") can't be probed directly — // there's no specific host to GET. Skip with "unknown" rather than // flagging as down. if strings.Contains(primary, "*") { return appHealthEntry{Status: "unknown", Error: "wildcard domain — no specific host to probe", CheckedAt: now} } // Preflight: if DNS from caddyui's vantage point resolves the domain // only to private/RFC1918 addresses (split-horizon DNS, /etc/hosts // override, Docker embedded DNS), any probe result here is a view of // caddyui's private network plumbing, not the public reachability the // user cares about. Bail early with "unknown" and a pointed tooltip // instead of showing a misleading red "down" for a site that's fine // from the internet. Handled before the outbound request so we don't // burn a 5s timeout on something we already know won't tell the truth. if ip, onlyPrivate := resolvesOnlyPrivate(primary); onlyPrivate { return appHealthEntry{ Status: "unknown", Error: fmt.Sprintf("DNS from caddyui points to %s (private) — probe from here would be misleading; check your browser", ip), CheckedAt: now, } } // Decide scheme: we always use https because Caddy auto-upgrades, but // fall back to http if the host explicitly disables SSL (SSLEnabled=false // + no custom cert). Most installs will be https. scheme := "https" if !h.SSLEnabled && h.CertificateID == 0 { scheme = "http" } // v2.28.0 (issue #39): resolve the per-host probe overrides. An "auto" // host yields exactly the values that were hardcoded before this change, // so its behaviour is unchanged. probePath, probeMethod, expectStatus, _, probeTimeout := h.MonitorSettings(appHealthInterval, appHealthProbeTO) probeURL := (&url.URL{Scheme: scheme, Host: primary, Path: probePath}).String() client := &http.Client{ Timeout: probeTimeout, Transport: &http.Transport{ // Skip TLS verification: we care whether the app responds, not // about cert validity (that's tracked separately by the cert // notifier). Self-signed / expired certs shouldn't mask app // status. TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // nolint:gosec // probe-only // Be polite: reuse one connection per host, don't hold it open. DisableKeepAlives: true, }, CheckRedirect: func(req *http.Request, via []*http.Request) error { if len(via) >= appHealthMaxRedirect { return http.ErrUseLastResponse } return nil }, } probeCtx, cancel := context.WithTimeout(ctx, probeTimeout) defer cancel() req, err := http.NewRequestWithContext(probeCtx, probeMethod, probeURL, nil) if err != nil { return appHealthEntry{Status: "down", Error: err.Error(), CheckedAt: now} } req.Header.Set("User-Agent", "caddyui-app-health/1.0") start := time.Now() resp, err := client.Do(req) latency := time.Since(start).Milliseconds() if err != nil { // DNS resolution failure → unknown, not down: the caddyui container // may legitimately not resolve an edge-only domain even though the // wider internet does. if isDNSError(err) || isNetDNSError(err) { return appHealthEntry{Status: "unknown", Error: err.Error(), LatencyMS: latency, CheckedAt: now} } // If the dial failed against a private IP, the preflight check // above should've caught this — but DNS caches and IPv6 vs IPv4 // races can still slip one through. Re-check the error string for // an embedded private IP (Go error format is // `dial tcp 192.168.x.x:443: connect: connection refused`). // Treat it the same as the preflight case: "unknown", not "down". if ip := privateIPFromErr(err); ip != "" { return appHealthEntry{ Status: "unknown", Error: fmt.Sprintf("caddyui reached %s (private) — probe from here would be misleading; check your browser", ip), LatencyMS: latency, CheckedAt: now, } } return appHealthEntry{Status: "down", Error: err.Error(), LatencyMS: latency, CheckedAt: now} } defer resp.Body.Close() code := resp.StatusCode entry := appHealthEntry{Code: code, LatencyMS: latency, CheckedAt: now} // v2.28.0 (issue #39): when the host declares an exact expected status, // that verdict replaces the heuristic below entirely. This is what makes // deliberately-unusual endpoints reportable — a health path that answers // 204, or an app that correctly returns 418 on its probe route, is "ok" // only if the operator said so, and anything else is unambiguously down. if expectStatus > 0 { if code == expectStatus { entry.Status = "ok" } else { entry.Status = "down" entry.Error = fmt.Sprintf("HTTP %d (expected %d)", code, expectStatus) } return entry } switch { case code >= 500: entry.Status = "degraded" entry.Error = fmt.Sprintf("HTTP %d", code) case code >= 200 && code < 400, code == http.StatusUnauthorized, code == http.StatusForbidden: entry.Status = "ok" default: // 4xx other than 401/403 — treat as degraded so the user notices; // 404 on "/" often means the app is misconfigured. entry.Status = "degraded" entry.Error = fmt.Sprintf("HTTP %d", code) } return entry } // isNetDNSError catches net.DNSError values that the simpler string-match // isDNSError() might miss (e.g. wrapped errors). Shared with the existing // isDNSError helper so callers get the same "unknown not down" semantics // whichever error shape bubbles up. func isNetDNSError(err error) bool { var dnsErr *net.DNSError return errors.As(err, &dnsErr) } // resolvesOnlyPrivate reports whether DNS from caddyui's vantage point // returns only non-public addresses for host — i.e. RFC1918 IPv4 // (10/8, 172.16/12, 192.168/16), IPv4/IPv6 loopback, link-local, or // IPv6 ULA (fc00::/7). Returns the first offending IP as a string for // tooltip context. Used to short-circuit the App probe in split-horizon // DNS setups where caddyui's answer is an internal IP that doesn't // match what a public client would see. // // A "mixed" result (some public, some private) returns false — Go's // stdlib dialer will happily fail over to the public address, so the // probe can still give a meaningful result. func resolvesOnlyPrivate(host string) (string, bool) { ips, err := net.LookupIP(host) if err != nil || len(ips) == 0 { // DNS failure is already handled downstream as "unknown" via // isDNSError. Don't claim "onlyPrivate" here just because lookup // bombed — let the real probe run and surface the DNSError. return "", false } firstPrivate := "" for _, ip := range ips { if !isPrivateOrLocalIP(ip) { return "", false } if firstPrivate == "" { firstPrivate = ip.String() } } return firstPrivate, true } // isPrivateOrLocalIP is IsPrivate ∪ loopback ∪ link-local ∪ ULA. We treat // all of these as "caddyui's private view" — a probe hitting any of them // isn't a trustworthy signal of public reachability. func isPrivateOrLocalIP(ip net.IP) bool { if ip == nil { return false } if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsPrivate() { return true } // IsPrivate covers RFC1918 + ULA in Go 1.17+. IsUnspecified catches 0.0.0.0 / :: // which dialers should never return from LookupIP but defensively // don't count as "public." if ip.IsUnspecified() { return true } return false } // privateIPFromErr tries to pull a private IP literal out of a Go dial // error. Format from the net stdlib looks like: // // "dial tcp 192.168.112.7:443: connect: connection refused" // "dial tcp [fc00::1]:443: connect: connection refused" // // Returns "" if no private IP is found. Intentionally string-based // because Go doesn't expose the dialed address as a typed field on // net.OpError.Err in a stable way — the error message is the stable // contract most callers rely on. func privateIPFromErr(err error) string { if err == nil { return "" } msg := err.Error() // Look for "dial tcp :" token. const marker = "dial tcp " i := strings.Index(msg, marker) if i < 0 { return "" } rest := msg[i+len(marker):] // IPv6 bracketed form first. if strings.HasPrefix(rest, "[") { end := strings.Index(rest, "]") if end < 0 { return "" } ipStr := rest[1:end] if ip := net.ParseIP(ipStr); ip != nil && isPrivateOrLocalIP(ip) { return ipStr } return "" } // IPv4: up to the next colon. end := strings.IndexAny(rest, ":,] ") if end < 0 { return "" } ipStr := rest[:end] if ip := net.ParseIP(ipStr); ip != nil && isPrivateOrLocalIP(ip) { return ipStr } return "" }