package lists import ( "context" "crypto/sha256" "encoding/hex" "errors" "fmt" "io" "net/http" "os" "path/filepath" "sync" ) // cacheFilePath returns the on-disk path for a source URL: /. func cacheFilePath(dir, url string) string { sum := sha256.Sum256([]byte(url)) return filepath.Join(dir, hex.EncodeToString(sum[:])) } // openCached returns a reader over the on-disk body for link, or an error if absent. func openCached(dir, link string) (io.ReadCloser, error) { return os.Open(cacheFilePath(dir, link)) } // cacheValidators holds the HTTP validators last seen for a source URL. type cacheValidators struct { etag string lastModified string } // cachingDownloader decorates an httpDownloader with an on-disk body cache and // HTTP conditional requests. Validators are kept in memory only (lost on restart). type cachingDownloader struct { inner *httpDownloader dir string mu sync.Mutex validators map[string]cacheValidators } func newCachingDownloader(inner *httpDownloader, dir string) *cachingDownloader { return &cachingDownloader{ inner: inner, dir: dir, validators: make(map[string]cacheValidators), } } func (c *cachingDownloader) setValidators(link, etag, lastModified string) { if etag == "" && lastModified == "" { return } c.mu.Lock() c.validators[link] = cacheValidators{etag: etag, lastModified: lastModified} c.mu.Unlock() } func (c *cachingDownloader) conditionalHeader(link string) http.Header { c.mu.Lock() v, ok := c.validators[link] c.mu.Unlock() if !ok { return nil } // setValidators only stores an entry when at least one validator is non-empty, // so a present entry always yields at least one conditional header. h := make(http.Header, 2) if v.etag != "" { h.Set("If-None-Match", v.etag) } if v.lastModified != "" { h.Set("If-Modified-Since", v.lastModified) } return h } func (c *cachingDownloader) DownloadFile(ctx context.Context, link string) (io.ReadCloser, error) { resp, err := c.inner.download(ctx, link, c.conditionalHeader(link)) if err != nil { // Only serve the stale cached copy when the source was unreachable // (connection/timeout/DNS error). If the host answered with an HTTP error // status (4xx/5xx) it is reachable, so we surface the error and let the // existing in-memory cache stand rather than masking a removed/changed source. var statusErr *httpStatusError if !errors.As(err, &statusErr) { if cached, openErr := openCached(c.dir, link); openErr == nil { logger().WithField("link", link).WithError(err).Warn("download failed, using cached copy") return cached, nil } } return nil, err } if resp.statusCode == http.StatusNotModified { cached, openErr := openCached(c.dir, link) if openErr == nil { logger().WithField("link", link).Debug("source not modified, using cached copy") return cached, nil } // Validators say unchanged but the cache file is gone: force a full download. resp, err = c.inner.download(ctx, link, nil) if err != nil { return nil, err } if resp.body == nil { // A forced unconditional GET sends no validators, so a 304 (nil body) is impossible. return nil, fmt.Errorf("unexpected 304 Not Modified on forced unconditional request for '%s'", link) } } return c.serveAndStore(link, resp) } // serveAndStore returns a reader over the 200 body that tees into a temp file and, // once fully read, atomically renames it to the cache file and stores the validators. func (c *cachingDownloader) serveAndStore(link string, resp *downloadResponse) (io.ReadCloser, error) { tmp, err := os.CreateTemp(c.dir, "dl-*.tmp") if err != nil { logger().WithError(err).Warnf("cannot create temp cache file in %s, serving without caching", c.dir) return resp.body, nil } finalPath := cacheFilePath(c.dir, link) etag := resp.header.Get("ETag") lastModified := resp.header.Get("Last-Modified") tw := &tolerantFileWriter{w: tmp} fin := &cacheFinalizer{ body: resp.body, tmp: tmp, } fin.tee = io.TeeReader(resp.body, tw) fin.finalize = func() { if tw.failed { // a mid-stream write error already abandoned the cache copy _ = tmp.Close() _ = os.Remove(tmp.Name()) return } // No fsync: the cache is regenerable disposable state. A torn write after a // crash is tolerated by the parser and corrected by the next refresh, so the // durability cost of fsync-per-download is not worth it. _ = tmp.Close() if err := os.Rename(tmp.Name(), finalPath); err != nil { logger().WithError(err).Warnf("cannot finalize cache file %s", finalPath) _ = os.Remove(tmp.Name()) return } c.setValidators(link, etag, lastModified) } return fin, nil } // tolerantFileWriter writes to an underlying writer but never propagates a write // error to its caller. Used with io.TeeReader so that a mid-stream disk write // failure (e.g. the disk fills up during a download) does NOT surface as a read // error to the list parser: the download/parse still succeeds over the network and // only the on-disk cache copy is abandoned. The first failure is logged and sets // failed; subsequent writes are silently skipped. type tolerantFileWriter struct { w io.Writer failed bool } func (w *tolerantFileWriter) Write(p []byte) (int, error) { if w.failed { return len(p), nil } if _, err := w.w.Write(p); err != nil { w.failed = true logger().WithError(err).Warn("cache write failed, abandoning on-disk copy") } return len(p), nil } // cacheFinalizer tees the network body into a temp file and finalizes (rename + // store validators) exactly once when the whole body has been teed to disk. // If the consumer stops before EOF (e.g. the parser aborts after too many errors), // Close drains the rest of the body into the temp file so the cache still captures // the full download and the HTTP connection can be reused; a read error during the // drain (e.g. context cancellation or a mid-stream network failure) leaves the temp // file discarded and any previous cached copy untouched. // It is not safe for concurrent use; it follows the single-consumer io.ReadCloser contract. type cacheFinalizer struct { body io.ReadCloser tee io.Reader tmp *os.File finalize func() done bool } func (f *cacheFinalizer) Read(p []byte) (int, error) { n, err := f.tee.Read(p) if errors.Is(err, io.EOF) && !f.done { f.done = true f.finalize() } return n, err } func (f *cacheFinalizer) Close() error { if !f.done { // Drain whatever the consumer left unread through the tee so the cache file // holds the complete body. io.Copy treats io.EOF as success; any other error // (cancelled context, mid-stream reset) leaves done=false and the temp file is // discarded below. if _, err := io.Copy(io.Discard, f.tee); err == nil { f.done = true f.finalize() } } err := f.body.Close() if !f.done { _ = f.tmp.Close() _ = os.Remove(f.tmp.Name()) } return err } // Ensure cachingDownloader satisfies the FileDownloader interface. var _ FileDownloader = (*cachingDownloader)(nil)