// Package handler provides HTTP protocol handlers for package manager proxying. package handler import ( "bytes" "context" "crypto/sha256" "database/sql" "encoding/hex" "errors" "fmt" "io" "log/slog" "net/http" "net/url" "strconv" "strings" "sync" "sync/atomic" "time" "github.com/git-pkgs/artifacts" "github.com/git-pkgs/cooldown" "github.com/git-pkgs/integrity" "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/denylist" "github.com/git-pkgs/proxy/internal/metrics" "github.com/git-pkgs/proxy/internal/packageurl" "github.com/git-pkgs/proxy/internal/scanner" "github.com/git-pkgs/proxy/internal/storage" "github.com/git-pkgs/purl" "github.com/git-pkgs/registries/fetch" "github.com/opencontainers/go-digest" ) // containsPathTraversal returns true if the path contains ".." segments // that could be used to escape the intended directory. It checks the path // as given and after URL-decoding, and treats backslashes as separators. func containsPathTraversal(path string) bool { if hasDotDotSegment(path) { return true } if decoded, err := url.PathUnescape(path); err == nil && decoded != path { return hasDotDotSegment(decoded) } return false } // rejectQueryChars refuses a decoded path containing '?' or '#', which would cut // the upstream URL short and cache one file under another's name. func rejectQueryChars(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if strings.ContainsAny(r.URL.Path, "?#") { http.Error(w, "invalid path", http.StatusBadRequest) return } next.ServeHTTP(w, r) }) } func hasDotDotSegment(path string) bool { path = strings.ReplaceAll(path, "\\", "/") for segment := range strings.SplitSeq(path, "/") { if segment == ".." { return true } } return false } func configuredUpstreamURL(value, defaultValue string) string { if value == "" { value = defaultValue } return strings.TrimRight(value, "/") } const defaultHTTPTimeout = 30 * time.Second const artifactCopyBufferSize = 32 << 10 var artifactCopyBufferPool = sync.Pool{ //nolint:gochecknoglobals // shared across artifact responses New: func() any { buffer := make([]byte, artifactCopyBufferSize) return &buffer }, } // canonicalPackagePURL returns a versionless PURL in canonical form so cooldown // lookups match keys produced by config.CooldownConfig.NormalizedPackages. func canonicalPackagePURL(ecosystem, name string) string { return packageurl.MakeString(ecosystem, name, "") } // canonicalVersionPURL returns a versioned PURL in canonical form, matching // the keys the artifact cache writes to the versions table. func canonicalVersionPURL(ecosystem, name, version string) string { return packageurl.MakeString(ecosystem, name, version) } var errUnsupportedPackageIdentity = errors.New("package identity cannot be represented as a PURL") func packagePURLStrings(ecosystem, name, version string) (string, string, error) { packagePURL := packageurl.MakeString(ecosystem, name, "") versionPURL := packageurl.MakeString(ecosystem, name, version) if packagePURL == "" || versionPURL == "" { return "", "", fmt.Errorf("%w: %s %q", errUnsupportedPackageIdentity, ecosystem, name) } return packagePURL, versionPURL, nil } const contentTypeJSON = "application/json" const ( headerAccept = "Accept" headerAcceptEncoding = "Accept-Encoding" headerContentType = "Content-Type" headerContentLength = "Content-Length" headerContentEncoding = "Content-Encoding" headerETag = "ETag" headerLastModified = "Last-Modified" ) // ifNoneMatchHits reports whether the given If-None-Match header value // matches the current entity tag using weak comparison, so "*" matches any // tag, W/ prefixes are ignored on both sides, and a comma-separated list is // scanned. An empty header or an empty stored tag never match. func ifNoneMatchHits(header, etag string) bool { if etag == "" || header == "" { return false } if header == "*" { return true } etag = strings.TrimPrefix(etag, "W/") for value := range strings.SplitSeq(header, ",") { if strings.TrimPrefix(strings.TrimSpace(value), "W/") == etag { return true } } return false } // defaultMetadataMaxSize is used when Proxy.MetadataMaxSize is unset. const defaultMetadataMaxSize = 100 << 20 // defaultMetadataStreamMaxSize is used when Proxy.MetadataStreamMaxSize is // unset. const defaultMetadataStreamMaxSize = 1 << 30 // ErrMetadataTooLarge is returned when upstream metadata exceeds the configured limit. var ErrMetadataTooLarge = errors.New("metadata response exceeds size limit") // ReadMetadata reads an upstream response body with a size limit to prevent OOM // from unexpectedly large responses. Returns ErrMetadataTooLarge if the response // is truncated by the limit. func (p *Proxy) ReadMetadata(r io.Reader) ([]byte, error) { return p.ReadMetadataSized(r, -1) } // ReadMetadataSized is ReadMetadata for a body expected to be size bytes // long, such as a cached document or a response with a Content-Length. // Reading into a buffer of that size avoids io.ReadAll growing its buffer // step by step, which peaks at more than twice the document size. size is // only a hint: a negative one, or one above the limit, reads as ReadMetadata // does, and a body of a different length is still read correctly. A body // read through upstreamBodyReader that upstream cuts off fails, even though // it ends like a body shorter than size. func (p *Proxy) ReadMetadataSized(r io.Reader, size int64) ([]byte, error) { limit := p.MetadataMaxSize if limit <= 0 { limit = defaultMetadataMaxSize } if size < 0 || size > limit { return readLimited(r, nil, limit) } // One byte more than expected, so a body of exactly size bytes ends in // the buffer and a longer one is noticed. buf := make([]byte, size+1) n, err := io.ReadFull(r, buf) var cut *upstreamBodyError switch { case errors.As(err, &cut): return nil, err case errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF): if int64(n) < size/2 { // The hint was far off; don't keep the oversized buffer alive. return bytes.Clone(buf[:n]), nil } return buf[:n], nil case err != nil: return nil, err } return readLimited(r, buf, limit) } // readLimited appends the rest of r to data, failing with // ErrMetadataTooLarge once the total exceeds limit. func readLimited(r io.Reader, data []byte, limit int64) ([]byte, error) { rest, err := io.ReadAll(io.LimitReader(r, limit+1-int64(len(data)))) if err != nil { return nil, err } if data == nil { data = rest } else { data = append(data, rest...) } if int64(len(data)) > limit { return nil, ErrMetadataTooLarge } return data, nil } // Proxy provides shared functionality for protocol handlers. type Proxy struct { DB *database.DB Storage storage.Storage Fetcher fetch.FetcherInterface Resolver *fetch.Resolver // RouteFor returns how to fetch a package that doesn't come from the // ecosystem's default upstream, such as an npm package in // upstream.npm_routes, or nil to use Resolver. RouteFor func(ecosystem, name string) *ArtifactRoute Logger *slog.Logger Cooldown CooldownPolicy // ShortestCooldown is the shortest cooldown configured anywhere. It // bounds how long a rewrite filtered by cooldown is cached. ShortestCooldown time.Duration Denylist *denylist.Policy CacheMetadata bool MetadataTTL time.Duration MetadataMaxSize int64 // MetadataStreamMaxSize caps a metadata document streamed from upstream // into storage, which metadata_max_size doesn't cover. Default 1 GB. MetadataStreamMaxSize int64 GradleReadOnly bool HostedEnabled bool GradleMaxUploadSize int64 // NPMFullMetadata requests full npm packuments from upstream even when // cooldown is disabled, so served metadata carries publish times. NPMFullMetadata bool DirectServe bool DirectServeTTL time.Duration HTTPClient *http.Client // RateLimits is the gate under the upstream clients. Artifact fetches // check it before calling Fetcher, so held hosts never reach its breaker. RateLimits UpstreamHolds // HostedNPMScopes and HostedComposerVendors name namespaces served from // pushed packages alone, without asking the upstream. HostedNPMScopes []string HostedComposerVendors []string // StreamArtifacts streams artifacts from upstream without storing them. // Each request fetches its own copy: there is no cache to check and // nothing for concurrent misses to share. StreamArtifacts bool // Scanners runs pre-cache artifact scanning (e.g. trivy, ClamAV, Wiz). // Nil or disabled means artifacts are cached without scanning. Scanners *scanner.Group // ScanSigningKey authenticates pull requests to the internal // /_internal/scan-fetch route used by scanners to retrieve staged // artifacts, for every storage backend. ScanSigningKey []byte // ScanFetchBaseURL is the address scanners use to reach this proxy to // pull staged artifacts. ScanFetchBaseURL string // inFlight coalesces concurrent cache misses for one artifact, so a single // upstream fetch serves every waiting caller. Keyed by artifactCoalesceKey. fetchMu sync.Mutex inFlight map[string]*inflightFetch // detachSlots caps the artifact fetches that keep running after their // client went away (see DetachUpstreamFetch). Guarded by fetchMu while it // is created. detachSlots chan struct{} // inFlightMeta does the same for metadata misses. Keyed by // metadataCoalesceKey. metaMu sync.Mutex inFlightMeta map[string]*inflightMetadata // metaDetachSlots caps the metadata fetches that keep running after // their caller went away (see coalesceMetadata). Guarded by metaMu // while it is created. metaDetachSlots chan struct{} // rewrites caches metadata documents after their handler rewrites them. // Nil leaves every request to rewrite its own copy. rewrites *rewriteCache // publishIndexes keeps publish times read out of large metadata // documents for the download cooldown check; see publishIndex. publishIndexes publishIndexCache // now is the clock rewritten metadata is dated by, and started the time // the process started; see metadataDate. now func() time.Time started time.Time // cooldownWatch holds the rewrites recording which versions cooldown // holds back; see cooldown_expiry.go. cooldownWatch cooldownWatch shared *Proxy // viewCooldown is set on the shared proxy once a view of it has cooldown // on; see anyCooldownEnabled. viewCooldown atomic.Bool } // cooldownEcosystems are the ecosystems whose handlers apply cooldown. Others // have no publish time to check, so a configured cooldown has no effect there. var cooldownEcosystems = map[string]bool{ "cargo": true, "composer": true, "conda": true, "gem": true, "golang": true, "helm": true, "hex": true, "maven": true, "npm": true, "nuget": true, "pub": true, "pypi": true, } // CooldownSupported reports whether the proxy applies cooldown to an // ecosystem. func CooldownSupported(ecosystem string) bool { return cooldownEcosystems[ecosystem] } // CooldownPolicy supplies version-age filtering and package-specific durations. type CooldownPolicy interface { IsAllowed(ecosystem, packagePURL string, publishedAt time.Time) bool Evaluate(ecosystem, packagePURL string, publishedAt, evaluatedAt time.Time) cooldown.Decision For(ecosystem, packagePURL string) time.Duration Enabled() bool } // NewProxy creates a new Proxy with the given dependencies. func NewProxy(db *database.DB, store storage.Storage, fetcher fetch.FetcherInterface, resolver *fetch.Resolver, logger *slog.Logger) *Proxy { if logger == nil { logger = slog.Default() } return &Proxy{ DB: db, Storage: store, Fetcher: fetcher, Resolver: resolver, Logger: logger, HTTPClient: &http.Client{ Timeout: defaultHTTPTimeout, }, now: time.Now, started: time.Now(), } } // CacheResult contains information about a cached or fetched artifact. type CacheResult struct { Reader io.ReadCloser RedirectURL string Artifact artifacts.Artifact Cached bool storagePath string // rangeable is set on a HEAD cache hit, which has no Reader, when the // stored object would open as a seekable reader. rangeable bool } // GetOrFetchArtifact retrieves an artifact from cache or fetches from upstream. func (p *Proxy) GetOrFetchArtifact(ctx context.Context, ecosystem, name, version, filename string) (*CacheResult, error) { // NuGet ids are case-insensitive and the NuGet handler stores them // lowercased, so a mirrored Newtonsoft.Json must land on the same record // clients are served. if ecosystem == "nuget" { name = strings.ToLower(name) } pkgPURL, versionPURL, err := packagePURLStrings(ecosystem, name, version) if err != nil { return nil, err } // Deny before resolution as well as before reading the cache. Mirror callers // do not supply a filename, and resolving it can contact an upstream registry. if p.Denylist.Denied(versionPURL) { return nil, fmt.Errorf("%w: %s", ErrVersionDenied, versionPURL) } var info *fetch.ArtifactInfo if filename == "" { info, err = p.resolveArtifact(ctx, ecosystem, name, version) if err != nil { return nil, err } filename = info.Filename if filename == "" { return nil, errors.New("resolved artifact has no filename") } if route := p.routeFor(ecosystem, name); route != nil { filename = route.FilenamePrefix + filename } } if p.StreamArtifacts { if info == nil { if info, err = p.resolveArtifact(ctx, ecosystem, name, version); err != nil { return nil, err } } return p.streamFromUpstream(ctx, ecosystem, name, version, filename, versionPURL, info.URL, "", "", func(fetchCtx context.Context) (*fetch.Artifact, error) { return p.Fetcher.Fetch(fetchCtx, info.URL) }) } if cached, err := p.checkCache(ctx, pkgPURL, versionPURL, filename); err != nil { return nil, err } else if cached != nil { return cached, nil } metrics.RecordArtifactCacheMiss(ecosystem) key := artifactCoalesceKey(versionPURL, filename, "", "", "") recheck := func() (artifacts.Artifact, string, bool) { return p.cachedArtifactRecord(pkgPURL, versionPURL, filename, "") } return p.coalesceFetch(ctx, key, recheck, func(fetchCtx context.Context) (artifacts.Artifact, string, error) { return p.fetchAndCache(fetchCtx, ecosystem, name, version, filename, pkgPURL, versionPURL, info) }) } // GetCachedArtifact retrieves an artifact from cache without contacting an upstream. // It returns nil when no usable cache entry exists. func (p *Proxy) GetCachedArtifact(ctx context.Context, ecosystem, name, version, filename string) (*CacheResult, error) { pkgPURL, versionPURL, err := packagePURLStrings(ecosystem, name, version) if err != nil { return nil, err } return p.checkCache(ctx, pkgPURL, versionPURL, filename) } // HasCachedArtifact reports whether an artifact is cached and its object is // stored, without opening it or counting a cache hit, so a mirror run does // not look like client traffic. func (p *Proxy) HasCachedArtifact(ctx context.Context, ecosystem, name, version, filename string) (bool, error) { pkgPURL, versionPURL, err := packagePURLStrings(ecosystem, name, version) if err != nil { return false, err } if p.Denylist.Denied(versionPURL) { return false, fmt.Errorf("%w: %s", ErrVersionDenied, versionPURL) } if p.StreamArtifacts { return false, nil } artifact, err := p.DB.GetCachedArtifact(pkgPURL, versionPURL, filename) if err != nil { return false, fmt.Errorf("checking artifact cache: %w", err) } if artifact == nil { return false, nil } // An unusable record is a miss; the fetch that follows clears it. if _, err := newIntegrityChecks(artifact.Artifact.Digest.Encoded(), artifact.Integrity.String); err != nil { return false, nil } return p.Storage.Exists(ctx, artifact.StoragePath) } // MarkMirrored records that a mirror run found an artifact cached, which // keeps it from retention without counting as a cache hit. func (p *Proxy) MarkMirrored(ecosystem, name, version, filename string) error { _, versionPURL, err := packagePURLStrings(ecosystem, name, version) if err != nil { return err } return p.DB.MarkArtifactMirrored(versionPURL, filename) } // ClearCachedArtifact clears an artifact cache record after an external // integrity check fails, and queues its stored bytes for deletion. func (p *Proxy) ClearCachedArtifact(ecosystem, name, version, filename string) error { if p.DB == nil || p.Storage == nil { return nil } pkgPURL, versionPURL, err := packagePURLStrings(ecosystem, name, version) if err != nil { return err } cached, err := p.DB.GetCachedArtifact(pkgPURL, versionPURL, filename) if err != nil { return fmt.Errorf("looking up cached artifact: %w", err) } if cached == nil { return nil } return p.DB.DiscardArtifact(versionPURL, filename, cached.StoragePath) } // checkCache looks up an artifact in the cache. Returns nil if not cached. // With StreamArtifacts set it always reports a miss, so entries stored before the // mode was enabled are never served. // // A miss on a pushed artifact is an error rather than a miss: the caller would // otherwise fetch the upstream copy and replace the pushed one with it. func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename string) (*CacheResult, error) { result, err := p.lookupCache(ctx, pkgPURL, versionPURL, filename) if err != nil || result != nil || p.StreamArtifacts { return result, err } return nil, p.localArtifactMiss(versionPURL, filename) } // localArtifactMiss returns ErrHostedArtifactUnavailable when the artifact // was pushed and is stored but couldn't be served, ErrHostedArtifactMissing // when it was pushed and its stored object is gone, and nil otherwise. func (p *Proxy) localArtifactMiss(versionPURL, filename string) error { record, err := p.DB.GetArtifact(versionPURL, filename) if err != nil { return fmt.Errorf("checking artifact source: %w", err) } if record == nil || record.Source != database.SourceLocal { return nil } if record.StoragePath.Valid { return fmt.Errorf("%w: %s %s", ErrHostedArtifactUnavailable, versionPURL, filename) } return fmt.Errorf("%w: %s %s", ErrHostedArtifactMissing, versionPURL, filename) } // lookupCache looks up an artifact in the cache. Returns nil if not cached. func (p *Proxy) lookupCache(ctx context.Context, pkgPURL, versionPURL, filename string) (*CacheResult, error) { if p.Denylist.Denied(versionPURL) { return nil, fmt.Errorf("%w: %s", ErrVersionDenied, versionPURL) } if p.StreamArtifacts { return nil, nil } if cached, err := p.readStored(ctx, pkgPURL, versionPURL, filename); err != nil || cached != nil { return cached, err } if moved, err := p.rekeyLegacyGoArtifact(pkgPURL, versionPURL, filename); err != nil || !moved { return nil, err } return p.readStored(ctx, pkgPURL, versionPURL, filename) } // readStored opens the stored object of an artifact, whether or not // StreamArtifacts is set. Returns nil if no usable object is stored. func (p *Proxy) readStored(ctx context.Context, pkgPURL, versionPURL, filename string) (*CacheResult, error) { artifact, err := p.DB.GetCachedArtifact(pkgPURL, versionPURL, filename) if err != nil { return nil, fmt.Errorf("checking artifact cache: %w", err) } if artifact == nil { return nil, nil } checks, err := newIntegrityChecks(artifact.Artifact.Digest.Encoded(), artifact.Integrity.String) if err != nil { p.rejectUnusableCacheRecord(artifact, versionPURL, filename, err) return nil, nil } result := &CacheResult{ Artifact: artifact.Artifact, Cached: true, storagePath: artifact.StoragePath, } if servingHead(ctx) { result.rangeable = readsSeekable(p.Storage) p.recordCacheHit(ctx, artifact.Ecosystem, versionPURL, filename) return result, nil } if p.DirectServe { signed, err := p.Storage.SignedURL(ctx, artifact.StoragePath, p.DirectServeTTL) if err == nil { result.RedirectURL = signed p.recordCacheHit(ctx, artifact.Ecosystem, versionPURL, filename) return result, nil } if !errors.Is(err, storage.ErrSignedURLUnsupported) { p.Logger.Warn("failed to sign storage URL, falling back to streaming", "path", artifact.StoragePath, "error", err) } } start := time.Now() reader, err := p.Storage.Open(ctx, artifact.StoragePath) metrics.RecordStorageOperation("read", time.Since(start)) if err != nil { metrics.RecordStorageError("read", metrics.FailureDetail{ PURL: versionPURL, Filename: filename, Path: artifact.StoragePath, Err: err}) p.Logger.Warn("cached artifact missing from storage, will refetch", "path", artifact.StoragePath, "error", err) return nil, nil } result.Reader, err = checks.forStoredObject().wrapFailOnMismatch(reader, func(reason string) { p.Logger.Error("cached artifact failed integrity check", "purl", versionPURL, "filename", filename, "path", artifact.StoragePath, "reason", reason) metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(artifact.Ecosystem), metrics.FailureDetail{ PURL: versionPURL, Filename: filename, Path: artifact.StoragePath, Err: errors.New(reason)}) if err := p.DB.DiscardArtifact(versionPURL, filename, artifact.StoragePath); err != nil { p.Logger.Warn("failed to clear corrupt artifact from cache", "error", err) } }) if err != nil { _ = reader.Close() p.rejectUnusableCacheRecord(artifact, versionPURL, filename, err) return nil, nil } p.recordCacheHit(ctx, artifact.Ecosystem, versionPURL, filename) return result, nil } // recordCacheHit counts a cache hit. A mirror run's read leaves the // artifact's hits and last access alone, so it doesn't look like a client // download, and keeps it from retention instead. func (p *Proxy) recordCacheHit(ctx context.Context, ecosystem, versionPURL, filename string) { if mirrorFetch(ctx) { _ = p.DB.KeepMirroredArtifact(versionPURL, filename) } else { _ = p.DB.RecordArtifactHit(versionPURL, filename) } metrics.RecordArtifactCacheHit(ecosystem) } func (p *Proxy) rejectUnusableCacheRecord(artifact *database.CachedArtifact, versionPURL, filename string, cause error) { p.Logger.Warn("cached artifact has unusable integrity metadata", "purl", versionPURL, "filename", filename, "path", artifact.StoragePath, "error", cause) metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(artifact.Ecosystem), metrics.FailureDetail{ PURL: versionPURL, Filename: filename, Path: artifact.StoragePath, Err: cause}) if err := p.DB.DiscardArtifact(versionPURL, filename, artifact.StoragePath); err != nil { p.Logger.Warn("failed to clear unusable artifact from cache", "error", err) } } // ArtifactRoute fetches a package from an upstream other than its // ecosystem's default one. type ArtifactRoute struct { // Resolver finds the package's download URLs on that upstream. Resolver *fetch.Resolver // FilenamePrefix qualifies the artifact filename the way the handler // does for the route, so clients are served what was fetched. FilenamePrefix string } func (p *Proxy) routeFor(ecosystem, name string) *ArtifactRoute { if p.RouteFor == nil { return nil } return p.RouteFor(ecosystem, name) } func (p *Proxy) resolveArtifact(ctx context.Context, ecosystem, name, version string) (*fetch.ArtifactInfo, error) { resolver := p.Resolver if route := p.routeFor(ecosystem, name); route != nil { resolver = route.Resolver } info, err := resolver.Resolve(ctx, ecosystem, name, version) if err != nil { if errors.Is(err, fetch.ErrNotFound) { return nil, ErrUpstreamNotFound } return nil, fmt.Errorf("resolving download URL: %w", err) } return info, nil } func (p *Proxy) fetchAndCache(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL string, info *fetch.ArtifactInfo) (artifacts.Artifact, string, error) { // Reuse the resolution used for an unnamed cache lookup. Named requests // still resolve only on a cache miss, inside the coalesced fetch. if info == nil { var err error info, err = p.resolveArtifact(ctx, ecosystem, name, version) if err != nil { return artifacts.Artifact{}, "", err } } if err := p.upstreamHeld(info.URL); err != nil { return artifacts.Artifact{}, "", err } p.Logger.Info("fetching from upstream", "ecosystem", ecosystem, "name", name, "version", version, "url", info.URL) // Fetch from upstream with timing fetchStart := time.Now() artifact, err := p.Fetcher.Fetch(ctx, info.URL) fetchDuration := time.Since(fetchStart) if err != nil { metrics.RecordUpstreamFetch(ecosystem, fetchDuration) metrics.RecordUpstreamError(ecosystem, "fetch_failed", metrics.FailureDetail{ PURL: versionPURL, Filename: filename, URL: info.URL, Err: err}) if errors.Is(err, fetch.ErrNotFound) { return artifacts.Artifact{}, "", ErrUpstreamNotFound } return artifacts.Artifact{}, "", fmt.Errorf("fetching from upstream: %w", err) } metrics.RecordUpstreamFetch(ecosystem, fetchDuration) return p.storeFetched(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, info.URL, "", "", artifact) } // storeArtifact writes a fetched artifact to storage, verifies it against // upstreamHash and declared, the integrity the registry's metadata declares, // if they are non-empty, runs it through the scan gate if scanning is enabled, // and commits it to the cache database, recording declared with the version // so cache hits are verified against it too. // // The scan gate sits between Storage.Store and updateCacheDB: updateCacheDB // is the only thing that makes an artifact visible to clients (checkCache // looks up its row before touching Storage), so deferring it until after a // verdict means a blocked artifact was never reachable by any client. On // block, the just-stored bytes are deleted and ErrArtifactBlocked is // returned; updateCacheDB is never called. // // It returns the artifact and its storage path, not a reader; callers get one // from openStoredArtifact. func (p *Proxy) storeArtifact(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL, upstreamURL, upstreamHash, declared string, artifact *fetch.Artifact) (artifacts.Artifact, string, error) { storagePath := storage.FetchPath(ecosystem, name, version, storage.NewFetchID(), filename) body, verified, err := verifyIntegrity(artifact.Body, declared) if err != nil { _ = artifact.Body.Close() return artifacts.Artifact{}, "", err } storeStart := time.Now() size, hash, err := p.Storage.Store(ctx, storagePath, body) _ = artifact.Body.Close() metrics.RecordStorageOperation("write", time.Since(storeStart)) if err != nil { metrics.RecordStorageError("write", metrics.FailureDetail{ PURL: versionPURL, Filename: filename, URL: upstreamURL, Path: storagePath, Err: err}) return artifacts.Artifact{}, "", fmt.Errorf("storing artifact: %w", err) } if !artifactHashMatches(hash, upstreamHash) { if delErr := p.Storage.Delete(ctx, storagePath); delErr != nil { p.Logger.Warn("failed to discard artifact with mismatched checksum", "path", storagePath, "error", delErr) } return artifacts.Artifact{}, "", fmt.Errorf("%w: upstream declared %s, got %s", ErrArtifactDigestMismatch, upstreamHash, hash) } if err := verified(); err != nil { p.Logger.Error("fetched artifact failed integrity check", "purl", versionPURL, "filename", filename, "url", upstreamURL, "error", err) metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(ecosystem), metrics.FailureDetail{ PURL: versionPURL, Filename: filename, URL: upstreamURL, Err: err}) if delErr := p.Storage.Delete(ctx, storagePath); delErr != nil { p.Logger.Warn("failed to discard artifact with mismatched integrity", "path", storagePath, "error", delErr) } return artifacts.Artifact{}, "", err } if p.Scanners != nil && p.Scanners.Enabled() { if err := p.runScan(ctx, ecosystem, name, version, filename, versionPURL, storagePath, size, artifact.ContentType); err != nil { // Detached from ctx: a client disconnecting must not abort // cleanup of a genuinely blocked artifact and leave its bytes // orphaned in storage with no DB row pointing at them. if delErr := p.Storage.Delete(context.WithoutCancel(ctx), storagePath); delErr != nil { p.Logger.Warn("failed to delete blocked artifact from storage", "path", storagePath, "error", delErr) } return artifacts.Artifact{}, "", err } } sharedArtifact := artifacts.Artifact{ PURL: versionPURL, Digest: digest.Digest("sha256:" + hash), Size: size, Filename: filename, MediaType: artifact.ContentType, } // Update database if err := p.updateCacheDB(ecosystem, name, pkgPURL, upstreamURL, storagePath, sharedArtifact, declared); err != nil { if errors.Is(err, database.ErrLocalArtifactExists) { p.Logger.Info("artifact was pushed while it was fetched; keeping the pushed copy", "purl", versionPURL, "filename", filename) } else { p.Logger.Warn("failed to update cache database", "error", err) } // Continue anyway - we have the file. Queue it for deletion in case // no record points at it; the queue skips it while one does. if qErr := p.DB.QueuePendingDelete(storagePath); qErr != nil && !p.recordedAt(versionPURL, filename, storagePath) { p.Logger.Warn("failed to queue unrecorded artifact for deletion, deleting it once served", "path", storagePath, "error", qErr) return sharedArtifact, storagePath, errUnrecorded } } else if mirrorFetch(ctx) { if err := p.DB.MarkArtifactMirrored(versionPURL, filename); err != nil { p.Logger.Warn("failed to record mirrored artifact", "error", err) } } return sharedArtifact, storagePath, nil } // recordedAt reports whether the artifact's record points at storagePath. A // commit can apply and still report an error, so storeArtifact checks before // treating an object as unrecorded. A failed lookup counts as not recorded. func (p *Proxy) recordedAt(versionPURL, filename, storagePath string) bool { a, err := p.DB.GetArtifact(versionPURL, filename) return err == nil && a != nil && a.StoragePath.Valid && a.StoragePath.String == storagePath } // errUnrecorded comes back from storeArtifact together with a usable artifact // and path when neither a record nor a queue entry could be written for it. // Nothing else would ever delete the object, so the fetch serves it to // everyone sharing it and deletes it when the last of them is done. var errUnrecorded = errors.New("stored artifact has no record") // unrecordedObject deletes an unrecorded object once every caller sharing // its fetch has released it. type unrecordedObject struct { refs atomic.Int32 delete func() } func (u *unrecordedObject) release() { if u.refs.Add(-1) == 0 { u.delete() } } // releaseOnClose releases an unrecorded object once its reader is closed. type releaseOnClose struct { io.ReadCloser release func() } func (r *releaseOnClose) Close() error { err := r.ReadCloser.Close() r.release() return err } // seekableReleaseOnClose keeps range requests working on a seekable reader. type seekableReleaseOnClose struct { releaseOnClose io.Seeker } // serveUnrecorded gives one caller its own reader over an unrecorded object // and releases the caller's hold on it when that reader closes, or right away // if it can't be opened. func (p *Proxy) serveUnrecorded(ctx context.Context, u *unrecordedObject, artifact artifacts.Artifact, path string) (*CacheResult, error) { res, err := p.openStoredArtifact(ctx, artifact, path) if err != nil { u.release() return nil, err } // Once, so a second Close can't drop another caller's hold. r := releaseOnClose{ReadCloser: res.Reader, release: sync.OnceFunc(u.release)} if s, ok := res.Reader.(io.Seeker); ok { res.Reader = &seekableReleaseOnClose{releaseOnClose: r, Seeker: s} } else { res.Reader = &r } return res, nil } // openStoredArtifact gives one caller its own reader over just-committed // bytes. Callers sharing a fetch cannot share a handle: it has one read // position, so they would consume each other's bytes and the first Close would // break the rest. func (p *Proxy) openStoredArtifact(ctx context.Context, artifact artifacts.Artifact, storagePath string) (*CacheResult, error) { readStart := time.Now() reader, err := p.Storage.Open(ctx, storagePath) metrics.RecordStorageOperation("read", time.Since(readStart)) if err != nil { metrics.RecordStorageError("read", metrics.FailureDetail{ PURL: artifact.PURL, Filename: artifact.Filename, Path: storagePath, Err: err}) return nil, fmt.Errorf("opening cached artifact: %w", err) } return &CacheResult{ Reader: reader, Artifact: artifact, Cached: false, }, nil } // artifactCoalesceKey identifies one artifact fetch. downloadURL, upstreamHash // and declared are included so callers expecting different bytes (multiple // upstreams, or a re-published version) never share a fetch. The hash is // lowercased because artifactHashMatches compares case-insensitively, so one // digest in two casings describes one artifact and must not split the fetch. func artifactCoalesceKey(versionPURL, filename, downloadURL, upstreamHash, declared string) string { return strings.Join([]string{versionPURL, filename, downloadURL, strings.ToLower(upstreamHash), declared}, "\x00") } // cachedArtifactRecord reports an artifact already committed to the cache, // without opening it. A caller checks the cache before it gets here, so a // concurrent fetch can commit the same artifact in between; rechecking the // record keeps that caller from fetching it a second time. A lookup error is // reported as a miss, which costs a redundant fetch rather than a failure. func (p *Proxy) cachedArtifactRecord(pkgPURL, versionPURL, filename, upstreamHash string) (artifacts.Artifact, string, bool) { record, err := p.DB.GetCachedArtifact(pkgPURL, versionPURL, filename) if err != nil || record == nil { return artifacts.Artifact{}, "", false } if !artifactHashMatches(record.Artifact.Digest.Encoded(), upstreamHash) { return artifacts.Artifact{}, "", false } return record.Artifact, record.StoragePath, true } // errSharedFetchAbandoned is what waiters see if the caller running a shared // fetch panicked out of it. var errSharedFetchAbandoned = errors.New("shared upstream fetch did not complete") // inflightFetch is one upstream fetch that concurrent callers share. val, err // and unrecorded are written before done closes and read only after, so the // close is the handoff. done closes under fetchMu, so a waiter leaving early // knows whether it was counted in waiters. type inflightFetch struct { done chan struct{} val fetchedArtifact err error unrecorded *unrecordedObject // waiters counts callers waiting on the fetch, guarded by fetchMu. // Tests read it to know a caller has joined. waiters int } // coalesceFetch runs commit at most once for concurrent callers sharing key, // then gives each its own reader over the stored bytes. // // The first caller in runs the fetch and the rest wait on it. Roles are // decided under fetchMu rather than inferred afterwards, because the two need // different cancellation behaviour: a waiter may leave when its own client goes // away, while the caller running the fetch must see it through so // storeArtifact's scan-on-disconnect handling still decides the outcome. // // Before fetching, that caller rechecks the cache through recheck: its own // lookup happened before it took the key, so a fetch that committed in // between would otherwise be repeated. A hit fills the shared value as a // fetch would. // // commit runs on that caller's context, so cancellation behaves as it did // uncoalesced and mirroring still relies on it aborting the fetch. If that // caller goes away, everyone sharing the fetch gets its error and the key is // released for a later retry. A context marked with DetachUpstreamFetch is the // exception: there commit keeps running after the caller goes away, so the // artifact still lands in the cache. // // Every sharing caller still records a cache miss, so the gap between // proxy_cache_misses_total and upstream fetch observations is what coalescing // saved. func (p *Proxy) coalesceFetch(ctx context.Context, key string, recheck func() (artifacts.Artifact, string, bool), commit func(context.Context) (artifacts.Artifact, string, error)) (*CacheResult, error) { shared := p.sharedProxy() shared.fetchMu.Lock() if shared.inFlight == nil { shared.inFlight = make(map[string]*inflightFetch) } f, joined := shared.inFlight[key] if joined { f.waiters++ } else { f = &inflightFetch{done: make(chan struct{})} shared.inFlight[key] = f } shared.fetchMu.Unlock() if !joined { if streamsCacheMiss(ctx) { return p.runSharedFetchStreaming(ctx, key, f, recheck, commit) } return p.runSharedFetch(ctx, key, f, recheck, commit) } select { case <-ctx.Done(): // This caller gave up; the fetch continues for everyone else. p.leaveFetch(f) return nil, ctx.Err() case <-f.done: } if f.err != nil { return nil, f.err } if f.unrecorded != nil { return p.serveUnrecorded(ctx, f.unrecorded, f.val.artifact, f.val.storagePath) } // A client served by a fetch it joined downloaded the artifact, even // when a mirror run started the fetch. if !mirrorFetch(ctx) { _ = p.DB.RecordArtifactHit(f.val.artifact.PURL, f.val.artifact.Filename) } return p.openStoredArtifact(ctx, f.val.artifact, f.val.storagePath) } // leaveFetch withdraws a waiter that gave up. Once the fetch has finished the // waiter was counted as holding any unrecorded object, so it releases it. func (p *Proxy) leaveFetch(f *inflightFetch) { shared := p.sharedProxy() shared.fetchMu.Lock() select { case <-f.done: shared.fetchMu.Unlock() if f.unrecorded != nil { f.unrecorded.release() } default: f.waiters-- shared.fetchMu.Unlock() } } // runSharedFetch performs the fetch that joined callers are waiting on. It is // never abandoned early, and always releases the key and wakes the waiters. func (p *Proxy) runSharedFetch(ctx context.Context, key string, f *inflightFetch, recheck func() (artifacts.Artifact, string, bool), commit func(context.Context) (artifacts.Artifact, string, error)) (*CacheResult, error) { shared := p.sharedProxy() // Set before running so a panicking commit leaves waiters with an error // rather than a zero-valued artifact. f.err = errSharedFetchAbandoned defer func() { shared.fetchMu.Lock() delete(shared.inFlight, key) if f.unrecorded != nil { // Hand each waiter a hold, then drop the one kept while the // waiters were still unknown. f.unrecorded.refs.Add(int32(f.waiters)) defer f.unrecorded.release() } close(f.done) shared.fetchMu.Unlock() }() // A caller checks the cache before reaching here, so a fetch that finished // in between would otherwise be repeated. Serve that record only if its // bytes are still present: a record can outlive them, and refetching is // the same recovery the cache lookup makes. served := func(ctx context.Context) (*CacheResult, bool) { stored, path, ok := recheck() if !ok { return nil, false } res, err := p.openStoredArtifact(ctx, stored, path) if err != nil { return nil, false } f.val, f.err = fetchedArtifact{artifact: stored, storagePath: path}, nil return res, true } if res, ok := served(ctx); ok { return res, nil } // Another instance sharing the database may be fetching the same // artifact. Wait for it to store the artifact rather than fetch it too. // The wait makes no upstream request, so a detached fetch waits past // its client too, bounded by the holder's fetch and the lease expiry; // otherwise callers that joined this one would get its cancellation. waitCtx := ctx if upstreamFetchDetached(ctx) { waitCtx = context.WithoutCancel(ctx) } var waitedFor *CacheResult release, outcome, err := p.awaitFetchLease(waitCtx, metrics.CacheKindArtifact, key, func() bool { res, ok := served(waitCtx) waitedFor = res return ok }) if err != nil { f.err = err return nil, err } if outcome == leaseStored { return waitedFor, nil } // With leaseFailed, release is a no-op and this instance fetches on its // own, so a failing upstream costs each instance one attempt in // parallel. A rate-limited holder has published its hold by now, and // rereading it makes commit refuse the fetch with the holder's // Retry-After instead of sending one more request to the limited host. if outcome == leaseFailed { p.reloadUpstreamHolds() } failed := true defer func() { release(failed) }() commitCtx := ctx if upstreamFetchDetached(ctx) && p.acquireDetachSlot() { defer p.releaseDetachSlot() commitCtx = context.WithoutCancel(ctx) } stored, path, err := commit(commitCtx) unrecorded := errors.Is(err, errUnrecorded) if unrecorded { err = nil } // Nothing an instance waiting on the lease could find was recorded, so // an unrecorded object counts as failed for it. failed = err != nil || unrecorded f.val, f.err = fetchedArtifact{artifact: stored, storagePath: path}, err if err != nil { return nil, err } // A caller handed the stream already has the bytes; opening the stored // copy would cost a read it never uses, a GET on S3 or GCS. h := handoffFrom(ctx) handedOver := h != nil && h.handedOver() if unrecorded { f.unrecorded = &unrecordedObject{delete: func() { p.deleteUnrecorded(path) }} f.unrecorded.refs.Add(1) // until the waiters are counted if handedOver { h.holdUnrecorded(f.unrecorded) return nil, nil } f.unrecorded.refs.Add(1) // this caller return p.serveUnrecorded(ctx, f.unrecorded, stored, path) } if handedOver { return nil, nil } return p.openStoredArtifact(ctx, stored, path) } func (p *Proxy) deleteUnrecorded(path string) { if err := p.Storage.Delete(context.Background(), path); err != nil { p.Logger.Warn("failed to delete unrecorded artifact", "path", path, "error", err) } } type servingHeadKey struct{} // ServingHead marks ctx as serving a HEAD request. A cache hit under it is // answered from the artifact's database record without opening the stored // object: on S3 or GCS opening it issues a GET whose unread body would only // be thrown away. The server sets it on every HEAD request. func ServingHead(ctx context.Context) context.Context { return context.WithValue(ctx, servingHeadKey{}, true) } func servingHead(ctx context.Context) bool { head, _ := ctx.Value(servingHeadKey{}).(bool) return head } // readsSeekable reports whether store opens objects as seekable readers, so a // HEAD response can advertise byte ranges exactly as the GET would. func readsSeekable(store storage.Storage) bool { s, ok := store.(storage.SeekableReads) return ok && s.ReadsSeekable() } type mirrorFetchKey struct{} // MirrorFetch marks ctx as a mirror run's rather than a client's. An // artifact fetched under it is recorded as mirrored, so it doesn't count as // downloaded by a client. func MirrorFetch(ctx context.Context) context.Context { return context.WithValue(ctx, mirrorFetchKey{}, true) } func mirrorFetch(ctx context.Context) bool { mirror, _ := ctx.Value(mirrorFetchKey{}).(bool) return mirror } type detachUpstreamFetchKey struct{} // DetachUpstreamFetch marks ctx so that an artifact fetch started under it // runs to completion even when ctx is cancelled. The server sets it on client // requests: a client that gives up on a slow cache miss (pacman aborts after // 10 s without data) would otherwise cancel the fetch, and every retry would // start again from zero, so a large artifact never reaches the cache. The // fetch stays bounded by the artifact HTTP client's timeout, and at most // maxDetachedFetches run detached at once. Mirror jobs leave it unset so // cancelling a job still aborts its fetches. func DetachUpstreamFetch(ctx context.Context) context.Context { return context.WithValue(ctx, detachUpstreamFetchKey{}, true) } func upstreamFetchDetached(ctx context.Context) bool { detached, _ := ctx.Value(detachUpstreamFetchKey{}).(bool) return detached } // maxDetachedFetches caps the artifact fetches that may outlive their client. // Without a cap a client could start any number of large uncached downloads // and hang up straight away, leaving each to run to the artifact client's // timeout. Fetches beyond the cap stay tied to their client, as before // detaching existed. const maxDetachedFetches = 64 func (p *Proxy) acquireDetachSlot() bool { shared := p.sharedProxy() return trySlot(&shared.fetchMu, &shared.detachSlots, maxDetachedFetches) } func (p *Proxy) releaseDetachSlot() { shared := p.sharedProxy() <-shared.detachSlots } // maxDetachedMetadataFetches caps the metadata fetches that may outlive their // caller, for the same reason as maxDetachedFetches: a client could request // any number of uncached documents and hang up, leaving each fetch to run to // the HTTP client's timeout. Fetches beyond the cap end with their caller. const maxDetachedMetadataFetches = 64 func (p *Proxy) acquireMetadataDetachSlot() bool { shared := p.sharedProxy() return trySlot(&shared.metaMu, &shared.metaDetachSlots, maxDetachedMetadataFetches) } func (p *Proxy) releaseMetadataDetachSlot() { shared := p.sharedProxy() <-shared.metaDetachSlots } // trySlot takes one of size slots without waiting, creating the slots under // mu on first use. It reports whether a slot was free. func trySlot(mu *sync.Mutex, slots *chan struct{}, size int) bool { mu.Lock() if *slots == nil { *slots = make(chan struct{}, size) } ch := *slots mu.Unlock() select { case ch <- struct{}{}: return true default: return false } } // fetchedArtifact is what a shared fetch hands its callers: metadata and a // storage path, neither holding reader state. type fetchedArtifact struct { artifact artifacts.Artifact storagePath string } // runScan generates a signed fetch URL for the just-staged artifact and // asks the configured scanners for a verdict. Returns a wrapped // ErrArtifactBlocked if any scanner blocks, or a scan-infrastructure error. // // The scan call runs on a context detached from ctx's cancellation // (context.WithoutCancel): ctx is the original client request's context, and // a client disconnecting mid-scan must not be indistinguishable from a real // scanner verdict. Group.Scan still bounds the call with its own configured // timeout, so a detached context cannot hang forever. func (p *Proxy) runScan(ctx context.Context, ecosystem, name, version, filename, purlStr, storagePath string, size int64, contentType string) error { fetchURL := p.scanFetchURL(storagePath, p.Scanners.Timeout()) result := p.Scanners.Scan(context.WithoutCancel(ctx), scanner.Request{ Ecosystem: ecosystem, Name: name, Version: version, Filename: filename, PURL: purlStr, FetchURL: fetchURL, Size: size, ContentType: contentType, }) if !result.Allowed { p.Logger.Warn("artifact blocked by security scan", "ecosystem", ecosystem, "name", name, "version", version, "filename", filename, "scanner", result.ScannerName, "reason", result.Reason, "infra_error", result.InfraError) reason := result.Reason if result.InfraError { // result.Reason may contain raw scanner-infrastructure details // (internal hostnames, ports, connection errors) that must not // reach an untrusted client via the 403 response body. reason = "scan could not be completed" } return fmt.Errorf("%w: %s", ErrArtifactBlocked, reason) } return nil } func (p *Proxy) updateCacheDB(ecosystem, name, pkgPURL, upstreamURL, storagePath string, artifact artifacts.Artifact, declared string) error { now := time.Now() // Upsert package pkg := &database.Package{ PURL: pkgPURL, Ecosystem: ecosystem, Name: name, RegistryURL: sql.NullString{String: upstreamURL, Valid: true}, EnrichedAt: sql.NullTime{Time: now, Valid: true}, } // Upsert version ver := &database.Version{ PURL: artifact.PURL, PackagePURL: pkgPURL, Integrity: sql.NullString{String: declared, Valid: declared != ""}, EnrichedAt: sql.NullTime{Time: now, Valid: true}, } // Upsert artifact art := &database.Artifact{ VersionPURL: artifact.PURL, Filename: artifact.Filename, UpstreamURL: upstreamURL, StoragePath: sql.NullString{String: storagePath, Valid: true}, ContentHash: sql.NullString{String: artifact.Digest.Encoded(), Valid: true}, Size: sql.NullInt64{Int64: artifact.Size, Valid: true}, ContentType: sql.NullString{String: artifact.MediaType, Valid: true}, FetchedAt: sql.NullTime{Time: now, Valid: true}, } // One transaction, so a push that landed while this was fetched keeps // both its artifact and its version's integrity. if err := p.DB.RecordCachedArtifact(pkg, ver, art); err != nil { return fmt.Errorf("recording artifact: %w", err) } return nil } // ServeArtifact writes a CacheResult to an HTTP response. func ServeArtifact(w http.ResponseWriter, result *CacheResult) { serveArtifact(w, http.MethodGet, result) } // ServeArtifactRequest writes a CacheResult to an HTTP response using request // headers such as Range and If-Range. func ServeArtifactRequest(w http.ResponseWriter, r *http.Request, result *CacheResult) { if r == nil { ServeArtifact(w, result) return } serveArtifactResponse(w, r.Method, r, result) } func serveArtifact(w http.ResponseWriter, method string, result *CacheResult) { serveArtifactResponse(w, method, nil, result) } func serveArtifactResponse(w http.ResponseWriter, method string, request *http.Request, result *CacheResult) { contentHash := "" if result.Artifact.Digest != "" { contentHash = result.Artifact.Digest.Encoded() } if result.RedirectURL != "" { if contentHash != "" { w.Header().Set(headerETag, `"`+contentHash+`"`) } w.Header().Set("Location", result.RedirectURL) w.WriteHeader(http.StatusFound) return } if result.Reader != nil { defer func() { _ = result.Reader.Close() }() } if result.Artifact.MediaType != "" { w.Header().Set(headerContentType, result.Artifact.MediaType) } if contentHash != "" { w.Header().Set(headerETag, `"`+contentHash+`"`) } var seeker io.Seeker if result.Reader != nil { seeker, _ = result.Reader.(io.Seeker) } if (seeker != nil || result.rangeable) && result.Artifact.Size >= 0 { w.Header().Set("Accept-Ranges", "bytes") if seeker != nil && request != nil && serveArtifactRange(w, request, result, seeker) { return } } if result.Artifact.Size > 0 || (method == http.MethodHead && result.Artifact.Size == 0) { w.Header().Set(headerContentLength, strconv.FormatInt(result.Artifact.Size, 10)) } w.WriteHeader(http.StatusOK) if method != http.MethodHead && result.Reader != nil { copyArtifactBody(w, result.Reader, result.Artifact.Size, false) } } type parsedByteRange struct { start int64 end int64 } func serveArtifactRange(w http.ResponseWriter, request *http.Request, result *CacheResult, seeker io.Seeker) bool { if request.Method != http.MethodGet { return false } ranges := request.Header.Values("Range") if len(ranges) != 1 || !ifRangeMatches(request, w.Header().Get(headerETag)) { return false } byteRange, valid, satisfiable := parseByteRange(ranges[0], result.Artifact.Size) if !valid { return false } if !satisfiable { w.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", result.Artifact.Size)) w.Header().Set(headerContentLength, "0") w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) return true } if _, err := seeker.Seek(byteRange.start, io.SeekStart); err != nil { w.Header().Del(headerContentType) w.Header().Del(headerContentLength) w.Header().Del(headerETag) w.Header().Del("Accept-Ranges") http.Error(w, "failed to seek cached artifact", http.StatusInternalServerError) return true } length := byteRange.end - byteRange.start + 1 w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", byteRange.start, byteRange.end, result.Artifact.Size)) w.Header().Set(headerContentLength, strconv.FormatInt(length, 10)) w.WriteHeader(http.StatusPartialContent) copyArtifactBody(w, result.Reader, length, true) return true } func ifRangeMatches(request *http.Request, etag string) bool { values := request.Header.Values("If-Range") if len(values) == 0 { return true } return len(values) == 1 && etag != "" && strings.TrimSpace(values[0]) == etag } func parseByteRange(value string, size int64) (parsedByteRange, bool, bool) { unit, spec, ok := strings.Cut(strings.TrimSpace(value), "=") if !ok || !strings.EqualFold(strings.TrimSpace(unit), "bytes") { return parsedByteRange{}, false, false } spec = strings.TrimSpace(spec) if spec == "" || strings.Contains(spec, ",") { return parsedByteRange{}, false, false } first, last, ok := strings.Cut(spec, "-") if !ok { return parsedByteRange{}, false, false } if first == "" { suffixLength, ok := parseRangeNumber(last) if !ok { return parsedByteRange{}, false, false } if size == 0 || suffixLength == 0 { return parsedByteRange{}, true, false } if suffixLength >= size { return parsedByteRange{start: 0, end: size - 1}, true, true } return parsedByteRange{start: size - suffixLength, end: size - 1}, true, true } start, ok := parseRangeNumber(first) if !ok { return parsedByteRange{}, false, false } if last == "" { if size == 0 || start >= size { return parsedByteRange{}, true, false } return parsedByteRange{start: start, end: size - 1}, true, true } end, ok := parseRangeNumber(last) if !ok || start > end { return parsedByteRange{}, false, false } if size == 0 || start >= size { return parsedByteRange{}, true, false } if end >= size { end = size - 1 } return parsedByteRange{start: start, end: end}, true, true } func parseRangeNumber(value string) (int64, bool) { if value == "" { return 0, false } for _, digit := range value { if digit < '0' || digit > '9' { return 0, false } } number, err := strconv.ParseInt(value, 10, 64) return number, err == nil } func copyArtifactBody(w http.ResponseWriter, reader io.Reader, expectedSize int64, bounded bool) { buffer := artifactCopyBufferPool.Get().(*[]byte) defer artifactCopyBufferPool.Put(buffer) source := reader if bounded { source = io.LimitReader(reader, expectedSize) } // Hide optional ReaderFrom methods so io.CopyBuffer uses the pooled buffer. written, err := io.CopyBuffer(struct{ io.Writer }{w}, source, *buffer) if err != nil || (bounded && written != expectedSize) || (!bounded && expectedSize > 0 && written != expectedSize) { // Headers are already committed, so an error status is no longer // possible. Aborting leaves the response unterminated so clients discard // a truncated or unverified artifact. // net/http recovers ErrAbortHandler for this request and keeps the server // running; on HTTP/1.x it may close this connection as well. panic(http.ErrAbortHandler) } } // upstreamRequest sends r's method to upstreamURL, copying only the // forwardHeaders that r sets. func (p *Proxy) upstreamRequest(r *http.Request, upstreamURL string, forwardHeaders []string) (*http.Response, error) { req, err := http.NewRequestWithContext(r.Context(), r.Method, upstreamURL, nil) if err != nil { return nil, err } for _, header := range forwardHeaders { if v := r.Header.Get(header); v != "" { req.Header.Set(header, v) } } return p.HTTPClient.Do(req) } // ProxyUpstream forwards a request to an upstream URL without caching. // It copies the request, forwards specified headers, and streams the response back. // forwardHeaders controls the request headers sent upstream. End-to-end response // headers and trailers are relayed independently of that list. func (p *Proxy) ProxyUpstream(w http.ResponseWriter, r *http.Request, upstreamURL string, forwardHeaders []string) { p.Logger.Debug("proxying to upstream", "url", upstreamURL) resp, err := p.upstreamRequest(r, upstreamURL, forwardHeaders) p.relayUpstream(w, r, resp, err) } // relayUpstream writes the result of an upstream request: the response, or // a 502 when the request failed. func (p *Proxy) relayUpstream(w http.ResponseWriter, r *http.Request, resp *http.Response, err error) { if err != nil { if writeUpstreamHeld(w, err) { return } p.Logger.Error("upstream request failed", "error", err) http.Error(w, "upstream request failed", http.StatusBadGateway) return } defer func() { _ = resp.Body.Close() }() p.relayResponse(w, r, resp, nil) } // ProxyFile forwards a file request, relaying end-to-end headers and trailers. func (p *Proxy) ProxyFile(w http.ResponseWriter, r *http.Request, upstreamURL string) { req, err := http.NewRequestWithContext(r.Context(), r.Method, upstreamURL, nil) if err != nil { http.Error(w, "failed to create request", http.StatusInternalServerError) return } resp, err := p.HTTPClient.Do(req) if err != nil { if writeUpstreamHeld(w, err) { return } http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) return } defer func() { _ = resp.Body.Close() }() p.relayResponse(w, r, resp, nil) } // JSONError writes a JSON error response. func JSONError(w http.ResponseWriter, status int, message string) { w.Header().Set(headerContentType, contentTypeJSON) w.WriteHeader(status) _, _ = fmt.Fprintf(w, `{"error":%q}`, message) } // staleWarning marks a response served from the metadata cache because the // upstream could not be reached. const staleWarning = `110 - "Response is Stale"` // markStale sets the stale warning on a metadata response when stale is set. func markStale(w http.ResponseWriter, stale bool) { if stale { w.Header().Set("Warning", staleWarning) } } // ErrUpstreamNotFound indicates the upstream returned 404. var ErrUpstreamNotFound = fmt.Errorf("upstream: %w", fetch.ErrNotFound) // upstreamNotFoundBodyLimit caps the part of an upstream 404 body kept in // upstreamNotFoundError. const upstreamNotFoundBodyLimit = 4 << 10 // upstreamStatusBodyLimit bounds the body kept of another refusal of a // metadata request. const upstreamStatusBodyLimit = 4 << 10 // upstreamNotFoundError is ErrUpstreamNotFound with the start of the // upstream's 404 body, for handlers that relay the reason to clients. type upstreamNotFoundError struct { body []byte } func (e *upstreamNotFoundError) Error() string { return ErrUpstreamNotFound.Error() } func (e *upstreamNotFoundError) Unwrap() error { return ErrUpstreamNotFound } // ErrArtifactBlocked indicates a pre-cache security scan blocked the artifact. var ErrArtifactBlocked = errors.New("artifact blocked by security scan") // ErrVersionDenied indicates an operator explicitly denied this package version. var ErrVersionDenied = errors.New("package version blocked by denylist") // ErrVersionNotFound indicates the requested version is not available through // the proxy, including when cooldown or other filtering removed it. var ErrVersionNotFound = errors.New("version not found") // ErrPackageNotFound indicates the requested package metadata is not available // through the proxy or upstream registry. var ErrPackageNotFound = errors.New("package not found") // serveArtifactError writes response for a failed fetch: // 404 when upstream reports artifact missing, 403 when a security scan or // denylist blocked the artifact, 502 otherwise. func (p *Proxy) serveArtifactError(w http.ResponseWriter, err error, clientMsg string) { if errors.Is(err, ErrUpstreamNotFound) { http.Error(w, "not found", http.StatusNotFound) return } if errors.Is(err, ErrArtifactBlocked) || errors.Is(err, ErrVersionDenied) { JSONError(w, http.StatusForbidden, err.Error()) return } if errors.Is(err, ErrHostedArtifactMissing) { http.Error(w, "not found", http.StatusNotFound) return } if errors.Is(err, ErrHostedArtifactUnavailable) { p.Logger.Error("failed to read pushed artifact", "error", err) http.Error(w, "pushed package is temporarily unavailable", http.StatusServiceUnavailable) return } if writeUpstreamHeld(w, err) { return } if p.serveUpstreamBlock(w, err) { return } p.Logger.Error("failed to get artifact", "error", err) http.Error(w, clientMsg, http.StatusBadGateway) } // errStale304 is returned when upstream sends 304 but the cached file is missing. var errStale304 = fmt.Errorf("upstream returned 304 but cached file is missing") // FetchOrCacheMetadata fetches metadata from upstream with caching. // On success it returns the raw response bytes and content type. // If upstream fails and a cached copy exists, the cached version is returned. // cacheKey is typically the package name but can include subpath components. // Optional acceptHeaders specify the Accept header(s) to send; defaults to application/json. func (p *Proxy) FetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL string, acceptHeaders ...string) ([]byte, string, error) { return p.fetchOrCacheMetadata(ctx, ecosystem, cacheKey, upstreamURL, "", nil, acceptHeaders...) } // fetchOrCacheMetadata implements FetchOrCacheMetadata. acceptEncoding controls // the upstream Accept-Encoding: an empty string leaves it unset so Go // transparently decompresses (for direct callers that parse or rewrite the // body); any non-empty value is sent verbatim, which disables Go's // decompression so the wire bytes and their Content-Encoding are stored and // replayed as sent. The ProxyCached path uses "identity" for signed indexes and // "gzip" where both hops should stay compressed. // validate, when supplied, runs before caching or serving a document. Validation // failures follow the same stale-cache fallback path as upstream failures. It // runs for every caller, including one that shares another caller's fetch, so // it can also decode the document into request-local state; a caller that // joined a fetch gets its own validation error directly. It must not modify the // body, which joined callers share. func (p *Proxy) fetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, acceptEncoding string, validate func([]byte) error, acceptHeaders ...string) ([]byte, string, error) { accept := contentTypeJSON if len(acceptHeaders) > 0 && acceptHeaders[0] != "" { accept = acceptHeaders[0] } res := p.lookupMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, validate) return res.body, res.contentType, res.err } // fetchMetadataDocument is FetchOrCacheMetadata for callers that rewrite the // document and need to know which one they got: the result carries its // digest along with the body. validate, which may be nil, is as for // fetchOrCacheMetadata. func (p *Proxy) fetchMetadataDocument(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, validate func([]byte) error) metadataResult { return p.lookupMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, "", validate) } // lookupMetadata implements fetchOrCacheMetadata and fetchMetadataDocument. func (p *Proxy) lookupMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding string, validate func([]byte) error) metadataResult { if containsPathTraversal(cacheKey) { return metadataResult{err: fmt.Errorf("invalid cache key: %q", cacheKey)} } // Serve from cache if within TTL (skip upstream entirely) if _, hit := p.cachedMetadataState(ctx, ecosystem, cacheKey, validate); hit != nil { metrics.RecordMetadataCacheHit(ecosystem, len(hit.body)) return *hit } p.recordMetadataCacheMiss(ecosystem) return p.coalescedMetadataMiss(ctx, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, validate) } // coalescedMetadataMiss handles a metadata cache miss, sharing one upstream // fetch among concurrent callers with the same key. func (p *Proxy) coalescedMetadataMiss(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding string, validate func([]byte) error) metadataResult { key := metadataCoalesceKey(ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, validate != nil) res, shared := p.coalesceMetadata(ctx, key, func(fetchCtx context.Context) metadataResult { // The caller's lookup ran before it took the key, so a fetch that // finished in between has already refreshed the row. Recheck it // rather than fetching again, and revalidate against the row as it // is now. entry, hit := p.cachedMetadataState(fetchCtx, ecosystem, cacheKey, validate) if hit != nil { return *hit } // Another instance sharing the database may be fetching the same // document. Only a stored copy within TTL can be served after // waiting, so without that there is nothing to wait for. release := func(bool) {} if p.CacheMetadata && p.MetadataTTL > 0 { var outcome leaseOutcome var err error release, outcome, err = p.awaitFetchLease(fetchCtx, metrics.CacheKindMetadata, key, func() bool { entry, hit = p.cachedMetadataState(fetchCtx, ecosystem, cacheKey, validate) return hit != nil }) if err != nil { return metadataResult{err: err} } switch outcome { case leaseStored: return *hit case leaseFailed: // Serve the stale copy at once, as the holder did, rather // than trying the failing upstream again. Without one, fetch, // unless the holder was rate limited: its hold, read here, // refuses the fetch with its Retry-After. p.reloadUpstreamHolds() if entry != nil { if res := p.staleMetadata(fetchCtx, ecosystem, cacheKey, validate, entry, errFetchFailedElsewhere); res.err == nil { return res } } case leaseFetch: } } res := metadataResult{err: errSharedFetchAbandoned} defer func() { release(res.err != nil || res.stale) }() res = p.fetchMetadataFromUpstream(fetchCtx, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, validate, entry) return res }) // The fetch ran the first caller's validate. A caller that joined it runs // its own on the shared bytes, since validate may also decode them for it. if shared && res.err == nil && validate != nil { if err := validate(res.body); err != nil { return metadataResult{err: err} } } return res } // cachedMetadataState reads the cache row for a metadata lookup. hit is set // when the row is within TTL and its bytes are usable. Otherwise entry is the // row to revalidate against with its ETag: nil when there is none, or when // validate rejected the cached body, since revalidating an unusable body would // keep it. func (p *Proxy) cachedMetadataState(ctx context.Context, ecosystem, cacheKey string, validate func([]byte) error) (*database.MetadataCacheEntry, *metadataResult) { if !p.CacheMetadata || p.DB == nil { return nil, nil } entry, _ := p.DB.GetMetadataCache(ecosystem, cacheKey) entry = p.migrateLegacyMetadataEntry(ctx, ecosystem, cacheKey, entry) if entry != nil && p.MetadataTTL > 0 && entry.FetchedAt.Valid && time.Since(entry.FetchedAt.Time) < p.MetadataTTL { data, ct, err := p.readCachedMetadata(ctx, entry, validate) if err == nil { hit := metadataResult{body: data, contentType: ct, contentEncoding: entry.ContentEncoding.String, fromCache: true}.describedBy(entry) return entry, &hit } if validate != nil { return nil, nil } // Cache file missing/unreadable, fall through to upstream } return entry, nil } // refreshMetadata is lookupMetadata for a document whose cached copy is known // to be out of date although it is within its TTL. It skips the TTL and asks // upstream, revalidating the cached copy with its validators, and falls back // to that copy if upstream fails, which the result reports as stale. // Concurrent refreshes of one document share a fetch. func (p *Proxy) refreshMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding string, validate func([]byte) error) metadataResult { if containsPathTraversal(cacheKey) { return metadataResult{err: fmt.Errorf("invalid cache key: %q", cacheKey)} } key := metadataCoalesceKey(ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, validate != nil) + "\x00refresh" res, shared := p.coalesceMetadata(ctx, key, func(fetchCtx context.Context) metadataResult { var entry *database.MetadataCacheEntry if p.CacheMetadata && p.DB != nil { entry, _ = p.DB.GetMetadataCache(ecosystem, cacheKey) } return p.fetchMetadataFromUpstream(fetchCtx, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, validate, entry) }) if shared && res.err == nil && validate != nil { if err := validate(res.body); err != nil { return metadataResult{err: err} } } return res } // fetchMetadataFromUpstream fetches metadata after a cache miss, caches it, // and falls back to the cached copy if upstream fails. entry is the row to // revalidate against, or nil. func (p *Proxy) fetchMetadataFromUpstream(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding string, validate func([]byte) error, entry *database.MetadataCacheEntry) metadataResult { fetch := func(entry *database.MetadataCacheEntry) (*upstreamMetadata, error) { meta, err := p.fetchUpstreamMetadata(ctx, upstreamURL, entry, accept, acceptEncoding) if err == nil && validate != nil { err = validate(meta.body) } return meta, err } meta, err := fetch(entry) var digest string // Validated first, so a 304 for a rejected copy doesn't mark it fresh. if err == nil && p.CacheMetadata { digest, err = p.refreshMetadataBlob(ecosystem, cacheKey, meta) } if errors.Is(err, errStale304) { // 304 but the cached copy is gone or was replaced; retry without // validators. meta, err = fetch(nil) } if err == nil { if p.CacheMetadata && digest == "" { digest = p.cacheMetadataBlob(ctx, ecosystem, cacheKey, metadataStoragePath(ecosystem, cacheKey), meta) } return fetchedResult(meta, digest) } return p.staleMetadata(ctx, ecosystem, cacheKey, validate, entry, err) } // errFetchFailedElsewhere is the upstream error behind a stale copy served // because another instance's fetch for it failed. var errFetchFailedElsewhere = errors.New("upstream fetch failed on another instance") // staleMetadata falls back to the cached copy in entry after the upstream // fetch failed with err. func (p *Proxy) staleMetadata(ctx context.Context, ecosystem, cacheKey string, validate func([]byte) error, entry *database.MetadataCacheEntry, err error) metadataResult { if !p.CacheMetadata || entry == nil { return metadataResult{err: fmt.Errorf("upstream failed and no cached metadata: %w", err)} } p.Logger.Warn("upstream metadata fetch failed, checking cache", "ecosystem", ecosystem, "key", cacheKey, "error", err) // Re-read the row so the encoding describes the blob as it is now: a // concurrent refetch may have replaced both since entry was read above // (an identity blob swapped for a gzip one during rollout). Bytes that // don't match the row were replaced after it was read, so the row is // read once more; if they still don't match, no row describes them. for attempt := 0; ; attempt++ { entry = p.currentMetadataEntry(ecosystem, cacheKey, entry) data, ct, readErr := p.readCachedMetadata(ctx, entry, validate) if readErr == nil && !matchesEntryDigest(entry, data) { if attempt == 0 { continue } readErr = errors.New("replaced while being read") } if readErr != nil { return metadataResult{err: fmt.Errorf("upstream failed and cached metadata unusable (%v): %w", readErr, err)} } p.Logger.Info("serving metadata from cache", "ecosystem", ecosystem, "key", cacheKey) return metadataResult{body: data, contentType: ct, contentEncoding: entry.ContentEncoding.String, stale: true, upstreamErr: err}.describedBy(entry) } } // metadataResult is what a metadata lookup hands back. A shared fetch gives // every waiter the same body, so callers must treat it as read-only. type metadataResult struct { body []byte contentType string contentEncoding string // digest identifies body when the metadata cache stored it, and is // empty otherwise. digest string err error // etag and lastModified are the validators of body: upstream's for a // fetched document, the cache row's for a stored one. etag string lastModified time.Time // stale is set when the body is the cached copy served because upstream // failed, and upstreamErr is that failure. stale bool upstreamErr error // fromCache is set when the body is a stored copy within its TTL, read // without asking upstream. fromCache bool // entry is the cache row of the stored document, for a lookup that // leaves the document in storage (see storedMetadataDocument). body is // nil then. entry *database.MetadataCacheEntry } // describedBy returns res with the digest and validators that entry, the // cache row its body was read for, records. func (res metadataResult) describedBy(entry *database.MetadataCacheEntry) metadataResult { if entry.ContentDigest.Valid { res.digest = entry.ContentDigest.String } res.etag = entry.ETag.String if entry.LastModified.Valid { res.lastModified = entry.LastModified.Time } return res } // fetchedResult returns the result for meta, a document from upstream (or // the stored copy it answered 304 for), cached under digest. func fetchedResult(meta *upstreamMetadata, digest string) metadataResult { return metadataResult{ body: meta.body, contentType: meta.contentType, contentEncoding: meta.contentEncoding, digest: digest, etag: meta.etag, lastModified: meta.lastModified, } } // cachedMeta returns the validators and staleness of res's body. func (res metadataResult) cachedMeta() cachedMeta { return cachedMeta{etag: res.etag, lastModified: res.lastModified, stale: res.stale} } // inflightMetadata is one metadata fetch that concurrent callers share. res is // written before done closes and read only after, so the close is the handoff. type inflightMetadata struct { done chan struct{} res metadataResult // waiters counts callers that joined the fetch, guarded by metaMu. Tests // read it to know every caller has joined before the fetch finishes. waiters int } // metadataCoalesceKey identifies metadata requests that can share one fetch. // Accept and Accept-Encoding are part of it because they change the bytes // upstream returns: npm serves an abbreviated or a full document for the same // package, and the cached-proxy paths ask for identity or gzip. Whether the // caller validates is part of it so an unvalidated caller is never handed a // result that skipped validation, or the reverse. func metadataCoalesceKey(ecosystem, cacheKey, upstreamURL, accept, acceptEncoding string, validated bool) string { return strings.Join([]string{ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, strconv.FormatBool(validated)}, "\x00") } // coalesceMetadata runs fetch at most once for concurrent callers sharing key // and gives each the result, reporting whether this caller joined another's // fetch rather than running it. It follows coalesceFetch with one difference: // fetch runs on a context detached from the first caller's cancellation. A // metadata fetch has no scan or mirror that depends on the caller aborting it, // and CI jobs asking for the same Composer or npm metadata would otherwise all // fail when the first of them disconnects. It stays bounded by the HTTP // client's timeout, and at most maxDetachedMetadataFetches run detached at // once; beyond that, fetch ends with the first caller. Waiters still leave // when their own context ends. func (p *Proxy) coalesceMetadata(ctx context.Context, key string, fetch func(context.Context) metadataResult) (metadataResult, bool) { shared := p.sharedProxy() shared.metaMu.Lock() if shared.inFlightMeta == nil { shared.inFlightMeta = make(map[string]*inflightMetadata) } f, joined := shared.inFlightMeta[key] if joined { f.waiters++ } else { f = &inflightMetadata{done: make(chan struct{})} shared.inFlightMeta[key] = f } shared.metaMu.Unlock() if joined { select { case <-ctx.Done(): return metadataResult{err: ctx.Err()}, true case <-f.done: return f.res, true } } // Set before running so a panicking fetch leaves waiters with an error // rather than an empty body. f.res = metadataResult{err: errSharedFetchAbandoned} defer func() { shared.metaMu.Lock() delete(shared.inFlightMeta, key) shared.metaMu.Unlock() close(f.done) }() fetchCtx := ctx if p.acquireMetadataDetachSlot() { defer p.releaseMetadataDetachSlot() fetchCtx = context.WithoutCancel(ctx) } f.res = fetch(fetchCtx) return f.res, false } func (p *Proxy) readCachedMetadata(ctx context.Context, entry *database.MetadataCacheEntry, validate func([]byte) error) ([]byte, string, error) { cached, err := p.Storage.Open(ctx, entry.StoragePath) if err != nil { return nil, "", err } defer func() { _ = cached.Close() }() data, err := p.ReadMetadataSized(cached, metadataEntrySize(entry)) if err == nil && validate != nil { err = validate(data) } if err != nil { return nil, "", err } p.recordMetadataRead(entry) ct := contentTypeJSON if entry.ContentType.Valid { ct = entry.ContentType.String } return data, ct, nil } // metadataEntrySize is the recorded size of a cached metadata document, or // -1 when none is recorded. func metadataEntrySize(entry *database.MetadataCacheEntry) int64 { if !entry.Size.Valid { return -1 } return entry.Size.Int64 } // recordMetadataRead marks a cached metadata document as used, so size // eviction drops the documents nobody asks for first. func (p *Proxy) recordMetadataRead(entry *database.MetadataCacheEntry) { if err := p.DB.RecordMetadataHit(entry.Ecosystem, entry.Name); err != nil { p.Logger.Warn("failed to record metadata cache hit", "ecosystem", entry.Ecosystem, "key", entry.Name, "error", err) } } func (p *Proxy) recordMetadataCacheMiss(ecosystem string) { if p.CacheMetadata { metrics.RecordMetadataCacheMiss(ecosystem) } } // upstreamMetadata holds a fetched metadata response in upstream byte form. type upstreamMetadata struct { body []byte contentType string contentEncoding string etag string lastModified time.Time // url is the upstream URL the document was fetched from. url string // notModified is the cache row upstream answered 304 for, whose stored // bytes body holds. It is nil for a document upstream sent. notModified *database.MetadataCacheEntry } // fetchUpstreamMetadata fetches metadata from upstream, using ETag for conditional revalidation. // When acceptEncoding is non-empty it is sent as the Accept-Encoding header, which disables Go's // transparent decompression (it only applies when the transport adds the header itself), so the // returned bytes are exactly what the upstream sent and any Content-Encoding it applied is reported // alongside for the caller to store and replay. An empty acceptEncoding leaves Go to negotiate and // decompress transparently. func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, entry *database.MetadataCacheEntry, accept, acceptEncoding string) (*upstreamMetadata, error) { resp, err := p.requestUpstreamMetadata(ctx, upstreamURL, entry, accept, acceptEncoding) if err != nil { return nil, err } defer func() { _ = resp.Body.Close() }() // 304 Not Modified -- our cached copy is still good if resp.StatusCode == http.StatusNotModified && entry != nil { cached, readErr := p.Storage.Open(ctx, entry.StoragePath) if readErr != nil { return nil, errStale304 } defer func() { _ = cached.Close() }() data, readErr := p.ReadMetadataSized(cached, metadataEntrySize(entry)) if readErr != nil || !matchesEntryDigest(entry, data) { // Gone, or replaced by another instance since entry was read. return nil, errStale304 } meta := &upstreamMetadata{body: data, contentType: contentTypeJSON, etag: entry.ETag.String, url: upstreamURL, notModified: entry} if entry.ContentType.Valid { meta.contentType = entry.ContentType.String } if entry.ContentEncoding.Valid { meta.contentEncoding = entry.ContentEncoding.String } if entry.LastModified.Valid { meta.lastModified = entry.LastModified.Time } return meta, nil } // A Terraform module registry may answer a download with the location // in a header, usually with no content, which Terraform reads before // the body. It is stored as the JSON document other registries answer // with. if location := resp.Header.Get(headerTerraformGet); location != "" && (resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusNoContent) { return &upstreamMetadata{body: terraformModuleDownloadDocument(location), contentType: contentTypeJSON, url: upstreamURL}, nil } if err := upstreamMetadataStatus(resp); err != nil { return nil, err } // ContentLength is -1 when the transport decompresses the body, and // otherwise the length of the bytes read here. body, err := p.ReadMetadataSized(upstreamBodyReader{resp.Body}, resp.ContentLength) if err != nil { return nil, fmt.Errorf("reading response: %w", err) } meta := &upstreamMetadata{ body: body, contentType: resp.Header.Get(headerContentType), contentEncoding: resp.Header.Get(headerContentEncoding), etag: resp.Header.Get(headerETag), url: upstreamURL, } if meta.contentType == "" { meta.contentType = contentTypeJSON } if lm := resp.Header.Get(headerLastModified); lm != "" { meta.lastModified, _ = http.ParseTime(lm) } return meta, nil } // requestUpstreamMetadata sends the request for a metadata document, // revalidating entry when it is given. func (p *Proxy) requestUpstreamMetadata(ctx context.Context, upstreamURL string, entry *database.MetadataCacheEntry, accept, acceptEncoding string) (*http.Response, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, upstreamURL, nil) if err != nil { return nil, fmt.Errorf("creating request: %w", err) } req.Header.Set("Accept", accept) if acceptEncoding != "" { req.Header.Set(headerAcceptEncoding, acceptEncoding) } if entry != nil && entry.ETag.Valid { req.Header.Set("If-None-Match", entry.ETag.String) } else if entry != nil && entry.LastModified.Valid { req.Header.Set("If-Modified-Since", entry.LastModified.Time.UTC().Format(http.TimeFormat)) } resp, err := p.HTTPClient.Do(req) if err != nil { return nil, fmt.Errorf("fetching metadata: %w", err) } return resp, nil } // matchesEntryDigest reports whether data is the document entry records, // when entry has a sha256 digest to check it by. func matchesEntryDigest(entry *database.MetadataCacheEntry, data []byte) bool { want, ok := strings.CutPrefix(entry.ContentDigest.String, "sha256:") if !entry.ContentDigest.Valid || !ok { return true } sum := sha256.Sum256(data) return hex.EncodeToString(sum[:]) == want } // upstreamMetadataStatus returns the error for a metadata response other // than 200. Callers handle a 304 they asked for before. func upstreamMetadataStatus(resp *http.Response) error { switch resp.StatusCode { case http.StatusOK: return nil case http.StatusNotFound: body, _ := io.ReadAll(io.LimitReader(resp.Body, upstreamNotFoundBodyLimit)) return &upstreamNotFoundError{body: body} } // The status, body and type are kept, so a policy denial can be // forwarded as upstream sent it. body, _ := io.ReadAll(io.LimitReader(resp.Body, upstreamStatusBodyLimit)) return &fetch.StatusError{Code: resp.StatusCode, Body: string(body), ContentType: resp.Header.Get(headerContentType)} } // refreshMetadataBlob records that upstream answered 304 for the cached // document meta was read from, by moving the row's fetched_at on. The bytes in // storage are unchanged, so they are neither written nor hashed again. It // returns the row's digest, or "" when the caller has to store meta as for a // new document: when upstream sent one or the row records no digest. When // a concurrent fetch has replaced the document since it was read, it returns // errStale304: storing meta would put the old document back under the new // one. A failed database write only leaves fetched_at behind, so the // confirmed bytes are still served. func (p *Proxy) refreshMetadataBlob(ecosystem, cacheKey string, meta *upstreamMetadata) (string, error) { entry := meta.notModified if entry == nil || p.DB == nil || !entry.ContentDigest.Valid { return "", nil } refreshed, err := p.DB.RefreshMetadataCache(ecosystem, cacheKey, entry.ContentDigest.String, time.Now()) if err != nil { // The bytes were checked against the row and upstream confirmed // them; only fetched_at stays behind, so they are still served. p.Logger.Warn("failed to refresh cached metadata", "ecosystem", ecosystem, "key", cacheKey, "error", err) return entry.ContentDigest.String, nil } if !refreshed { return "", errStale304 } p.recordMetadataUpstreamURL(ecosystem, cacheKey, entry, meta.url) p.recordMetadataRead(entry) return entry.ContentDigest.String, nil } // recordMetadataUpstreamURL records upstreamURL on the cache row entry, a // row upstream confirmed with 304, when the row has none yet. func (p *Proxy) recordMetadataUpstreamURL(ecosystem, cacheKey string, entry *database.MetadataCacheEntry, upstreamURL string) { if entry.UpstreamURL.Valid || upstreamURL == "" { return } if err := p.DB.SetMetadataUpstreamURL(ecosystem, cacheKey, upstreamURL); err != nil { p.Logger.Warn("failed to record metadata upstream URL", "ecosystem", ecosystem, "key", cacheKey, "error", err) } } // cacheMetadataBlob stores metadata bytes in storage and updates the // database. It returns the digest it recorded for the bytes, or "" when they // were not cached. func (p *Proxy) cacheMetadataBlob(ctx context.Context, ecosystem, cacheKey, storagePath string, meta *upstreamMetadata) string { digest, _ := p.storeMetadataBlob(ctx, ecosystem, cacheKey, storagePath, bytes.NewReader(meta.body), meta) return digest } // storeMetadataBlob is cacheMetadataBlob for a document read from body, // which meta describes. It also returns the error that kept it from caching // the document. func (p *Proxy) storeMetadataBlob(ctx context.Context, ecosystem, cacheKey, storagePath string, body io.Reader, meta *upstreamMetadata) (string, error) { if p.DB == nil || p.Storage == nil { return "", errMetadataNotStored } size, hash, err := p.Storage.Store(ctx, storagePath, body) if err != nil { p.Logger.Warn("failed to cache metadata", "ecosystem", ecosystem, "key", cacheKey, "error", err) return "", err } var digest string if hash != "" { digest = "sha256:" + hash } err = p.DB.UpsertMetadataCache(&database.MetadataCacheEntry{ Ecosystem: ecosystem, Name: cacheKey, StoragePath: storagePath, ETag: sql.NullString{String: meta.etag, Valid: meta.etag != ""}, ContentType: sql.NullString{String: meta.contentType, Valid: meta.contentType != ""}, ContentEncoding: sql.NullString{String: meta.contentEncoding, Valid: meta.contentEncoding != ""}, // The digest identifies the stored bytes, so a rewrite cached for them // can be found without reading them back (see storedRewrite). ContentDigest: sql.NullString{String: digest, Valid: digest != ""}, UpstreamURL: sql.NullString{String: meta.url, Valid: meta.url != ""}, Size: sql.NullInt64{Int64: size, Valid: true}, LastModified: sql.NullTime{Time: meta.lastModified, Valid: !meta.lastModified.IsZero()}, FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, }) if err != nil { // The blob is written but the row describing it is not, so a later // TTL hit or stale fallback would serve these bytes with the previous // row's encoding. Drop the blob so row and bytes can never disagree; // the next request refetches instead. p.Logger.Warn("failed to record cached metadata, discarding blob", "ecosystem", ecosystem, "key", cacheKey, "error", err) if delErr := p.Storage.Delete(ctx, storagePath); delErr != nil { p.Logger.Warn("failed to discard metadata blob", "ecosystem", ecosystem, "key", cacheKey, "error", delErr) } return "", err } return digest, nil } // currentMetadataEntry re-reads the metadata cache row and returns it, or // fallback when the row cannot be read. Used before serving a stored blob so // its encoding comes from the row as it is now rather than from a snapshot // taken before the upstream fetch. func (p *Proxy) currentMetadataEntry(ecosystem, cacheKey string, fallback *database.MetadataCacheEntry) *database.MetadataCacheEntry { if fresh, err := p.DB.GetMetadataCache(ecosystem, cacheKey); err == nil && fresh != nil { return fresh } return fallback } // cachedMeta holds cache validators and freshness state from a metadata cache entry. type cachedMeta struct { etag string lastModified time.Time stale bool } // cachedMetaFor returns the cache validators and freshness state of entry. func (p *Proxy) cachedMetaFor(entry *database.MetadataCacheEntry) cachedMeta { var cm cachedMeta if entry.ETag.Valid { cm.etag = entry.ETag.String } if entry.LastModified.Valid { cm.lastModified = entry.LastModified.Time } // If FetchedAt is older than TTL, upstream must have failed and // we served from stale cache (successful fetches update FetchedAt). if p.MetadataTTL > 0 && entry.FetchedAt.Valid && time.Since(entry.FetchedAt.Time) > p.MetadataTTL { cm.stale = true } return cm } // ProxyCached fetches metadata from upstream (with optional caching for offline fallback) // and writes it to the response. Optional acceptHeaders specify the Accept header to send. // When metadata caching is disabled, the response is streamed directly to avoid buffering // large metadata responses (e.g. npm packages with many versions) in memory. func (p *Proxy) ProxyCached(w http.ResponseWriter, r *http.Request, upstreamURL, ecosystem, cacheKey string, acceptHeaders ...string) { p.proxyCachedWithEncoding(w, r, upstreamURL, ecosystem, cacheKey, "identity", nil, acceptHeaders...) } // metadataHooks are optional callbacks for proxyCachedWithEncoding: hit runs // after a fresh cached row is served, with that row and when it was read, and // fetched runs with a fetched document before it is written. type metadataHooks struct { hit func(entry *database.MetadataCacheEntry, seen time.Time) fetched func(body []byte, contentType, contentEncoding string) } // proxyCachedWithEncoding is ProxyCached with an explicit upstream Accept-Encoding. // "identity" preserves signed index bytes (the default); "gzip" keeps both hops // compressed for large, non-hash-pinned metadata whose clients decode gzip // (Homebrew API and Conda repodata). The stored bytes and Content-Encoding are // replayed verbatim either way. func (p *Proxy) proxyCachedWithEncoding(w http.ResponseWriter, r *http.Request, upstreamURL, ecosystem, cacheKey, acceptEncoding string, hooks *metadataHooks, acceptHeaders ...string) { if !p.CacheMetadata { // Stream directly without buffering when caching is off. p.proxyMetadataStream(w, r, upstreamURL, acceptEncoding, acceptHeaders...) return } if hooks != nil && hooks.hit != nil { if entry := p.freshMetadataEntry(ecosystem, cacheKey); entry != nil { seen := time.Now() if p.serveCachedMetadata(w, r, ecosystem, cacheKey) { hooks.hit(entry, seen) return } } } else if p.serveCachedMetadata(w, r, ecosystem, cacheKey) { return } accept := contentTypeJSON if len(acceptHeaders) > 0 && acceptHeaders[0] != "" { accept = acceptHeaders[0] } if hooks == nil && p.relayStoredMetadata(w, r, upstreamURL, ecosystem, cacheKey, accept, acceptEncoding) { return } res := p.lookupMetadata(r.Context(), ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, nil) if res.err != nil { p.writeMetadataFetchError(w, res.err) return } if hooks != nil && hooks.fetched != nil { hooks.fetched(res.body, res.contentType, res.contentEncoding) } writeMetadataResult(w, r, res) } // writeMetadataFetchError answers a failed metadata fetch. func (p *Proxy) writeMetadataFetchError(w http.ResponseWriter, err error) { if errors.Is(err, ErrUpstreamNotFound) { http.Error(w, "not found", http.StatusNotFound) return } if p.writeUpstreamRefusal(w, err) { return } p.Logger.Error("metadata fetch failed", "error", err) http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) } // writeMetadataResult writes the document res holds with its own validators // and staleness, rather than the cache row's, which may describe other bytes // when the cache write failed or another instance replaced it. func writeMetadataResult(w http.ResponseWriter, r *http.Request, res metadataResult) { writeMetadataBody(w, r, res.cachedMeta(), res.body, res.contentType, res.contentEncoding) } // writeMetadataBody writes a metadata document with the validators and // staleness in cm. func writeMetadataBody(w http.ResponseWriter, r *http.Request, cm cachedMeta, body []byte, contentType, contentEncoding string) { if writeMetadataValidators(w, r, cm) { return } w.Header().Set(headerContentType, contentType) w.Header().Set(headerContentLength, strconv.Itoa(len(body))) if contentEncoding != "" { w.Header().Set(headerContentEncoding, contentEncoding) } markStale(w, cm.stale) w.WriteHeader(http.StatusOK) if r.Method != http.MethodHead { _, _ = w.Write(body) } } // writeMetadataValidators sets cm's ETag and Last-Modified on w and answers // a conditional request they satisfy with 304, reporting whether it did. func writeMetadataValidators(w http.ResponseWriter, r *http.Request, cm cachedMeta) bool { if cm.etag != "" { w.Header().Set(headerETag, cm.etag) } if !cm.lastModified.IsZero() { w.Header().Set(headerLastModified, cm.lastModified.UTC().Format(http.TimeFormat)) } if ifNoneMatchHits(r.Header.Get("If-None-Match"), cm.etag) { w.WriteHeader(http.StatusNotModified) return true } if !cm.lastModified.IsZero() { if ims := r.Header.Get("If-Modified-Since"); ims != "" { if t, err := http.ParseTime(ims); err == nil && !cm.lastModified.After(t) { w.WriteHeader(http.StatusNotModified) return true } } } return false } // proxyMetadataStream forwards an upstream metadata response by streaming it to the client // without buffering the full body in memory. func (p *Proxy) proxyMetadataStream(w http.ResponseWriter, r *http.Request, upstreamURL, acceptEncoding string, acceptHeaders ...string) { req, err := http.NewRequestWithContext(r.Context(), r.Method, upstreamURL, nil) if err != nil { http.Error(w, "failed to create request", http.StatusInternalServerError) return } accept := contentTypeJSON if len(acceptHeaders) > 0 && acceptHeaders[0] != "" { accept = acceptHeaders[0] } req.Header.Set("Accept", accept) // Set Accept-Encoding explicitly (identity, or gzip for compressible // verbatim metadata) so Go does not transparently decompress and strip the // Content-Encoding of the bytes we forward, regardless of what the client // negotiated. An empty value leaves the header unset, as in // fetchUpstreamMetadata. if acceptEncoding != "" { req.Header.Set(headerAcceptEncoding, acceptEncoding) } for _, header := range []string{"If-Modified-Since", "If-None-Match"} { if v := r.Header.Get(header); v != "" { req.Header.Set(header, v) } } resp, err := p.HTTPClient.Do(req) if err != nil { if writeUpstreamHeld(w, err) { return } http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) return } defer func() { _ = resp.Body.Close() }() p.relayResponse(w, r, resp, func(dst, src http.Header) { for _, header := range []string{headerContentType, headerContentLength, headerContentEncoding, headerLastModified, headerETag} { if v := src.Get(header); v != "" { dst.Set(header, v) } } }) } // GetOrFetchArtifactFromURL retrieves an artifact from cache or fetches from a specific URL. // This is useful for registries where download URLs are determined from metadata. func (p *Proxy) GetOrFetchArtifactFromURL(ctx context.Context, ecosystem, name, version, filename, downloadURL string) (*CacheResult, error) { return p.getOrFetchArtifactFromURL(ctx, ecosystem, name, version, filename, downloadURL, nil, "", "") } // cacheFilenameKeepingLegacy returns legacy when an artifact recorded under // it was fetched from downloadURL, and filename otherwise. func (p *Proxy) cacheFilenameKeepingLegacy(ecosystem, name, version, filename, legacy, downloadURL string) (string, error) { _, versionPURL, err := packagePURLStrings(ecosystem, name, version) if err != nil { return "", err } record, err := p.DB.GetArtifact(versionPURL, legacy) if err != nil { return "", fmt.Errorf("checking artifact cache: %w", err) } if record != nil && record.UpstreamURL == downloadURL { return legacy, nil } return filename, nil } // GetOrFetchArtifactFromURLWithHeaders retrieves an artifact from cache or fetches from a URL // with additional request-specific HTTP headers. func (p *Proxy) GetOrFetchArtifactFromURLWithHeaders(ctx context.Context, ecosystem, name, version, filename, downloadURL string, headers http.Header) (*CacheResult, error) { return p.getOrFetchArtifactFromURL(ctx, ecosystem, name, version, filename, downloadURL, headers, "", "") } // GetOrFetchArtifactFromURLWithDigest retrieves an artifact and verifies a // newly fetched response against expected, an OCI-style digest such as // "sha256:" or "sha512:", before serving or caching it. A digest // whose algorithm can't be verified is refused with ErrUnsupportedDigest // rather than fetched unverified. An empty expected fetches without // verification. func (p *Proxy) GetOrFetchArtifactFromURLWithDigest(ctx context.Context, ecosystem, name, version, filename, downloadURL, expected string) (*CacheResult, error) { if expected == "" { return p.getOrFetchArtifactFromURL(ctx, ecosystem, name, version, filename, downloadURL, nil, "", "") } upstreamHash, declared, err := digestChecks(expected) if err != nil { return nil, err } return p.getOrFetchArtifactFromURL(ctx, ecosystem, name, version, filename, downloadURL, nil, upstreamHash, declared) } // ErrUnsupportedDigest is returned for a digest whose algorithm or encoding // the proxy can't verify. var ErrUnsupportedDigest = errors.New("unsupported digest") // digestChecks turns an OCI-style digest into the checks the artifact fetch // runs: the hex of a sha256 digest, which is also how cached artifacts are // keyed, or for another algorithm the integrity string verifyIntegrity // checks. func digestChecks(expected string) (upstreamHash, declared string, _ error) { // The sha256 hex is compared as given, ignoring case; a malformed one // fails as a mismatch once the body has been hashed. if hash, ok := strings.CutPrefix(expected, "sha256:"); ok { return hash, "", nil } d, err := digest.Parse(expected) if err != nil { return "", "", fmt.Errorf("%w %q: %w", ErrUnsupportedDigest, expected, err) } var algorithm integrity.Algorithm switch d.Algorithm() { case digest.SHA384: algorithm = integrity.SHA384 case digest.SHA512: algorithm = integrity.SHA512 default: return "", "", fmt.Errorf("%w %q", ErrUnsupportedDigest, expected) } sum, err := integrity.ParseHex(algorithm, d.Encoded()) if err != nil { return "", "", fmt.Errorf("%w %q: %w", ErrUnsupportedDigest, expected, err) } return "", sum.SRI(), nil } // verifyDigest checks body against expected, an OCI-style digest. A digest // whose algorithm can't be computed is an ErrUnsupportedDigest, and one that // doesn't match is an ErrArtifactDigestMismatch. func verifyDigest(expected string, body []byte) error { d, err := digest.Parse(expected) if err != nil { return fmt.Errorf("%w %q: %w", ErrUnsupportedDigest, expected, err) } if actual := d.Algorithm().FromBytes(body); actual != d { return fmt.Errorf("%w: expected %s, got %s", ErrArtifactDigestMismatch, d, actual) } return nil } // GetOrFetchArtifactFromURLWithIntegrity retrieves an artifact and verifies a // newly fetched response against declared, the integrity the registry's // metadata declares, before serving or caching it: a Subresource Integrity // string, or a single algorithm-prefixed hex checksum such as "sha1-". // declared is recorded with the cached version, so cache hits are verified // against it as well. An empty declared fetches without verification. func (p *Proxy) GetOrFetchArtifactFromURLWithIntegrity(ctx context.Context, ecosystem, name, version, filename, downloadURL, declared string) (*CacheResult, error) { return p.getOrFetchArtifactFromURL(ctx, ecosystem, name, version, filename, downloadURL, nil, "", declared) } func (p *Proxy) getOrFetchArtifactFromURL(ctx context.Context, ecosystem, name, version, filename, downloadURL string, headers http.Header, upstreamHash, declared string) (*CacheResult, error) { pkgPURL, versionPURL, err := packagePURLStrings(ecosystem, name, version) if err != nil { return nil, err } return p.getOrFetchArtifactFromURLWithCachePURLs( ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL, headers, upstreamHash, declared, ) } func (p *Proxy) getOrFetchArtifactFromURLWithCachePURLs(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL string, headers http.Header, upstreamHash, declared string) (*CacheResult, error) { // Some protocols use qualified cache keys. Check their package identity // too, and the cache key itself for those, such as Swift, whose name has // no PURL of its own. if p.versionDenied(ecosystem, name, version) { return nil, fmt.Errorf("%w: %s", ErrVersionDenied, canonicalVersionPURL(ecosystem, name, version)) } if p.Denylist.Denied(versionPURL) { return nil, fmt.Errorf("%w: %s", ErrVersionDenied, versionPURL) } if p.StreamArtifacts { return p.streamFromUpstream(ctx, ecosystem, name, version, filename, versionPURL, downloadURL, upstreamHash, declared, func(fetchCtx context.Context) (*fetch.Artifact, error) { return p.fetchUpstream(fetchCtx, downloadURL, headers) }) } if cached, err := p.getCachedArtifactWithUpstreamHash(ctx, pkgPURL, versionPURL, filename, upstreamHash); err != nil { return nil, err } else if cached != nil { return cached, nil } metrics.RecordArtifactCacheMiss(ecosystem) return p.coalescedFetchFromURL(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL, headers, upstreamHash, declared) } // coalescedFetchFromURL fetches an artifact the cache could not serve, sharing // the fetch with concurrent callers. The caller running it discards a stale // entry under the key, where it cannot delete a fetch that just replaced it. func (p *Proxy) coalescedFetchFromURL(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL string, headers http.Header, upstreamHash, declared string) (*CacheResult, error) { key := artifactCoalesceKey(versionPURL, filename, downloadURL, upstreamHash, declared) recheck := func() (artifacts.Artifact, string, bool) { return p.cachedArtifactRecord(pkgPURL, versionPURL, filename, upstreamHash) } return p.coalesceFetch(ctx, key, recheck, func(fetchCtx context.Context) (artifacts.Artifact, string, error) { p.discardStaleArtifact(pkgPURL, versionPURL, filename, upstreamHash) return p.fetchAndCacheFromURL(fetchCtx, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL, headers, upstreamHash, declared) }) } // cacheStoredCopy caches a copy of the object stored at source under a new // name, instead of downloading the same bytes again. It shares the coalescing // key a download from upstreamURL would use, so the two never run at once. func (p *Proxy) cacheStoredCopy(ctx context.Context, ecosystem, name, version, filename, upstreamURL, upstreamHash string, source *database.StoredContent) (*CacheResult, error) { pkgPURL, versionPURL, err := packagePURLStrings(ecosystem, name, version) if err != nil { return nil, err } if p.versionDenied(ecosystem, name, version) || p.Denylist.Denied(versionPURL) { return nil, fmt.Errorf("%w: %s", ErrVersionDenied, versionPURL) } key := artifactCoalesceKey(versionPURL, filename, upstreamURL, upstreamHash, "") recheck := func() (artifacts.Artifact, string, bool) { return p.cachedArtifactRecord(pkgPURL, versionPURL, filename, upstreamHash) } return p.coalesceFetch(ctx, key, recheck, func(fetchCtx context.Context) (artifacts.Artifact, string, error) { body, err := p.Storage.Open(fetchCtx, source.StoragePath) if err != nil { return artifacts.Artifact{}, "", fmt.Errorf("opening stored copy: %w", err) } p.Logger.Info("caching a stored copy", "ecosystem", ecosystem, "name", name, "version", version, "source", source.StoragePath) return p.storeArtifact(fetchCtx, ecosystem, name, version, filename, pkgPURL, versionPURL, upstreamURL, upstreamHash, "", &fetch.Artifact{Body: body, Size: source.Size, ContentType: source.ContentType.String}) }) } // getCachedArtifactWithUpstreamHash returns a cached artifact whose recorded // content hash matches the checksum the upstream currently declares for it. // This detects an upstream re-publishing under the same version, which the // stream integrity check in checkCache cannot: that check only verifies the // stored blob against the hash recorded when it was cached. A stale entry is // a miss and is left in place: the fetch that replaces it discards it under // the coalescing key. func (p *Proxy) getCachedArtifactWithUpstreamHash(ctx context.Context, pkgPURL, versionPURL, filename, upstreamHash string) (*CacheResult, error) { cached, err := p.checkCache(ctx, pkgPURL, versionPURL, filename) if err != nil || cached == nil { return cached, err } if artifactHashMatches(cached.Artifact.Digest.Encoded(), upstreamHash) { return cached, nil } if cached.Reader != nil { _ = cached.Reader.Close() } return nil, nil } // refetchArtifactFromURL fetches a file upstream replaces in place, which // a cached copy can't stand in for while upstream answers. The new copy // replaces the cached one only once it is stored, and the cached one is // served when the fetch fails for any reason but the file being missing or // blocked. Concurrent callers share one fetch. func (p *Proxy) refetchArtifactFromURL(ctx context.Context, ecosystem, name, version, filename, downloadURL string) (*CacheResult, error) { pkgPURL, versionPURL, err := packagePURLStrings(ecosystem, name, version) if err != nil { return nil, err } // Streaming never serves a cached copy, and a denied version is refused // there. if p.StreamArtifacts || p.versionDenied(ecosystem, name, version) || p.Denylist.Denied(versionPURL) { return p.getOrFetchArtifactFromURL(ctx, ecosystem, name, version, filename, downloadURL, nil, "", "") } metrics.RecordArtifactCacheMiss(ecosystem) key := artifactCoalesceKey(versionPURL, filename, downloadURL, "", "") noRecheck := func() (artifacts.Artifact, string, bool) { return artifacts.Artifact{}, "", false } result, err := p.coalesceFetch(ctx, key, noRecheck, func(fetchCtx context.Context) (artifacts.Artifact, string, error) { return p.fetchAndCacheFromURL(fetchCtx, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL, nil, "", "") }) if err == nil || errors.Is(err, ErrUpstreamNotFound) || errors.Is(err, ErrArtifactBlocked) || ctx.Err() != nil { return result, err } cached, cacheErr := p.checkCache(ctx, pkgPURL, versionPURL, filename) if cacheErr != nil || cached == nil { return nil, err } p.Logger.Warn("upstream fetch failed, serving cached copy", "purl", versionPURL, "filename", filename, "error", err) return cached, nil } // discardStaleArtifact clears the cached entry when its digest disagrees with // upstreamHash, queueing its object for deletion. It runs under the coalescing // key, after the recheck, so an entry a previous fetch refreshed is kept. func (p *Proxy) discardStaleArtifact(pkgPURL, versionPURL, filename, upstreamHash string) { record, err := p.DB.GetCachedArtifact(pkgPURL, versionPURL, filename) if err != nil { p.Logger.Warn("failed to read cache record before refetch", "purl", versionPURL, "filename", filename, "error", err) return } if record == nil || artifactHashMatches(record.Artifact.Digest.Encoded(), upstreamHash) { return } p.Logger.Warn("cached artifact hash disagrees with upstream metadata, discarding", "purl", versionPURL, "filename", filename, "cached", record.Artifact.Digest.Encoded(), "upstream", upstreamHash) if err := p.DB.DiscardArtifact(versionPURL, filename, record.StoragePath); err != nil { p.Logger.Warn("failed to clear artifact cache record", "purl", versionPURL, "filename", filename, "error", err) } } func (p *Proxy) fetchAndCacheFromURL(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL string, headers http.Header, upstreamHash, declared string) (artifacts.Artifact, string, error) { if err := p.upstreamHeld(downloadURL); err != nil { return artifacts.Artifact{}, "", err } p.Logger.Info("fetching from upstream", "ecosystem", ecosystem, "name", name, "version", version, "url", downloadURL) fetchStart := time.Now() artifact, err := p.fetchUpstream(ctx, downloadURL, headers) metrics.RecordUpstreamFetch(ecosystem, time.Since(fetchStart)) if err != nil { metrics.RecordUpstreamError(ecosystem, "fetch_failed", metrics.FailureDetail{ PURL: versionPURL, Filename: filename, URL: downloadURL, Err: err}) if errors.Is(err, fetch.ErrNotFound) { return artifacts.Artifact{}, "", ErrUpstreamNotFound } return artifacts.Artifact{}, "", fmt.Errorf("fetching from upstream: %w", err) } return p.storeFetched(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL, upstreamHash, declared, artifact) } type upstreamLastModifiedKey struct{} // WithUpstreamLastModified returns a context under which an artifact fetched // from upstream reports its Last-Modified to fn, so a handler can learn a // version's publish time from the download itself rather than with a // request of its own. fn is not called for a cache hit, or when the // response has no usable Last-Modified. func WithUpstreamLastModified(ctx context.Context, fn func(time.Time)) context.Context { return context.WithValue(ctx, upstreamLastModifiedKey{}, fn) } // observedFetcher is the part of fetch.Fetcher and its circuit breaker that // reports response headers. type observedFetcher interface { FetchObservedWithHeaders(ctx context.Context, url string, headers http.Header) (*fetch.ObservedArtifact, error) } type signedDownloadKey struct{} // signedDownload is a download URL carrying a signature, and the URL without // its query that stands for it everywhere but the request itself. size, when // not negative, is the exact size the download must have. type signedDownload struct { signed, recorded string size int64 } // errDownloadSize is a download larger or smaller than the size it was // expected to have. var errDownloadSize = errors.New("download does not have the expected size") // withSignedDownload returns a context under which a fetch of the returned // URL requests signedURL instead. The returned URL, signedURL without its // query and fragment, is what is logged, recorded with the artifact and used // to coalesce fetches, so a signature that grants access to the object // stays out of logs and the database. A download of other than size bytes // is cut off with errDownloadSize, unless size is negative. func withSignedDownload(ctx context.Context, signedURL string, size int64) (context.Context, string) { u, err := url.Parse(signedURL) if err != nil { return ctx, signedURL } u.User = nil u.RawQuery, u.ForceQuery = "", false u.Fragment, u.RawFragment = "", "" recorded := u.String() return context.WithValue(ctx, signedDownloadKey{}, signedDownload{signed: signedURL, recorded: recorded, size: size}), recorded } // redactedFetchError is a fetch error whose text names the recorded URL // rather than the signed one. type redactedFetchError struct { err error download signedDownload } func (e *redactedFetchError) Error() string { return strings.ReplaceAll(e.err.Error(), e.download.signed, e.download.recorded) } func (e *redactedFetchError) Unwrap() error { return e.err } // fetchUpstream fetches downloadURL, or the signed URL ctx gives for it, // reporting its Last-Modified when ctx asks for it. func (p *Proxy) fetchUpstream(ctx context.Context, downloadURL string, headers http.Header) (*fetch.Artifact, error) { download, ok := ctx.Value(signedDownloadKey{}).(signedDownload) if !ok || download.recorded != downloadURL { return p.fetchUpstreamURL(ctx, downloadURL, headers) } artifact, err := p.fetchUpstreamURL(ctx, download.signed, headers) if err != nil { return nil, &redactedFetchError{err: err, download: download} } if download.size < 0 { return artifact, nil } if artifact.Size >= 0 && artifact.Size != download.size { _ = artifact.Body.Close() return nil, fmt.Errorf("%w: upstream announced %d bytes, want %d", errDownloadSize, artifact.Size, download.size) } artifact.Body = &exactSizeBody{ReadCloser: artifact.Body, left: download.size} return artifact, nil } // exactSizeBody fails a body that turns out longer or shorter than left // bytes, without reading more than one byte past it. type exactSizeBody struct { io.ReadCloser left int64 } func (b *exactSizeBody) Read(p []byte) (int, error) { if int64(len(p)) > b.left+1 { p = p[:b.left+1] } n, err := b.ReadCloser.Read(p) b.left -= int64(n) switch { case b.left < 0: return n, fmt.Errorf("%w: upstream sent more bytes", errDownloadSize) case errors.Is(err, io.EOF) && b.left > 0: return n, fmt.Errorf("%w: upstream sent %d bytes too few", errDownloadSize, b.left) } return n, err } func (p *Proxy) fetchUpstreamURL(ctx context.Context, downloadURL string, headers http.Header) (*fetch.Artifact, error) { report, _ := ctx.Value(upstreamLastModifiedKey{}).(func(time.Time)) observer, ok := p.Fetcher.(observedFetcher) if report == nil || !ok { return p.Fetcher.FetchWithHeaders(ctx, downloadURL, headers) } observed, err := observer.FetchObservedWithHeaders(ctx, downloadURL, headers) if err != nil { return nil, err } if observed.Observation != nil { if t, err := http.ParseTime(observed.Observation.Headers.Get(headerLastModified)); err == nil { report(t) } } return observed.Artifact, nil } // streamFromUpstream fetches an artifact when StreamArtifacts is set and hands its // body to the caller without storing it. // // With upstreamHash or the declared integrity set the body is verified as it // streams. The size is left // unknown so the response goes out chunked: a mismatch only shows at EOF, // after the bytes have been sent, and an unterminated chunked response is the // only way left to make the client reject them (see serveArtifact). func (p *Proxy) streamFromUpstream(ctx context.Context, ecosystem, name, version, filename, versionPURL, upstreamURL, upstreamHash, declared string, fetchArtifact func(context.Context) (*fetch.Artifact, error)) (*CacheResult, error) { if err := p.upstreamHeld(upstreamURL); err != nil { return nil, err } p.Logger.Info("streaming from upstream", "ecosystem", ecosystem, "name", name, "version", version, "url", upstreamURL) fetchStart := time.Now() artifact, err := fetchArtifact(ctx) metrics.RecordUpstreamFetch(ecosystem, time.Since(fetchStart)) if err != nil { metrics.RecordUpstreamError(ecosystem, "fetch_failed", metrics.FailureDetail{ PURL: versionPURL, Filename: filename, URL: upstreamURL, Err: err}) if errors.Is(err, fetch.ErrNotFound) { return nil, ErrUpstreamNotFound } return nil, fmt.Errorf("fetching from upstream: %w", err) } body := &streamErrorLogger{ ReadCloser: artifact.Body, onError: func(read int64, err error) { p.Logger.Warn("streaming artifact from upstream failed", "purl", versionPURL, "filename", filename, "url", upstreamURL, "bytes", read, "error", err) metrics.RecordUpstreamError(ecosystem, "stream_failed", metrics.FailureDetail{ PURL: versionPURL, Filename: filename, URL: upstreamURL, Err: err}) }, } result := &CacheResult{ Reader: body, Artifact: artifacts.Artifact{ PURL: versionPURL, Size: artifact.Size, Filename: filename, MediaType: artifact.ContentType, }, } if upstreamHash == "" && declared == "" { return result, nil } hash := strings.ToLower(upstreamHash) checks, err := newIntegrityChecks(hash, declared) if err != nil { _ = artifact.Body.Close() return nil, fmt.Errorf("parsing upstream digest: %w", err) } result.Reader, err = checks.wrapFailOnMismatch(body, func(reason string) { p.Logger.Error("streamed artifact failed integrity check", "purl", versionPURL, "filename", filename, "url", upstreamURL, "reason", reason) metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(ecosystem), metrics.FailureDetail{ PURL: versionPURL, Filename: filename, URL: upstreamURL, Err: errors.New(reason)}) }) if err != nil { _ = artifact.Body.Close() return nil, err } if hash != "" { result.Artifact.Digest = digest.Digest("sha256:" + hash) } result.Artifact.Size = -1 return result, nil } // streamErrorLogger reports the first read error of a streamed upstream body. // The error itself still reaches serveArtifact, which aborts the response. type streamErrorLogger struct { io.ReadCloser onError func(read int64, err error) read int64 logged bool } func (r *streamErrorLogger) Read(p []byte) (int, error) { n, err := r.ReadCloser.Read(p) r.read += int64(n) if err != nil && err != io.EOF && !r.logged { r.logged = true r.onError(r.read, err) } return n, err } // ErrArtifactDigestMismatch indicates that fetched bytes did not match the // checksum the upstream declared and were not recorded in the cache database. var ErrArtifactDigestMismatch = errors.New("artifact digest mismatch") func artifactHashMatches(got, expected string) bool { return expected == "" || strings.EqualFold(got, expected) }