// Package storage provides artifact storage backends for the proxy cache. // // Storage backends are accessed via gocloud.dev/blob URLs: // // - file:///path/to/dir - Local filesystem storage // - s3://bucket-name - Amazon S3 // - s3://bucket?endpoint=http://localhost:9000 - S3-compatible (MinIO) // - gs://bucket-name - Google Cloud Storage (supports GKE Workload Identity // via Application Default Credentials) // - azblob://container-name - Azure Blob Storage // // Use OpenBucket to create a storage backend from a URL. package storage import ( "context" "crypto/rand" "encoding/hex" "errors" "io" "time" ) const dirPermissions = 0755 var ( ErrNotFound = errors.New("artifact not found") // ErrSignedURLUnsupported is returned by SignedURL when the backend // cannot generate presigned URLs (e.g. local filesystem). ErrSignedURLUnsupported = errors.New("signed URLs not supported by storage backend") ) // ObjectInfo contains metadata for a stored object. type ObjectInfo struct { Path string Size int64 ModTime time.Time } // Storage defines the interface for artifact storage backends. type Storage interface { // Store writes content from r to the given path. // Returns the number of bytes written and the SHA256 hash of the content. Store(ctx context.Context, path string, r io.Reader) (size int64, hash string, err error) // Open returns a reader for the content at path. // The caller must close the reader when done. // The reader may also implement io.Seeker when seeking is efficient for the // storage backend. Backends must not expose io.Seeker when seeking requires // reading and discarding the bytes before the requested position. // Returns ErrNotFound if the path does not exist. Open(ctx context.Context, path string) (io.ReadCloser, error) // Exists returns true if content exists at path. Exists(ctx context.Context, path string) (bool, error) // Delete removes the content at path. // Returns nil if the path does not exist. Delete(ctx context.Context, path string) error // Size returns the size in bytes of content at path. // Returns ErrNotFound if the path does not exist. Size(ctx context.Context, path string) (int64, error) // SignedURL returns a presigned URL granting time-limited GET access to path. // Returns ErrSignedURLUnsupported if the backend cannot generate presigned URLs. SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) // UsedSpace returns the total bytes used by all stored content. UsedSpace(ctx context.Context) (int64, error) // URL returns the storage backend URL (e.g. "file:///path" or "s3://bucket"). URL() string // Close releases any resources held by the storage backend. Close() error } // SeekableReads is implemented by backends whose Open always returns a // reader that implements io.Seeker. It tells a caller that byte ranges can be // served from an object without opening it first. type SeekableReads interface { ReadsSeekable() bool } // FetchPath builds the storage path for one fetch of an artifact: // {ecosystem}/{name}/{version}/{fetchID}/{filename}. Each fetch writes its own // object, so no fetch overwrites or deletes another's. func FetchPath(ecosystem, name, version, fetchID, filename string) string { return ecosystem + "/" + name + "/" + version + "/" + fetchID + "/" + filename } // fetchIDBytes sizes a fetch id: 16 hex characters keeps paths short, which // matters on Windows, and a collision between fetches of one artifact version // is out of reach. const fetchIDBytes = 8 // NewFetchID returns a random id for FetchPath. func NewFetchID() string { b := make([]byte, fetchIDBytes) _, _ = rand.Read(b) return hex.EncodeToString(b) } // isFetchDir reports whether a directory name has the shape of a fetch id. // Only one fetch ever writes into such a directory. func isFetchDir(name string) bool { if len(name) != 2*fetchIDBytes { return false } for _, c := range name { if (c < '0' || c > '9') && (c < 'a' || c > 'f') { return false } } return true }