package gofakes3
import (
"encoding/hex"
"errors"
"io"
"time"
)
const (
DefaultBucketVersionKeys = 1000
)
// Object contains the data retrieved from a backend for the specified bucket
// and object key.
//
// You MUST always call Contents.Close() otherwise you may leak resources.
type Object struct {
Name string
Metadata map[string]string
Size int64
Contents io.ReadCloser
Hash []byte
Range *ObjectRange
// VersionID will be empty if bucket versioning has not been enabled.
VersionID VersionID
// If versioning is enabled for the bucket, this is true if this object version
// is a delete marker.
IsDeleteMarker bool
}
type ObjectList struct {
CommonPrefixes []CommonPrefix
Contents []*Content
IsTruncated bool
NextMarker string
// prefixes maintains an index of prefixes that have already been seen.
// This is a convenience for backend implementers like s3bolt and s3mem,
// which operate on a full, flat list of keys.
prefixes map[string]bool
}
func NewObjectList() *ObjectList {
return &ObjectList{}
}
func (b *ObjectList) Add(item *Content) {
b.Contents = append(b.Contents, item)
}
func (b *ObjectList) AddPrefix(prefix string) {
if b.prefixes == nil {
b.prefixes = map[string]bool{}
} else if b.prefixes[prefix] {
return
}
b.prefixes[prefix] = true
b.CommonPrefixes = append(b.CommonPrefixes, CommonPrefix{Prefix: prefix})
}
type ObjectDeleteResult struct {
// Specifies whether the versioned object that was permanently deleted was
// (true) or was not (false) a delete marker. In a simple DELETE, this
// header indicates whether (true) or not (false) a delete marker was
// created.
IsDeleteMarker bool
// Returns the version ID of the delete marker created as a result of the
// DELETE operation. If you delete a specific object version, the value
// returned by this header is the version ID of the object version deleted.
VersionID VersionID
}
type ListBucketVersionsPage struct {
// Specifies the key in the bucket that you want to start listing from.
// If HasKeyMarker is true, this must be non-empty.
KeyMarker string
HasKeyMarker bool
// Specifies the object version you want to start listing from. If
// HasVersionIDMarker is true, this must be non-empty.
VersionIDMarker VersionID
HasVersionIDMarker bool
// Sets the maximum number of keys returned in the response body. The
// response might contain fewer keys, but will never contain more. If
// additional keys satisfy the search criteria, but were not returned
// because max-keys was exceeded, the response contains
// true. To return the additional keys, see
// key-marker and version-id-marker.
//
// MaxKeys MUST be > 0, otherwise it is ignored.
MaxKeys int64
}
type ListBucketPage struct {
// Specifies the key in the bucket that represents the last item in
// the previous page. The first key in the returned page will be the
// next lexicographically (UTF-8 binary) sorted key after Marker.
// If HasMarker is true, this must be non-empty.
Marker string
HasMarker bool
// Sets the maximum number of keys returned in the response body. The
// response might contain fewer keys, but will never contain more. If
// additional keys satisfy the search criteria, but were not returned
// because max-keys was exceeded, the response contains
// true. To return the additional keys, see
// key-marker and version-id-marker.
//
// MaxKeys MUST be > 0, otherwise it is ignored.
MaxKeys int64
}
func (p ListBucketPage) IsEmpty() bool {
return p == ListBucketPage{}
}
type PutObjectResult struct {
// If versioning is enabled on the bucket, this should be set to the
// created version ID. If versioning is not enabled, this should be
// empty.
VersionID VersionID
}
// PutConditions represents the conditional headers for S3 PutObject operations.
// These conditions are checked atomically before writing the object.
type PutConditions struct {
// IfMatch specifies that the object should only be written if its ETag
// matches this value. If the ETag doesn't match, ErrPreconditionFailed
// should be returned.
IfMatch *string
// IfNoneMatch specifies that the object should only be written if the
// object key doesn't already exist. The value should be "*".
// If the object exists, ErrPreconditionFailed should be returned.
IfNoneMatch *string
}
// ConditionalObjectInfo represents the current state of an object for conditional checking.
// Backends should provide this information to the shared CheckPutConditions function.
type ConditionalObjectInfo struct {
// Exists indicates whether the object exists
Exists bool
// Hash is the MD5 hash of the object content (used for ETag comparison)
// Only required if Exists is true
Hash []byte
}
// ObjectInfo is a deprecated alias for ConditionalObjectInfo.
// Use ConditionalObjectInfo instead.
type ObjectInfo = ConditionalObjectInfo
// FormatETag formats a hash as an S3 ETag with quotes
func FormatETag(hash []byte) string {
return `"` + hex.EncodeToString(hash) + `"`
}
// CheckPutConditions validates conditional headers for PutObject operations.
// This is a shared implementation that all backends can use.
func CheckPutConditions(conditions *PutConditions, objectInfo *ConditionalObjectInfo) error {
// Check If-None-Match: object should not exist
if conditions.IfNoneMatch != nil {
if *conditions.IfNoneMatch == "*" && objectInfo.Exists {
return ErrorMessage(ErrPreconditionFailed, "The object already exists")
}
}
// Check If-Match: if specified, object must exist and ETag must match
if conditions.IfMatch != nil {
if !objectInfo.Exists {
return ErrorMessage(ErrPreconditionFailed, "The object does not exist")
}
expectedETag := *conditions.IfMatch
// Remove quotes if present for comparison
if len(expectedETag) >= 2 && expectedETag[0] == '"' && expectedETag[len(expectedETag)-1] == '"' {
expectedETag = expectedETag[1 : len(expectedETag)-1]
}
actualETag := hex.EncodeToString(objectInfo.Hash)
if expectedETag != actualETag {
return ErrorMessage(ErrPreconditionFailed, "The ETag does not match")
}
}
return nil
}
// Backend provides a set of operations to be implemented in order to support
// gofakes3.
//
// The Backend API is not yet stable; if you create your own Backend, breakage
// is likely until this notice is removed.
type Backend interface {
// ListBuckets returns a list of all buckets owned by the authenticated
// sender of the request.
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTServiceGET.html
ListBuckets() ([]BucketInfo, error)
// ListBucket returns the contents of a bucket. Backends should use the
// supplied prefix to limit the contents of the bucket and to sort the
// matched items into the Contents and CommonPrefixes fields.
//
// ListBucket must return a gofakes3.ErrNoSuchBucket error if the bucket
// does not exist. See gofakes3.BucketNotFound() for a convenient way to create one.
//
// The prefix MUST be correctly handled for the backend to be valid. Each
// item you consider returning should be checked using prefix.Match(name),
// even if the prefix is empty. The Backend MUST treat a nil prefix
// identically to a zero prefix.
//
// At this stage, implementers MAY return gofakes3.ErrInternalPageNotImplemented
// if the page argument is non-empty. In this case, gofakes3 may or may
// not, depending on how it was configured, retry the same request with no page.
// We have observed (though not yet confirmed) that simple clients tend to
// work fine if you ignore the pagination request, but this may not suit
// your application. Not all backends bundled with gofakes3 correctly
// support this pagination yet, but that will change.
ListBucket(name string, prefix *Prefix, page ListBucketPage) (*ObjectList, error)
// CreateBucket creates the bucket if it does not already exist. The name
// should be assumed to be a valid name.
//
// If the bucket already exists, a gofakes3.ResourceError with
// gofakes3.ErrBucketAlreadyExists MUST be returned.
CreateBucket(name string) error
// BucketExists should return a boolean indicating the bucket existence, or
// an error if the backend was unable to determine existence.
BucketExists(name string) (exists bool, err error)
// DeleteBucket deletes a bucket if and only if it is empty.
//
// If the bucket is not empty, gofakes3.ResourceError with
// gofakes3.ErrBucketNotEmpty MUST be returned.
//
// If the bucket does not exist, gofakes3.ErrNoSuchBucket MUST be returned.
//
// AWS does not validate the bucket's name for anything other than existence.
DeleteBucket(name string) error
// ForceDeleteBucket must delete a bucket and all its contents, regardless of
// whether the bucket is empty or not. This is useful for testing purposes
// where you need to clean up after yourself.
ForceDeleteBucket(name string) error
// GetObject must return a gofakes3.ErrNoSuchKey error if the object does
// not exist. See gofakes3.KeyNotFound() for a convenient way to create
// one.
//
// If the returned Object is not nil, you MUST call Object.Contents.Close(),
// otherwise you will leak resources. Implementers should return a no-op
// implementation of io.ReadCloser.
//
// If rnge is nil, it is assumed you want the entire object. If rnge is not
// nil, but the underlying backend does not support range requests,
// implementers MUST return ErrNotImplemented.
//
// If the backend is a VersionedBackend, GetObject retrieves the latest version.
GetObject(bucketName, objectName string, rangeRequest *ObjectRangeRequest) (*Object, error)
// HeadObject fetches the Object from the backend, but reading the Contents
// will return io.EOF immediately.
//
// If the returned Object is not nil, you MUST call Object.Contents.Close(),
// otherwise you will leak resources. Implementers should return a no-op
// implementation of io.ReadCloser.
//
// HeadObject should return a NotFound() error if the object does not
// exist.
HeadObject(bucketName, objectName string) (*Object, error)
// DeleteObject deletes an object from the bucket.
//
// If the backend is a VersionedBackend and versioning is enabled, this
// should introduce a delete marker rather than actually delete the object.
//
// DeleteObject must return a gofakes3.ErrNoSuchBucket error if the bucket
// does not exist. See gofakes3.BucketNotFound() for a convenient way to create one.
// FIXME: confirm with S3 whether this is the correct behaviour.
//
// DeleteObject must not return an error if the object does not exist. Source:
// https://docs.aws.amazon.com/sdk-for-go/api/service/s3/#S3.DeleteObject:
//
// Removes the null version (if there is one) of an object and inserts a
// delete marker, which becomes the latest version of the object. If there
// isn't a null version, Amazon S3 does not remove any objects.
//
DeleteObject(bucketName, objectName string) (ObjectDeleteResult, error)
// PutObject should assume that the key is valid. The map containing meta
// may be nil.
//
// The size can be used if the backend needs to read the whole reader; use
// gofakes3.ReadAll() for this job rather than ioutil.ReadAll().
//
// If conditions is not nil, the backend should check the conditions before
// writing the object. If conditions fail, it should return ErrPreconditionFailed
// or ErrConditionalRequestConflict as appropriate.
PutObject(bucketName, key string, meta map[string]string, input io.Reader, size int64, conditions *PutConditions) (PutObjectResult, error)
DeleteMulti(bucketName string, objects ...string) (MultiDeleteResult, error)
CopyObject(srcBucket, srcKey, dstBucket, dstKey string, meta map[string]string) (CopyObjectResult, error)
}
// VersionedBackend may be optionally implemented by a Backend in order to support
// operations on S3 object versions.
//
// If you don't implement VersionedBackend, requests to GoFakeS3 that attempt to
// make use of versions will return ErrNotImplemented if GoFakesS3 is unable to
// find another way to satisfy the request.
type VersionedBackend interface {
// VersioningConfiguration must return a gofakes3.ErrNoSuchBucket error if the bucket
// does not exist. See gofakes3.BucketNotFound() for a convenient way to create one.
//
// If the bucket has never had versioning enabled, VersioningConfiguration MUST return
// empty strings (S300001).
VersioningConfiguration(bucket string) (VersioningConfiguration, error)
// SetVersioningConfiguration must return a gofakes3.ErrNoSuchBucket error if the bucket
// does not exist. See gofakes3.BucketNotFound() for a convenient way to create one.
SetVersioningConfiguration(bucket string, v VersioningConfiguration) error
// GetObject must return a gofakes3.ErrNoSuchKey error if the object does
// not exist. See gofakes3.KeyNotFound() for a convenient way to create
// one.
//
// If the returned Object is not nil, you MUST call Object.Contents.Close(),
// otherwise you will leak resources. Implementers should return a no-op
// implementation of io.ReadCloser.
//
// GetObject must return gofakes3.ErrNoSuchVersion if the version does not
// exist.
//
// If versioning has been enabled on a bucket, but subsequently suspended,
// GetObjectVersion should still return the object version (S300001).
//
// FIXME: s3assumer test; what happens when versionID is empty? Does it
// return the latest?
GetObjectVersion(
bucketName, objectName string,
versionID VersionID,
rangeRequest *ObjectRangeRequest) (*Object, error)
// HeadObjectVersion fetches the Object version from the backend, but the Contents will be
// a no-op ReadCloser.
//
// If the returned Object is not nil, you MUST call Object.Contents.Close(),
// otherwise you will leak resources. Implementers should return a no-op
// implementation of io.ReadCloser.
//
// HeadObjectVersion should return a NotFound() error if the object does not
// exist.
HeadObjectVersion(bucketName, objectName string, versionID VersionID) (*Object, error)
// DeleteObjectVersion permanently deletes a specific object version.
//
// DeleteObjectVersion must return a gofakes3.ErrNoSuchBucket error if the bucket
// does not exist. See gofakes3.BucketNotFound() for a convenient way to create one.
//
// If the bucket exists and either the object does not exist (S300003) or
// the version does not exist (S300002), you MUST return an empty
// ObjectDeleteResult and a nil error.
DeleteObjectVersion(bucketName, objectName string, versionID VersionID) (ObjectDeleteResult, error)
// DeleteMultiVersions permanently deletes all of the specified Object Versions
DeleteMultiVersions(bucketName string, objects ...ObjectID) (MultiDeleteResult, error)
// Backend implementers can assume the ListBucketVersionsPage is valid:
// KeyMarker and VersionIDMarker will either both be set, or both be unset. No
// other combination will be present (S300004).
//
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGETVersion.html
//
// This MUST return the list of current versions with an empty VersionID
// even if versioning has never been enabled for the bucket (S300005).
//
// The Backend MUST treat a nil prefix identically to a zero prefix, and a
// nil page identically to a zero page.
ListBucketVersions(bucketName string, prefix *Prefix, page *ListBucketVersionsPage) (*ListBucketVersionsResult, error)
}
// MultipartBackend may be optionally implemented by a Backend in order to
// support S3 multiplart uploads.
// If you don't implement MultipartBackend, GoFakeS3 will fall back to an
// in-memory implementation which holds all parts in memory until the upload
// gets finalised and pushed to the backend.
type MultipartBackend interface {
CreateMultipartUpload(bucket, object string, meta map[string]string) (UploadID, error)
UploadPart(bucket, object string, id UploadID, partNumber int, contentLength int64, input io.Reader) (etag string, err error)
ListMultipartUploads(bucket string, marker *UploadListMarker, prefix Prefix, limit int64) (*ListMultipartUploadsResult, error)
ListParts(bucket, object string, uploadID UploadID, marker int, limit int64) (*ListMultipartUploadPartsResult, error)
AbortMultipartUpload(bucket, object string, id UploadID) error
CompleteMultipartUpload(bucket, object string, id UploadID, input *CompleteMultipartUploadRequest) (versionID VersionID, etag string, err error)
}
// CopyObject is a helper function useful for quickly implementing CopyObject on
// a backend that already supports GetObject and PutObject. This isn't very
// efficient so only use this if performance isn't important.
func CopyObject(db Backend, srcBucket, srcKey, dstBucket, dstKey string, meta map[string]string) (result CopyObjectResult, err error) {
c, err := db.GetObject(srcBucket, srcKey, nil)
if err != nil {
return
}
defer c.Contents.Close()
_, err = db.PutObject(dstBucket, dstKey, meta, c.Contents, c.Size, nil)
if err != nil {
return
}
return CopyObjectResult{
ETag: `"` + hex.EncodeToString(c.Hash) + `"`,
LastModified: NewContentTime(time.Now()),
}, nil
}
func MergeMetadata(db Backend, bucketName string, objectName string, meta map[string]string) error {
// get potential existing object to potentially carry metadata over
existingObj, err := db.GetObject(bucketName, objectName, nil)
if err != nil {
var nsk *ErrorResponse
if errors.As(err, &nsk) && nsk.Code != "NoSuchKey" {
return err
}
}
// carry over metadata if it exists
if existingObj != nil {
for k, v := range existingObj.Metadata {
// new metadata overwrites old but keep the rest
// TODO: check how metadata can be deleted?!
if _, ok := meta[k]; !ok {
meta[k] = v
}
}
}
return nil
}