/** * DynamoModel — Schema annotations, date schemas, and model configuration for effect-dynamodb. * * Provides: * - `configure` — per-field DynamoDB overrides (field rename, storage encoding, immutable, identifier, ref) * - `Hidden` annotation — marks fields as excluded from default decode (asModel/asRecord) * - `DynamoEncoding` annotation — controls how date fields are stored in DynamoDB * - Date schemas — wire ↔ domain ↔ storage transformations for DateTime.Utc, DateTime.Zoned, Date * - `storedAs` modifier — overrides DynamoDB storage format via annotation * - `SparseMap()` — opt a `Schema.Record` field into flattened storage (one * top-level attribute per entry) * - `Packed()` — store an object/array-shaped field (or EventStore data/state) * as one JSON string, optionally compressed over a threshold, * and (`external`) offloaded to the blob store over another * - `External()` — store a binary field's content in the blob store * * See docs/design-dynamo-model.md for the three-layer model (wire → domain → storage). */ import { DateTime, Effect, Schema, SchemaAST, SchemaGetter, SchemaIssue } from "effect" import { ownValue, setOwn } from "./internal/OwnNames.js" import { getSchemaFields, recordValueAst, recordValueAstFromAst, } from "./internal/SchemaAccessors.js" import { zonedIsoText } from "./internal/StoredForm.js" import { parseZonedIso } from "./internal/ZonedIso.js" // --------------------------------------------------------------------------- // Identifier annotation // --------------------------------------------------------------------------- /** Unique symbol for the Identifier annotation */ const IdentifierId: unique symbol = Symbol.for("effect-dynamodb/Identifier") /** * Mark a schema field as the primary identifier for ref resolution. * * Entities that can be referenced via `DynamoModel.ref` must annotate exactly * one field with `identifier`. The annotated field should be a string or * branded string type. */ export const identifier = (schema: S): S => schema.pipe(Schema.annotate({ [IdentifierId]: true })) as S /** * Check if a schema field has the Identifier annotation. */ export const isIdentifier = (schema: Schema.Top): boolean => ((SchemaAST.resolve(schema.ast) as globalThis.Record | undefined)?.[ IdentifierId ] as boolean) ?? false /** * Find the identifier field in a schema with `.fields` (Schema.Class or Schema.Struct). * Checks both schema-level `DynamoModel.identifier` annotations and ConfiguredModel * `identifier: true` overrides. * * Returns the field name and schema, or undefined if no field has the identifier annotation. */ export const getIdentifierField = ( model: Schema.Top, ): { readonly name: string; readonly schema: Schema.Top } | undefined => { // Check ConfiguredModel attributes first (configure-level overrides take precedence) if (isConfiguredModel(model)) { const configured = model as ConfiguredModel for (const [name, attr] of Object.entries(configured.attributes)) { if (attr.identifier) { const fields = getSchemaFields(configured.model) if (fields) { const fieldSchema = ownValue(fields, name) if (fieldSchema) return { name, schema: fieldSchema } } // Fallback: return with a generic string schema return { name, schema: Schema.String as unknown as Schema.Top } } } // Fall through to check schema-level annotations on the inner model return getIdentifierField(configured.model) } const fields = getSchemaFields(model) if (!fields) return undefined for (const [name, fieldSchema] of Object.entries(fields)) { if (isIdentifier(fieldSchema)) return { name, schema: fieldSchema } } return undefined } // --------------------------------------------------------------------------- // Ref annotation // --------------------------------------------------------------------------- /** Unique symbol for the Ref annotation */ const RefId: unique symbol = Symbol.for("effect-dynamodb/Ref") /** Annotation metadata stored on ref-annotated fields */ export interface RefAnnotation { readonly _tag: "Ref" /** Schema.Class identifier of the referenced type (e.g., "Team", "Player") */ readonly refSchemaId?: string | undefined } /** * Mark a schema field as a denormalized reference to another entity. * * Ref fields: * - In `Entity.Input`, become ID fields (e.g., `team: Team` → `teamId: string`) * - In `Entity.Record`, remain the full entity domain type * - In `Entity.Update`, become optional ID fields * - In DynamoDB, store the entity's core domain data as an embedded map * * The referenced entity's model must have exactly one `DynamoModel.identifier`-annotated field. */ export const ref = (schema: S): S => { // Extract Schema.Class identifier if available (e.g., "Team" from Schema.Class("Team")) // In Effect v4, Schema.Class exposes `identifier` (not `_tag`) as the class name const s = schema as globalThis.Record const refSchemaId = "identifier" in schema && typeof s.identifier === "string" ? s.identifier : "_tag" in schema && typeof s._tag === "string" ? s._tag : undefined return schema.pipe( Schema.annotate({ [RefId]: { _tag: "Ref", refSchemaId } satisfies RefAnnotation }), ) as S } /** * Check if a schema field has the Ref annotation. */ export const isRef = (schema: Schema.Top): boolean => { const ann = (SchemaAST.resolve(schema.ast) as globalThis.Record | undefined)?.[ RefId ] return ann != null && typeof ann === "object" && (ann as RefAnnotation)._tag === "Ref" } /** * Check if a field is a ref, considering both schema-level annotations and * ConfiguredModel overrides. Use this when you have access to the configured model. */ export const isRefField = (fieldName: string, model: Schema.Top): boolean => { // Check ConfiguredModel attributes if (isConfiguredModel(model)) { const configured = model as ConfiguredModel if (ownValue(configured.attributes, fieldName)?.ref) return true // Fall through to check schema-level annotation on the inner model return isRefField(fieldName, configured.model) } // Check schema-level annotation const fields = getSchemaFields(model) if (fields) { const fieldSchema = ownValue(fields, fieldName) if (fieldSchema) return isRef(fieldSchema) } return false } /** * Get the full Ref annotation metadata from a schema field. * Returns undefined if the field is not ref-annotated. */ export const getRefAnnotation = (schema: Schema.Top): RefAnnotation | undefined => { const ann = (SchemaAST.resolve(schema.ast) as globalThis.Record | undefined)?.[ RefId ] if (ann != null && typeof ann === "object" && (ann as RefAnnotation)._tag === "Ref") { return ann as RefAnnotation } return undefined } // --------------------------------------------------------------------------- // Hidden annotation // --------------------------------------------------------------------------- /** Unique symbol for the Hidden annotation */ const HiddenId: unique symbol = Symbol.for("effect-dynamodb/Hidden") /** * Mark a schema field as hidden from default decode modes (asModel, asRecord). * * Hidden fields: * - Stored in DynamoDB normally * - Visible in asItem and asNative decode modes * - Stripped from asModel and asRecord decode modes * * Use for internal/infrastructure fields that shouldn't leak into application code. */ export const Hidden = (schema: S): S => schema.pipe(Schema.annotate({ [HiddenId]: true })) as S /** * Check if a schema field has the Hidden annotation. */ export const isHidden = (schema: Schema.Top): boolean => ((SchemaAST.resolve(schema.ast) as globalThis.Record | undefined)?.[ HiddenId ] as boolean) ?? false // --------------------------------------------------------------------------- // DynamoEncoding annotation // --------------------------------------------------------------------------- /** Unique symbol for the DynamoEncoding annotation */ export const DynamoEncodingKey: unique symbol = Symbol.for("effect-dynamodb/DynamoEncoding") /** * Describes how a date field is stored in DynamoDB. * - `storage` — the DynamoDB attribute format * - `domain` — the domain type (for deserialization dispatch) */ export interface DynamoEncoding { readonly storage: "string" | "epochMs" | "epochSeconds" readonly domain: "DateTime.Utc" | "DateTime.Zoned" | "Date" } /** * Read the DynamoEncoding annotation from a schema, if present. */ export const getEncoding = (schema: Schema.Top): DynamoEncoding | undefined => (SchemaAST.resolve(schema.ast) as globalThis.Record | undefined)?.[ DynamoEncodingKey ] as DynamoEncoding | undefined // --------------------------------------------------------------------------- // Date schemas — DateTime.Utc domain // --------------------------------------------------------------------------- /** * Wire: ISO 8601 string ↔ Domain: DateTime.Utc * * Default storage: ISO string (`S`). * This is a thin wrapper around `Schema.DateTimeUtcFromString` with the DynamoEncoding annotation. */ export const DateString = Schema.DateTimeUtcFromString.pipe( Schema.annotate({ [DynamoEncodingKey]: { storage: "string", domain: "DateTime.Utc" } satisfies DynamoEncoding, }), ) /** * Wire: epoch milliseconds (number) ↔ Domain: DateTime.Utc * * Default storage: epoch ms (`N`). */ export const DateEpochMs = Schema.Number.pipe( Schema.decodeTo(Schema.DateTimeUtc, { decode: SchemaGetter.transform((n: number) => DateTime.makeUnsafe(n)), encode: SchemaGetter.transform((dt: DateTime.Utc) => DateTime.toEpochMillis(dt)), }), Schema.annotate({ [DynamoEncodingKey]: { storage: "epochMs", domain: "DateTime.Utc" } satisfies DynamoEncoding, }), ) /** * Wire: epoch seconds (number) ↔ Domain: DateTime.Utc * * Default storage: epoch seconds (`N`). * DynamoDB TTL requires this format. */ export const DateEpochSeconds = Schema.Number.pipe( Schema.decodeTo(Schema.DateTimeUtc, { decode: SchemaGetter.transform((n: number) => DateTime.makeUnsafe(n * 1000)), encode: SchemaGetter.transform((dt: DateTime.Utc) => DateTime.toEpochSeconds(dt)), }), Schema.annotate({ [DynamoEncodingKey]: { storage: "epochSeconds", domain: "DateTime.Utc", } satisfies DynamoEncoding, }), ) /** * Wire: auto-detect epoch ms or seconds ↔ Domain: DateTime.Utc * * Uses a minimum threshold to disambiguate: if the value interpreted as ms * produces a date before `minimum`, it tries seconds instead. * * @param options.minimum - Threshold date for disambiguation * @param options.encode - Epoch schema for wire encoding (default: DateEpochMs) */ export const DateEpoch = (options: { readonly minimum: string | DateTime.DateTime.Input readonly encode?: typeof DateEpochMs | typeof DateEpochSeconds }) => { const minEpochMs = typeof options.minimum === "string" ? new Date(options.minimum).getTime() : DateTime.toEpochMillis(DateTime.makeUnsafe(options.minimum)) const encodeSchema = options.encode ?? DateEpochMs const encodeEncoding = getEncoding(encodeSchema) return Schema.Number.pipe( Schema.decodeTo(Schema.DateTimeUtc, { decode: SchemaGetter.transformEffect((n: number) => { // Try milliseconds first try { const asMs = DateTime.makeUnsafe(n) if (DateTime.toEpochMillis(asMs) >= minEpochMs) { return Effect.succeed(asMs) } } catch { // fall through } // Fall back to seconds try { const asSec = DateTime.makeUnsafe(n * 1000) if (DateTime.toEpochMillis(asSec) >= minEpochMs) { return Effect.succeed(asSec) } } catch { // fall through } return Effect.fail(new SchemaIssue.InvalidType(Schema.Number.ast, n)) }), encode: SchemaGetter.transform((dt: DateTime.Utc) => Schema.encodeSync(encodeSchema)(dt as any), ), }), Schema.annotate({ [DynamoEncodingKey]: { storage: encodeEncoding?.storage ?? "epochMs", domain: "DateTime.Utc", } satisfies DynamoEncoding, }), ) } // --------------------------------------------------------------------------- // Date schemas — DateTime.Zoned domain // --------------------------------------------------------------------------- /** * Wire: ISO 8601 string with offset/zone ↔ Domain: DateTime.Zoned * * Default storage: extended ISO with zone (`S`). * Format: `2024-01-01T15:00:00+09:00[Asia/Tokyo]` * * Cannot use epoch storage — timezone information would be lost (enforced at type level via storedAs). */ export const DateTimeZoned = Schema.String.pipe( Schema.decodeTo(Schema.DateTimeZoned, { decode: SchemaGetter.transformEffect((s: string) => { // `…+09:00[Asia/Tokyo]` (named zone) or `…+05:00` (offset zone), each // rebuilt in the zone it was written with (#133). try { return Effect.succeed(parseZonedIso(s)) } catch { return Effect.fail(new SchemaIssue.InvalidType(Schema.String.ast, s)) } }), encode: SchemaGetter.transform((dt: DateTime.Zoned) => zonedIsoText(dt)), }), Schema.annotate({ [DynamoEncodingKey]: { storage: "string", domain: "DateTime.Zoned" } satisfies DynamoEncoding, }), ) // --------------------------------------------------------------------------- // Unsafe date schemas — native Date domain // --------------------------------------------------------------------------- /** * Wire: ISO 8601 string ↔ Domain: native Date (mutable) * * Default storage: ISO string (`S`). * For interop with non-Effect code that expects native JS Date objects. */ export const UnsafeDateString = Schema.String.pipe( Schema.decodeTo(Schema.Date, { decode: SchemaGetter.transform((s: string) => new Date(s)), encode: SchemaGetter.transform((d: Date) => d.toISOString()), }), Schema.annotate({ [DynamoEncodingKey]: { storage: "string", domain: "Date" } satisfies DynamoEncoding, }), ) /** * Wire: epoch milliseconds (number) ↔ Domain: native Date (mutable) * * Default storage: epoch ms (`N`). */ export const UnsafeDateEpochMs = Schema.Number.pipe( Schema.decodeTo(Schema.Date, { decode: SchemaGetter.transform((ms: number) => new Date(ms)), encode: SchemaGetter.transform((d: Date) => d.getTime()), }), Schema.annotate({ [DynamoEncodingKey]: { storage: "epochMs", domain: "Date" } satisfies DynamoEncoding, }), ) /** * Wire: epoch seconds (number) ↔ Domain: native Date (mutable) * * Default storage: epoch seconds (`N`). */ export const UnsafeDateEpochSeconds = Schema.Number.pipe( Schema.decodeTo(Schema.Date, { decode: SchemaGetter.transform((s: number) => new Date(s * 1000)), encode: SchemaGetter.transform((d: Date) => Math.floor(d.getTime() / 1000)), }), Schema.annotate({ [DynamoEncodingKey]: { storage: "epochSeconds", domain: "Date", } satisfies DynamoEncoding, }), ) // --------------------------------------------------------------------------- // TTL alias // --------------------------------------------------------------------------- /** * Alias for `DateEpochSeconds`. DynamoDB TTL requires epoch seconds. * A named alias makes intent clear at the call site. */ export const TTL = DateEpochSeconds // --------------------------------------------------------------------------- // storedAs modifier // --------------------------------------------------------------------------- /** * Override the DynamoDB storage format for a date field. * * `storedAs` reads the DynamoEncoding annotation from the target storage schema * and applies its storage format to the field schema. The domain type must match * (enforced at the type level via the `A` constraint). * * @example * ```typescript * // A model field: a DateTime.Utc stored as epoch seconds (for TTL) * Schema.DateTimeUtc.pipe(DynamoModel.storedAs(DynamoModel.DateEpochSeconds)) * * // A timestamp's `schema`: wire ISO string, DynamoDB epoch seconds * DynamoModel.DateString.pipe(DynamoModel.storedAs(DynamoModel.DateEpochSeconds)) * ``` * * A date transform model field (`DynamoModel.DateString`, `Schema.DateTimeUtcFromString`) owns its * wire form: `Entity.make` refuses one whose `storedAs` names the other wire kind, and under * `Schema.optional` the transform's own form is stored (#232). Annotate a self schema instead. * * Type-safe: a storage schema of another domain type is a compile error. No `DateTime.Zoned` is * stored as an epoch, which has no zone: forced through a cast, `Entity.make` refuses this one (a * transform), under `Schema.optional` it keeps its ISO form, and a `Schema.DateTimeZoned` field so * annotated refuses a zoned value at the write. * ```typescript * // Type error — DateTime.Zoned ≠ DateTime.Utc * DynamoModel.DateTimeZoned.pipe(DynamoModel.storedAs(DynamoModel.DateEpochMs)) * ``` * * `A` is the domain type, as it always was (`storedAs(DateEpochSeconds)`). `W`, the * storage schema's encoded type, is inferred from it; the schema returned carries it as * {@link StoredAs} `` (#236). Given `A` alone, or a storage schema typed `Schema.Schema`, `W` * is `unknown` and the schema returned is the field schema's own type. */ export const storedAs = ( storageSchema: Schema.Schema & { readonly Encoded: W }, ): (>(fieldSchema: S) => WithStoredAs) => { const storageEncoding = getEncoding(storageSchema as Schema.Top) if (!storageEncoding) { throw new Error("storedAs: target schema has no DynamoEncoding annotation") } return >(fieldSchema: S): WithStoredAs => { const currentEncoding = getEncoding(fieldSchema as Schema.Top) return (fieldSchema as Schema.Top).pipe( Schema.annotate({ [DynamoEncodingKey]: { storage: storageEncoding.storage, domain: currentEncoding?.domain ?? storageEncoding.domain, } satisfies DynamoEncoding, }), ) as unknown as WithStoredAs } } /** * What the type of a schema {@link storedAs} returns carries: the encoded type of the storage * schema it was given (`number` for `DateEpochSeconds`, `string` for `DateString`). It exists only * in the type, never at run time. The entity types read it to tell a `Schema.optional` date whose * `storedAs` names the other wire kind than its transform's, which is the caller's and never * stamped, from a date the library stamps (#236). The property is optional, so the schema returned * and the field schema are each assignable to the other's type, as they were when `storedAs` * returned the field schema's type. */ export interface StoredAs { readonly "~effect-dynamodb/storedAs"?: W } /** The type {@link storedAs} returns: `S`, carrying {@link StoredAs} `` when `W` is known. */ export type WithStoredAs = unknown extends W ? S : S & StoredAs // --------------------------------------------------------------------------- // ConfiguredModel // --------------------------------------------------------------------------- /** Unique symbol identifying a ConfiguredModel wrapper */ export const ConfiguredModelTag: unique symbol = Symbol.for("effect-dynamodb/ConfiguredModel") /** * Sparse-map configuration carried on a `ConfiguredModel` attribute when the * field is marked `storedAs: DynamoModel.SparseMap()`. * * - `prefix` — top-level attribute name prefix; defaults to the model field name. * Each Record entry is stored as a separate top-level attribute named * `#`. The default prefix can be overridden by passing * `{ prefix }` to {@link SparseMap}. */ export interface SparseConfig { readonly prefix: string } /** Brand symbol identifying a {@link SparseMapConfig} produced by {@link SparseMap}. */ const SparseMapTag: unique symbol = Symbol.for("effect-dynamodb/SparseMap") /** * Typed config object returned by {@link SparseMap}. Pass as the `storedAs` * value on a `Schema.Record` field in {@link configure} to opt into sparse * map storage — each Record entry is flattened into a top-level DynamoDB * attribute named `#`. */ export interface SparseMapConfig { readonly [SparseMapTag]: true readonly prefix?: string | undefined } /** * Opt a `Schema.Record` field into sparse map storage. * * Each Record entry becomes an independently addressable top-level DynamoDB * attribute (`#`) — concurrent writers to disjoint buckets * never race, counters can `ADD` on a fresh item with no parent-map dance, * and `attribute_exists`/`attribute_not_exists` work natively per entry. * * @example * ```ts * DynamoModel.configure(Page, { * metrics: { storedAs: DynamoModel.SparseMap() }, * totals: { storedAs: DynamoModel.SparseMap({ prefix: "t" }) }, * }) * ``` * * @param options.prefix - Override the top-level attribute name prefix. * Defaults to the model field name. Distinct sparse fields on the same * entity must have distinct prefixes (EDD-9023). * * See `DESIGN.md §7.5 Sparse Map Storage`. */ export const SparseMap = (options?: { readonly prefix?: string | undefined }): SparseMapConfig => { const cfg: SparseMapConfig = { [SparseMapTag]: true } if (options?.prefix !== undefined) { return { ...cfg, prefix: options.prefix } } return cfg } /** Type guard — returns true iff `value` is a {@link SparseMapConfig}. */ export const isSparseMapConfig = (value: unknown): value is SparseMapConfig => typeof value === "object" && value !== null && (value as globalThis.Record)[SparseMapTag] === true // --------------------------------------------------------------------------- // Packed storage (JSON string, optional compression) — DESIGN.md §7 "Packed Storage" // --------------------------------------------------------------------------- /** * The serialisation of a packed value. Only `"json"` is accepted; `"binary"` * (a future SchemaBinary encoding) is reserved in the stored header and refused * here until it is implemented. */ export type PackedFormat = "json" /** A compression codec a packed value may be stored with. */ export type CompressionCodec = "gzip" | "brotli" | "zstd" /** Every {@link CompressionCodec}, in header-code order. */ export const compressionCodecs: ReadonlyArray = ["gzip", "brotli", "zstd"] /** * Default `compression.threshold`: 1,024 bytes of UTF-8 JSON. DynamoDB rounds * write capacity per item at 1 KB, so compressing a smaller value saves no WCU * and costs a decompression on every read. */ export const DEFAULT_COMPRESSION_THRESHOLD = 1024 /** * Default `external.threshold`: 65,536 stored bytes. Below roughly 8 KB an * object-store `PUT` costs more than the DynamoDB write units it saves, and * every offloaded read adds a round trip, so only genuinely large values go * out by default. Tune it per workload. */ export const DEFAULT_EXTERNAL_THRESHOLD = 65_536 /** The `external` option of {@link Packed}: offload large values to the blob store. */ export interface ExternalOptions { /** * Stored bytes of the packed value (the `S` UTF-8 length, or the `B` length * after compression) at or above which it is written to the blob store * instead. A non-negative safe integer; `0` always offloads. Default 65,536. */ readonly threshold?: number | undefined } /** Options accepted by {@link Packed}. */ export interface PackedOptions { readonly format: PackedFormat readonly compression?: | { readonly codec: CompressionCodec /** * UTF-8 byte length of the JSON at or above which compression is tried. * A non-negative safe integer; `0` always tries. Default 1024. */ readonly threshold?: number | undefined } | undefined /** * Not accepted here: offloading is only reachable through the * {@link ExternalPackedOptions} overload of {@link Packed}, so options typed * `PackedOptions` never return a config the type level would not see as * offloaded. */ readonly external?: never | undefined } /** * Options of an overflow field (`Packed({ external })`): packed inline below * `external.threshold`, offloaded to the blob store at or above it. */ export interface ExternalPackedOptions extends Omit { readonly external: ExternalOptions } /** Brand symbol identifying a {@link PackedConfig} produced by {@link Packed}. */ const PackedTag: unique symbol = Symbol.for("effect-dynamodb/Packed") /** * The validated, frozen storage config returned by {@link Packed} without * `external` — the value of an entity field's `storedAs`, an EventStore * stream's `storedAs.data`, or a snapshot's `storedAs`, always stored inline. * * Disjoint from {@link ExternalPackedConfig} (#179): an overflow config is not * assignable to this type, so it cannot be widened into one the type level * would read as inline (and so drop its blob-store errors). Every API that * takes a packed config takes either: `PackedConfig | ExternalPackedConfig`. */ export interface PackedConfig { readonly [PackedTag]: true readonly format: PackedFormat readonly compression: { readonly codec: CompressionCodec; readonly threshold: number } | undefined /** Always `undefined`: a plain packed value is never offloaded. */ readonly external: undefined } /** * The config `Packed({ external })` returns — an overflow value, packed * inline below `external.threshold` and offloaded to the blob store at or * above it. What the type level keys on ({@link ExtractOffloadFields}); not * assignable to {@link PackedConfig}. */ export interface ExternalPackedConfig { readonly [PackedTag]: true readonly format: PackedFormat readonly compression: { readonly codec: CompressionCodec; readonly threshold: number } | undefined /** Offload settings, resolved: stored bytes at or above `threshold` are offloaded. */ readonly external: { readonly threshold: number } } const packedError = (message: string): Error => new Error(`[EDD-9071] DynamoModel.Packed: ${message}`) const externalError = (api: "Packed" | "External", message: string): Error => new Error(`[EDD-9141] DynamoModel.${api}: ${message}`) /** The resolved `external` option of {@link Packed}, or `undefined` when absent (EDD-9141). */ const resolveExternalOption = (external: unknown): { readonly threshold: number } | undefined => { if (external === undefined) return undefined if (typeof external !== "object" || external === null || Array.isArray(external)) { throw externalError( "Packed", `external must be an object, e.g. { threshold: ${DEFAULT_EXTERNAL_THRESHOLD} }; received ` + `${external === null ? "null" : Array.isArray(external) ? "an array" : typeof external}`, ) } for (const key of Object.keys(external)) { if (key !== "threshold") { throw externalError( "Packed", `unknown external option "${key}" (expected optional "threshold")`, ) } } const threshold = (external as { readonly threshold?: unknown }).threshold if ( threshold !== undefined && (typeof threshold !== "number" || !Number.isSafeInteger(threshold) || threshold < 0) ) { throw externalError( "Packed", `external.threshold must be a non-negative safe integer (stored bytes), got ${String(threshold)}`, ) } return Object.freeze({ threshold: (threshold as number | undefined) ?? DEFAULT_EXTERNAL_THRESHOLD, }) } /** * Store a value packed into one attribute: its wire form as a JSON string * (`S`), or — when `compression` is set, the JSON is at least `threshold` * UTF-8 bytes long, and the compressed bytes plus a one-byte header are * smaller — as compressed binary (`B`). * * The stored form is self-describing: readers dispatch on what is stored (a * native map, `S`, or `B` with its header), never on the current config, so * switching a field or stream between native, packed and compressed storage * needs no rewrite. * * Compression codecs are pluggable: register implementations through the * `Compression` service (`node:zlib`-backed ones ship from `effect-dynamodb/compression/zlib`). * * @example * ```ts * DynamoModel.configure(Order, { * lines: { storedAs: DynamoModel.Packed({ format: "json" }) }, * details: { * storedAs: DynamoModel.Packed({ format: "json", compression: { codec: "zstd" } }), * }, * }) * ``` * * **Overflow fields** (`external`, #179): a value whose stored bytes (after * compression) reach `external.threshold` (default 65,536) is written to the * blob store and the attribute holds a pointer to the object version; smaller * values stay inline, byte-identical to a plain `Packed` field. This overload * returns an {@link ExternalPackedConfig}, which is what marks the field as * offloaded at the type level. See `DESIGN.md §7 "Offloaded Storage"`. * * @example * ```ts * DynamoModel.configure(Document, { * body: { * storedAs: DynamoModel.Packed({ * format: "json", * compression: { codec: "zstd" }, * external: { threshold: 65_536 }, * }), * }, * }) * ``` * * @throws `[EDD-9071]` on invalid options: a `format` other than `"json"`, an * unknown `compression.codec`, a `threshold` that is not a non-negative safe * integer, or an unknown option key. * @throws `[EDD-9141]` on an invalid `external` option: not an object, an * unknown key, or a `threshold` that is not a non-negative safe integer. */ export function Packed(options: ExternalPackedOptions): ExternalPackedConfig export function Packed(options: PackedOptions): PackedConfig export function Packed( options: PackedOptions | ExternalPackedOptions, ): PackedConfig | ExternalPackedConfig { if (typeof options !== "object" || options === null) { throw packedError('options must be an object, e.g. { format: "json" }') } for (const key of Object.keys(options)) { if (key !== "format" && key !== "compression" && key !== "external") { throw packedError( `unknown option "${key}" (expected "format", optional "compression" and optional "external")`, ) } } const format = (options as { readonly format?: unknown }).format if (format === "binary") { throw packedError( `format "binary" is reserved for a future SchemaBinary encoding and is not supported ` + `by this version — use format: "json"`, ) } if (format !== "json") { throw packedError(`unknown format ${JSON.stringify(format)} — the supported format is "json"`) } const compression = (options as { readonly compression?: unknown }).compression const external = resolveExternalOption((options as { readonly external?: unknown }).external) if (compression === undefined) { return Object.freeze({ [PackedTag]: true as const, format, compression: undefined, external }) } if (typeof compression !== "object" || compression === null) { throw packedError('compression must be an object, e.g. { codec: "zstd" }') } for (const key of Object.keys(compression)) { if (key !== "codec" && key !== "threshold") { throw packedError( `unknown compression option "${key}" (expected "codec" and optional "threshold")`, ) } } const { codec, threshold } = compression as { readonly codec?: unknown readonly threshold?: unknown } if (!compressionCodecs.includes(codec as CompressionCodec)) { throw packedError( `unknown compression codec ${JSON.stringify(codec)} — expected one of ` + compressionCodecs.map((c) => `"${c}"`).join(", "), ) } if ( threshold !== undefined && (typeof threshold !== "number" || !Number.isSafeInteger(threshold) || threshold < 0) ) { throw packedError( `compression.threshold must be a non-negative safe integer (bytes of UTF-8 JSON), ` + `got ${String(threshold)}`, ) } return Object.freeze({ [PackedTag]: true as const, format, compression: Object.freeze({ codec: codec as CompressionCodec, threshold: (threshold as number | undefined) ?? DEFAULT_COMPRESSION_THRESHOLD, }), external, }) } /** * Type guard — returns true iff `value` is a config {@link Packed} returned: * a {@link PackedConfig} or an {@link ExternalPackedConfig}. */ export const isPackedConfig = (value: unknown): value is PackedConfig | ExternalPackedConfig => typeof value === "object" && value !== null && (value as globalThis.Record)[PackedTag] === true /** * Type guard — returns true iff `value` is an {@link ExternalPackedConfig} * (an overflow field's config, `Packed({ external })`). */ export const isExternalPackedConfig = (value: unknown): value is ExternalPackedConfig => isPackedConfig(value) && value.external !== undefined // --------------------------------------------------------------------------- // Binary fields offloaded to the blob store — DESIGN.md §7 "Offloaded Storage" // --------------------------------------------------------------------------- /** Default `contentType` of an {@link External} field's objects. */ export const DEFAULT_EXTERNAL_CONTENT_TYPE = "application/octet-stream" /** Bytes of the longest content type an object may be stored with (a blob pointer's limit). */ const MAX_CONTENT_TYPE_BYTES = 255 /** Options accepted by {@link External}. */ export interface ExternalBinaryOptions { /** * Content-Type of the stored object when a write does not name one. A * non-empty printable US-ASCII string (no control character, which no HTTP * header carries) of at most 255 bytes. Default * `"application/octet-stream"`. */ readonly contentType?: string | undefined } /** Brand symbol identifying an {@link ExternalConfig} produced by {@link External}. */ const ExternalTag: unique symbol = Symbol.for("effect-dynamodb/External") /** * The validated, frozen storage config returned by {@link External} — the * `storedAs` of a binary field whose content always lives in the blob store. */ export interface ExternalConfig { readonly [ExternalTag]: true readonly contentType: string } /** Printable US-ASCII (0x20–0x7e): what an HTTP `Content-Type` header can carry. */ const isPrintableAscii = (text: string): boolean => { for (let i = 0; i < text.length; i++) { const code = text.charCodeAt(i) if (code < 0x20 || code > 0x7e) return false } return true } /** * Store a binary field (encoded side `Uint8Array`) in the blob store: every * write uploads the bytes as a new object version, and the attribute holds a * pointer to it (an `S` value, so readers that predate offloading fail loudly * instead of returning the pointer's bytes). Reads return a lazy `BlobHandle` * by default. A value stored natively before `External` was configured still * reads. See `DESIGN.md §7 "Offloaded Storage"`. * * Admitted by {@link configure} only on fields whose domain type is a * `Uint8Array` (optionally `undefined` / `null`); `Entity.make` checks the * encoded side (`[EDD-9111]`). `field` renames and `immutable` combine with it. * * @example * ```ts * DynamoModel.configure(Document, { * scan: { storedAs: DynamoModel.External({ contentType: "application/pdf" }) }, * }) * ``` * * @throws `[EDD-9141]` on invalid options: not an object, an unknown key, or a * `contentType` that is not a non-empty printable US-ASCII string of at most * 255 bytes. */ export const External = (options?: ExternalBinaryOptions): ExternalConfig => { if (options === undefined) { return Object.freeze({ [ExternalTag]: true as const, contentType: DEFAULT_EXTERNAL_CONTENT_TYPE, }) } if (typeof options !== "object" || options === null || Array.isArray(options)) { throw externalError( "External", 'options must be an object, e.g. { contentType: "application/pdf" }; received ' + `${options === null ? "null" : Array.isArray(options) ? "an array" : typeof options}`, ) } for (const key of Object.keys(options)) { if (key !== "contentType") { throw externalError("External", `unknown option "${key}" (expected optional "contentType")`) } } const contentType = (options as { readonly contentType?: unknown }).contentType if (contentType === undefined) { return Object.freeze({ [ExternalTag]: true as const, contentType: DEFAULT_EXTERNAL_CONTENT_TYPE, }) } if ( typeof contentType !== "string" || contentType.length === 0 || contentType.length > MAX_CONTENT_TYPE_BYTES || !isPrintableAscii(contentType) ) { throw externalError( "External", `contentType must be a non-empty printable US-ASCII string of at most ${MAX_CONTENT_TYPE_BYTES} ` + `bytes, got ${typeof contentType === "string" ? JSON.stringify(contentType) : String(contentType)}`, ) } return Object.freeze({ [ExternalTag]: true as const, contentType }) } /** Type guard — returns true iff `value` is an {@link ExternalConfig}. */ export const isExternalConfig = (value: unknown): value is ExternalConfig => typeof value === "object" && value !== null && (value as globalThis.Record)[ExternalTag] === true /** * Resolve the binary offload config for every field on a model configured * `storedAs: DynamoModel.External(...)` via {@link configure}. * * Returns `Record` — empty when the model is * not a `ConfiguredModel` or configures no `External` field. */ export const getExternalFields = ( model: Schema.Top | ConfiguredModel, ): globalThis.Record => { if (!isConfiguredModel(model)) return {} const out: globalThis.Record = {} for (const [name, attr] of Object.entries(model.attributes)) { if (attr.external) out[name] = attr.external } return out } /** * Resolve the packed storage config for every field on a model configured * `storedAs: DynamoModel.Packed(...)` via {@link configure}. * * Returns `Record` — * overflow fields included — empty when the model is not a `ConfiguredModel` * or configures no packed field. Like sparse maps, packing is a storage * concern: it is read from `ConfiguredModel.attributes` only. */ export const getPackedFields = ( model: Schema.Top | ConfiguredModel, ): globalThis.Record => { // Null-prototype: a field is looked up by name, and a model field may be named `constructor`. const out: globalThis.Record = Object.create(null) if (!isConfiguredModel(model)) return out for (const [name, attr] of Object.entries(model.attributes)) { if (attr.packed) setOwn(out, name, attr.packed) } return out } /** * The domain field names a `ConfiguredModel` stores packed (`never` when none), * overflow fields included. */ export type ExtractPackedFields = Model extends ConfiguredModel ? { [K in keyof A & string]: A[K] extends { readonly storedAs: PackedConfig | ExternalPackedConfig } ? K : never }[keyof A & string] : never /** * The domain field names a `ConfiguredModel` marks immutable or stores as a * sparse map, which `.remove()` does not take (`never` when none): an update * can't change an immutable field, and a sparse map is cleared with * `.clearMap()`. */ export type ExtractUnremovableFields = Model extends ConfiguredModel ? { [K in keyof A & string]: A[K] extends | { readonly immutable: true } | { readonly storedAs: SparseMapConfig } ? K : never }[keyof A & string] : never /** * The domain field names a member offloads to the blob store, by kind: * `overflow` (`Packed({ external })`) and `binary` (`External()`). */ export interface OffloadFields { readonly overflow: string readonly binary: string } /** A member that offloads nothing. */ export type NoOffload = { readonly overflow: never; readonly binary: never } /** * The {@link OffloadFields} of a `ConfiguredModel` — {@link NoOffload} for a * model that configures neither kind (or is not configured at all). * * A field is overflow when its `storedAs` may be an {@link ExternalPackedConfig} * — a union-typed config (`PackedConfig | ExternalPackedConfig`) included — so * the blob-store errors are never under-reported. {@link PackedConfig} is * disjoint from it, so an overflow config cannot be widened to look inline. */ export type ExtractOffloadFields = Model extends ConfiguredModel ? { readonly overflow: { [K in keyof A & string]: A[K] extends { readonly storedAs: infer S } ? [Extract] extends [never] ? never : K : never }[keyof A & string] readonly binary: { [K in keyof A & string]: A[K] extends { readonly storedAs: ExternalConfig } ? K : never }[keyof A & string] } : NoOffload /** * A model wrapped with DynamoDB-specific attribute overrides. * * Carries the original model plus per-field configuration: * - `field` — rename domain field → DynamoDB attribute name * - `storedAs` — override DynamoDB storage encoding (resolved to DynamoEncoding) * OR a {@link SparseMapConfig} (via {@link SparseMap}) to * flatten a `Schema.Record` field into per-entry top-level * attributes (see {@link SparseConfig}) * OR a {@link PackedConfig} (via {@link Packed}) to store the * field as one JSON string / compressed binary value * OR an {@link ExternalConfig} (via {@link External}) to store a * binary field's content in the blob store. * - `immutable` — mark field as read-only after creation (excluded from Entity.Update) * - `identifier` — mark field as the identity field for ref resolution * - `ref` — mark field as a denormalized reference to another entity */ export interface ConfiguredModel< M extends Schema.Top, A extends ConfigureAttributes = ConfigureAttributes, > { readonly [ConfiguredModelTag]: true readonly model: M readonly _attributes: A readonly attributes: globalThis.Record } /** The resolved per-field overrides a {@link ConfiguredModel} carries. */ interface ResolvedAttributeConfig { readonly field?: string readonly encoding?: DynamoEncoding readonly immutable?: boolean readonly identifier?: boolean readonly ref?: boolean readonly sparse?: SparseConfig readonly packed?: PackedConfig | ExternalPackedConfig readonly external?: ExternalConfig } /** Extract the identifier field name from a ConfiguredModel's attributes type. */ export type ExtractIdentifier = Model extends ConfiguredModel ? { [K in keyof A & string]: A[K] extends { readonly identifier: true } ? K : never }[keyof A & string] : undefined /** * Check if a value is a ConfiguredModel. */ export const isConfiguredModel = (value: unknown): value is ConfiguredModel => typeof value === "object" && value !== null && (value as globalThis.Record)[ConfiguredModelTag] === true /** * Resolve sparse-map configuration for every field on a model that is marked * `storedAs: DynamoModel.SparseMap()` via {@link configure}. * * Returns `Record`. Empty when no sparse fields are * configured. * * Walks `ConfiguredModel.attributes` only — schema-level annotations are not * supported for sparse (it's a deployment concern, not a domain one). */ export const getSparseFields = ( model: Schema.Top | ConfiguredModel, ): globalThis.Record => { // Null-prototype: a field is looked up by name (`field in sparseFields`), and a model field may // be named like an `Object.prototype` member (`valueOf`, `constructor`), which a plain object // reports as a sparse map whose prefix is `undefined` (#238). const out: globalThis.Record = Object.create(null) if (!isConfiguredModel(model)) return out for (const [name, attr] of Object.entries(model.attributes)) { if (attr.sparse) setOwn(out, name, attr.sparse) } return out } /** * Inspect a Schema field to determine whether it is shaped like a `Schema.Record`. * * Effect v4 `Schema.Record(K, V)` produces an AST `Objects` node with no * `propertySignatures` and a single `indexSignatures` entry (string-keyed). * * Returns the value AST on match, otherwise `undefined`. Used by * `Entity.make()` to validate sparse-map fields and detect nested sparse * (a Record whose value is itself a Record). */ export const isRecordSchema = (schema: Schema.Top): { readonly valueAst: unknown } | undefined => { const valueAst = recordValueAst(schema) if (valueAst === undefined) return undefined return { valueAst } } /** True iff `schema` is shaped like `Schema.Record(K, V)`. */ export const isRecord = (schema: Schema.Top): boolean => isRecordSchema(schema) !== undefined /** True iff `valueAst` (raw AST from a Record value position) is itself a Record AST. */ export const isRecordAst = (valueAst: unknown): boolean => recordValueAstFromAst(valueAst as SchemaAST.AST | undefined) !== undefined /** * Attribute override for a single field in `DynamoModel.configure`. * * - `field` — Rename the domain field to a different DynamoDB attribute name. * - `storedAs` — Override DynamoDB storage format via a DynamoModel date schema, * or {@link SparseMap}() to flatten a `Schema.Record` into * per-entry top-level attributes (see {@link SparseConfig}). * - `immutable` — Mark as read-only after creation (excluded from Entity.Update). * - `identifier` — Mark as the identity field for ref resolution. * - `ref` — Mark as a denormalized reference field. * * Or {@link Packed}(...) to store an object/array-shaped field as one JSON * string or compressed binary value (with `external`, offloaded to the blob * store at or over a threshold), or {@link External}(...) to store a * `Uint8Array` field's content in the blob store. * * Note: the `prefix` option moved into the {@link SparseMap} config in 1.6 — * pass `storedAs: DynamoModel.SparseMap({ prefix })` instead of declaring * `prefix` as a sibling here. The sibling form had no meaning for non-sparse * `storedAs` values and was a footgun. */ type AttributeConfig = { readonly field?: string readonly storedAs?: | Schema.Schema | SparseMapConfig | PackableStorage | ExternalBinaryStorage readonly immutable?: boolean readonly identifier?: boolean readonly ref?: boolean } /** * A packed config (`PackedConfig` or, overflow, `ExternalPackedConfig`) is * admitted only on fields whose domain type is object- or array-shaped (or * `unknown`); a scalar field is a compile error. The runtime shape rule at * `Entity.make()` (EDD-9073), which inspects the encoded side, stays * authoritative. */ type PackableStorage = unknown extends A ? PackedConfig | ExternalPackedConfig : [NonNullable] extends [object] ? PackedConfig | ExternalPackedConfig : never /** * `ExternalConfig` is admitted only on fields whose domain type is a * `Uint8Array` (optionally `undefined` / `null`). The runtime rule at * `Entity.make()` (EDD-9111), which requires the encoded side to be a * `Uint8Array`, stays authoritative. */ type ExternalBinaryStorage = unknown extends A ? never : [NonNullable] extends [never] ? never : [NonNullable] extends [Uint8Array] ? ExternalConfig : never /** * Type-safe attribute overrides map: keys constrained to model fields, * `storedAs` constrained to matching domain type. */ type ConfigureAttributes = { readonly [K in keyof Schema.Schema.Type]?: AttributeConfig[K]> } /** * Compile-time validation: detects `field` rename targets that collide * with existing model field names. Replaces the `field` type with an * error tuple to produce a readable type error at the call site. */ type ValidateFieldRenames = { [K in keyof A]: A[K] extends { readonly field: infer F extends string } ? F extends Exclude ? { readonly field: [`EDD-9007: "${F}" collides with existing model field`] } & Omit< A[K], "field" > : A[K] : A[K] } /** * Compile-time validation: a key that is not a model field takes only an * error message, so a misspelled field is a type error on that key. * (`ConfigureAttributes` alone lets an extra key through beside a valid one.) */ type OnlyModelFields = unknown extends Schema.Schema.Type ? unknown : { readonly [K in Exclude>]: `${K & string} is not a field of the model` } /** * Create a configured model with per-field DynamoDB overrides. * * Separates DynamoDB infrastructure (storage format, field renaming) from * the domain model, enabling pure models that work across storage backends. * * @example * ```typescript * class Order extends Schema.Class("Order")({ * orderId: Schema.String, * placedAt: Schema.DateTimeUtcFromString, * expiresAt: Schema.DateTimeUtcFromString, * }) {} * * const OrderModel = DynamoModel.configure(Order, { * expiresAt: { field: "ttl", storedAs: DynamoModel.DateEpochSeconds }, * }) * ``` */ export const configure = >( model: M, attributes: A & ValidateFieldRenames & string, A> & OnlyModelFields, ): ConfiguredModel => { const resolved: globalThis.Record = {} for (const [key, config] of Object.entries( attributes as globalThis.Record>, )) { const entry: { -readonly [K in keyof ResolvedAttributeConfig]: ResolvedAttributeConfig[K] } = {} if (config.field) entry.field = config.field if (config.immutable) entry.immutable = true if (config.identifier) entry.identifier = true if (config.ref) entry.ref = true if (isSparseMapConfig(config.storedAs)) { // Sparse Map storage: each entry is stored as a top-level attribute named // `#`. Validation that the field is a Schema.Record happens // at Entity.make() time (EDD-9020) — configure() can't introspect the // model's schema field types reliably without committing to a single // representation, so we keep configure() permissive and defer enforcement. const prefix = config.storedAs.prefix ?? key if (typeof prefix !== "string" || prefix.length === 0) { throw new Error( `DynamoModel.configure: sparse field "${key}" must have a non-empty string prefix`, ) } if (prefix.includes("#")) { throw new Error( `DynamoModel.configure: sparse field "${key}" prefix "${prefix}" must not contain '#'`, ) } entry.sparse = { prefix } } else if (isPackedConfig(config.storedAs)) { // Packed storage: the field is stored as one JSON string (S) or // compressed binary (B). Shape, key, unique and ref guards run at // Entity.make() time (EDD-9072–9075), where the model and its indexes // are known. entry.packed = config.storedAs } else if (isExternalConfig(config.storedAs)) { // Binary field offloaded to the blob store. The encoded-side rule // (EDD-9111) and the key / index / unique / ref guards (EDD-9142) run at // Entity.make() time, where the model and its indexes are known. entry.external = config.storedAs } else if (config.storedAs !== undefined) { const encoding = getEncoding(config.storedAs as Schema.Top) if (!encoding) { throw new Error( `DynamoModel.configure: storedAs schema for "${key}" has no DynamoEncoding annotation`, ) } entry.encoding = encoding } setOwn(resolved, key, entry) } // Validate: no field rename targets collide with existing model field names const modelFields = new Set() const fields = getSchemaFields(model) if (fields) { for (const fieldName of Object.keys(fields)) { modelFields.add(fieldName) } } if (modelFields.size > 0) { for (const [sourceField, config] of Object.entries(resolved)) { if (config.field && config.field !== sourceField && modelFields.has(config.field)) { throw new Error( `[EDD-9007] DynamoModel.configure: renaming "${sourceField}" to "${config.field}" collides with existing model field "${config.field}". ` + `This would cause both fields to map to the same DynamoDB attribute.`, ) } } } return { [ConfiguredModelTag]: true as const, model, _attributes: attributes, attributes: resolved, } as ConfiguredModel }