import { deepEquals } from '../utils' import { SortedMap } from '../SortedMap' import { enrichRowWithVirtualProps } from '../virtual-props.js' import { DuplicateTransactionIdError, SyncQueueInvariantError, SyncTransactionAbortedError, } from '../errors.js' import type { DuplicateKeySyncError } from '../errors.js' import type { VirtualOrigin, VirtualRowProps, WithVirtualProps, } from '../virtual-props.js' import type { Transaction } from '../transactions' import type { StandardSchemaV1 } from '@standard-schema/spec' import type { ChangeMessage, CollectionConfig, OperationType, OptimisticChangeMessage, PendingMutation, } from '../types' import type { CollectionImpl } from './index.js' import type { CollectionLifecycleManager } from './lifecycle' import type { CollectionChangesManager } from './changes' import type { CollectionIndexesManager } from './indexes' import type { CollectionEventsManager } from './events' import type { Deferred } from '../deferred' type PendingSyncOperation< T extends object, TKey extends string | number, > = OptimisticChangeMessage & { /** Preserve adapter intent when queue changes require reclassification. */ originalSyncType?: `insert` /** A partial update admitted against a row must not become an upsert. */ } type PendingSyncedKeyState = { exists: boolean value: T | undefined } type PendingSyncedProjection = { states: Map> truncated: boolean } type PendingInsertDisposition = `insert` | `update` | `duplicate` interface PendingSyncedTransaction< T extends object = Record, TKey extends string | number = string | number, > { committed: boolean applicationStarted: boolean layoutChanged: boolean operations: Array> truncate?: boolean truncateMarkReady?: boolean rowMetadataWrites: Map /** The last explicit write per key, after `position` operations. */ explicitRowMetadataWrites: Map< TKey, { position: number; write: PendingMetadataWrite } > collectionMetadataWrites: Map /** Resolves after application and rejects if canceled before application. */ applied: Deferred preserveHydrationSeedKeys?: boolean /** Builds the error for an open transaction whose insert a replay invalidates. */ duplicateKeyError?: (key: TKey) => DuplicateKeySyncError /** * Set when a replay invalidates an open transaction, which happens when a * later transaction begun inside it commits first. Its commit rejects. */ invalidationError?: Error } export type PendingMetadataWrite = { type: `set`; value: unknown } | { type: `delete` } /** The row metadata a sync operation writes unless the adapter set it explicitly. */ export function automaticRowMetadataWrite( operation: Pick, `type` | `metadata`>, ): PendingMetadataWrite | undefined { if (operation.metadata !== undefined && operation.type !== `delete`) { return { type: `set`, value: operation.metadata } } return operation.type === `update` ? undefined : { type: `delete` } } type InternalChangeMessage< T extends object = Record, TKey extends string | number = string | number, > = ChangeMessage & { __virtualProps?: { value?: VirtualRowProps previousValue?: VirtualRowProps } } export class CollectionStateManager< TOutput extends object = Record, TKey extends string | number = string | number, TSchema extends StandardSchemaV1 = StandardSchemaV1, TInput extends object = TOutput, > { public config!: CollectionConfig public collection!: CollectionImpl public lifecycle!: CollectionLifecycleManager public changes!: CollectionChangesManager public indexes!: CollectionIndexesManager private _events!: CollectionEventsManager // Core state - make public for testing public transactions: SortedMap> public pendingSyncedTransactions: Array< PendingSyncedTransaction > = [] private pendingSyncedProjection: PendingSyncedProjection = { states: new Map(), truncated: false, } public syncedData: SortedMap public syncedMetadata = new Map() public syncedCollectionMetadata = new Map() public hydrationSeedKeys = new Set() public hydratedKeys = new Set() // DbClient may receive a stale hydration chunk after an adapter delete. // Only DbClient collections retain these absences until cleanup because // hydration has no completion boundary. private appliedAdapterDeletedKeys?: Set private hasAppliedAdapterTruncate = false // Optimistic state tracking - make public for testing public optimisticUpserts = new Map() public optimisticDeletes = new Set() // A completed transaction's optimistic row is held only while a queued sync // transaction touches its key, so the drop and that sync transaction // publish together. // The newest completed transaction owns a held key; no row means a delete. public heldOptimisticRows = new Map< TKey, { owner: Transaction; row?: TOutput } >() /** * Tracks Collection attribution for applied source rows. A same-key local * mutation can make an independent source write appear 'local'; this map * does not identify the source client. Optimistic rows are separately 'local'. * Used for the $origin virtual property. */ public rowOrigins = new Map() /** * Tracks keys of pending or persisting local mutations for source-write attribution. * A same-key source write can receive 'local' even when a peer supplied it. */ public pendingLocalChanges = new Set() // A successful mutation attributes only the sync writes committed before its // optimistic state dropped: those held at its completion boundary. Active or // failed mutations must not add to, or erase a sibling's entry in, this set. public pendingLocalOrigins = new Set() // Keyed by row key, not row object: adding a WeakMap entry for each // published row cost more than the copy it saves. Sync writes, deletes, // and cleanup drop a key's entry. private virtualPropsCache = new Map< TKey, { row: TOutput synced: boolean origin: VirtualOrigin collectionId: string enriched: WithVirtualProps } >() // Cached size for performance public size = 0 // State used for computing the change events // Hidden keys are captured as undefined so a later recompute cannot become // the baseline for a sync commit that must publish that key's change. public preSyncVisibleState = new Map() public preSyncVirtualState = new Map>() public recentlySyncedKeys = new Set() public hasReceivedFirstCommit = false public isCommittingSyncTransactions = false private isDrainingSyncTransactions = false private syncRunGeneration = 0 public isLocalOnly = false /** * Set by a local-only Collection for operation types without a user handler. * Their direct mutations can be written as synced rows at once. */ public localOnlyDirectWrite: | { types: ReadonlySet write: (mutations: Array>) => void } | undefined /** * Creates a new CollectionState manager */ constructor(config: CollectionConfig) { this.config = config this.transactions = new SortedMap>((a, b) => a.compareCreatedAt(b), ) // Set up data storage - always use SortedMap for deterministic iteration. // If a custom compare function is provided, use it; otherwise entries are sorted by key only. this.syncedData = new SortedMap(config.compare) } setDeps(deps: { collection: CollectionImpl lifecycle: CollectionLifecycleManager changes: CollectionChangesManager indexes: CollectionIndexesManager events: CollectionEventsManager }) { this.collection = deps.collection this.lifecycle = deps.lifecycle this.changes = deps.changes this.indexes = deps.indexes this._events = deps.events } /** * Checks whether this row currently has no pending local optimistic writes. * * This is local mutation status, not backend confirmation: `true` means the * row is not currently affected by an optimistic transaction in this * collection's visible state. * * Used to compute the $synced virtual property. */ public isRowSynced(key: TKey): boolean { if (this.isLocalOnly) { return true } return !this.optimisticUpserts.has(key) && !this.optimisticDeletes.has(key) } /** * Gets Collection attribution for a row. An optimistic row is 'local'; * otherwise the applied source row retains its timing-based attribution. * Used to compute the $origin virtual property. */ public getRowOrigin(key: TKey): VirtualOrigin { if (this.isLocalOnly) { return 'local' } // If there are optimistic changes, they're local if (this.optimisticUpserts.has(key) || this.optimisticDeletes.has(key)) { return 'local' } // Otherwise, return the confirmed origin (defaults to 'remote' for synced data) return this.rowOrigins.get(key) ?? 'remote' } private createVirtualPropsSnapshot( key: TKey, overrides?: Partial>, ): VirtualRowProps { const synced = overrides?.$synced ?? this.isRowSynced(key) return { $hasPendingWrites: !synced, $synced: synced, $origin: overrides?.$origin ?? this.getRowOrigin(key), $key: overrides?.$key ?? key, $collectionId: overrides?.$collectionId ?? this.collection.id, } } private getVirtualPropsSnapshotForState( key: TKey, options?: { rowOrigins?: ReadonlyMap optimisticUpserts?: Pick, 'has'> optimisticDeletes?: Pick, 'has'> }, ): VirtualRowProps { if (this.isLocalOnly) { return this.createVirtualPropsSnapshot(key, { $synced: true, $origin: 'local', }) } const optimisticUpserts = options?.optimisticUpserts ?? this.optimisticUpserts const optimisticDeletes = options?.optimisticDeletes ?? this.optimisticDeletes const hasOptimisticChange = optimisticUpserts.has(key) || optimisticDeletes.has(key) return this.createVirtualPropsSnapshot(key, { $synced: !hasOptimisticChange, $origin: hasOptimisticChange ? 'local' : ((options?.rowOrigins ?? this.rowOrigins).get(key) ?? 'remote'), }) } private snapshotRowOriginsForKeys( keys: Iterable, ): Map { const rowOrigins = new Map() for (const key of keys) { const origin = this.rowOrigins.get(key) if (origin !== undefined) { rowOrigins.set(key, origin) } } return rowOrigins } private enrichWithVirtualPropsSnapshot( row: TOutput, virtualProps: VirtualRowProps, ): WithVirtualProps { const existingRow = row as Partial> const synced = existingRow.$synced ?? virtualProps.$synced const origin = existingRow.$origin ?? virtualProps.$origin const resolvedKey = existingRow.$key ?? virtualProps.$key const collectionId = existingRow.$collectionId ?? virtualProps.$collectionId const cached = this.virtualPropsCache.get(resolvedKey) if ( cached && cached.row === row && cached.synced === synced && cached.origin === origin && cached.collectionId === collectionId ) { return cached.enriched } const enriched = { ...row, $hasPendingWrites: !synced, $synced: synced, $origin: origin, $key: resolvedKey, $collectionId: collectionId, } as WithVirtualProps this.virtualPropsCache.set(resolvedKey, { row, synced, origin, collectionId, enriched, }) return enriched } private clearOriginTrackingState(): void { this.virtualPropsCache.clear() this.rowOrigins.clear() this.pendingLocalChanges.clear() this.pendingLocalOrigins.clear() } /** * Enriches a row with virtual properties using the "add-if-missing" pattern. * If the row already has virtual properties (from an upstream collection), * they are preserved. Otherwise, new values are computed. */ public enrichWithVirtualProps( row: TOutput, key: TKey, ): WithVirtualProps { return this.enrichWithVirtualPropsSnapshot( row, this.createVirtualPropsSnapshot(key), ) } /** * Visible entries whose stored row passes `prefilter`, enriched with virtual * properties. Rows that fail are never enriched. */ public *entriesPassing( prefilter: (row: object) => boolean, ): IterableIterator<[TKey, WithVirtualProps]> { // Without optimistic state, the visible rows are the synced rows in order. const rows = this.optimisticUpserts.size === 0 && this.optimisticDeletes.size === 0 ? this.syncedData : this.entries() for (const [key, row] of rows) { if (prefilter(row)) yield [key, this.enrichWithVirtualProps(row, key)] } } /** * Creates a change message with virtual properties. * Uses the "add-if-missing" pattern so that pass-through from upstream * collections works correctly. */ public enrichChangeMessage( change: ChangeMessage, ): ChangeMessage, TKey> { const { __virtualProps } = change as InternalChangeMessage // The cache holds one row per key, so the previous row goes first and // the published value stays the row that later reads return. const enrichedPreviousValue = change.previousValue ? __virtualProps?.previousValue ? this.enrichWithVirtualPropsSnapshot( change.previousValue, __virtualProps.previousValue, ) : this.enrichWithVirtualProps(change.previousValue, change.key) : undefined const enrichedValue = __virtualProps?.value ? this.enrichWithVirtualPropsSnapshot(change.value, __virtualProps.value) : this.enrichWithVirtualProps(change.value, change.key) // A deleted key, such as a rolled-back insert, has no row to read again. if (change.type === `delete`) this.virtualPropsCache.delete(change.key) return { key: change.key, type: change.type, value: enrichedValue, previousValue: enrichedPreviousValue, metadata: change.metadata, } as ChangeMessage, TKey> } /** * Get the current value for a key enriched with virtual properties. */ public getWithVirtualProps( key: TKey, ): WithVirtualProps | undefined { const value = this.get(key) if (value === undefined) { return undefined } return this.enrichWithVirtualProps(value, key) } /** * Get the current value for a key (virtual derived state) */ public get(key: TKey): TOutput | undefined { const { optimisticDeletes, optimisticUpserts, syncedData } = this // Check if optimistically deleted if (optimisticDeletes.has(key)) { return undefined } // Check optimistic upserts first if (optimisticUpserts.has(key)) { return optimisticUpserts.get(key) } // Fall back to synced data return syncedData.get(key) } /** * Check if a key exists in the collection (virtual derived state) */ public has(key: TKey): boolean { const { optimisticDeletes, optimisticUpserts, syncedData } = this // Check if optimistically deleted if (optimisticDeletes.has(key)) { return false } // Check optimistic upserts first if (optimisticUpserts.has(key)) { return true } // Fall back to synced data return syncedData.has(key) } /** * Get all keys (virtual derived state) */ public *keys(): IterableIterator { const { syncedData, optimisticDeletes, optimisticUpserts } = this // Yield keys from synced data, skipping any that are deleted. for (const key of syncedData.keys()) { if (!optimisticDeletes.has(key)) { yield key } } // Yield keys from upserts that were not already in synced data. for (const key of optimisticUpserts.keys()) { if (!syncedData.has(key) && !optimisticDeletes.has(key)) { // The optimisticDeletes check is technically redundant if inserts/updates always remove from deletes, // but it's safer to keep it. yield key } } } /** * Get all values (virtual derived state) */ public *values(): IterableIterator { for (const key of this.keys()) { const value = this.get(key) if (value !== undefined) { yield value } } } /** * Get all entries (virtual derived state) */ public *entries(): IterableIterator<[TKey, TOutput]> { for (const key of this.keys()) { const value = this.get(key) if (value !== undefined) { yield [key, value] } } } /** * Get all entries (virtual derived state) */ public *[Symbol.iterator](): IterableIterator<[TKey, TOutput]> { for (const [key, value] of this.entries()) { yield [key, value] } } /** * Execute a callback for each entry in the collection */ public forEach( callbackfn: (value: TOutput, key: TKey, index: number) => void, ): void { let index = 0 for (const [key, value] of this.entries()) { callbackfn(value, key, index++) } } /** * Create a new array with the results of calling a function for each entry in the collection */ public map( callbackfn: (value: TOutput, key: TKey, index: number) => U, ): Array { const result: Array = [] let index = 0 for (const [key, value] of this.entries()) { result.push(callbackfn(value, key, index++)) } return result } /** * Check if the given collection is this collection * @param collection The collection to check * @returns True if the given collection is this collection, false otherwise */ private isThisCollection( collection: CollectionImpl, ): boolean { return collection === this.collection } /** * Recompute optimistic state from active transactions */ public recomputeOptimisticState( triggeredByUserAction: boolean = false, ): void { // Skip redundant recalculations when we're in the middle of committing sync transactions // While the sync pipeline is replaying a large batch we still want to honour // fresh optimistic mutations from the UI. Only skip recompute for the // internal sync-driven redraws; user-triggered work (triggeredByUserAction) // must run so live queries stay responsive during long commits. if (this.isCommittingSyncTransactions && !triggeredByUserAction) { return } const previousState = new Map(this.optimisticUpserts) const previousDeletes = new Set(this.optimisticDeletes) const previousRowOrigins = this.rowOrigins // Hold completed optimistic rows, and their local attribution, only while // an accepted, queued sync transaction touches them; an open one is not // accepted yet. That sync was committed before // the optimistic state dropped; a later one is remote. const pendingSyncKeys = new Set() for (const transaction of this.pendingSyncedTransactions) { if (!transaction.committed) continue for (const operation of transaction.operations) { pendingSyncKeys.add(operation.key as TKey) } } for (const transaction of this.transactions.values()) { if (transaction.state !== `completed`) continue for (const mutation of transaction.mutations) { if ( !this.isThisCollection(mutation.collection) || !pendingSyncKeys.has(mutation.key) ) continue this.pendingLocalOrigins.add(mutation.key) if (!mutation.optimistic) continue // A completed transaction leaves `transactions` before a newer one // settles, so keep the newest owner's row. const held = this.heldOptimisticRows.get(mutation.key) if (held && held.owner.compareCreatedAt(transaction) > 0) continue this.heldOptimisticRows.set(mutation.key, { owner: transaction, row: mutation.type === `delete` ? undefined : mutation.modified, }) } } // Clear current optimistic state this.optimisticUpserts.clear() this.optimisticDeletes.clear() this.pendingLocalChanges.clear() for (const [key, { row }] of this.heldOptimisticRows) { if (!pendingSyncKeys.has(key)) { this.heldOptimisticRows.delete(key) this.pendingLocalOrigins.delete(key) } else if (row === undefined) this.optimisticDeletes.add(key) else this.optimisticUpserts.set(key, row) } this.overlayActiveTransactions() // Update cached size this.size = this.calculateSize() // Collect events for changes const events: Array> = [] this.collectOptimisticChanges( previousState, previousDeletes, previousRowOrigins, events, ) // A user action publishes all its events. Otherwise the sync drain // already published the recently synced keys. const filteredEvents = triggeredByUserAction ? events : events.filter((event) => !this.recentlySyncedKeys.has(event.key)) if (filteredEvents.length > 0) this.indexes.updateIndexes(filteredEvents) this.changes.emitEvents(filteredEvents, triggeredByUserAction) } /** * Tracks `transaction`. Returns `false` when this Collection already tracks * it. A different unsettled transaction with the same id is a contract * violation: ids are unique. */ public trackTransaction(transaction: Transaction): boolean { const tracked = this.transactions.get(transaction.id) if (tracked === transaction) return false if (tracked && tracked.state !== `completed` && tracked.state !== `failed`) throw new DuplicateTransactionIdError(transaction.id) this.transactions.set(transaction.id, transaction) transaction.collections.add(this.collection) return true } /** * Overlay still-active optimistic mutations on the current layers and * record their keys as pending local changes for $origin tracking. * * A settled transaction leaves `transactions` here, by its own entry. A * recompute calls this after it records held rows, and a sync commit calls * it after it skips those recomputes. */ private overlayActiveTransactions(): void { const settled: Array = [] for (const [id, transaction] of this.transactions) { if (transaction.state === `completed` || transaction.state === `failed`) { settled.push(id) continue } for (const mutation of transaction.mutations) { if (!this.isThisCollection(mutation.collection)) continue this.pendingLocalChanges.add(mutation.key) if (!mutation.optimistic) continue if (mutation.type === `delete`) { this.optimisticUpserts.delete(mutation.key) this.optimisticDeletes.add(mutation.key) } else { this.optimisticUpserts.set(mutation.key, mutation.modified) this.optimisticDeletes.delete(mutation.key) } } } for (const id of settled) this.transactions.delete(id) } /** * Calculate the current size based on synced data and optimistic changes */ private calculateSize(): number { const syncedSize = this.syncedData.size const deletesFromSynced = Array.from(this.optimisticDeletes).filter( (key) => this.syncedData.has(key) && !this.optimisticUpserts.has(key), ).length const upsertsNotInSynced = Array.from(this.optimisticUpserts.keys()).filter( (key) => !this.syncedData.has(key), ).length return syncedSize - deletesFromSynced + upsertsNotInSynced } /** * Collect events for optimistic changes */ private collectOptimisticChanges( previousUpserts: Map, previousDeletes: Set, previousRowOrigins: ReadonlyMap, events: Array>, ): void { const allKeys = new Set([ ...previousUpserts.keys(), ...this.optimisticUpserts.keys(), ...previousDeletes, ...this.optimisticDeletes, ]) for (const key of allKeys) { const currentValue = this.get(key) const previousValue = this.getPreviousValue( key, previousUpserts, previousDeletes, ) const previousVirtualProps = this.getVirtualPropsSnapshotForState(key, { rowOrigins: previousRowOrigins, optimisticUpserts: previousUpserts, optimisticDeletes: previousDeletes, }) const nextVirtualProps = this.getVirtualPropsSnapshotForState(key) if (previousValue !== undefined && currentValue === undefined) { events.push({ type: `delete`, key, value: previousValue, __virtualProps: { value: previousVirtualProps, }, }) } else if (previousValue === undefined && currentValue !== undefined) { events.push({ type: `insert`, key, value: currentValue, __virtualProps: { value: nextVirtualProps, }, }) } else if ( previousValue !== undefined && currentValue !== undefined && (!deepEquals(previousValue, currentValue) || previousVirtualProps.$origin !== nextVirtualProps.$origin || previousVirtualProps.$synced !== nextVirtualProps.$synced) ) { events.push({ type: `update`, key, value: currentValue, previousValue, __virtualProps: { value: nextVirtualProps, previousValue: previousVirtualProps, }, }) } } } /** Build once per output flush; queued membership excludes optimistic edits. */ createSyncedKeyLookup(): (key: TKey) => boolean { if (this.pendingSyncedTransactions.length === 0) return (key) => this.syncedData.has(key) const queued = new Map() let truncated = false for (const transaction of this.pendingSyncedTransactions) { if (!transaction.committed) continue if (transaction.truncate) { queued.clear() truncated = true } for (const operation of transaction.operations) { queued.set(operation.key as TKey, operation.type !== `delete`) } } return (key) => queued.get(key) ?? (!truncated && this.syncedData.has(key)) } enableHydrationAuthorityTracking(): void { this.appliedAdapterDeletedKeys ??= new Set() } /** A late hydration seed cannot supersede committed adapter work. */ createAdapterAuthorityLookup(): (key: TKey) => boolean { const queuedKeys = new Set() let queuedTruncate = false for (const transaction of this.pendingSyncedTransactions) { if (!transaction.committed || transaction.preserveHydrationSeedKeys) continue if (transaction.truncate) queuedTruncate = true for (const operation of transaction.operations) { queuedKeys.add(operation.key as TKey) } } return (key) => this.hasAppliedAdapterTruncate || queuedTruncate || queuedKeys.has(key) || this.appliedAdapterDeletedKeys?.has(key) === true || (this.syncedData.has(key) && !this.hydrationSeedKeys.has(key)) } private getProjectedSyncedKeyState( projection: PendingSyncedProjection, key: TKey, ): PendingSyncedKeyState { const state = projection.states.get(key) if (state !== undefined) return state if (projection.truncated) return { exists: false, value: undefined } return { exists: this.syncedData.has(key), value: this.syncedData.get(key), } } /** * The synced row once every accepted sync transaction applies. A queued * transaction is accepted before it is visible, so a direct write must * read this rather than `syncedData`. */ getAcceptedSyncedRow(key: TKey): TOutput | undefined { return this.getProjectedSyncedKeyState(this.pendingSyncedProjection, key) .value } /** * Accept a committed seed transaction, such as a DbClient hydration chunk, * ahead of any still-open source transaction, so that source's `commit()` * still targets its own writes. */ acceptSeedTransaction( transaction: PendingSyncedTransaction, ): void { const open = this.pendingSyncedTransactions.findIndex( (pending) => !pending.committed, ) if (open === -1) this.pendingSyncedTransactions.push(transaction) else this.pendingSyncedTransactions.splice(open, 0, transaction) this.refreshPendingSyncedProjection() this.commitPendingTransactions() } /** Every synced row once every accepted sync transaction applies. */ *acceptedSyncedEntries(): IterableIterator<[TKey, TOutput]> { const { states, truncated } = this.pendingSyncedProjection if (!truncated) { for (const entry of this.syncedData.entries()) { if (!states.has(entry[0])) yield entry } } for (const [key, state] of states) { if (state.exists) yield [key, state.value!] } } private classifyProjectedInsert( projection: PendingSyncedProjection, key: TKey, value: TOutput, ): PendingInsertDisposition { const current = this.getProjectedSyncedKeyState(projection, key) if (!current.exists) return `insert` if ( (current.value !== undefined && deepEquals(current.value, value)) || this.hydrationSeedKeys.has(key) ) { return `update` } return `duplicate` } /** Classify an adapter insert against retained and queued source state. */ classifyPendingSyncedInsert( key: TKey, value: TOutput, ): PendingInsertDisposition { return this.classifyProjectedInsert( this.pendingSyncedProjection, key, value, ) } private applyPendingSyncOperation( projection: PendingSyncedProjection, operation: PendingSyncOperation, ): void { const key = operation.key as TKey const rowUpdateMode = this.config.sync.rowUpdateMode || `partial` if (operation.type === `delete`) { projection.states.set(key, { exists: false, value: undefined }) } else if (operation.type === `update` && rowUpdateMode === `partial`) { const current = this.getProjectedSyncedKeyState(projection, key) projection.states.set(key, { exists: true, // Spread defines own fields; assignment would run a `__proto__` setter. value: { ...current.value, ...operation.value }, }) } else { projection.states.set(key, { exists: true, value: operation.value }) } } /** Extend the queued projection after admitting one sync operation. */ stagePendingSyncOperation( operation: PendingSyncOperation, ): void { this.applyPendingSyncOperation(this.pendingSyncedProjection, operation) } /** * Get the previous value for a key given previous optimistic state */ private getPreviousValue( key: TKey, previousUpserts: Map, previousDeletes: Set, ): TOutput | undefined { if (previousDeletes.has(key)) { return undefined } if (previousUpserts.has(key)) { return previousUpserts.get(key) } return this.syncedData.get(key) } private rebuildAutomaticRowMetadataWrites( transaction: PendingSyncedTransaction, ): void { // A rebuild can reclassify an operation. Last write wins: each key's // last explicit write holds unless a later operation writes the key. const explicit = transaction.explicitRowMetadataWrites const writes = transaction.rowMetadataWrites for (const operation of transaction.operations) writes.delete(operation.key as TKey) for (const [key, last] of explicit) writes.set(key, last.write) transaction.operations.forEach((operation, position) => { const key = operation.key as TKey if (position < (explicit.get(key)?.position ?? 0)) return const metadataWrite = automaticRowMetadataWrite(operation) if (metadataWrite) writes.set(key, metadataWrite) }) } /** * Rebuild the queued projection after truncate, application, or an * accepted seed changes queue history. An open transaction can become * invalid when a later transaction begun inside it commits first; it then * holds an invalidation error that its commit returns. Only the open last * transaction can be canceled, so a replay that invalidates a committed * transaction is an invariant failure. */ private rebuildPendingSyncedProjection(): void { const projection: PendingSyncedProjection = { states: new Map(), truncated: false, } for (const transaction of this.pendingSyncedTransactions) { if (transaction.invalidationError !== undefined) continue const statesBeforeTransaction = new Map(projection.states) const truncatedBeforeTransaction = projection.truncated if (transaction.truncate) { projection.states = new Map() projection.truncated = true } let invalidKey: TKey | undefined for (const operation of transaction.operations) { const key = operation.key as TKey if ( operation.originalSyncType === `insert` && operation.type !== `delete` ) { const disposition = this.classifyProjectedInsert( projection, key, operation.value, ) if (disposition === `duplicate`) { invalidKey = key break } operation.type = disposition } this.applyPendingSyncOperation(projection, operation) } if (invalidKey !== undefined) { if (transaction.committed) throw new SyncQueueInvariantError( `replay made an accepted insert a duplicate`, ) projection.states = statesBeforeTransaction projection.truncated = truncatedBeforeTransaction transaction.invalidationError = transaction.duplicateKeyError?.(invalidKey) ?? new SyncTransactionAbortedError() continue } this.rebuildAutomaticRowMetadataWrites(transaction) } this.pendingSyncedProjection = projection } /** Rebuild after truncate, application, or an accepted seed. */ refreshPendingSyncedProjection(): void { this.rebuildPendingSyncedProjection() } /** * Attempts to commit pending synced transactions if there are no active transactions * This method processes operations from pending transactions and applies them to the synced data */ commitPendingTransactions = () => { if (this.isDrainingSyncTransactions) return this.isDrainingSyncTransactions = true let failed = false let firstError: unknown try { let result: { processed: boolean; failure?: { error: unknown } } do { result = this.commitNextPendingTransactionBatch() if (result.failure && !failed) { failed = true firstError = result.failure.error } } while (result.processed) } finally { this.isDrainingSyncTransactions = false } if (failed) throw firstError } hasPersistingTransaction(): boolean { for (const transaction of this.transactions.values()) { if (transaction.state === `persisting`) return true } return false } private commitNextPendingTransactionBatch(): { processed: boolean failure?: { error: unknown } } { const syncRunGeneration = this.syncRunGeneration const hasPersistingTransaction = this.hasPersistingTransaction() // pending synced transactions could be either `committed` or still open. // we only want to process `committed` transactions here const committedSyncedTransactions: Array< PendingSyncedTransaction > = [] const uncommittedSyncedTransactions: Array< PendingSyncedTransaction > = [] let hasTruncateSync = false let truncateMarksReady = false let layoutChanged = false for (const t of this.pendingSyncedTransactions) { if (t.committed) { committedSyncedTransactions.push(t) layoutChanged ||= t.layoutChanged if (t.truncate) { hasTruncateSync = true truncateMarksReady = t.truncateMarkReady !== false } } else { uncommittedSyncedTransactions.push(t) } } if (committedSyncedTransactions.length === 0) { return { processed: false } } // A persisting transaction holds sync transactions until it settles. A // truncate applies at once, with every committed transaction before it, // so later application cannot overwrite newer state. if (!hasPersistingTransaction || hasTruncateSync) { const previousLayout = layoutChanged ? [...this.keys()] : undefined this.pendingSyncedTransactions = uncommittedSyncedTransactions // Application is now the point of no return. Event listeners run before // the receipts resolve, so a signal aborted from one of those listeners // must not cancel writes that are already becoming visible. const deferOrder = committedSyncedTransactions.reduce( (count, transaction) => count + transaction.operations.length, 0, ) >= 512 for (const transaction of committedSyncedTransactions) { transaction.applicationStarted = true } // Set flag to prevent redundant optimistic state recalculations this.isCommittingSyncTransactions = true // First collect all keys that will be affected by sync operations const changedKeys = new Set() const syncedInsertedOrUpdatedKeys = new Set() const firstSyncOperations = new Map< TKey, OptimisticChangeMessage >() for (const transaction of committedSyncedTransactions) { for (const operation of transaction.operations) { const key = operation.key as TKey changedKeys.add(key) if (!firstSyncOperations.has(key)) firstSyncOperations.set(key, operation) if (operation.type !== `delete`) syncedInsertedOrUpdatedKeys.add(key) } for (const [key] of transaction.rowMetadataWrites) { changedKeys.add(key) } } const previousRowOrigins = this.snapshotRowOriginsForKeys(changedKeys) const previousOptimisticUpserts = new Map(this.optimisticUpserts) const previousOptimisticDeletes = new Set(this.optimisticDeletes) // Use pre-captured state if available (from optimistic scenarios), // otherwise capture current state (for pure sync scenarios) let currentVisibleState = this.preSyncVisibleState if (currentVisibleState.size === 0) { // No pre-captured state, capture it now for pure sync operations currentVisibleState = new Map() for (const key of changedKeys) { const currentValue = this.get(key) if (currentValue !== undefined) { currentVisibleState.set(key, currentValue) } } } const events: Array> = [] let reappliedKeys: ReadonlySet | undefined if (hasTruncateSync) { // All queued transactions publish as one batch. Its clear prefix must // describe the prior visible rows, not intermediate queued writes. // Freeze metadata before replacement writes change row attribution. for (const [key, value] of this.entries()) { events.push({ type: `delete`, key, value: this.enrichWithVirtualPropsSnapshot( value, this.getVirtualPropsSnapshotForState(key), ), }) } } const rowUpdateMode = this.config.sync.rowUpdateMode || `partial` for (const transaction of committedSyncedTransactions) { // Handle truncate operations first if (transaction.truncate) { // TRUNCATE PHASE // Clear the authoritative synced base. Subsequent server ops in this // same commit will rebuild the base atomically. // Keep active local mutations for later source transactions on keys // this truncate does not write. Completed attribution is one-use. this.syncedData.clear() this.syncedMetadata.clear() this.hydrationSeedKeys.clear() this.hydratedKeys.clear() if ( !transaction.preserveHydrationSeedKeys && this.appliedAdapterDeletedKeys ) { this.hasAppliedAdapterTruncate = true this.appliedAdapterDeletedKeys.clear() } this.virtualPropsCache.clear() this.rowOrigins.clear() this.pendingLocalOrigins.clear() // Clear currentVisibleState for truncated keys to ensure subsequent operations // are compared against the post-truncate state (undefined) rather than pre-truncate state // This ensures that re-inserted keys are emitted as INSERT events, not UPDATE events for (const key of changedKeys) { currentVisibleState.delete(key) } // Emit truncate event so subscriptions can reset their cursor tracking state this._events.emit(`truncate`, { type: `truncate`, collection: this.collection, }) } // Attribution belongs to the whole atomic batch. A repeated write must // not forget the local acknowledgement consumed by its first operation. const localKeys = new Set() for (const operation of transaction.operations) { const key = operation.key as TKey // Attribute this source write from active or held same-key mutations. const origin: VirtualOrigin = this.isLocalOnly || this.pendingLocalChanges.has(key) || this.pendingLocalOrigins.has(key) || localKeys.has(key) ? 'local' : 'remote' if (origin === `local`) localKeys.add(key) // A sync source may reuse a live-reading row object, making an // enriched snapshot cached for an earlier publication stale. this.virtualPropsCache.delete(key) if (operation.type === `delete`) { this.syncedData.delete(key, deferOrder) this.syncedMetadata.delete(key) this.rowOrigins.delete(key) } else { this.syncedData.set( key, operation.type === `update` && rowUpdateMode === `partial` ? { ...this.syncedData.get(key), ...operation.value } : operation.value, deferOrder, ) this.rowOrigins.set(key, origin) } // Sync has confirmed the key, so local tracking for it ends. this.pendingLocalChanges.delete(key) this.pendingLocalOrigins.delete(key) this.heldOptimisticRows.delete(key) if (!transaction.preserveHydrationSeedKeys) { this.hydrationSeedKeys.delete(key) this.hydratedKeys.delete(key) if (operation.type === `delete`) this.appliedAdapterDeletedKeys?.add(key) else this.appliedAdapterDeletedKeys?.delete(key) } } for (const [key, metadataWrite] of transaction.rowMetadataWrites) { if (metadataWrite.type === `delete`) { this.syncedMetadata.delete(key) continue } this.syncedMetadata.set(key, metadataWrite.value) } for (const [ key, metadataWrite, ] of transaction.collectionMetadataWrites) { if (metadataWrite.type === `delete`) { this.syncedCollectionMetadata.delete(key) continue } this.syncedCollectionMetadata.set(key, metadataWrite.value) } } this.syncedData.restoreOrder() // The retained base now includes every removed committed transaction. // Rebuild only the projection for still-open transactions. this.refreshPendingSyncedProjection() // Maintain optimistic state appropriately // Clear optimistic state since sync operations will now provide the authoritative data. // Any still-active user transactions will be re-applied below in recompute. this.optimisticUpserts.clear() this.optimisticDeletes.clear() // Reset flag and recompute optimistic state for any remaining active transactions this.isCommittingSyncTransactions = false // Always overlay any still-active optimistic transactions so mutations that started // after the truncate snapshot are preserved. Truncate clears attribution // with the old base, not the still-live local requests. this.overlayActiveTransactions() // After applying synced operations, if this commit included a truncate, // re-apply optimistic mutations on top of the fresh synced base. This ensures // the UI preserves local intent while respecting server rebuild semantics. // Ordering: deletes (above) -> server ops (just applied) -> optimistic upserts. if (hasTruncateSync) { // Events use the same rebuilt overlay as synchronous reads. Until // this point the batch holds only the truncate delete prefix, so each // re-applied upsert publishes as an insert after it. A key is never // both an optimistic upsert and an optimistic delete. for (const [key, value] of this.optimisticUpserts) { events.push({ type: `insert`, key, value }) } // The changed-key loop below must not publish these keys again. reappliedKeys = new Set(this.optimisticUpserts.keys()) } // Now check what actually changed in the final visible state for (const key of changedKeys) { // Truncate already published each re-applied upsert as an insert. if (reappliedKeys?.has(key)) continue const firstSyncOperation = firstSyncOperations.get(key) // A live-reading source can change a reused row before this commit // captures it. Later writes must not substitute an intermediate value. const syncPreviousValue = firstSyncOperation?.type === `update` && currentVisibleState.get(key) === firstSyncOperation.value ? firstSyncOperation.previousValue : undefined const previousVisibleValue = !hasTruncateSync && !previousOptimisticUpserts.has(key) && !previousOptimisticDeletes.has(key) && syncPreviousValue !== undefined ? syncPreviousValue : currentVisibleState.get(key) const newVisibleValue = this.get(key) // This returns the new derived state const previousVirtualProps = this.preSyncVirtualState.get(key) ?? this.getVirtualPropsSnapshotForState(key, { rowOrigins: previousRowOrigins, optimisticUpserts: previousOptimisticUpserts, optimisticDeletes: previousOptimisticDeletes, }) const nextVirtualProps = this.getVirtualPropsSnapshotForState(key) const virtualChanged = previousVirtualProps.$synced !== nextVirtualProps.$synced || previousVirtualProps.$origin !== nextVirtualProps.$origin const withPreviousVirtualProps = (value: TOutput) => enrichRowWithVirtualProps( value, key, this.collection.id, () => previousVirtualProps.$synced, () => previousVirtualProps.$origin, ) if (previousVisibleValue === undefined) { if (newVisibleValue === undefined) continue // Subscribers last saw this key absent, whatever a completed // optimistic request held. Batching composes a buffered delete with // this insert into an update. events.push({ type: `insert`, key, value: newVisibleValue }) } else if (newVisibleValue === undefined) { events.push({ type: `delete`, key, value: withPreviousVirtualProps(previousVisibleValue), }) } else if ( virtualChanged || !deepEquals(previousVisibleValue, newVisibleValue) ) { events.push({ type: `update`, key, value: newVisibleValue, previousValue: withPreviousVirtualProps(previousVisibleValue), }) } } // Update cached size after synced data changes this.size = this.calculateSize() // Update indexes for all events before emitting if (events.length > 0) { this.indexes.updateIndexes(events) } // End batching and emit all events (combines any batched events with sync events) // Subscribers and ready callbacks can throw. Keep the first error so // the applied receipts below still settle. let failure: { error: unknown } | undefined const capture = (step: () => void) => { try { step() } catch (error) { failure ??= { error } } } const visibleLayoutChanged = previousLayout !== undefined && (previousLayout.length !== this.size || [...this.keys()].some((key, index) => key !== previousLayout[index])) // markReady may skip its transition, and then the batch still emits. let emitted = false as boolean const emit = () => { emitted = true capture(() => this.changes.emitEvents(events, true, visibleLayoutChanged), ) } // A truncate that makes the Collection ready publishes its batch after // the status reads ready and before status listeners and ready // callbacks run, so their writes follow the batch they describe. if (truncateMarksReady && this.lifecycle.status !== `ready`) capture(() => this.lifecycle.markReady(emit)) if (!emitted) emit() if (this.syncRunGeneration === syncRunGeneration) { this.preSyncVisibleState.clear() this.preSyncVirtualState.clear() Promise.resolve().then(() => { if (this.syncRunGeneration === syncRunGeneration) { this.recentlySyncedKeys.clear() } }) this.hasReceivedFirstCommit = true } for (const transaction of committedSyncedTransactions) { transaction.applied.resolve() } return { processed: true, failure } } return { processed: false } } /** * Abandons the open last sync transaction before acceptance. Only * `commit(signal)` with an aborted signal reaches this, so no other * transaction can depend on the one canceled. */ public cancelPendingSyncedTransaction( transaction: PendingSyncedTransaction, reason: Error = new SyncTransactionAbortedError(), ): void { if ( transaction.committed || this.pendingSyncedTransactions.at(-1) !== transaction ) throw new SyncQueueInvariantError( `only the open last sync transaction can be canceled`, ) this.pendingSyncedTransactions.pop() const canceledKeys = new Set() for (const operation of transaction.operations) { canceledKeys.add(operation.key as TKey) } transaction.applied.reject(reason) this.rebuildPendingSyncedProjection() const remainingPendingKeys = new Set() for (const pending of this.pendingSyncedTransactions) { if (pending.invalidationError !== undefined) continue for (const operation of pending.operations) { remainingPendingKeys.add(operation.key as TKey) } } for (const key of canceledKeys) { if (!remainingPendingKeys.has(key)) { this.recentlySyncedKeys.delete(key) this.preSyncVisibleState.delete(key) this.preSyncVirtualState.delete(key) } } if (this.pendingSyncedTransactions.length === 0) { this.preSyncVisibleState.clear() this.preSyncVirtualState.clear() this.recentlySyncedKeys.clear() this.changes.emitEvents([], true) } else { // Recompute after removing the canceled keys so optimistic cleanup is // no longer suppressed by a sync transaction that will never publish. this.recomputeOptimisticState(false) } } /** * Capture visible state for keys that will be affected by pending sync operations * This must be called BEFORE onTransactionStateChange clears optimistic state */ public capturePreSyncVisibleState(): void { if (this.pendingSyncedTransactions.length === 0) return // Get all keys that will be affected by sync operations, including // metadata-only writes, which the drain also compares. const syncedKeys = new Set() for (const transaction of this.pendingSyncedTransactions) { // An open transaction does not publish in this drain. if (!transaction.committed) continue for (const operation of transaction.operations) { syncedKeys.add(operation.key as TKey) } for (const key of transaction.rowMetadataWrites.keys()) { syncedKeys.add(key) } } // Mark keys as about to be synced to suppress intermediate events from recomputeOptimisticState for (const key of syncedKeys) { this.recentlySyncedKeys.add(key) } // Only capture current visible state for keys that will be affected by sync operations // This is much more efficient than capturing the entire collection state // Only capture keys that haven't been captured yet to preserve earlier captures for (const key of syncedKeys) { if (!this.preSyncVisibleState.has(key)) { const currentValue = this.get(key) this.preSyncVisibleState.set(key, currentValue) if (currentValue !== undefined) { this.preSyncVirtualState.set( key, this.getVirtualPropsSnapshotForState(key), ) } } } } /** * Trigger a recomputation when transactions change * This method should be called by the Transaction class when state changes */ public onTransactionStateChange(): void { // Batch only when the next sync drain can actually publish. A persisting // sibling can keep normal sync queued; it must not hide this rollback. const hasPersistingTransaction = this.hasPersistingTransaction() this.changes.shouldBatchEvents = this.pendingSyncedTransactions.some( (transaction) => transaction.committed && (!hasPersistingTransaction || transaction.truncate), ) // CRITICAL: Capture visible state BEFORE clearing optimistic state if (this.changes.shouldBatchEvents) this.capturePreSyncVisibleState() this.recomputeOptimisticState(false) } /** * Clean up the collection by stopping sync and clearing data * This can be called manually or automatically by garbage collection */ public cleanup(): void { this.syncRunGeneration++ for (const transaction of this.pendingSyncedTransactions) { transaction.applied.reject(new SyncTransactionAbortedError()) } this.syncedData.clear() this.syncedMetadata.clear() this.syncedCollectionMetadata.clear() this.optimisticUpserts.clear() this.optimisticDeletes.clear() this.heldOptimisticRows.clear() this.hydrationSeedKeys.clear() this.hydratedKeys.clear() this.appliedAdapterDeletedKeys?.clear() this.hasAppliedAdapterTruncate = false this.clearOriginTrackingState() this.isLocalOnly = false this.localOnlyDirectWrite = undefined this.size = 0 this.pendingSyncedTransactions = [] this.pendingSyncedProjection = { states: new Map(), truncated: false } this.preSyncVisibleState.clear() this.preSyncVirtualState.clear() this.recentlySyncedKeys.clear() this.hasReceivedFirstCommit = false } }