import { ErrorCode, ServiceError } from '@powersync/lib-services-framework'; import { InternalOpId, storage } from '@powersync/service-core'; import { ParameterIndexId, SqliteJsonValue } from '@powersync/service-sync-rules'; import { event_types } from '@powersync/service-types'; import * as bson from 'bson'; import type { CurrentDataDocument, SourceTableDocumentV1 } from './v1/models.js'; import type { CurrentDataDocumentV3, RecordedLookupV3 } from './v3/models.js'; /** * Replica id uniquely identifying a row on the source database. * * Can be any value serializable to BSON. * * If the value is an entire document, the data serialized to a v5 UUID may be a good choice here. */ export type ReplicaId = bson.UUID | bson.Document | any; export interface SourceKey { /** group_id */ g: number; /** source table id */ t: bson.ObjectId; /** source key */ k: ReplicaId; } export interface SourceTableKey { /** source table id */ t: bson.ObjectId; /** source key */ k: ReplicaId; } export interface BucketDataKey { /** bucket name */ b: string; /** op_id */ o: bigint; } export interface CurrentBucket { bucket: string; table: string; id: string; } export interface BucketParameterDocumentBase { _id: bigint; key: TKey; lookup: bson.Binary; bucket_parameters: Record[]; } export interface TaggedBucketParameterDocument extends BucketParameterDocumentBase { index: ParameterIndexId; } export function bucketParameterDocumentToTagged( document: BucketParameterDocumentBase, index: ParameterIndexId ): TaggedBucketParameterDocument { return { ...document, index }; } export type OpType = 'PUT' | 'REMOVE' | 'MOVE' | 'CLEAR'; /** * Common properties for storage V1, storage V3 and in-memory BucketDataDoc. */ export interface BucketDataProperties { op: OpType; source_table?: bson.ObjectId; source_key?: ReplicaId; /** * Stable identifier for the source record in the client protocol. * * This is derived from (source_table, source_key), so persistence is optional but preferred. */ subkey?: string; table?: string; row_id?: string; checksum: bigint; data: string | null; /** * V1-only. * * V3 stores this on the BucketDataDocumentV3 instead of the individual ops. */ target_op?: bigint | null; } export interface BucketDataDocumentBase extends BucketDataProperties { _id: { b: string }; } /** * Internal-only tag used for v1 bucket_data rows before they are converted to the v1 on-disk shape. */ export const LEGACY_BUCKET_DATA_DEFINITION_ID = '0'; /** * Internal-only tag used for v1 bucket_parameters rows before they are converted to the v1 on-disk shape. */ export const LEGACY_BUCKET_PARAMETER_INDEX_ID = '0'; export interface SourceTableDocument { _id: bson.ObjectId; connection_id: number; relation_id: number | string | undefined; schema_name: string; table_name: string; replica_id_columns: string[] | null; replica_id_columns2: { name: string; type_oid?: number; type?: string }[] | undefined; snapshot_done: boolean | undefined; snapshot_status: SourceTableDocumentSnapshotStatus | undefined; /** * Source-specific metadata. Absent for legacy records. */ source_metadata?: storage.JsonValue; } export interface SourceTableDocumentSnapshotStatus { total_estimated_count: number; replicated_count: number; last_key: bson.Binary | null; } /** * Record the state of each bucket. * * The primary use case is to track when buckets are updated, for efficient incremental sync. * * The secondary use case is to track operation counts to determine whether or not a bucket should be compacted. * * Note: For storage V1, there is no migration to populate this collection from existing data - it is only * populated by new updates. * * For storage V3, these will always be present. */ export interface BucketStateDocumentBase { _id: { b: string; }; /** * Important: There is an unique index on last_op per logical stream. * That means the last_op must match an actual op in the bucket, and not the commit checkpoint. */ last_op: bigint; /** * If set, this can be treated as "cache" of a checksum at a specific point. * Can be updated periodically, for example by the compact job. */ compacted_state?: { op_id: InternalOpId; count: number; checksum: bigint; bytes: number | bigint | null; }; estimate_since_compact?: { count: number; bytes: number | bigint; }; } export interface IdSequenceDocument { _id: string; op_id: bigint; } /** * Base for sync_rules collection. */ export interface SyncRuleDocumentBase { _id: number; state: storage.SyncRuleState; /** * This is now used for "resumeLsn". * * If snapshot_done = false, this may be the lsn at which we started the snapshot. * * This can be used for resuming the snapshot after a restart. * * If snapshot_done is true, this is treated as the point to restart replication from. * * More specifically, we resume replication from max(snapshot_lsn, last_checkpoint_lsn). */ snapshot_lsn: string | undefined; slot_name: string | null; /** * Last time we persisted a checkpoint. * * This may be old if no data is incoming. */ last_checkpoint_ts: Date | null; /** * Last time we persisted a checkpoint or keepalive. * * This should stay fairly current while replicating. */ last_keepalive_ts: Date | null; /** * If an error is stopping replication, it will be stored here. */ last_fatal_error: string | null; last_fatal_error_ts: Date | null; lock?: { id: string; expires_at: Date; } | null; storage_version?: number; /** Optional operator-supplied label identifying the sync config version. */ version_label?: string | null; /** * Incremental parameter compaction state for the replication stream. * * Operation ids are allocated across all parameter indexes of a stream, so this is * stream-level state, shared by all sync configs of the stream. */ parameter_compaction?: { /** * The exclusive operation-id boundary through which every parameter index in this stream has * been compacted. */ compacted_before: InternalOpId; /** * Parameter entries below this boundary may no longer be available for checkpoint change * detection, since compaction may have deleted them. * * This is advanced before the first delete of a compaction pass, while * {@link compacted_before} is only advanced after every delete of the pass completed. * A checkpoint transition starting below this boundary must conservatively invalidate all * parameter buckets, since the individual changes may no longer be available. */ checkpoint_changes_invalid_before?: InternalOpId; }; } export interface SyncRuleCheckpointFields { /** * The last consistent checkpoint. * * There may be higher OpIds used in the database if we're in the middle of replicating a large transaction. */ last_checkpoint: bigint | null; /** * The LSN associated with the last consistent checkpoint. */ last_checkpoint_lsn: string | null; /** * If set, no new checkpoints may be created < this value. */ no_checkpoint_before: string | null; } export interface StorageConfig extends storage.StorageVersionConfig { /** * When true, bucket_data.checksum is guaranteed to be persisted as a Long. * * When false, it could also have been persisted as an Int32 or Double, in which case it must be converted to * a Long before summing. */ longChecksums: boolean; /** * Enables v3 MongoDB storage behavior used for incremental reprocessing. */ incrementalReprocessing: boolean; } const LONG_CHECKSUMS_STORAGE_VERSION = 2; const INCREMENTAL_REPROCESSING_STORAGE_VERSION = storage.STORAGE_VERSION_3; export function getMongoStorageConfig(storageVersion: number): StorageConfig { const baseConfig = storage.STORAGE_VERSION_CONFIG[storageVersion]; if (baseConfig == null) { throw new ServiceError(ErrorCode.PSYNC_S1005, `Unsupported storage version ${storageVersion}`); } return { ...baseConfig, longChecksums: storageVersion >= LONG_CHECKSUMS_STORAGE_VERSION, incrementalReprocessing: storageVersion >= INCREMENTAL_REPROCESSING_STORAGE_VERSION }; } export interface CheckpointEventDocument { _id: bson.ObjectId; } export interface CustomWriteCheckpointDocument { _id: bson.ObjectId; user_id: string; checkpoint: bigint; // Uses the legacy field name "sync_rules_id" for backwards-compatibility. See AGENTS.md. sync_rules_id: number; /** * Unlike managed write checkpoints, custom write checkpoints are flushed together with * normal ops. This means we can assign an op_id for ordering / correlating with read checkpoints. * * This is not unique - multiple write checkpoints can have the same op_id. */ op_id?: InternalOpId; /** * Set when this checkpoint was created from a client-supplied checkpoint * request rather than a persistent custom write checkpoint. */ checkpoint_requested_at?: Date | null; } export interface WriteCheckpointDocument { _id: bson.ObjectId; user_id: string; lsns: Record; client_id: bigint; /** * This is set to the checkpoint lsn when the checkpoint lsn >= this lsn. * This is used to make it easier to determine what write checkpoints have been processed * between two checkpoints. */ processed_at_lsn: string | null; /** * Set when this checkpoint was created from a client-supplied checkpoint request * rather than a server-generated id. A missing/null value means this is not a * checkpoint request. */ checkpoint_requested_at?: Date | null; } export interface InstanceDocument { // The instance UUID _id: string; } export interface ClientConnectionDocument extends event_types.ClientConnection {} export type CurrentDataDocumentId = CurrentDataDocument['_id'] | CurrentDataDocumentV3['_id']; export type CommonCurrentBucket = CurrentBucket; export type CommonCurrentLookup = bson.Binary | RecordedLookupV3; export type CommonSourceTableDocument = Pick;