import { PgManager } from '@module/replication/PgManager.js'; import { PUBLICATION_NAME, WalStream, WalStreamOptions } from '@module/replication/WalStream.js'; import { NormalizedPostgresConnectionConfig } from '@module/types/types.js'; import { ReplicationAbortedError } from '@powersync/lib-services-framework'; import { BucketStorageFactory, createCoreReplicationMetrics, initializeCoreReplicationMetrics, InternalOpId, LEGACY_STORAGE_VERSION, settledPromise, storage, SyncRulesBucketStorage, unsettledPromise, updateSyncRulesFromYaml } from '@powersync/service-core'; import { bucketRequest, getTestStorage, METRICS_HELPER, StorageDataHelpers, test_utils } from '@powersync/service-core-tests'; import * as pgwire from '@powersync/service-jpgwire'; import { clearTestDb, getClientCheckpoint, TEST_CONNECTION_OPTIONS } from './util.js'; export class WalStreamTestContext implements AsyncDisposable { private _walStream?: WalStream; private abortController = new AbortController(); private syncRulesContent?: storage.PersistedSyncConfigContent; public storage?: SyncRulesBucketStorage; private settledReplicationPromise?: Promise>; /** * Tests operating on the wal stream need to configure the stream and manage asynchronous * replication, which gets a little tricky. * * This configures all the context, and tears it down afterwards. */ static async open( factory: (options: storage.TestStorageOptions) => Promise, options?: { doNotClear?: boolean; storageVersion?: number; walStreamOptions?: Partial; /** * Source database to replicate from. Defaults to TEST_CONNECTION_OPTIONS. */ connectionOptions?: NormalizedPostgresConnectionConfig; } ) { const f = await factory({ doNotClear: options?.doNotClear }); const connectionManager = new PgManager(options?.connectionOptions ?? TEST_CONNECTION_OPTIONS, {}); if (!options?.doNotClear) { await clearTestDb(connectionManager.pool); } const storageVersion = options?.storageVersion ?? LEGACY_STORAGE_VERSION; return new WalStreamTestContext(f, connectionManager, options?.walStreamOptions, storageVersion); } constructor( public factory: BucketStorageFactory, public connectionManager: PgManager, private walStreamOptions?: Partial, private storageVersion: number = LEGACY_STORAGE_VERSION ) { createCoreReplicationMetrics(METRICS_HELPER.metricsEngine); initializeCoreReplicationMetrics(METRICS_HELPER.metricsEngine); } async [Symbol.asyncDispose]() { await this.dispose(); } async dispose() { this.abortController.abort(); try { await this.settledReplicationPromise; await this.connectionManager.destroy(); await this.factory?.[Symbol.asyncDispose](); } catch (e) { // Throwing here may result in SuppressedError. The underlying errors often don't show up // in the test output, so we log it here. // If we could get vitest to log SuppressedError.error and SuppressedError.suppressed, we // could remove this. console.error('Error during WalStreamTestContext dispose', e); throw e; } } get pool() { return this.connectionManager.pool; } get connectionTag() { return this.connectionManager.connectionTag; } get publicationName() { return PUBLICATION_NAME; } async updateSyncRules(content: string) { const replicationStream = await this.factory.updateSyncRules( updateSyncRulesFromYaml(content, { validate: true, storageVersion: this.storageVersion }) ); this.syncRulesContent = replicationStream.syncConfigContent[0]; this.storage = await getTestStorage(this.factory, replicationStream); return this.storage!; } async loadNextSyncRules() { const syncConfig = await this.factory.getDeployingSyncConfig(); if (syncConfig == null) { throw new Error(`Next replication stream not available`); } this.syncRulesContent = syncConfig.content; this.storage = await getTestStorage(this.factory, syncConfig.replicationStream); return this.storage!; } async loadActiveSyncRules() { const syncConfig = await this.factory.getActiveSyncConfig(); if (syncConfig == null) { throw new Error(`Active replication stream not available`); } this.syncRulesContent = syncConfig.content; this.storage = await getTestStorage(this.factory, syncConfig.replicationStream); return this.storage!; } private getSyncConfigContent(): storage.PersistedSyncConfigContent { if (this.syncRulesContent == null) { throw new Error('Sync config not configured - call updateSyncRules() first'); } return this.syncRulesContent; } get walStream() { if (this.storage == null) { throw new Error('updateSyncRules() first'); } if (this._walStream) { return this._walStream; } const options: WalStreamOptions = { storage: this.storage, metrics: METRICS_HELPER.metricsEngine, connections: this.connectionManager, abort_signal: this.abortController.signal, ...this.walStreamOptions }; this._walStream = new WalStream(options); return this._walStream!; } /** * Replicate a snapshot, start streaming, and wait for a consistent checkpoint. */ async initializeReplication() { await this.replicateSnapshot(); // Make sure we're up to date await this.getCheckpoint(); } /** * Replicate the initial snapshot, and start streaming. */ async replicateSnapshot() { // Use a settledPromise to avoid unhandled rejections this.settledReplicationPromise = settledPromise(this.walStream.replicate()); try { await Promise.race([unsettledPromise(this.settledReplicationPromise), this.walStream.waitForInitialSnapshot()]); } catch (e) { if (e instanceof ReplicationAbortedError && e.cause != null) { // Edge case for tests: replicate() can throw an error, but we'd receive the ReplicationAbortedError from // waitForInitialSnapshot() first. In that case, prioritize the cause, e.g. MissingReplicationSlotError. // This is not a concern for production use, since we only use waitForInitialSnapshot() in tests. throw e.cause; } throw e; } } async getCheckpoint(options?: { timeout?: number }) { let checkpoint = await Promise.race([ getClientCheckpoint(this.pool, this.factory, { timeout: options?.timeout ?? 15_000 }), unsettledPromise(this.settledReplicationPromise!) ]); if (checkpoint == null) { // This indicates an issue with the test setup - replicationPromise completed instead // of getClientCheckpoint() throw new Error('Test failure - replicationPromise completed'); } return checkpoint; } async getBucketsDataBatch(buckets: Record, options?: { timeout?: number }) { const helpers = new StorageDataHelpers(this.storage!, this.getSyncConfigContent()); const checkpoint = await this.getCheckpoint(options); return helpers.getBucketsDataBatch(buckets, checkpoint); } /** * This waits for a client checkpoint. */ async getBucketData(bucket: string, start?: InternalOpId | string | undefined, options?: { timeout?: number }) { const helpers = new StorageDataHelpers(this.storage!, this.getSyncConfigContent()); const checkpoint = await this.getCheckpoint(options); return helpers.getBucketData(bucket, checkpoint, start); } async getChecksums(buckets: string[], options?: { timeout?: number }) { const checkpoint = await this.getCheckpoint(options); const syncConfigContent = this.getSyncConfigContent(); const versionedBuckets = buckets.map((bucket) => bucketRequest(syncConfigContent, bucket, 0n)); const checksums = await this.storage!.getChecksums(checkpoint, versionedBuckets); const unversioned = new Map(); for (let i = 0; i < buckets.length; i++) { unversioned.set(buckets[i], checksums.get(versionedBuckets[i].bucket)!); } return unversioned; } async getChecksum(bucket: string, options?: { timeout?: number }) { const checksums = await this.getChecksums([bucket], options); return checksums.get(bucket); } /** * This does not wait for a client checkpoint. */ async getCurrentBucketData(bucket: string, start?: InternalOpId | string | undefined) { start ??= 0n; if (typeof start == 'string') { start = BigInt(start); } const syncConfigContent = this.getSyncConfigContent(); const checkpoint = await this.storage!.getCheckpoint(); const map = [bucketRequest(syncConfigContent, bucket, start)]; const batch = this.storage!.getBucketDataBatch(checkpoint, map); const batches = await test_utils.getBatchArray(batch); return batches[0]?.chunkData.data ?? []; } /** * Get resolved tables for testing table-level configuration. * * There is no "list all tables" storage method, so we re-run the same resolution the WAL stream * does during the initial snapshot: for every source-table pattern, read the relation (including * its replica identity) from Postgres and resolve it via a writer. Resolution is idempotent, so * this returns the persisted SourceTable for each table, with the computed `storeCurrentData`. */ async getResolvedTables(): Promise { if (this.storage == null) { throw new Error('updateSyncRules() first'); } const db = await this.connectionManager.snapshotConnection(); // Release the backend per call - the connection manager otherwise holds it until dispose. await using _ = { [Symbol.asyncDispose]: () => db.end() }; await using writer = await this.storage.createWriter(test_utils.BATCH_OPTIONS); const result: storage.SourceTable[] = []; for (const tablePattern of this.walStream.sync_rules.getSourceTables()) { const tables = await this.walStream.getQualifiedTableNames(writer, db, tablePattern); result.push(...tables); } return result; } } export async function withMaxWalSize(db: pgwire.PgClient, size: string) { try { const r1 = await db.query(`SHOW max_slot_wal_keep_size`); await setMaxWalSize(db, size); const oldSize = String(r1.results[0].rows[0].decodeWithoutCustomTypes(0)); return { [Symbol.asyncDispose]: async () => { await setMaxWalSize(db, oldSize); } }; } catch (e) { const err = new Error(`Failed to configure max_slot_wal_keep_size for test`); err.cause = e; throw err; } } async function setMaxWalSize(db: pgwire.PgClient, size: string) { const start = Date.now(); await db.query(`ALTER SYSTEM SET max_slot_wal_keep_size = '${size}'`); await db.query(`SELECT pg_reload_conf()`); // pg_reload_conf() returns before the new value is applied const timeout = 5_000; while (Date.now() - start < timeout) { const [row] = pgwire.pgwireRows( await db.query({ statement: `SELECT pg_size_bytes(current_setting('max_slot_wal_keep_size'))::text AS current, pg_size_bytes($1)::text AS expected`, params: [{ type: 'varchar', value: size }] }) ); if (row.current == row.expected) { return; } await new Promise((resolve) => setTimeout(resolve, 5)); } throw new Error('Timeout while waiting for max_slot_wal_keep_size'); }